feat(backend): add service factories by default

This change explicitely adds the default service factories to the
backend statically to prevent dynamic plugins from being able to
override them by default.  It's possible to override each statically
added service factory via an environment variable derived from the
service factory ID.  So for example to add a "core.rootHttpService"
service factory configuration from a dynamic plugin, set
ENABLE_CORE_ROOTHTTPSERVICE_OVERRIDE to "true".  This change also adds a
logger to the backend main.  Finally, a unit test has been added that
checks the installed backend-defaults value for the
defaultServiceFactories list against what this change adds to catch
future regressions.

Signed-off-by: Stan Lewis <[email protected]>

Author: gashcrumb

docs/dynamic-plugins/override-core-services.md

@@ -0,0 +1,59 @@
+# Overriding Core Backend Service Configuration
+
+## Overview
+
+The Backstage backend platform consists of a number of core services that are well encapsulated.  The configuration of these core services is normally done by directly customizing the backend source code and rebuilding.  However the dynamic plugin functionality adds the ability for core service customization via installing it as a `BackendFeature`.  The Developer Hub backend normally installs all of these default core services statically during initialization.  Environment variables can configure the backend to avoid statically installing a given default core service, allowing for dynamic plugin installation.
+
+## An Example
+
+Some use cases may be easier solved at a lower level service than what's available in the Backstage backend plugin API.  Adding a middleware function to handle all incoming requests can be done by installing a custom `configure` function for the root HTTP router backend service, which allows access to the underlying Express app.
+
+```typescript
+// Create the BackendFeature
+export const customRootHttpServerFactory: BackendFeature =
+  rootHttpRouterServiceFactory({
+    configure: ({ app, routes, middleware, logger }) => {
+      logger.info(
+        'Using custom root HttpRouterServiceFactory configure function',
+      );
+      app.use(middleware.helmet());
+      app.use(middleware.cors());
+      app.use(middleware.compression());
+      app.use(middleware.logging());
+      // Add a the custom middleware function before all
+      // of the route handlers
+      app.use(addTestHeaderMiddleware({ logger }));
+      app.use(routes);
+      app.use(middleware.notFound());
+      app.use(middleware.error());
+    },
+  });
+
+// Export the BackendFeature as the default entrypoint
+export default customRootHttpServerFactory;
+```
+
+This `BackendFeature` overrides the default HTTP router service factory.  Because this is overriding the default implementation of a core service, the above example would need the `ENABLE_CORE_ROOTHTTPROUTER_OVERRIDE` environment variable set to `true` so that the Developer Hub does not install the default implementation automatically.
+
+## Override Environment Variables
+
+To allow a dynamic plugin to load a core service override, start the Developer Hub backend with the environment variable set that corresponds with the core service ID to be overridden.  Here is a list of the available environment variables and core service IDs:
+
+
+- `ENABLE_CORE_AUTH_OVERRIDE` - allow overriding the `core.auth` service
+- `ENABLE_CORE_CACHE_OVERRIDE` - allow overriding the `core.cache` service
+- `ENABLE_CORE_ROOTCONFIG_OVERRIDE` - allow overriding the `core.rootConfig` service
+- `ENABLE_CORE_DATABASE_OVERRIDE` - allow overriding the `core.database` service
+- `ENABLE_CORE_DISCOVERY_OVERRIDE` - allow overriding the `core.discovery` service
+- `ENABLE_CORE_HTTPAUTH_OVERRIDE` - allow overriding the `core.httpAuth` service
+- `ENABLE_CORE_HTTPROUTER_OVERRIDE` - allow overriding the `core.httpRouter` service
+- `ENABLE_CORE_LIFECYCLE_OVERRIDE` - allow overriding the `core.lifecycle` service
+- `ENABLE_CORE_LOGGER_OVERRIDE` - allow overriding the `core.logger` service
+- `ENABLE_CORE_PERMISSIONS_OVERRIDE` - allow overriding the `core.permissions` service
+- `ENABLE_CORE_ROOTHEALTH_OVERRIDE` - allow overriding the `core.rootHealth` service
+- `ENABLE_CORE_ROOTHTTPROUTER_OVERRIDE` - allow overriding the `core.rootHttpRouter` service
+- `ENABLE_CORE_ROOTLIFECYCLE_OVERRIDE` - allow overriding the `core.rootLifecycle` service
+- `ENABLE_CORE_SCHEDULER_OVERRIDE` - allow overriding the `core.scheduler` service
+- `ENABLE_CORE_USERINFO_OVERRIDE` - allow overriding the `core.userInfo` service
+- `ENABLE_CORE_URLREADER_OVERRIDE` - allow overriding the `core.urlReader` service
+- `ENABLE_EVENTS_SERVICE_OVERRIDE` - allow overriding the `events.service` service

packages/backend/package.json

@@ -40,6 +40,7 @@
     "@backstage/plugin-catalog-backend-module-openapi": "0.2.3",
     "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": "0.2.1",
     "@backstage/plugin-events-backend": "0.3.15",
+    "@backstage/plugin-events-node": "^0.4.4",
     "@backstage/plugin-proxy-backend": "0.5.7",
     "@backstage/plugin-scaffolder-backend": "1.26.2",
     "@backstage/plugin-search-backend": "1.6.1",

packages/backend/src/defaultServiceFactories.test.ts

@@ -0,0 +1,38 @@
+import { ServiceFactory } from '@backstage/backend-plugin-api';
+
+import { defaultServiceFactories } from './defaultServiceFactories';
+
+// explicitly check this against the module inside the installed package
+const {
+  defaultServiceFactories: upstreamDefaultServiceFactories,
+  // eslint-disable-next-line
+} = require('../../../node_modules/@backstage/backend-defaults/dist/CreateBackend.cjs.js');
+
+function findDifference(a1: string[], a2: string[]) {
+  const set = new Set(a2);
+  return a1.filter(i => !set.has(i));
+}
+
+function findSymmetricDifference(a1: string[], a2: string[]) {
+  return [...new Set([...findDifference(a1, a2), ...findDifference(a2, a1)])];
+}
+
+/**
+ * Validate that the installed backend-defaults package contains the expected
+ * list of default service factories.  A failure in this test indicates that
+ * either the export was removed, the list was moved, or the list in
+ * "defaultServiceFactories" should be updated.
+ */
+describe('Default service factory list comparison', () => {
+  it('Should produce an expected difference of service factories as compared to the upstream implementation', () => {
+    const upstreamServiceFactoryIds = upstreamDefaultServiceFactories.map(
+      (serviceFactory: ServiceFactory) => serviceFactory.service.id,
+    );
+    const serviceFactoryIds = defaultServiceFactories.map(
+      (serviceFactory: ServiceFactory) => serviceFactory.service.id,
+    );
+    expect(
+      findSymmetricDifference(upstreamServiceFactoryIds, serviceFactoryIds),
+    ).toEqual(['core.rootLogger']);
+  });
+});

packages/backend/src/defaultServiceFactories.ts

@@ -0,0 +1,64 @@
+import { authServiceFactory } from '@backstage/backend-defaults/auth';
+import { cacheServiceFactory } from '@backstage/backend-defaults/cache';
+import { databaseServiceFactory } from '@backstage/backend-defaults/database';
+import { discoveryServiceFactory } from '@backstage/backend-defaults/discovery';
+import { httpAuthServiceFactory } from '@backstage/backend-defaults/httpAuth';
+import { httpRouterServiceFactory } from '@backstage/backend-defaults/httpRouter';
+import { lifecycleServiceFactory } from '@backstage/backend-defaults/lifecycle';
+import { loggerServiceFactory } from '@backstage/backend-defaults/logger';
+import { permissionsServiceFactory } from '@backstage/backend-defaults/permissions';
+import { rootConfigServiceFactory } from '@backstage/backend-defaults/rootConfig';
+import { rootHealthServiceFactory } from '@backstage/backend-defaults/rootHealth';
+import { rootHttpRouterServiceFactory } from '@backstage/backend-defaults/rootHttpRouter';
+import { rootLifecycleServiceFactory } from '@backstage/backend-defaults/rootLifecycle';
+import { WinstonLogger } from '@backstage/backend-defaults/rootLogger';
+import { schedulerServiceFactory } from '@backstage/backend-defaults/scheduler';
+import { urlReaderServiceFactory } from '@backstage/backend-defaults/urlReader';
+import { userInfoServiceFactory } from '@backstage/backend-defaults/userInfo';
+import { eventsServiceFactory } from '@backstage/plugin-events-node';
+
+/**
+ * Service factories that are added to the backend statically by default.  This
+ * should be kept up to date with the upstream package code, which is currently
+ * not exported.
+ */
+export const defaultServiceFactories = [
+  authServiceFactory,
+  cacheServiceFactory,
+  rootConfigServiceFactory,
+  databaseServiceFactory,
+  discoveryServiceFactory,
+  httpAuthServiceFactory,
+  httpRouterServiceFactory,
+  lifecycleServiceFactory,
+  loggerServiceFactory,
+  permissionsServiceFactory,
+  rootHealthServiceFactory,
+  rootHttpRouterServiceFactory,
+  rootLifecycleServiceFactory,
+  // rootLoggerServiceFactory,
+  schedulerServiceFactory,
+  userInfoServiceFactory,
+  urlReaderServiceFactory,
+  eventsServiceFactory,
+];
+
+export const getDefaultServiceFactories = ({
+  logger,
+}: {
+  logger: WinstonLogger;
+}) => {
+  return defaultServiceFactories.filter(serviceFactory => {
+    const envName = `ENABLE_${serviceFactory.service.id.toLocaleUpperCase().replace('.', '_')}_OVERRIDE`;
+    if ((process.env[envName] || '').toLocaleLowerCase() !== 'true') {
+      logger.debug(
+        `Adding service factory "${serviceFactory.service.id}", to override set "${envName}" to "true"`,
+      );
+      return true;
+    }
+    logger.warn(
+      `Allowing override for service factory "${serviceFactory.service.id}"`,
+    );
+    return false;
+  });
+};

packages/backend/src/index.ts

@@ -5,19 +5,31 @@ import { PackageRoles } from '@backstage/cli-node';
 import * as path from 'path';
 
 import { configureCorporateProxyAgent } from './corporate-proxy';
+import { getDefaultServiceFactories } from './defaultServiceFactories';
 import { CommonJSModuleLoader } from './loader';
-import { transports } from './logger';
+import { createStaticLogger, transports } from './logger';
 import {
   healthCheckPlugin,
   pluginIDProviderService,
   rbacDynamicPluginsProvider,
 } from './modules';
 
+// Create a logger to cover logging static initialization tasks
+const staticLogger = createStaticLogger({ service: 'developer-hub-init' });
+staticLogger.info('Starting Developer Hub backend');
+
 // RHIDP-2217: adds support for corporate proxy
 configureCorporateProxyAgent();
 
 const backend = createBackend();
 
+const defaultServiceFactories = getDefaultServiceFactories({
+  logger: staticLogger,
+});
+defaultServiceFactories.forEach(serviceFactory => {
+  backend.add(serviceFactory);
+});
+
 backend.add(
   dynamicPluginsFeatureLoader({
     schemaLocator(pluginPackage) {

packages/backend/src/logger/customLogger.ts

@@ -1,3 +1,4 @@
+import { WinstonLogger } from '@backstage/backend-defaults/rootLogger';
 import type { Config } from '@backstage/config';
 
 import * as winston from 'winston';
@@ -73,3 +74,18 @@ export const transports = {
     ];
   },
 };
+
+export const createStaticLogger = ({ service }: { service: string }) => {
+  const logger = WinstonLogger.create({
+    meta: {
+      service,
+    },
+    level: process.env.LOG_LEVEL || 'info',
+    format:
+      process.env.NODE_ENV === 'production'
+        ? defaultFormat
+        : WinstonLogger.colorFormat(),
+    transports: transports.log,
+  });
+  return logger;
+};

yarn.lock

@@ -6764,7 +6764,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@backstage/plugin-events-node@npm:^0.4.1":
+"@backstage/plugin-events-node@npm:^0.4.1, @backstage/plugin-events-node@npm:^0.4.4":
   version: 0.4.4
   resolution: "@backstage/plugin-events-node@npm:0.4.4"
   dependencies:
@@ -22368,6 +22368,7 @@ __metadata:
     "@backstage/plugin-catalog-backend-module-openapi": 0.2.3
     "@backstage/plugin-catalog-backend-module-scaffolder-entity-model": 0.2.1
     "@backstage/plugin-events-backend": 0.3.15
+    "@backstage/plugin-events-node": ^0.4.4
     "@backstage/plugin-proxy-backend": 0.5.7
     "@backstage/plugin-scaffolder-backend": 1.26.2
     "@backstage/plugin-search-backend": 1.6.1