;
+ let packagesUpdated = 0;
+ let iteration = 0;
+
+ // Loop through batches of packages to update.
+ // We batch here so that we can pause and check that we're still within the
+ // maxImportDuration, and use small enough batches so that we can ensure at
+ // least one batch in that time.
+ do {
+ // getPackagesToUpdate() queries the first N (default 100) packages that
+ // have not been updated since the update interval (default 6 hours).
+ // When a package is imported it's lastUpdate date will be updated and the
+ // next call to getPackagesToUpdate() will return the next 100 packages.
+ // This way we don't need a DB cursor to make progress through the
+ // package list.
+ packagesToUpdate = await catalog.getPackagesToUpdate(notUpdatedSince);
+
+ if (packagesToUpdate.length === 0) {
+ // No more packages to update
+ if (iteration === 0) {
+ console.log('No packages to update');
+ }
+ break;
+ }
+
+ await Promise.allSettled(
+ packagesToUpdate.map(async (pkg) => {
+ try {
+ return await catalog.importPackage(pkg.name, packageUpdateInterval);
+ } catch (e) {
+ console.error(e);
+ throw e;
+ }
+ })
+ );
+ packagesUpdated += packagesToUpdate.length;
+
+ const now = Temporal.Now.instant();
+ const timeSinceStart = now.since(startInstant);
+ // If the time since the update started is not less than that max import
+ // duration, stop.
+ // TODO (justinfagnani): we need a way to test this
+ if (Temporal.Duration.compare(timeSinceStart, maxImportDuration) !== -1) {
+ break;
+ }
+ } while (true);
+ console.log(`Updated ${packagesUpdated} packages`);
+
+ if (packagesToUpdate.length > 0) {
+ // TODO (justinfagnani): kick off new update request
+ console.log(`Not all packages were updated (${packagesToUpdate.length})`);
+ }
+
+ context.status = 200;
+ context.type = 'html';
+ context.body = `
+ Update Results
+ Updated ${packagesUpdated} package
+ `;
+ };
diff --git a/packages/catalog-server/src/lib/server/server.ts b/packages/catalog-server/src/lib/server/server.ts
index 1b0fc92d..36400f99 100644
--- a/packages/catalog-server/src/lib/server/server.ts
+++ b/packages/catalog-server/src/lib/server/server.ts
@@ -1,6 +1,6 @@
/**
* @license
- * Copyright 2021 Google LLC
+ * Copyright 2022 Google LLC
* SPDX-License-Identifier: BSD-3-Clause
*/
@@ -17,6 +17,7 @@ import {NpmAndUnpkgFiles} from '@webcomponents/custom-elements-manifest-tools/li
import {makeGraphQLRoute} from './routes/graphql.js';
import {makeBootstrapPackagesRoute} from './routes/bootstrap-packages.js';
+import {makeUpdatePackagesRoute} from './routes/update-packages.js';
export const makeServer = async () => {
const files = new NpmAndUnpkgFiles();
@@ -32,6 +33,8 @@ export const makeServer = async () => {
router.get('/bootstrap-packages', makeBootstrapPackagesRoute(catalog));
+ router.get('/update-packages', makeUpdatePackagesRoute(catalog));
+
router.get('/', async (ctx) => {
ctx.status = 200;
ctx.type = 'html';