Rearchitect Updater into a registry (#428)

Signed-off-by: John Blackbourn <john@johnblackbourn.com>
Signed-off-by: Andy Fragen <andy@thefragens.com>
Signed-off-by: Carrie Dils <carriedils@gmail.com>
Signed-off-by: Norcross <andrew.norcross@gmail.com>
Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Signed-off-by: joedolson <joedolson@users.noreply.github.com>
Signed-off-by: Joe Dolson <design@joedolson.com>
Signed-off-by: Shadi Sharaf <shady@sharaf.me>
Co-authored-by: Chuck Adams <chaz@chaz.works>
Co-authored-by: Andy Fragen <andy@thefragens.com>
Co-authored-by: Norcross <andrew.norcross@gmail.com>
Co-authored-by: Carrie Dils <cdils@users.noreply.github.com>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: rmccue <21655+rmccue@users.noreply.github.com>
Co-authored-by: cdils <3099408+cdils@users.noreply.github.com>
Co-authored-by: joedolson <joedolson@users.noreply.github.com>
Co-authored-by: Joe Dolson <design@joedolson.com>
Co-authored-by: Shady Sharaf <shady@sharaf.me>

Author: 9 people

.github/workflows/releases.yml

@@ -37,6 +37,18 @@ jobs:
             /tmp/${{ github.event.repository.name }}-${{ steps.tag.outputs.tag }}.zip
             /tmp/fair-dist/*
 
+      - name: Upload to Fastly Object Storage
+        # note the plugin filename is always 'fair-connect', regardless of the repo name or slug
+        run: |
+          aws s3 cp /tmp/${{ github.event.repository.name }}-${{ steps.tag.outputs.tag }}.zip s3://download.fair.pm/release/fair-connect-${{ steps.tag.outputs.tag }}.zip
+          aws s3 cp --recursive --exclude '*'  --include '*.zip' /tmp/fair-dist s3://download.fair.pm/release/
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.DOWNLOAD_BUCKET_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.DOWNLOAD_BUCKET_SECRET_KEY }}
+          AWS_DEFAULT_REGION: 'us-west'
+          AWS_ENDPOINT_URL: 'https://us-west.object.fastlystorage.app'
+          AWS_REQUEST_CHECKSUM_CALCULATION: 'WHEN_REQUIRED'
+
       - name: Build provenance attestation
         uses: actions/attest-build-provenance@v2
         with:

bin/bundle.sh

@@ -19,6 +19,22 @@ touch /tmp/fair-dist/SHA1SUMS
 touch /tmp/fair-dist/SHA256SUMS
 touch /tmp/fair-dist/SHA384SUMS
 
+# Auto-detect and hash plugin zip.
+PLUGIN_VERSION=$(get_plugin_header "Version")
+REPO_NAME=$(basename "$GITHUB_REPOSITORY")
+PLUGIN_ZIP="/tmp/${REPO_NAME}-${PLUGIN_VERSION}.zip"
+
+if [ -f "$PLUGIN_ZIP" ]; then
+	echo "Hashing plugin zip: $PLUGIN_ZIP" >&2
+	# Change to /tmp so hash files contain just filename, not full path
+	cd /tmp
+	md5sum -b "$(basename "$PLUGIN_ZIP")" >> /tmp/fair-dist/MD5SUMS
+	sha1sum -b "$(basename "$PLUGIN_ZIP")" >> /tmp/fair-dist/SHA1SUMS
+	sha256sum -b "$(basename "$PLUGIN_ZIP")" >> /tmp/fair-dist/SHA256SUMS
+	sha384sum -b "$(basename "$PLUGIN_ZIP")" >> /tmp/fair-dist/SHA384SUMS
+	cd - > /dev/null
+fi
+
 # Bundle our plugin first.
 [ -d /tmp/fair-temp ] && rm -rf /tmp/fair-temp
 mkdir -p /tmp/fair-temp/wordpress/wp-content/plugins/fair-plugin
@@ -54,6 +70,12 @@ for VERSION in $AVAILABLE_VERSIONS; do
 	curl -sSL "$WP_ZIP_URL" -o "$WP_ZIP_FILE"
 	EXPECTED_HASH=$(curl -sSL "$WP_ZIP_URL.sha1")
 
+	# Skip if we can't get a valid hash.
+	if [[ ! "$EXPECTED_HASH" =~ ^[a-z0-9]{40}$ ]]; then
+		echo "Failed to fetch valid hash for $VERSION" >&2
+		continue
+	fi
+
 	# Verify the checksum.
 	# (sha1 is suboptimal, but it's all we've got.)
 	echo "  Verifying checksum" >&2

inc/default-repo/namespace.php

@@ -53,6 +53,7 @@ function replace_repo_api_urls( $status, $args, $url ) {
 	if (
 		! str_contains( $url, 'api.wordpress.org/plugins/' )
 		&& ! str_contains( $url, 'api.wordpress.org/themes/' )
+		&& ! str_contains( $url, 'api.wordpress.org/translations/' )
 		&& ! str_contains( $url, 'api.wordpress.org/core/version-check/' )
 	) {
 		return $status;

inc/namespace.php

@@ -11,6 +11,7 @@
 
 const CACHE_BASE = 'fair-';
 const CACHE_LIFETIME = 12 * HOUR_IN_SECONDS;
+const CACHE_LIFETIME_FAILURE = HOUR_IN_SECONDS;
 const NS_SEPARATOR = '\\';
 
 /**

inc/packages/namespace.php

@@ -9,6 +9,7 @@
 
 use const FAIR\CACHE_BASE;
 use const FAIR\CACHE_LIFETIME;
+use const FAIR\CACHE_LIFETIME_FAILURE;
 use FAIR\Packages\DID\Document as DIDDocument;
 use FAIR\Packages\DID\PLC;
 use FAIR\Packages\DID\Web;
@@ -22,12 +23,33 @@
 const CACHE_KEY = CACHE_BASE . 'packages-';
 const CACHE_METADATA_DOCUMENTS = CACHE_BASE . 'metadata-documents-';
 const CACHE_RELEASE_PACKAGES = CACHE_BASE . 'release-packages';
+const CACHE_UPDATE_ERRORS = CACHE_BASE . 'update-errors-';
 const CACHE_DID_FOR_INSTALL = 'fair-install-did';
 const CONTENT_TYPE = 'application/json+fair';
 const SERVICE_ID = 'FairPackageManagementRepo';
 
 // phpcs:disable WordPress.NamingConventions.ValidVariableName
 
+/**
+ * Cache an update error for a package.
+ *
+ * @param string   $did   DID of the package.
+ * @param WP_Error $error The error to cache.
+ */
+function cache_update_error( string $did, WP_Error $error ): void {
+	$error->add_data( [ 'timestamp' => time() ], $error->get_error_code() );
+	set_site_transient( CACHE_UPDATE_ERRORS . $did, $error, CACHE_LIFETIME_FAILURE );
+}
+
+/**
+ * Clear a cached update error for a package.
+ *
+ * @param string $did DID of the package.
+ */
+function clear_update_error( string $did ): void {
+	delete_site_transient( CACHE_UPDATE_ERRORS . $did );
+}
+
 /**
  * Bootstrap.
  *
@@ -93,6 +115,12 @@ function get_did_hash( string $id ) {
  * @return DIDDocument|WP_Error
  */
 function get_did_document( string $id ) {
+	// Check for cached error from previous failed request.
+	$cached_error = get_site_transient( CACHE_UPDATE_ERRORS . $id );
+	if ( is_wp_error( $cached_error ) ) {
+		return $cached_error;
+	}
+
 	$cached = get_site_transient( CACHE_METADATA_DOCUMENTS . $id );
 	if ( $cached ) {
 		return $cached;
@@ -101,13 +129,18 @@ function get_did_document( string $id ) {
 	// Parse the DID, then fetch the details.
 	$did = parse_did( $id );
 	if ( is_wp_error( $did ) ) {
+		cache_update_error( $id, $did );
 		return $did;
 	}
 
 	$document = $did->fetch_document();
 	if ( is_wp_error( $document ) ) {
+		cache_update_error( $id, $document );
 		return $document;
 	}
+
+	// Clear any previous error on success.
+	clear_update_error( $id );
 	set_site_transient( CACHE_METADATA_DOCUMENTS . $id, $document, CACHE_LIFETIME );
 
 	return $document;
@@ -177,30 +210,38 @@ function fetch_package_metadata( string $id ) {
 	// Fetch data from the repository.
 	$service = $document->get_service( SERVICE_ID );
 	if ( empty( $service ) ) {
-		return new WP_Error( 'fair.packages.fetch_metadata.no_service', __( 'DID is not a valid package to fetch metadata for.', 'fair' ) );
+		$error = new WP_Error( 'fair.packages.fetch_metadata.no_service', __( 'DID is not a valid package to fetch metadata for.', 'fair' ) );
+		cache_update_error( $id, $error );
+		return $error;
 	}
 	$repo_url = $service->serviceEndpoint;
 
-	$metadata = fetch_metadata_doc( $repo_url );
+	$metadata = fetch_metadata_doc( $repo_url, $id );
 
 	if ( is_wp_error( $metadata ) ) {
 		return $metadata;
 	}
 
 	if ( $metadata->id !== $id ) {
-		return new WP_Error( 'fair.packages.fetch_metadata.mismatch', __( 'Fetched metadata does not match the requested DID.', 'fair' ) );
+		$error = new WP_Error( 'fair.packages.fetch_metadata.mismatch', __( 'Fetched metadata does not match the requested DID.', 'fair' ) );
+		cache_update_error( $id, $error );
+		return $error;
 	}
 
+	// Clear any previous error on success.
+	clear_update_error( $id );
+
 	return $metadata;
 }
 
 /**
  * Fetch the metadata document for a package.
  *
  * @param string $url URL for the metadata document.
+ * @param string $did DID of the package.
  * @return MetadataDocument|WP_Error
  */
-function fetch_metadata_doc( string $url ) {
+function fetch_metadata_doc( string $url, string $did ) {
 	$cache_key = CACHE_KEY . md5( $url );
 	$response = get_site_transient( $cache_key );
 	$response = fetch_metadata_from_local( $response, $url );
@@ -219,9 +260,15 @@ function fetch_metadata_doc( string $url ) {
 		$response = wp_remote_get( $url, $options );
 		$code = wp_remote_retrieve_response_code( $response );
 		if ( is_wp_error( $response ) ) {
+			cache_update_error( $did, $response );
 			return $response;
 		} elseif ( $code !== 200 ) {
-			return new WP_Error( 'fair.packages.metadata.failure', __( 'HTTP error code received', 'fair' ) );
+			$error = new WP_Error(
+				'fair.packages.metadata.http_error',
+				sprintf( __( 'HTTP %d error received', 'fair' ), $code )
+			);
+			cache_update_error( $did, $error );
+			return $error;
 		}
 
 		// Reorder sections before caching.
@@ -727,6 +774,9 @@ function cache_did_for_install( array $options ): array {
 		$did = array_find_key(
 			$releases,
 			function ( $release ) use ( $options ) {
+				if ( ! is_array( $release->artifacts->package ) ) {
+					return false;
+				}
 				$artifact = pick_artifact_by_lang( $release->artifacts->package );
 				return $artifact && $artifact->url === $options['package'];
 			}
@@ -754,7 +804,7 @@ function delete_cached_did_for_install(): void {
  *
  * This is commonly required for packages from Git hosts.
  *
- * @param string $source        Path of $source.
+ * @param string|WP_Error $source Path of $source, or a WP_Error object.
  * @param string $remote_source Path of $remote_source.
  * @param WP_Upgrader $upgrader An Upgrader object.
  * @param array $hook_extra     Array of hook data.

inc/updater/class-lite.php

@@ -125,6 +125,13 @@ public function run() {
 			);
 			$response = get_site_transient( "git-updater-lite_{$this->file}" );
 			if ( ! $response ) {
+				/* Apply filter to API URL.
+				 * Add `channel=development` query arg to URL to get pre-release versions.
+				 *
+				 * @param string $url The API URL.
+				 * @param string $slug The plugin/theme slug
+				 */
+				$url = apply_filters( 'git_updater_lite_api_url', $url, $this->slug );
 				$response = wp_remote_get( $url );
 				if ( is_wp_error( $response ) || wp_remote_retrieve_response_code( $response ) === 404 ) {
 					return $response;
@@ -136,11 +143,9 @@ public function run() {
 				}
 				$this->api_data->file = $this->file;
 
-				/*
-				* Set transient for 5 minutes as AWS sets 5 minute timeout
-				* for release asset redirect.
-				*/
-				set_site_transient( "git-updater-lite_{$this->file}", $this->api_data, 5 * \MINUTE_IN_SECONDS );
+				// Set timeout for transient via filter.
+				$timeout = apply_filters( 'git_updater_lite_transient_timeout', 6 * HOUR_IN_SECONDS, $this->file );
+				set_site_transient( "git-updater-lite_{$this->file}", $this->api_data, $timeout );
 			} else {
 				if ( property_exists( $response, 'error' ) ) {
 					return new WP_Error( 'repo-no-exist', 'Specified repo does not exist' );
@@ -178,18 +183,23 @@ function () {
 		/**
 		 * Correctly rename dependency for activation.
 		 *
-		 * @param string      $source        Path of $source.
-		 * @param string      $remote_source Path of $remote_source.
-		 * @param WP_Upgrader $upgrader      An Upgrader object.
-		 * @param array       $hook_extra    Array of hook data.
+		 * @param string|WP_Error $source        Path of $source, or a WP_Error object.
+		 * @param string          $remote_source Path of $remote_source.
+		 * @param WP_Upgrader     $upgrader      An Upgrader object.
+		 * @param array           $hook_extra    Array of hook data.
 		 *
 		 * @throws TypeError If the type of $upgrader is not correct.
 		 *
 		 * @return string|WP_Error
 		 */
-		public function upgrader_source_selection( string $source, string $remote_source, WP_Upgrader $upgrader, $hook_extra = null ) {
+		public function upgrader_source_selection( $source, string $remote_source, WP_Upgrader $upgrader, $hook_extra = null ) {
 			global $wp_filesystem;
 
+			// Exit early for errors.
+			if ( is_wp_error( $source ) ) {
+				return $source;
+			}
+
 			$new_source = $source;
 
 			// Exit if installing.

inc/updater/class-package.php

@@ -0,0 +1,109 @@
+<?php
+/**
+ * Package data container.
+ *
+ * @package FAIR
+ */
+
+namespace FAIR\Updater;
+
+use FAIR\Packages;
+
+/**
+ * Represents a registered FAIR package (plugin or theme).
+ */
+abstract class Package {
+
+	/**
+	 * The DID of the package.
+	 *
+	 * @var string
+	 */
+	public string $did;
+
+	/**
+	 * Absolute path to the main file.
+	 *
+	 * @var string
+	 */
+	public string $filepath;
+
+	/**
+	 * Current installed version.
+	 *
+	 * @var string|null
+	 */
+	public ?string $local_version;
+
+	/**
+	 * Cached metadata document.
+	 *
+	 * @var \FAIR\Packages\MetadataDocument|null
+	 */
+	private $metadata = null;
+
+	/**
+	 * Cached release document.
+	 *
+	 * @var \FAIR\Packages\ReleaseDocument|null
+	 */
+	private $release = null;
+
+	/**
+	 * Constructor.
+	 *
+	 * @param string $did      The DID of the package.
+	 * @param string $filepath Absolute path to the main file.
+	 */
+	public function __construct( string $did, string $filepath ) {
+		$this->did           = $did;
+		$this->filepath      = $filepath;
+		$this->local_version = $filepath ? get_file_data( $filepath, [ 'Version' => 'Version' ] )['Version'] : null;
+	}
+
+	/**
+	 * Get the package slug.
+	 *
+	 * @return string The slug (directory name for plugins, stylesheet for themes).
+	 */
+	abstract public function get_slug(): string;
+
+	/**
+	 * Get the relative path used in update transients.
+	 *
+	 * @return string The relative path.
+	 */
+	abstract public function get_relative_path(): string;
+
+	/**
+	 * Get the metadata document, fetching and caching if needed.
+	 *
+	 * @return \FAIR\Packages\MetadataDocument|\WP_Error|null
+	 */
+	final public function get_metadata() {
+		if ( $this->metadata === null ) {
+			$metadata = Packages\fetch_package_metadata( $this->did );
+			if ( ! is_wp_error( $metadata ) ) {
+				$this->metadata = $metadata;
+			}
+			return $metadata;
+		}
+		return $this->metadata;
+	}
+
+	/**
+	 * Get the release document, fetching and caching if needed.
+	 *
+	 * @return \FAIR\Packages\ReleaseDocument|\WP_Error|null
+	 */
+	final public function get_release() {
+		if ( $this->release === null ) {
+			$release = Packages\get_latest_release_from_did( $this->did );
+			if ( ! is_wp_error( $release ) ) {
+				$this->release = $release;
+			}
+			return $release;
+		}
+		return $this->release;
+	}
+}