DGI9-645: Filter destination properties based on available source properties.

README.md

@@ -72,11 +72,26 @@ dgi-migrate:rollback beer_user --idlist=5 --user=islandora
 
 ## Configuration
 
+### Search API, direct/immediate indexing
+
 The suppression `search_api`'s "immediate indexing"/`index_directly` functionality (which can cause instability in long processes, when `search_api` attempts to index potentially large sets of entities at the end of the request) is enabled by default, but can be configured by multiple means.
 
 - The `DGI_MIGRATE_SUPPRESS_DIRECT_INDEXING_DURING_MIGRATIONS` environment variables takes precedence if set. The string `true` should enable (case-sensitive!); while any other non-empty value should disable.
 - In config: `dgi_migrate.settings:suppress_direct_indexing_during_migrations`, as a boolean flag.
 
+### Entity update process
+
+Migrations can update existing entities (even without the `--update` flag), should the process identify an existing entity; however, during this process, if all the source properties are not provided for the destination rows that are mapped, it might end up erasing the related fields/properties from the entities in the database.
+
+To permit partial sources to be provided, we have implemented a process to try to track which destination properties did not have a corresponding source property, by implementing some handling around the `get` plugin. Where there is no source properties for an existing entity, existing values should be left intact. If a source property is provided with an empty value, the existing value should be erased.
+
+This functionality is enabled by default; however:
+
+* it only affects migrations making use of our `dgi_revisioned_entity` destination plugin
+* it might be disabled by specifying an environment variable `DGI_MIGRATE_TRACKING_GET_DISABLED=true`.
+
+NOTE: There may be other plugins that access properties from the row by means other than using the `get` plugin, to which the current implementation is blind, for example, `dgi_migrate.process.entity_query`, via its `conditions` key.
+
 ## Troubleshooting/Issues
 
 Having problems or solved a problem? Contact

dgi_migrate.module

@@ -6,6 +6,7 @@
  */
 
 use Drupal\dgi_migrate\Plugin\migrate\process\LockingMigrationLookup;
+use Drupal\dgi_migrate\Plugin\migrate\process\TrackingGet;
 
 /**
  * Implements hook_migration_plugins_alter().
@@ -32,4 +33,15 @@ function dgi_migrate_migrate_process_info_alter(&$definitions) {
   $lookup =& $definitions['migration_lookup'];
   $lookup['class'] = LockingMigrationLookup::class;
   $lookup['provider'] = 'dgi_migrate';
+
+  if (getenv('DGI_MIGRATE_TRACKING_GET_DISABLED') !== 'true') {
+    $definitions['dgi_migrate_original_get'] = [
+      'provider' => 'dgi_migrate',
+      'id' => 'dgi_migrate_original_get',
+    ] + $definitions['get'];
+
+    $get =& $definitions['get'];
+    $get['class'] = TrackingGet::class;
+    $get['provider'] = 'dgi_migrate';
+  }
 }

src/Plugin/migrate/destination/DgiRevisionedEntity.php

@@ -3,6 +3,8 @@
 namespace Drupal\dgi_migrate\Plugin\migrate\destination;
 
 use Drupal\Core\Entity\ContentEntityInterface;
+use Drupal\Core\Entity\EntityInterface;
+use Drupal\dgi_migrate\Plugin\migrate\process\TrackingGet;
 use Drupal\migrate\MigrateException;
 use Drupal\migrate\Plugin\MigrateIdMapInterface;
 use Drupal\migrate\Plugin\MigrationInterface;
@@ -46,16 +48,50 @@ public static function create(ContainerInterface $container, array $configuratio
     $entity_type = $configuration['entity_type'] ?? 'node';
     $instance = parent::create($container, $configuration, 'entity:' . $entity_type, $plugin_definition, $migration);
     $instance->entityType = $entity_type;
-    $instance->migrationId = $migration->id();
+    $instance->migrationId = $migration?->id() ?? '(unknown; not provided)';
     return $instance;
   }
 
+  /**
+   * Helper; handle process disabling.
+   *
+   * @param \Drupal\migrate\Row $row
+   *   Get entity for given row.
+   * @param array $old_destination_id_values
+   *   Old destination ID values.
+   *
+   * @return \Drupal\Core\Entity\EntityInterface|null
+   *   The entity; otherwise, NULL.
+   *
+   * @see \Drupal\dgi_migrate\Plugin\migrate\destination\DgiRevisionedEntity::getEntity()
+   */
+  private function doGetEntity(Row $row, array $old_destination_id_values = []) : ?EntityInterface {
+    if (getenv('DGI_MIGRATE_TRACKING_GET_DISABLED') === 'true') {
+      return $this->getEntity($row, $old_destination_id_values);
+    }
+
+    $filtered_row = TrackingGet::filterRow($row);
+
+    $entity = $this->getEntity($filtered_row, $old_destination_id_values);
+
+    foreach ($row->getRawDestination() as $property => $values) {
+      $row->removeDestinationProperty($property);
+    }
+    foreach ($filtered_row->getRawDestination() as $property => $values) {
+      $row->setDestinationProperty($property, $values);
+    }
+    $row->setIdMap($filtered_row->getIdMap());
+
+    return $entity;
+  }
+
   /**
    * {@inheritdoc}
    */
   public function import(Row $row, array $old_destination_id_values = []) {
     $this->rollbackAction = MigrateIdMapInterface::ROLLBACK_DELETE;
-    $entity = $this->getEntity($row, $old_destination_id_values);
+
+    $entity = $this->doGetEntity($row, $old_destination_id_values);
 
     if (!$entity) {
       throw new MigrateException('Unable to get entity');

src/Plugin/migrate/process/TrackingGet.php

@@ -0,0 +1,160 @@
+<?php
+
+namespace Drupal\dgi_migrate\Plugin\migrate\process;
+
+use Drupal\Core\Plugin\ContainerFactoryPluginInterface;
+use Drupal\migrate\MigrateExecutableInterface;
+use Drupal\migrate\Plugin\MigrateProcessInterface;
+use Drupal\migrate\Plugin\MigrationInterface;
+use Drupal\migrate\ProcessPluginBase;
+use Drupal\migrate\Row;
+use Symfony\Component\DependencyInjection\ContainerInterface;
+
+/**
+ * Track where things came from.
+ *
+ * Effective looking to augment the core `get` plugin to track what destination
+ * properties had source properties.
+ *
+ * @see \Drupal\migrate\Plugin\migrate\process\Get
+ */
+class TrackingGet extends ProcessPluginBase implements MigrateProcessInterface, ContainerFactoryPluginInterface {
+
+  /**
+   * The name of the destination property in which to build our tracking info.
+   */
+  const PROPERTY_NAME = '_dgi_migrate_tracking_get_tracker';
+
+  /**
+   * Instance of the plugin we are overriding.
+   *
+   * @var \Drupal\migrate\Plugin\MigrateProcessInterface
+   */
+  protected MigrateProcessInterface $wrappedPlugin;
+
+  /**
+   * {@inheritDoc}
+   */
+  public static function create(ContainerInterface $container, array $configuration, $plugin_id, $plugin_definition, ?MigrationInterface $migration = NULL) : self {
+    /** @var \Drupal\migrate\Plugin\MigratePluginManagerInterface $process_plugin_manager */
+    $process_plugin_manager = $container->get('plugin.manager.migrate.process');
+    return (new static(
+      $configuration,
+      $plugin_id,
+      $plugin_definition,
+    ))
+      ->setWrappedPlugin($process_plugin_manager->createInstance('dgi_migrate_original_get', $configuration, $migration));
+  }
+
+  /**
+   * Set the "get" plugin instance we are to wrap.
+   *
+   * @param \Drupal\migrate\Plugin\MigrateProcessInterface $to_wrap
+   *   The original "get" plugin instance to wrap.
+   *
+   * @return $this
+   *   Fluent API.
+   */
+  public function setWrappedPlugin(MigrateProcessInterface $to_wrap) : self {
+    $this->wrappedPlugin = $to_wrap;
+    return $this;
+  }
+
+  /**
+   * {@inheritDoc}
+   */
+  public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) {
+    $tracker = $row->hasDestinationProperty(static::PROPERTY_NAME) ?
+      $row->getDestinationProperty(static::PROPERTY_NAME) :
+      [];
+
+    $source = $this->configuration['source'];
+    $properties = is_string($source) ? [$source] : $source;
+
+    $tracker[$destination_property] = static::any($properties, static function (string $property) use ($row, $tracker) {
+      // Adapted from the Row class.
+      // @see https://git.drupalcode.org/project/drupal/-/blob/4f22ed87387ed92e5b1c8be1814de354706f6623/core/modules/migrate/src/Row.php#L345-355
+      $is_source = TRUE;
+      if (str_starts_with($property, '@')) {
+        $property = preg_replace_callback('/^(@?)((?:@@)*)([^@]|$)/', static function ($matches) use (&$is_source) {
+          // If there are an odd number of @ in the beginning, it's a
+          // destination.
+          $is_source = empty($matches[1]);
+          // Remove the possible escaping and do not lose the terminating
+          // non-@ either.
+          return str_replace('@@', '@', $matches[2]) . $matches[3];
+        }, $property);
+      }
+      return $is_source ?
+        $row->hasSourceProperty($property) :
+        ($tracker[$property] ?? $row->hasDestinationProperty($property));
+    });
+    $row->setDestinationProperty(static::PROPERTY_NAME, $tracker);
+
+    return $this->wrappedPlugin->transform($value, $migrate_executable, $row, $destination_property);
+  }
+
+  /**
+   * Avoid dependency on PHP 8.4 polyfill for `\array_any()`.
+   *
+   * @param array $values
+   *   Values to test.
+   * @param callable $callback
+   *   The callback with which to test.
+   *
+   * @return bool
+   *   TRUE if an item returned TRUE; otherwise, FALSE.
+   *
+   * @see \array_any()
+   */
+  private static function any(array $values, callable $callback) : bool {
+    if (function_exists('array_any')) {
+      return array_any($values, $callback);
+    }
+
+    // Adapted from polyfill.
+    // @see https://github.com/symfony/polyfill-php84/blob/d8ced4d875142b6a7426000426b8abc631d6b191/Php84.php#L90-L99
+    foreach ($values as $key => $value) {
+      if ($callback($value, $key)) {
+        return TRUE;
+      }
+    }
+    return FALSE;
+  }
+
+  /**
+   * Produce a filtered copy of the row, filtered according to the tracked data.
+   *
+   * @param \Drupal\migrate\Row $row
+   *   The row to filter.
+   *
+   * @return \Drupal\migrate\Row
+   *   The filtered row.
+   */
+  public static function filterRow(Row $row) : Row {
+    if (!$row->hasDestinationProperty(static::PROPERTY_NAME)) {
+      // No tracker; do not do anything.
+      return $row;
+    }
+
+    $tracker = $row->getDestinationProperty(static::PROPERTY_NAME);
+
+    $copy = $row->cloneWithoutDestination();
+    $copy->setIdMap($row->getIdMap());
+    foreach ($row->getRawDestination() as $property => $value) {
+      $copy->setDestinationProperty($property, $value);
+    }
+    // If a source column was present, mark absent properties accordingly.
+    foreach (array_keys(array_filter($tracker)) as $property) {
+      if (!$copy->hasDestinationProperty($property)) {
+        $copy->setEmptyDestinationProperty($property);
+      }
+    }
+
+    // Remove our tracking info from the row, as it has served its purpose.
+    $copy->removeDestinationProperty(static::PROPERTY_NAME);
+
+    return $copy;
+  }
+
+}