diff --git a/README.md b/README.md index 63855e67..1faa8f00 100644 --- a/README.md +++ b/README.md @@ -72,11 +72,26 @@ dgi-migrate:rollback beer_user --idlist=5 --user=islandora ## Configuration +### Search API, direct/immediate indexing + The suppression `search_api`'s "immediate indexing"/`index_directly` functionality (which can cause instability in long processes, when `search_api` attempts to index potentially large sets of entities at the end of the request) is enabled by default, but can be configured by multiple means. - The `DGI_MIGRATE_SUPPRESS_DIRECT_INDEXING_DURING_MIGRATIONS` environment variables takes precedence if set. The string `true` should enable (case-sensitive!); while any other non-empty value should disable. - In config: `dgi_migrate.settings:suppress_direct_indexing_during_migrations`, as a boolean flag. +### Entity update process + +Migrations can update existing entities (even without the `--update` flag), should the process identify an existing entity; however, during this process, if all the source properties are not provided for the destination rows that are mapped, it might end up erasing the related fields/properties from the entities in the database. + +To permit partial sources to be provided, we have implemented a process to try to track which destination properties did not have a corresponding source property, by implementing some handling around the `get` plugin. Where there is no source properties for an existing entity, existing values should be left intact. If a source property is provided with an empty value, the existing value should be erased. + +This functionality is enabled by default; however: + +* it only affects migrations making use of our `dgi_revisioned_entity` destination plugin +* it might be disabled by specifying an environment variable `DGI_MIGRATE_TRACKING_GET_DISABLED=true`. + +NOTE: There may be other plugins that access properties from the row by means other than using the `get` plugin, to which the current implementation is blind, for example, `dgi_migrate.process.entity_query`, via its `conditions` key. + ## Troubleshooting/Issues Having problems or solved a problem? Contact diff --git a/dgi_migrate.module b/dgi_migrate.module index a715f556..ade8dd24 100644 --- a/dgi_migrate.module +++ b/dgi_migrate.module @@ -6,6 +6,7 @@ */ use Drupal\dgi_migrate\Plugin\migrate\process\LockingMigrationLookup; +use Drupal\dgi_migrate\Plugin\migrate\process\TrackingGet; /** * Implements hook_migration_plugins_alter(). @@ -32,4 +33,15 @@ function dgi_migrate_migrate_process_info_alter(&$definitions) { $lookup =& $definitions['migration_lookup']; $lookup['class'] = LockingMigrationLookup::class; $lookup['provider'] = 'dgi_migrate'; + + if (getenv('DGI_MIGRATE_TRACKING_GET_DISABLED') !== 'true') { + $definitions['dgi_migrate_original_get'] = [ + 'provider' => 'dgi_migrate', + 'id' => 'dgi_migrate_original_get', + ] + $definitions['get']; + + $get =& $definitions['get']; + $get['class'] = TrackingGet::class; + $get['provider'] = 'dgi_migrate'; + } } diff --git a/src/Plugin/migrate/destination/DgiRevisionedEntity.php b/src/Plugin/migrate/destination/DgiRevisionedEntity.php index bc53f604..3f2d0bb7 100644 --- a/src/Plugin/migrate/destination/DgiRevisionedEntity.php +++ b/src/Plugin/migrate/destination/DgiRevisionedEntity.php @@ -3,6 +3,8 @@ namespace Drupal\dgi_migrate\Plugin\migrate\destination; use Drupal\Core\Entity\ContentEntityInterface; +use Drupal\Core\Entity\EntityInterface; +use Drupal\dgi_migrate\Plugin\migrate\process\TrackingGet; use Drupal\migrate\MigrateException; use Drupal\migrate\Plugin\MigrateIdMapInterface; use Drupal\migrate\Plugin\MigrationInterface; @@ -46,16 +48,50 @@ public static function create(ContainerInterface $container, array $configuratio $entity_type = $configuration['entity_type'] ?? 'node'; $instance = parent::create($container, $configuration, 'entity:' . $entity_type, $plugin_definition, $migration); $instance->entityType = $entity_type; - $instance->migrationId = $migration->id(); + $instance->migrationId = $migration?->id() ?? '(unknown; not provided)'; return $instance; } + /** + * Helper; handle process disabling. + * + * @param \Drupal\migrate\Row $row + * Get entity for given row. + * @param array $old_destination_id_values + * Old destination ID values. + * + * @return \Drupal\Core\Entity\EntityInterface|null + * The entity; otherwise, NULL. + * + * @see \Drupal\dgi_migrate\Plugin\migrate\destination\DgiRevisionedEntity::getEntity() + */ + private function doGetEntity(Row $row, array $old_destination_id_values = []) : ?EntityInterface { + if (getenv('DGI_MIGRATE_TRACKING_GET_DISABLED') === 'true') { + return $this->getEntity($row, $old_destination_id_values); + } + + $filtered_row = TrackingGet::filterRow($row); + + $entity = $this->getEntity($filtered_row, $old_destination_id_values); + + foreach ($row->getRawDestination() as $property => $values) { + $row->removeDestinationProperty($property); + } + foreach ($filtered_row->getRawDestination() as $property => $values) { + $row->setDestinationProperty($property, $values); + } + $row->setIdMap($filtered_row->getIdMap()); + + return $entity; + } + /** * {@inheritdoc} */ public function import(Row $row, array $old_destination_id_values = []) { $this->rollbackAction = MigrateIdMapInterface::ROLLBACK_DELETE; - $entity = $this->getEntity($row, $old_destination_id_values); + + $entity = $this->doGetEntity($row, $old_destination_id_values); if (!$entity) { throw new MigrateException('Unable to get entity'); diff --git a/src/Plugin/migrate/process/TrackingGet.php b/src/Plugin/migrate/process/TrackingGet.php new file mode 100644 index 00000000..5b8e3c9a --- /dev/null +++ b/src/Plugin/migrate/process/TrackingGet.php @@ -0,0 +1,160 @@ +get('plugin.manager.migrate.process'); + return (new static( + $configuration, + $plugin_id, + $plugin_definition, + )) + ->setWrappedPlugin($process_plugin_manager->createInstance('dgi_migrate_original_get', $configuration, $migration)); + } + + /** + * Set the "get" plugin instance we are to wrap. + * + * @param \Drupal\migrate\Plugin\MigrateProcessInterface $to_wrap + * The original "get" plugin instance to wrap. + * + * @return $this + * Fluent API. + */ + public function setWrappedPlugin(MigrateProcessInterface $to_wrap) : self { + $this->wrappedPlugin = $to_wrap; + return $this; + } + + /** + * {@inheritDoc} + */ + public function transform($value, MigrateExecutableInterface $migrate_executable, Row $row, $destination_property) { + $tracker = $row->hasDestinationProperty(static::PROPERTY_NAME) ? + $row->getDestinationProperty(static::PROPERTY_NAME) : + []; + + $source = $this->configuration['source']; + $properties = is_string($source) ? [$source] : $source; + + $tracker[$destination_property] = static::any($properties, static function (string $property) use ($row, $tracker) { + // Adapted from the Row class. + // @see https://git.drupalcode.org/project/drupal/-/blob/4f22ed87387ed92e5b1c8be1814de354706f6623/core/modules/migrate/src/Row.php#L345-355 + $is_source = TRUE; + if (str_starts_with($property, '@')) { + $property = preg_replace_callback('/^(@?)((?:@@)*)([^@]|$)/', static function ($matches) use (&$is_source) { + // If there are an odd number of @ in the beginning, it's a + // destination. + $is_source = empty($matches[1]); + // Remove the possible escaping and do not lose the terminating + // non-@ either. + return str_replace('@@', '@', $matches[2]) . $matches[3]; + }, $property); + } + return $is_source ? + $row->hasSourceProperty($property) : + ($tracker[$property] ?? $row->hasDestinationProperty($property)); + }); + $row->setDestinationProperty(static::PROPERTY_NAME, $tracker); + + return $this->wrappedPlugin->transform($value, $migrate_executable, $row, $destination_property); + } + + /** + * Avoid dependency on PHP 8.4 polyfill for `\array_any()`. + * + * @param array $values + * Values to test. + * @param callable $callback + * The callback with which to test. + * + * @return bool + * TRUE if an item returned TRUE; otherwise, FALSE. + * + * @see \array_any() + */ + private static function any(array $values, callable $callback) : bool { + if (function_exists('array_any')) { + return array_any($values, $callback); + } + + // Adapted from polyfill. + // @see https://github.com/symfony/polyfill-php84/blob/d8ced4d875142b6a7426000426b8abc631d6b191/Php84.php#L90-L99 + foreach ($values as $key => $value) { + if ($callback($value, $key)) { + return TRUE; + } + } + return FALSE; + } + + /** + * Produce a filtered copy of the row, filtered according to the tracked data. + * + * @param \Drupal\migrate\Row $row + * The row to filter. + * + * @return \Drupal\migrate\Row + * The filtered row. + */ + public static function filterRow(Row $row) : Row { + if (!$row->hasDestinationProperty(static::PROPERTY_NAME)) { + // No tracker; do not do anything. + return $row; + } + + $tracker = $row->getDestinationProperty(static::PROPERTY_NAME); + + $copy = $row->cloneWithoutDestination(); + $copy->setIdMap($row->getIdMap()); + foreach ($row->getRawDestination() as $property => $value) { + $copy->setDestinationProperty($property, $value); + } + // If a source column was present, mark absent properties accordingly. + foreach (array_keys(array_filter($tracker)) as $property) { + if (!$copy->hasDestinationProperty($property)) { + $copy->setEmptyDestinationProperty($property); + } + } + + // Remove our tracking info from the row, as it has served its purpose. + $copy->removeDestinationProperty(static::PROPERTY_NAME); + + return $copy; + } + +} diff --git a/tests/src/Unit/TrackingGetTest.php b/tests/src/Unit/TrackingGetTest.php new file mode 100644 index 00000000..c0d69c8e --- /dev/null +++ b/tests/src/Unit/TrackingGetTest.php @@ -0,0 +1,203 @@ +mockExecutable = $this->createStub(MigrateExecutableInterface::class); + $this->testPropName = $this->randomMachineName(); + } + + /** + * Helper; build out the plugin instance. + * + * @param array $configuration + * Param to use during instantiation. + * + * @return \Drupal\dgi_migrate\Plugin\migrate\process\TrackingGet + * The plugin instance. + */ + protected function getInstance(array $configuration) : TrackingGet { + return (new TrackingGet($configuration, '', [])) + ->setWrappedPlugin($this->createStub(MigrateProcessInterface::class)); + } + + /** + * Test the presence of a source property is reflected. + */ + public function testPresentSource() : void { + $row = (new Row( + [ + 'a' => 'a', + 'b' => 'b', + ], + ['a' => 'a'], + )) + ->freezeSource(); + + $this->getInstance(['source' => 'b']) + ->transform(NULL, $this->mockExecutable, $row, $this->testPropName); + $row->setDestinationProperty($this->testPropName, 'b'); + $this->assertTrue($row->hasDestinationProperty(TrackingGet::PROPERTY_NAME)); + $this->assertTrue($row->getDestinationProperty(TrackingGet::PROPERTY_NAME)[$this->testPropName]); + + $filtered = TrackingGet::filterRow($row); + $this->assertNotContains($this->testPropName, $filtered->getEmptyDestinationProperties()); + } + + /** + * Test the presence of a source property that does "skip process". + */ + public function testPresentEmptySourceSkip() : void { + $row = (new Row( + [ + 'a' => 'a', + 'b' => '', + ], + ['a' => 'a'], + )) + ->freezeSource(); + + $this->getInstance(['source' => 'b']) + ->transform(NULL, $this->mockExecutable, $row, $this->testPropName); + $row->setEmptyDestinationProperty($this->testPropName); + $this->assertTrue($row->hasDestinationProperty(TrackingGet::PROPERTY_NAME)); + $this->assertTrue($row->getDestinationProperty(TrackingGet::PROPERTY_NAME)[$this->testPropName]); + + $filtered = TrackingGet::filterRow($row); + $this->assertContains($this->testPropName, $filtered->getEmptyDestinationProperties()); + } + + /** + * Test the presence of a source property that gets passed-through. + */ + public function testPresentEmptySourcePass() : void { + $row = (new Row( + [ + 'a' => 'a', + 'b' => '', + ], + ['a' => 'a'], + )) + ->freezeSource(); + + $this->getInstance(['source' => 'b']) + ->transform(NULL, $this->mockExecutable, $row, $this->testPropName); + $row->setDestinationProperty($this->testPropName, ''); + $this->assertTrue($row->hasDestinationProperty(TrackingGet::PROPERTY_NAME)); + $this->assertTrue($row->getDestinationProperty(TrackingGet::PROPERTY_NAME)[$this->testPropName]); + + $filtered = TrackingGet::filterRow($row); + $this->assertEquals('', $filtered->getDestinationProperty($this->testPropName)); + $this->assertNotContains($this->testPropName, $filtered->getEmptyDestinationProperties()); + } + + /** + * Test the absence of a source property is reflected. + */ + public function testAbsentSource() : void { + $row = (new Row( + ['a' => 'a'], + ['a' => 'a'], + )) + ->freezeSource(); + + $this->getInstance(['source' => 'b']) + ->transform(NULL, $this->mockExecutable, $row, $this->testPropName); + // Approximate behavior MigrateExecutable. + // @see https://git.drupalcode.org/project/drupal/-/blob/10.5.x/core/modules/migrate/src/MigrateExecutable.php?ref_type=heads#L476 + $row->setEmptyDestinationProperty($this->testPropName); + $this->assertTrue($row->hasDestinationProperty(TrackingGet::PROPERTY_NAME)); + $this->assertFalse($row->getDestinationProperty(TrackingGet::PROPERTY_NAME)[$this->testPropName]); + + $filtered = TrackingGet::filterRow($row); + $this->assertNull($filtered->getDestinationProperty($this->testPropName)); + $this->assertNotContains($this->testPropName, $filtered->getEmptyDestinationProperties()); + } + + /** + * Test transitive property existence. + */ + public function testTransitiveExistence() : void { + $row = (new Row( + [ + 'a' => 'a', + 'b' => 'b', + ], + ['a' => 'a'], + )) + ->freezeSource(); + + $transitive_prop = $this->randomMachineName(); + + $this->getInstance(['source' => 'b']) + ->transform(NULL, $this->mockExecutable, $row, $this->testPropName); + $row->setDestinationProperty($this->testPropName, 'b'); + $this->getInstance(['source' => "@{$this->testPropName}"]) + ->transform(NULL, $this->mockExecutable, $row, $transitive_prop); + $row->setDestinationProperty($transitive_prop, 'b'); + $this->assertTrue($row->getDestinationProperty(TrackingGet::PROPERTY_NAME)[$transitive_prop]); + + $filtered = TrackingGet::filterRow($row); + $this->assertNotContains($transitive_prop, $filtered->getEmptyDestinationProperties()); + } + + /** + * Test transitive property absence. + */ + public function testTransitiveAbsence() : void { + $row = (new Row( + ['a' => 'a'], + ['a' => 'a'], + )) + ->freezeSource(); + + $transitive_prop = $this->randomMachineName(); + + $this->getInstance(['source' => 'b']) + ->transform(NULL, $this->mockExecutable, $row, $this->testPropName); + // Approximate behavior MigrateExecutable. + // @see https://git.drupalcode.org/project/drupal/-/blob/10.5.x/core/modules/migrate/src/MigrateExecutable.php?ref_type=heads#L476 + $row->setEmptyDestinationProperty($this->testPropName); + $this->getInstance(['source' => "@{$this->testPropName}"]) + ->transform(NULL, $this->mockExecutable, $row, $transitive_prop); + $row->setEmptyDestinationProperty($transitive_prop); + $this->assertFalse($row->getDestinationProperty(TrackingGet::PROPERTY_NAME)[$transitive_prop]); + + $filtered = TrackingGet::filterRow($row); + $this->assertNull($filtered->getDestinationProperty($transitive_prop)); + $this->assertNotContains($transitive_prop, $filtered->getEmptyDestinationProperties()); + } + +}