Merge pull request #857 from City-of-Helsinki/ASU-1793

ASU-1793: replace ElasticSearch with HTTPS REST API

Author: leevi-identio

.github/workflows/test.yml

@@ -69,6 +69,16 @@ jobs:
       - name: Install Drupal
         run: |
           mysql --user=drupal --password=drupal --database=drupal --host=db --port=3306 -A < latest.sql
+          # The DB dump can occasionally contain an invalid view config row with
+          # an empty ID (config name "views.view." or other malformed variants).
+          # That breaks Drupal core's
+          # block_content post-update which iterates all views and expects every
+          # view entity to have a non-empty ID.
+          mysql --user=drupal --password=drupal --database=drupal --host=db --port=3306 -A -e "DELETE FROM config WHERE name = 'views.view.';"
+          drush php:eval '$storage = \Drupal::service("config.storage"); foreach ($storage->listAll("views.view.") as $name) { $data = $storage->read($name); if (empty($data["id"])) { $storage->delete($name); } }'
+          # deploy runs updatedb before cim; overlay sync first so DB matches
+          # conf/cmi (e.g. views without deprecated plugins) before post-updates.
+          drush cim -y --partial
           drush deploy
 
       - name: Start services

.github/workflows/update-config.yml

@@ -46,6 +46,11 @@ jobs:
         run: |
           composer install --no-interaction
           $(drush sql:connect) < latest.sql
+          # The DB dump can occasionally contain an invalid view config row with
+          # an empty ID (config name "views.view." or other malformed variants).
+          # Remove it before running updates that iterate over all view entities.
+          drush sql:query "DELETE FROM config WHERE name = 'views.view.';"
+          drush php:eval '$storage = \Drupal::service("config.storage"); foreach ($storage->listAll("views.view.") as $name) { $data = $storage->read($name); if (empty($data["id"])) { $storage->delete($name); } }'
           drush cim -y && drush updb -y && drush cr
           composer update drupal/helfi_* drupal/hdbt* -W --no-interaction
           drush cr

.gitignore

@@ -1,3 +1,8 @@
+# Ignore OAuth keys
+/keys
+*.key
+keys/*
+
 # Ignore project files that should not be in version control
 drush/sites/
 drush/Commands/*

AGENTS.md

@@ -9,6 +9,11 @@ This is an **Open Source Drupal project**. It serves as the content management s
 * **External Integrations:** Connects to a Django microservice for specific logic.
 * agent skills are located in the `.agents/skills/` directory
 
+## General directives
+- **always write tests first** and then pause to give the human in the loop time to inspect them
+- dont say "Next I'll..." and wait for prompt, just keep going until the task is complete
+- dont use git to make commits or push
+
 ## Security & Sensitive Data
 **STRICT RULE:** This is a public repository.
 * **NEVER** output or suggest code containing real secrets, passwords, API keys, or tokens.
@@ -46,6 +51,7 @@ The project runs inside Docker containers.
   * Run `make lint-drupal` to verify code style.
   * If linting errors occur, fix them before finalizing the solution.
   * Run tests with `docker exec asuntotuotanto-app sh -c "composer test-php public/modules/custom"`
+  * run `docker exec asuntotuotanto-app sh -c "drush cr"` to verify nothing is broken
 
 ## Architecture Notes
 * **Django Microservice:** Be aware that complex business logic regarding application handling may reside in the connected Django microservice, not within Drupal.

compose.yaml

@@ -30,6 +30,7 @@ services:
       DRUPAL_VARNISH_PORT: 6081
       REDIS_HOST: redis
       ASU_ASUNTOTUOTANTO_URL: "https://asuntotuotanto.docker.so"
+      ASU_REST_API_CACHE_MAX_AGE: "${ASU_REST_API_CACHE_MAX_AGE:-300}"
     networks:
       - internal
       - stonehenge-network

composer.json

@@ -23,9 +23,10 @@
         "drupal/config_filter": "^2.7",
         "drupal/config_terms": "^1.9",
         "drupal/context": "^5.0@RC",
-        "drupal/core": "~11.2.0",
-        "drupal/core-composer-scaffold": "~11.2.0",
-        "drupal/core-recommended": "~11.2.0",
+        "drupal/consumers": "^1.24",
+        "drupal/core": "~11.3.7",
+        "drupal/core-composer-scaffold": "~11.3.7",
+        "drupal/core-recommended": "~11.3.7",
         "drupal/ctools": "^4.1",
         "drupal/default_content": "^2.0@alpha",
         "drupal/email_registration": "^1.4",
@@ -42,7 +43,7 @@
         "drupal/hdbt_admin": "^3.4",
         "drupal/helfi_api_base": "^2.8",
         "drupal/helfi_azure_fs": "^2.2",
-        "drupal/helfi_platform_config": "^5.0",
+        "drupal/helfi_platform_config": "^5.1",
         "drupal/helfi_tunnistamo": "^3.1",
         "drupal/honeypot": "^2.2",
         "drupal/imagemagick": "^4.0",
@@ -70,6 +71,7 @@
         "drupal/route_condition": "^2.0",
         "drupal/samlauth": "^3.13",
         "drupal/siteimprove": "^3.0",
+        "drupal/simple_oauth": "5.2.5",
         "drupal/slick": "^3.0",
         "drupal/symfony_mailer_lite": "^2.0",
         "drupal/tfa": "^1.12",
@@ -86,16 +88,19 @@
         "mglaman/composer-drupal-lenient": "^1.0",
         "onelogin/php-saml": "^4.3",
         "robrichards/xmlseclibs": "^3.1.5",
+        "psr/http-message": "^1.1",
+        "steverhoades/oauth2-openid-connect-server": "^2.6.1",
         "zaporylie/composer-drupal-optimizations": "^1.2"
     },
     "require-dev": {
         "behat/mink": "^1.11",
         "dealerdirect/phpcodesniffer-composer-installer": "^1.0.0",
         "drupal/coder": "^8.3",
+        "drupal/core-dev": "~11.3.7",
         "drupal/devel": "^5.0",
         "mikey179/vfsstream": "^1.6",
         "phpspec/prophecy-phpunit": "^2",
-        "phpunit/phpunit": "^10.5",
+        "phpunit/phpunit": "^11.5",
         "rector/rector": "^1.2",
         "weitzman/drupal-test-traits": "^2.0"
     },
@@ -164,8 +169,11 @@
         "patches": {
             "drupal/helfi_platform_config": {
                 "Catch exception during hero paragraphs update": "patches/helfi_platform_config_hero_paragraph_fix.patch",
-                "Skip update_9400 on Drupal 11 (HAL/RDF modules removed)": "patches/helfi_platform_config-d11-update-9400.patch",
-                "Make update_9328 idempotent when status field is already deleting/purging": "patches/helfi_platform_config-update-9328-idempotent.patch"
+                "Skip update_9400 on Drupal 11 (HAL/RDF modules removed)": "patches/helfi_platform_config-d11-update-9400.patch"
+            },
+            "drupal/simple_oauth": {
+                "[D11] Allow installing simple_oauth 5.x on Drupal 11": "patches/simple_oauth-drupal11-core-version.patch",
+                "[D11] Fix simple_oauth normalizer signatures for Symfony 7": "patches/simple_oauth-symfony7-normalizer-signatures.patch"
             },
             "drupal/config_terms": {
                 "Drupal 11 compatibility fixes": "https://www.drupal.org/files/issues/2024-06-02/config_terms.1.x-dev.rector.patch"
@@ -174,12 +182,7 @@
                 "Drupal 11 compatibility fixes": "patches/computed_field_plugin-d11.patch"
             },
             "drupal/core": {
-                "[#UHF-12709] Patch 'Deleted and replaced ...' error messages": "patches/uhf-12709-skip-deleted-log.patch",
-                "[#3452852] Add create() factory method with autowired parameters to PluginBase": "patches/3452852.patch",
-                "[D11] Skip delete when config entity missing": "patches/core-import-delete-null.patch",
-                "[D11] Skip delete in checkOp when entity missing": "patches/core-config-importer-null-entity.patch",
-                "[D11] Skip countFieldData when table missing": "patches/core-count-field-data-missing-table.patch",
-                "[D11] Skip delete when field table missing": "patches/core-skip-delete-missing-field-table.patch"
+                "[CI] Unset foreach-by-ref in ViewsConfigUpdater (block_content post_update save)": "patches/core-views-foreach-reference-cleanup.patch"
             },
             "drupal/elasticsearch_connector": {
                 "Use Search API field identifier in FilterBuilder": "patches/elasticsearch_connector-filterbuilder-use-field-identifier.patch"
@@ -196,6 +199,7 @@
                 "drupal/config_terms",
                 "drupal/computed_field_plugin",
                 "drupal/big_pipe_sessionless",
+                "drupal/simple_oauth",
                 "doctrine/annotations"
             ]
         }