Merge pull request #83 from magento-commerce/develop

MCLOUD-8486: Release Cloud Tools

Author: BaDos

.github/.metadata.json

@@ -0,0 +1,26 @@
+{
+  "templateVersion": "0.1",
+  "product": {
+    "name": "Magento Commerce Cloud Deployment Tools",
+    "description": "ECE-Tools is a set of scripts and tools designed to manage and deploy Cloud projects"
+  },
+  "contacts": {
+    "team": {
+      "name": "Magic Mountain",
+      "DL": "Grp-magento-cloud-all",
+      "slackChannel": "magic_mountain"
+    }
+  },
+  "ticketTracker": {
+    "functionalJiraQueue": {
+      "projectKey": "MCLOUD"
+    },
+    "securityJiraQueue": {
+      "projectKey": "MAGREQ",
+      "component": "Magento Cloud Engineering"
+    }
+  },
+  "staticScan": {
+    "enable": false
+  }
+}

.github/workflows/ci.yaml

@@ -9,3 +9,4 @@
 /codeception.yml
 /_workdir
 /*.code-workspace
+**/.phpunit.result.cache

.gitignore

@@ -2,7 +2,7 @@
   "name": "magento/ece-tools",
   "description": "Provides tools to build and deploy Magento 2 Enterprise Edition",
   "type": "magento2-component",
-  "version": "2002.1.8",
+  "version": "2002.1.9",
   "license": "OSL-3.0",
   "repositories": {
     "repo.magento.com": {
@@ -11,7 +11,7 @@
     }
   },
   "require": {
-    "php": "^7.1.3",
+    "php": "^7.2 || ^8.0",
     "ext-PDO": "*",
     "ext-json": "*",
     "ext-sockets": "*",
@@ -20,7 +20,7 @@
     "composer/semver": "@stable",
     "graylog2/gelf-php": "^1.4.2",
     "guzzlehttp/guzzle": "^6.3||^7.3",
-    "illuminate/config": "^5.5",
+    "illuminate/config": "^5.5||^8.77",
     "magento/magento-cloud-components": "^1.0.8",
     "magento/magento-cloud-docker": "^1.0.0",
     "magento/magento-cloud-patches": "^1.0.11",
@@ -37,13 +37,17 @@
     "symfony/yaml": "^3.3 || ^4.0 || ^5.1"
   },
   "require-dev": {
-    "codeception/codeception": "^2.5.3",
+    "codeception/codeception": "^4.1",
+    "codeception/module-asserts": "^1.2",
+    "codeception/module-db": "^1.0",
+    "codeception/module-phpbrowser": "^1.0",
+    "codeception/module-rest": "^1.2",
     "consolidation/robo": "^1.2",
     "php-mock/php-mock-phpunit": "^2.0",
     "phpmd/phpmd": "@stable",
     "phpstan/phpstan": "^0.12",
-    "phpunit/php-code-coverage": "^6.0",
-    "phpunit/phpunit": "^7.2",
+    "phpunit/php-code-coverage": "^7.0 || ^9.2",
+    "phpunit/phpunit": "^8.5 || ^9.5",
     "squizlabs/php_codesniffer": "^3.0",
     "funkjedi/composer-include-files": "^1.0"
   },
@@ -91,7 +95,10 @@
     ]
   },
   "config": {
-    "sort-packages": true
+    "sort-packages": true,
+    "allow-plugins": {
+      "funkjedi/composer-include-files": true
+    }
   },
   "prefer-stable": true,
   "extra": {

composer.json

@@ -326,6 +326,11 @@
     suggestion: 'Check for valid Elasticsearch credentials and verify that the service is running'
     stage: deploy
     type: critical
+!php/const Magento\MagentoCloud\App\Error::DEPLOY_OS_CANNOT_CONNECT:
+    title: 'Can not connect to the Opensearch service'
+    suggestion: 'Check for valid Opensearch credentials and verify that the service is running'
+    stage: deploy
+    type: critical
 !php/const Magento\MagentoCloud\App\Error::DEPLOY_WRONG_BRAINTREE_VARIABLE:
     step: validate-config
     title: 'Remove Magento Braintree module configuration which is no longer supported in Magento 2.4 and later versions.'
@@ -338,6 +343,12 @@
     suggestion: 'Install Elasticsearch service'
     stage: deploy
     type: critical
+!php/const Magento\MagentoCloud\App\Error::DEPLOY_OS_SERVICE_NOT_INSTALLED:
+    step: validate-config
+    title: 'Magento 2.4.4 requires Opensearch or Elasticsearch service to be installed'
+    suggestion: 'Install Opensearch service'
+    stage: deploy
+    type: critical
 !php/const Magento\MagentoCloud\App\Error::DEPLOY_WRONG_SEARCH_ENGINE:
     step: validate-config
     title: 'The search engine must be set to Elasticsearch for Magento >= 2.4.0'
@@ -634,3 +645,15 @@
     title: 'Can not get system processor count:'
     type: warning
     stage: general
+!php/const Magento\MagentoCloud\App\Error::WARN_OS_ES_SERVICES_BOTH_INSTALLED:
+    title: 'Elasticsearch and Opensearch services are both installed at infrastructure layer. Magento 2.4.4 and higher uses Opensearch by default'
+    suggestion: 'Consider removing the Elasticsearch or Opensearch service from the infrastructure layer to optimize resource usage.'
+    stage: deploy
+    step: validate-config
+    type: warning
+!php/const Magento\MagentoCloud\App\Error::DEPLOY_MAGENTO_VERSION_DOES_NOT_SUPPORT_OS:
+    step: validate-config
+    title: 'Incorrect search engine'
+    suggestion: 'This Magento version does not support Opensearch. You must use versions 2.3.7-p3, 2.4.3-p2, or higher'
+    stage: deploy
+    type: critical

config/schema.error.yaml

@@ -561,7 +561,9 @@ variables:
                      process before terminating. Although we do not recommend it, you can use 0
                      to prevent the consumer from terminating.\n\n
       consumers — an array of strings specifying which consumer(s) to run.
-                  An empty array runs all consumers."
+                  An empty array runs all consumers.\n\n
+      multiple_processes - a number specifying the number of processes to spawn for each consumer.
+                           Supported in Magento 2.4.4 or greater."
     type: array
     stages:
       - deploy
@@ -576,6 +578,9 @@ variables:
               consumers:
                 - consumer1
                 - consumer2
+              multiple_processes:
+                consumer1: 4
+                consumer2: 3
   CONSUMERS_WAIT_FOR_MAX_MESSAGES:
     description: Use this variable to configure how consumers process messages. If this value is true, a consumer waits
       to process the number of messages (max_messages) from the message queue specified in the CRONS_CONSUMERS_RUNNER

config/schema.yaml

@@ -72,10 +72,13 @@ Critical errors indicate a problem with the Magento Commerce Cloud project confi
 | 130 | install-update: cache_type | Command failed: `php ./bin/magento cache:enable` | Command `php ./bin/magento cache:enable` runs only when Magento was installed but `./app/etc/env.php` file was absent or empty at the beginning of the deployment. Check the `cloud.log` for more information. Add `VERBOSE_COMMANDS: '-vvv'` into `.magento.env.yaml` for more detailed command output. |
 | 131 | install-update | The `crypt/key`  key value does not exist in the `./app/etc/env.php` file or the `CRYPT_KEY` cloud environment variable | This error occurs if the `./app/etc/env.php` file is not present when Magento deployment begins, or if the `crypt/key` value is undefined. If you migrated the database from another environment, retrieve the crypt key value from that environment. Then, add the value to the [CRYPT_KEY](https://devdocs.magento.com/cloud/env/variables-deploy.html#crypt_key) cloud environment variable in your current environment. See [Add the Magento encryption key](https://devdocs.magento.com/cloud/setup/first-time-setup-import-import.html#encryption-key). If you accidentally removed the `./app/etc/env.php` file, use the following command to restore it from the backup files created from a previous deployment: `./vendor/bin/ece-tools backup:restore` CLI command ." |
 | 132 |  | Can not connect to the Elasticsearch service | Check for valid Elasticsearch credentials and verify that the service is running |
+| 137 |  | Can not connect to the Opensearch service | Check for valid Opensearch credentials and verify that the service is running |
 | 133 | validate-config | Remove Magento Braintree module configuration which is no longer supported in Magento 2.4 and later versions. | Support for the Braintree module is no longer included with Magento 2.4.0 and later. Remove the CONFIG__STORES__DEFAULT__PAYMENT__BRAINTREE__CHANNEL variable from the variables section of the .magento.app.yaml file. For Braintree support, use an official Braintree Payments extension from the Magento Marketplace instead. |
 | 134 | validate-config | Magento 2.4.0 requires Elasticsearch service to be installed | Install Elasticsearch service |
+| 138 | validate-config | Magento 2.4.4 requires Opensearch or Elasticsearch service to be installed | Install Opensearch service |
 | 135 | validate-config | The search engine must be set to Elasticsearch for Magento >= 2.4.0 | Check the SEARCH_CONFIGURATION variable for the `engine` option. If it is configured, remove the option, or set the value to "elasticsearch". |
 | 136 | validate-config | Split Database was removed starting from Magento 2.5.0. | If you use split database you have to revert to or migrate to a single database or use an alternative approach. |
+| 139 | validate-config | Incorrect search engine | This Magento version does not support Opensearch. You must use versions 2.3.7-p3, 2.4.3-p2, or higher |
 
 ### Post-deploy stage
 
@@ -153,6 +156,7 @@ Warning errors indicate a problem with the Magento Commerce Cloud project config
 | 2026 | pre-deploy:restore-writable-dirs | Failed to restore some data generated during the build phase to the mounted directories | Check the `cloud.log` for more information. |
 | 2027 | validate-config:mage-mode-variable | Mode value for MAGE_MODE environment variable not supported | Remove the MAGE_MODE environment variable, or change its value to "production". Magento Cloud supports "production" mode only. |
 | 2028 | remote-storage | Remote storage could not be enabled. | Verify remote storage credentials. |
+| 2030 | validate-config | Elasticsearch and Opensearch services are both installed at infrastructure layer. Magento 2.4.4 and higher uses Opensearch by default | Consider removing the Elasticsearch or Opensearch service from the infrastructure layer to optimize resource usage. |
 
 ### Post-deploy stage
 

dist/error-codes.md

@@ -31,6 +31,7 @@
                     <item name="resource-configuration" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\ResourceConfiguration</item>
                     <item name="session-configuration" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\SessionConfiguration</item>
                     <item name="elasticsearch-integrity" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\ElasticSearchIntegrity</item>
+                    <item name="opensearch-integrity" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\OpenSearchIntegrity</item>
                     <item name="elasticsuite-integrity" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\ElasticSuiteIntegrity</item>
                     <item name="removed-split-db" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\RemovedSplitDb</item>
                 </item>
@@ -43,6 +44,7 @@
                     <item name="solr-integrity" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\SolrIntegrity</item>
                     <item name="es-usage" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\ElasticSearchUsage</item>
                     <item name="es-version" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\ElasticSearchVersion</item>
+                    <item name="es-os-installed-both" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\OpenSearchElasticSearchBothInstalled</item>
                     <item name="appropriate-version" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\AppropriateVersion</item>
                     <item name="scd-options-ignorance" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\ScdOptionsIgnorance</item>
                     <item name="deprecated-variables" xsi:type="object">Magento\MagentoCloud\Config\Validator\Deploy\DeprecatedVariables</item>
@@ -99,6 +101,7 @@
                 <item name="remote-storage" xsi:type="object" priority="1000">Magento\MagentoCloud\Step\Deploy\RemoteStorage</item>
                 <!-- Config update end -->
                 <item name="set-admin-url" xsi:type="object" priority="1000">Magento\MagentoCloud\Step\Deploy\InstallUpdate\Update\SetAdminUrl</item>
+                <item name="clean-cache" xsi:type="object" priority="1050">CleanCache.Deploy</item>
                 <item name="setup" xsi:type="object" priority="1100">Magento\MagentoCloud\Step\Deploy\InstallUpdate\Update\Setup</item>
                 <item name="split-db" xsi:type="object" priority="1200">Magento\MagentoCloud\Step\Deploy\SplitDbConnection</item>
                 <item name="cache-type" xsi:type="object" priority="1300">Magento\MagentoCloud\Step\Deploy\InstallUpdate\ConfigUpdate\CacheType</item>

scenario/deploy.xml

@@ -76,6 +76,9 @@ class Error
     public const DEPLOY_ES_SERVICE_NOT_INSTALLED = 134;
     public const DEPLOY_WRONG_SEARCH_ENGINE = 135;
     public const DEPLOY_REMOVED_SPLIT_DB = 136;
+    public const DEPLOY_OS_CANNOT_CONNECT = 137;
+    public const DEPLOY_OS_SERVICE_NOT_INSTALLED = 138;
+    public const DEPLOY_MAGENTO_VERSION_DOES_NOT_SUPPORT_OS = 139;
 
     public const PD_DEPLOY_IS_FAILED = 201;
     public const PD_ENV_PHP_IS_NOT_WRITABLE = 202;
@@ -135,6 +138,8 @@ class Error
     public const WARN_NOT_SUPPORTED_MAGE_MODE = 2027;
     public const WARN_REMOTE_STORAGE_CANNOT_BE_ENABLED = 2028;
     public const WARN_DEPRECATED_SPLIT_DB = 2029;
+    public const WARN_OS_ES_SERVICES_BOTH_INSTALLED = 2030;
+    public const WARN_REMOTE_STORAGE_CANNOT_BE_DISABLED = 2031;
 
     /**
      * Post-deploy

src/App/Error.php

@@ -12,7 +12,9 @@
 use Magento\MagentoCloud\Package\MagentoVersion;
 use Magento\MagentoCloud\Package\UndefinedPackageException;
 use Magento\MagentoCloud\Service\ElasticSearch;
+use Magento\MagentoCloud\Service\OpenSearch;
 use Magento\MagentoCloud\Service\ServiceException;
+use Magento\MagentoCloud\Service\Search\AbstractService as AbstractSearchService;
 
 /**
  * Returns search configuration.
@@ -36,6 +38,11 @@ class SearchEngine
      */
     private $elasticSearch;
 
+    /**
+     * @var OpenSearch
+     */
+    private $openSearch;
+
     /**
      * @var MagentoVersion
      */
@@ -60,6 +67,7 @@ class SearchEngine
      * @param Environment $environment
      * @param DeployInterface $stageConfig
      * @param ElasticSearch $elasticSearch
+     * @param OpenSearch $openSearch
      * @param ElasticSuite $elasticSuite
      * @param MagentoVersion $version
      * @param ConfigMerger $configMerger
@@ -68,13 +76,15 @@ public function __construct(
         Environment $environment,
         DeployInterface $stageConfig,
         ElasticSearch $elasticSearch,
+        OpenSearch $openSearch,
         ElasticSuite $elasticSuite,
         MagentoVersion $version,
         ConfigMerger $configMerger
     ) {
         $this->environment = $environment;
         $this->stageConfig = $stageConfig;
         $this->elasticSearch = $elasticSearch;
+        $this->openSearch = $openSearch;
         $this->elasticSuite = $elasticSuite;
         $this->magentoVersion = $version;
         $this->configMerger = $configMerger;
@@ -134,7 +144,7 @@ public function isESFamily(): bool
         $searchEngine = $this->getName();
 
         return (strpos($searchEngine, ElasticSearch::ENGINE_NAME) === 0)
-            || ($searchEngine === ElasticSuite::ENGINE_NAME);
+            || ($searchEngine === ElasticSuite::ENGINE_NAME) || ($searchEngine === OpenSearch::ENGINE_NAME);
     }
 
     /**
@@ -145,8 +155,12 @@ public function isESFamily(): bool
      */
     private function getSearchConfig(): array
     {
-        if ($esConfig = $this->elasticSearch->getConfiguration()) {
-            return $this->getElasticSearchConfiguration($esConfig);
+        if ($this->openSearch->getConfiguration()) {
+            return $this->getElasticSearchFamilyConfiguration($this->openSearch);
+        }
+
+        if ($this->elasticSearch->getConfiguration()) {
+            return $this->getElasticSearchFamilyConfiguration($this->elasticSearch);
         }
 
         $solrConfig = $this->environment->getRelationship('solr');
@@ -178,22 +192,23 @@ private function getSolrConfiguration(array $config): array
     /**
      * Returns ElasticSearch configuration
      *
-     * @param array $config Elasticsearch connection configuration
+     * @param AbstractSearchService $searchService
      * @return array
      *
      * @throws ServiceException
      */
-    private function getElasticSearchConfiguration(array $config): array
+    private function getElasticSearchFamilyConfiguration(AbstractSearchService $searchService): array
     {
-        $engine = $this->elasticSearch->getFullVersion();
+        $engine = $searchService->getFullEngineName();
+        $config = $searchService->getConfiguration();
 
         $elasticSearchConfig = [
             'engine' => $engine,
-            "{$engine}_server_hostname" => $config['host'],
+            "{$engine}_server_hostname" => $searchService->getHost(),
             "{$engine}_server_port" => $config['port'],
         ];
 
-        if ($this->elasticSearch->isAuthEnabled()) {
+        if ($searchService->isAuthEnabled()) {
             $elasticSearchConfig["{$engine}_enable_auth"] = 1;
             $elasticSearchConfig["{$engine}_username"] = $config['username'];
             $elasticSearchConfig["{$engine}_password"] = $config['password'];