Merge pull request #43 from magento/develop

Backport develop branch

Author: shiftedreality

.gitignore

@@ -4,3 +4,5 @@
 /composer.phar
 /composer.lock
 /auth.json
+/codeception.yml
+/_workdir

.travis.yml

@@ -3,16 +3,68 @@ dist: xenial
 git:
   depth: false
 
+addons:
+  hosts:
+    - magento2.docker
+
+services:
+  - docker
+
 language: php
-php:
-  - '7.0'
-  - '7.1'
-  - '7.2'
-  - '7.3'
 
-install: composer update
+stages:
+  - static-unit
+  - test
+
+jobs:
+  include:
+    - stage: static-unit
+      script: ./tests/travis/static-unit.sh;
+      php: '7.0'
+      env:
+        - TEST_SUITE=static-unit
+    - script: ./tests/travis/static-unit.sh;
+      php: '7.1'
+      env:
+        - TEST_SUITE=static-unit
+    - script: ./tests/travis/static-unit.sh;
+      php: '7.2'
+      env:
+        - TEST_SUITE=static-unit
+    - script: ./tests/travis/static-unit.sh;
+      php: '7.3'
+      env:
+        - TEST_SUITE=static-unit
+    - stage: test
+      php: '7.1'
+      env:
+        - TEST_SUITE=functional-ee
+    - php: '7.2'
+      env:
+        - TEST_SUITE=functional-ee
+    - php: '7.3'
+      env:
+        - TEST_SUITE=functional-ee
+    - php: '7.1'
+      env:
+        - TEST_SUITE=functional-ce
+    - php: '7.2'
+      env:
+        - TEST_SUITE=functional-ce
+    - php: '7.3'
+      env:
+        - TEST_SUITE=functional-ce
+
+install:
+  - composer config http-basic.repo.magento.com ${REPO_USERNAME_CE} ${REPO_PASSWORD_CE}
+  - composer config github-oauth.github.com ${GITHUB_TOKEN}
+  - if [ -n "${MCD_VERSION}" ] && [ $TRAVIS_PHP_VERSION != "7.0" ]; then composer config repositories.mcd git [email protected]:magento/magento-cloud-docker.git && composer require "magento/magento-cloud-docker:${MCD_VERSION}" --no-update; fi;
+  - composer update -n --no-suggest
+
+before_script:
+  - sudo sysctl -w vm.max_map_count=262144
+  - if [ $TEST_SUITE == "functional-ce" ]; then cp codeception.dist.yml codeception.yml && sed -i "s/REPO_USERNAME/REPO_USERNAME_CE/" codeception.yml && sed -i "s/REPO_PASSWORD/REPO_PASSWORD_CE/" codeception.yml; fi;
 
 script:
-  - ./vendor/bin/phpcs ./src --standard=./tests/static/phpcs-ruleset.xml -p -n
-  - ./vendor/bin/phpmd ./src xml ./tests/static/phpmd-ruleset.xml
-  - ./vendor/bin/phpunit --configuration ./tests/unit
+  - if [ $TRAVIS_SECURE_ENV_VARS == "true" ] && [ $TEST_SUITE == "functional-ee" ]; then ./tests/travis/functional-ee.sh; fi;
+  - if [ $TEST_SUITE == "functional-ce" ]; then ./tests/travis/functional-ce.sh; fi;

codeception.dist.yml

@@ -0,0 +1,37 @@
+paths:
+  tests: src/Test/Functional
+  output: tests/functional/_output
+  data: tests/functional/_data
+  support: vendor/magento/magento-cloud-docker/tests/functional/_support
+actor_suffix: Tester
+settings:
+  colors: true
+extensions:
+  enabled:
+    - Codeception\Extension\RunFailed
+    - Codeception\Extension\FailedInfo
+params:
+  - vendor/magento/magento-cloud-docker/tests/functional/configuration.dist.yml
+  - env
+modules:
+  config:
+    Magento\CloudDocker\Test\Functional\Codeception\TestInfrastructure:
+      template_repo: "https://github.com/magento/magento-cloud.git"
+      mcd_repo: "https://github.com/magento/magento-cloud-docker.git"
+      mcc_repo: "https://github.com/magento/magento-cloud-components.git"
+      mcp_repo: "https://github.com/magento/magento-cloud-patches.git"
+      composer_magento_username: "%REPO_USERNAME%"
+      composer_magento_password: "%REPO_PASSWORD%"
+      composer_github_token: "%GITHUB_TOKEN%"
+      printOutput: false
+    Magento\CloudDocker\Test\Functional\Codeception\Docker:
+      system_magento_dir: "%Magento.docker.settings.system.magento_dir%"
+      printOutput: false
+    PhpBrowser:
+      url: "%Magento.docker.settings.env.url.base%"
+    Magento\CloudDocker\Test\Functional\Codeception\MagentoDb:
+      dsn: "mysql:host=%Magento.docker.settings.db.host%;port=%Magento.docker.settings.db.port%;dbname=%Magento.docker.settings.db.path%"
+      user: "%Magento.docker.settings.db.username%"
+      password: "%Magento.docker.settings.db.password%"
+      exposed_port: "%Magento.docker.settings.db.port%"
+      reconnect: true

composer.json

@@ -15,6 +15,8 @@
         "symfony/process": "^2.1||^4.1"
     },
     "require-dev": {
+        "codeception/codeception": "^2.5.3",
+        "consolidation/robo": "^1.2",
         "phpmd/phpmd": "@stable",
         "phpunit/phpunit": "^6.2",
         "squizlabs/php_codesniffer": "^3.0"
@@ -24,7 +26,8 @@
     ],
     "autoload": {
         "psr-4": {
-            "Magento\\CloudPatches\\": "src/"
+            "Magento\\CloudPatches\\": "src/",
+            "Magento\\CloudPatches\\Test\\Functional\\": "tests/functional/"
         }
     },
     "scripts": {

patches.json

@@ -78,9 +78,6 @@
       "2.2.0 - 2.2.3": "SET-36__fix_oom_during_customer_import__2.2.0.patch",
       "2.2.4": "SET-36__fix_oom_during_customer_import__2.2.4.patch"
     },
-    "Add PayPal and Braintree TPV codes on checkout": {
-      "~2.1.8": "MAGETWO-53941__fix_enterprise_payment_codes__2.1.8.patch"
-    },
     "Fix Mview on staging environments": {
       "2.1.4": "MAGETWO-84444__fix_mview_on_staging__2.1.4.patch",
       "2.1.5 - 2.1.9": "MAGETWO-84444__fix_mview_on_staging__2.1.5.patch",
@@ -107,19 +104,14 @@
     "Unlock locale editing when SCD on demand is enabled": {
       "2.2.0 - 2.2.5": "MAGECLOUD-2159__unlock_locale_editing_when_scd_on_demand__2.2.0.patch"
     },
-    "Allow DB dumps done with the support module to complete": {
-      "2.2.0 - 2.2.5": "MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch"
-    },
     "Write Logs for Failed Process of Generating Factories in Extensions": {
       "2.2.0 - 2.2.5": "MAGECLOUD-2209__write_logs_for_failed_process_of_generating_factories_in_extensions__2.2.0.patch"
     },
     "Fix Problems with Consumer Runners on Cloud Clusters": {
-      "2.2.0 - 2.2.3": "MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch",
-      "2.2.4": "MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch",
-      "2.2.5": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5.patch",
-      "2.2.6": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6.patch",
-      "2.2.7": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7.patch",
-      "2.2.8 - 2.2.9": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8.patch",
+      "2.2.5": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5_ce.patch",
+      "2.2.6": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6_ce.patch",
+      "2.2.7": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7_ce.patch",
+      "2.2.8 - 2.2.9": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8_ce.patch",
       "2.3.0": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.0.patch",
       ">=2.3.1 <2.3.3": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.3.1.patch"
     },
@@ -186,7 +178,6 @@
       ">=2.3.0 <2.3.3": "MAGECLOUD-3806__error_code_fix_for_setup_upgrade__2.3.0.patch"
     },
     "Re-work consumers to terminate as soon as there is nothing left to process": {
-      "2.2.0 - 2.3.1": "MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch",
       ">=2.3.2 <2.3.4": "MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.3.2.patch"
     },
     "Backward Compatibility of new Mail Interfaces": {
@@ -224,9 +215,6 @@
       "2.2.0 - 2.2.8": "PRODSECBUG-2432__admin_path_disclosure_bug__2.2.0.patch",
       "2.3.0 - 2.3.1": "PRODSECBUG-2432__admin_path_disclosure_bug__2.3.0.patch"
     },
-    "Transaction MD5 hash field is removed by Authorize.net": {
-      "2.2.0 - 2.2.7": "MAGETWO-98129__transaction_field_is_removed_by_authorize_net__2.2.0.patch"
-    },
     "Customer attributes issue": {
       "2.2.6": "MAGETWO-95591__customer_attributes_issue__2.2.6.patch"
     },
@@ -239,12 +227,38 @@
     "Fix for multi-site configuration issue": {
       "2.2.4": "MAGETWO-92926__fix_for_multi-site_configuration_issue__2.2.4.patch"
     },
-    "Fix PayPal issue with region": {
-      "2.3.4": "MC-31387__fix_paypal_issue_with_region__2.3.4.patch"
-    },
     "FPC is getting disabled during deployments": {
       ">=2.3.2 <2.3.5": "MAGECLOUD-5069__fpc_is_getting_disabled_during_deployments__2.3.2.patch",
       ">=2.3.5 <2.4.0": "MAGECLOUD-5069__fpc_is_getting_disabled_during_deployments__2.3.5.patch"
+    },
+    "Hold deployment config after reading from file": {
+      ">=2.3.3 <2.3.6": "MCLOUD-5650__hold_deployment_config_after_reading_from_file.patch"
+    },
+    "Pagination Not working - product_list_limit=all": {
+      ">=2.3.2 <2.3.4": "MCLOUD-5684__pagination_not_working_product_list_limit_all__2.3.2.patch",
+      ">=2.3.4 <2.4.0": "MCLOUD-5684__pagination_not_working_product_list_limit_all__2.3.4.patch"
+    },
+    "Sitemap Generation Warnings": {
+      ">=2.3.0 <2.3.2": "MCLOUD-3025__sitemap_generation_warnings__2.3.0.patch",
+      ">=2.3.2 <2.3.5": "MCLOUD-3025__sitemap_generation_warnings__2.3.3.patch"
+    }
+  },
+  "magento/module-paypal": {
+    "Add PayPal TPV codes on checkout": {
+      "~100.1.6": "MAGETWO-53941__fix_enterprise_paypal_codes__100.1.6.patch"
+    },
+    "Fix PayPal issue with region": {
+      "100.3.4": "MC-31387__fix_paypal_issue_with_region__100.3.4.patch"
+    }
+  },
+  "magento/module-authorizenet": {
+    "Transaction MD5 hash field is removed by Authorize.net": {
+      "100.2.0 - 100.2.3": "MAGETWO-98129__transaction_field_is_removed_by_authorize_net__100.2.0.patch"
+    }
+  },
+  "magento/module-braintree": {
+    "Add Braintree TPV codes on checkout": {
+      "~100.1.6": "MAGETWO-53941__fix_enterprise_braintree_code__100.1.6.patch"
     }
   },
   "monolog/monolog": {
@@ -303,6 +317,20 @@
       "2.3.2": "PB-320__fix_pagebuilder_module__2.3.2.patch",
       ">=2.3.2-p1 <2.3.3": "PB-322__fix_pagebuilder_module__2.3.2-p1.patch",
       "2.3.3": "PB-323__fix_pagebuilder_module__2.3.3.patch"
+    },
+    "Allow DB dumps done with the support module to complete": {
+      "2.2.0 - 2.2.5": "MAGECLOUD-2033__prevent_deadlock_during_db_dump__2.2.0.patch"
+    },
+    "Fix Problems with Consumer Runners on Cloud Clusters": {
+      "2.2.0 - 2.2.3": "MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.0.patch",
+      "2.2.4": "MAGECLOUD-2464__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.4.patch",
+      "2.2.5": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5_ee.patch",
+      "2.2.6": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.6_ee.patch",
+      "2.2.7": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.7_ee.patch",
+      "2.2.8 - 2.2.9": "MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.8_ee.patch"
+    },
+    "Re-work consumers to terminate as soon as there is nothing left to process": {
+      ">=2.2.0 <2.3.2": "MAGECLOUD-4071__terminate_consumers_if_the_queue_is_empty__2.2.0.patch"
     }
   }
 }

patches/MAGECLOUD-3913__fix_problems_with_consumer_runners_on_cloud_clusters__2.2.5_ce.patch

@@ -0,0 +1,139 @@
+diff -Nuar a/vendor/magento/framework/Lock/Backend/Database.php b/vendor/magento/framework/Lock/Backend/Database.php
+--- a/vendor/magento/framework/Lock/Backend/Database.php
++++ b/vendor/magento/framework/Lock/Backend/Database.php
+@@ -3,8 +3,8 @@
+  * Copyright © Magento, Inc. All rights reserved.
+  * See COPYING.txt for license details.
+  */
+-
+ declare(strict_types=1);
++
+ namespace Magento\Framework\Lock\Backend;
+ 
+ use Magento\Framework\App\DeploymentConfig;
+@@ -14,20 +14,44 @@ use Magento\Framework\Exception\AlreadyExistsException;
+ use Magento\Framework\Exception\InputException;
+ use Magento\Framework\Phrase;
+ 
++/**
++ * Implementation of the lock manager on the basis of MySQL.
++ */
+ class Database implements \Magento\Framework\Lock\LockManagerInterface
+ {
+-    /** @var ResourceConnection */
++    /**
++     * Max time for lock is 1 week
++     *
++     * MariaDB does not support negative timeout value to get infinite timeout,
++     * so we set 1 week for lock timeout
++     */
++    const MAX_LOCK_TIME = 604800;
++
++    /**
++     * @var ResourceConnection
++     */
+     private $resource;
+ 
+-    /** @var DeploymentConfig */
++    /**
++     * @var DeploymentConfig
++     */
+     private $deploymentConfig;
+ 
+-    /** @var string Lock prefix */
++    /**
++     * @var string Lock prefix
++     */
+     private $prefix;
+ 
+-    /** @var string|false Holds current lock name if set, otherwise false */
++    /**
++     * @var string|false Holds current lock name if set, otherwise false
++     */
+     private $currentLock = false;
+ 
++    /**
++     * @param ResourceConnection $resource
++     * @param DeploymentConfig $deploymentConfig
++     * @param string|null $prefix
++     */
+     public function __construct(
+         ResourceConnection $resource,
+         DeploymentConfig $deploymentConfig,
+@@ -46,9 +70,13 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface
+      * @return bool
+      * @throws InputException
+      * @throws AlreadyExistsException
++     * @throws \Zend_Db_Statement_Exception
+      */
+     public function lock(string $name, int $timeout = -1): bool
+     {
++        if (!$this->deploymentConfig->isDbAvailable()) {
++            return true;
++        };
+         $name = $this->addPrefix($name);
+ 
+         /**
+@@ -59,7 +87,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface
+         if ($this->currentLock) {
+             throw new AlreadyExistsException(
+                 new Phrase(
+-                    'Current connection is already holding lock for $1, only single lock allowed',
++                    'Current connection is already holding lock for %1, only single lock allowed',
+                     [$this->currentLock]
+                 )
+             );
+@@ -67,7 +95,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface
+ 
+         $result = (bool)$this->resource->getConnection()->query(
+             "SELECT GET_LOCK(?, ?);",
+-            [(string)$name, (int)$timeout]
++            [$name, $timeout < 0 ? self::MAX_LOCK_TIME : $timeout]
+         )->fetchColumn();
+ 
+         if ($result === true) {
+@@ -83,9 +111,14 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface
+      * @param string $name lock name
+      * @return bool
+      * @throws InputException
++     * @throws \Zend_Db_Statement_Exception
+      */
+     public function unlock(string $name): bool
+     {
++        if (!$this->deploymentConfig->isDbAvailable()) {
++            return true;
++        };
++
+         $name = $this->addPrefix($name);
+ 
+         $result = (bool)$this->resource->getConnection()->query(
+@@ -106,14 +139,19 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface
+      * @param string $name lock name
+      * @return bool
+      * @throws InputException
++     * @throws \Zend_Db_Statement_Exception
+      */
+     public function isLocked(string $name): bool
+     {
++        if (!$this->deploymentConfig->isDbAvailable()) {
++            return false;
++        };
++
+         $name = $this->addPrefix($name);
+ 
+         return (bool)$this->resource->getConnection()->query(
+             "SELECT IS_USED_LOCK(?);",
+-            [(string)$name]
++            [$name]
+         )->fetchColumn();
+     }
+ 
+@@ -123,7 +161,7 @@ class Database implements \Magento\Framework\Lock\LockManagerInterface
+      * Limited to 64 characters in MySQL.
+      *
+      * @param string $name
+-     * @return string $name
++     * @return string
+      * @throws InputException
+      */
+     private function addPrefix(string $name): string