Feat various improvement

.github/workflows/qa.yml

@@ -27,3 +27,5 @@ jobs:
         run: php ./vendor/bin/php-cs-fixer fix --diff --dry-run src
       - name: Phpstan
         run: php ./vendor/bin/phpstan --memory-limit=1G analyse
+      - name: Phpunit
+        run: php ./vendor/bin/phpunit src

README.md

@@ -23,3 +23,18 @@ composer require gally/oro-plugin:dev-master
 - They should also appear in your Oro frontend when searching or browsing categories.
 - And you're done !
 
+## Devlopement
+
+If you need to update indexation behavior, you need to test various way to run the reindex process in oro
+Fisrt you need to test various combinations of these options of the reindex command :
+```shell
+bin/console oro:website-search:reindex
+bin/console oro:website-search:reindex --scheduled # Async reindex with message queue
+bin/console oro:website-search:reindex --website-id=3 # Specify the websites you want to reindex
+bin/console oro:website-search:reindex --ids=1000-2000 # Specify the ids range you want to reindex
+bin/console oro:website-search:reindex --class="Oro\Bundle\WebCatalogBundle\Entity\ContentNode" # Specify the entities you want to reindex
+```
+
+Then you need to test update of product of contentNode from oro backend (you need to have an active consumer for this).
+You can run a consumer with the command `bin/console oro:message-queue:consume`.
+

composer.json

@@ -20,7 +20,7 @@
     "require-dev": {
         "friendsofphp/php-cs-fixer": "*",
         "phpstan/phpstan": "*",
-        "phpunit/phpunit": "*"
+        "phpunit/phpunit": "~9"
     },
     "autoload": {
         "psr-4": {

src/Convertor/LocalizationConvertor.php

@@ -0,0 +1,37 @@
+<?php
+/**
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade Gally to newer versions in the future.
+ *
+ * @package   Gally
+ * @author    Gally Team <[email protected]>
+ * @copyright 2024-present Smile
+ * @license   Open Software License v. 3.0 (OSL-3.0)
+ */
+
+declare(strict_types=1);
+
+namespace Gally\OroPlugin\Convertor;
+
+use Oro\Bundle\LocaleBundle\Entity\Localization;
+
+class LocalizationConvertor
+{
+    public static function getLocaleFormattingCode(Localization $localization): string
+    {
+        $pattern = '/^[a-z]{2}_[A-Z]{2}$/';
+
+        if (preg_match($pattern, $localization->getFormattingCode())) {
+            return $localization->getFormattingCode();
+        }
+
+        $code = $localization->getParentLocalization()?->getFormattingCode();
+
+        if (preg_match($pattern, $code)) {
+            return $code;
+        }
+
+        return 'en_US';
+    }
+}

src/Decorator/AddIndexNameInContext.php

@@ -0,0 +1,84 @@
+<?php
+/**
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade Gally to newer versions in the future.
+ *
+ * @package   Gally
+ * @author    Gally Team <[email protected]>
+ * @copyright 2024-present Smile
+ * @license   Open Software License v. 3.0 (OSL-3.0)
+ */
+
+declare(strict_types=1);
+
+namespace Gally\OroPlugin\Decorator;
+
+use Gally\OroPlugin\Indexer\Registry\IndexRegistry;
+use Oro\Bundle\WebsiteSearchBundle\Engine\AsyncMessaging\ReindexMessageGranularizer;
+use Oro\Bundle\WebsiteSearchBundle\Engine\Context\ContextTrait;
+
+/**
+ * Add index name and message count in message before add them in the queue.
+ */
+class AddIndexNameInContext extends ReindexMessageGranularizer
+{
+    use ContextTrait;
+
+    public function __construct(
+        private ReindexMessageGranularizer $decorated,
+        private IndexRegistry $indexRegistry,
+    ) {
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function setChunkSize($chunkSize)
+    {
+        $this->decorated->setChunkSize($chunkSize);
+    }
+
+    /**
+     * {@inheritDoc}
+     */
+    public function process($entities, array $websites, array $context): iterable
+    {
+        $entityIds = $this->getContextEntityIds($context);
+        $isFullIndexation = empty($entityIds);
+
+        // Add index names in queue message in order to be able to update existing index.
+        if (!isset($context['indices_by_locale'])) {
+            $context['indices_by_locale'] = $this->indexRegistry->getIndicesByLocale();
+        }
+
+        $messageCount = [];
+        $childMessages = [];
+        foreach ($this->decorated->process($entities, $websites, $context) as $childMessage) {
+            $entityClass = reset($childMessage['class']);
+
+            foreach ($childMessage['context']['websiteIds'] ?? [] as $websiteId) {
+                if (!isset($messageCount[$entityClass][$websiteId])) {
+                    $messageCount[$entityClass][$websiteId] = 0;
+                }
+                ++$messageCount[$entityClass][$websiteId];
+            }
+
+            if (!isset($messageCount[$entityClass]['global'])) {
+                $messageCount[$entityClass]['global'] = 0;
+            }
+            ++$messageCount[$entityClass]['global'];
+
+            $childMessages[] = $childMessage;
+        }
+
+        foreach ($childMessages as $childMessage) {
+            $entityClass = reset($childMessage['class']);
+            $childMessage['context']['indices_by_locale'] = $context['indices_by_locale'];
+            $childMessage['context']['message_count'] = $messageCount[$entityClass];
+            $childMessage['context']['is_full_indexation'] = $isFullIndexation;
+
+            yield $childMessage;
+        }
+    }
+}

src/Decorator/AddIndexNameInReindexMessage.php

@@ -0,0 +1,78 @@
+<?php
+/**
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade Gally to newer versions in the future.
+ *
+ * @package   Gally
+ * @author    Gally Team <[email protected]>
+ * @copyright 2024-present Smile
+ * @license   Open Software License v. 3.0 (OSL-3.0)
+ */
+
+declare(strict_types=1);
+
+namespace Gally\OroPlugin\Decorator;
+
+use Oro\Bundle\WebsiteSearchBundle\Engine\Context\ContextTrait;
+use Oro\Bundle\WebsiteSearchBundle\Engine\IndexerInputValidator;
+use Symfony\Component\OptionsResolver\OptionsResolver;
+
+/**
+ * Add gally specific data validation for reindex messages.
+ */
+class AddIndexNameInReindexMessage extends IndexerInputValidator
+{
+    use ContextTrait;
+
+    public function __construct(
+        private IndexerInputValidator $decorated,
+    ) {
+    }
+
+    public function validateRequestParameters(array|string|null $classOrClasses, array $context): array
+    {
+        $parameters = $this->validateClassAndContext(['class' => $classOrClasses, 'context' => $context]);
+
+        return [$parameters['class'], $this->getContextWebsiteIds($parameters['context'])];
+    }
+
+    public function validateClassAndContext(array $parameters): array
+    {
+        $resolver = $this->decorated->getOptionResolver();
+        $this->configureClassOptions($resolver);
+        $this->configureGranulizeOptions($resolver);
+        $this->configureContextOptions($resolver);
+
+        return $resolver->resolve($parameters);
+    }
+
+    public function configureContextOptions(OptionsResolver $optionsResolver): void
+    {
+        $this->decorated->configureContextOptions($optionsResolver);
+
+        $optionsResolver->setDefault('context', function (OptionsResolver $resolver) {
+            $resolver->setDefined('indices_by_locale');
+            $resolver->setDefined('message_count');
+            $resolver->setDefined('is_full_indexation');
+            $resolver->setAllowedTypes('indices_by_locale', ['array']);
+            $resolver->setAllowedTypes('message_count', ['array']);
+            $resolver->setAllowedTypes('is_full_indexation', ['bool']);
+        });
+    }
+
+    public function configureClassOptions(OptionsResolver $optionsResolver): void
+    {
+        $this->decorated->configureClassOptions($optionsResolver);
+    }
+
+    public function configureEntityOptions(OptionsResolver $optionsResolver): void
+    {
+        $this->decorated->configureEntityOptions($optionsResolver);
+    }
+
+    public function configureGranulizeOptions(OptionsResolver $optionsResolver): void
+    {
+        $this->decorated->configureGranulizeOptions($optionsResolver);
+    }
+}

src/Decorator/ProductIndexFieldsProvider.php

@@ -35,7 +35,6 @@ public function addForceIndexed(string $field): void
 
     public function isForceIndexed(string $field): bool
     {
-        //        return true;
         return $this->contextProvider->isGallyContext() || $this->productIndexAttributeProvider->isForceIndexed($field);
     }
 }

src/DependencyInjection/GallyOroExtension.php

@@ -31,6 +31,7 @@ public function load(array $configs, ContainerBuilder $container)
         $loader->load('services.yml');
         $loader->load('controllers.yml');
 
+        // Use class path in a string because this class might not exist if enterprise bundles are not installed.
         if (class_exists('Oro\Bundle\WarehouseBundle\Provider\EnabledWarehousesProvider')) {
             $loader->load('services/enterprise.yml');
         }

src/Entity/IndexBatchCount.php

@@ -0,0 +1,62 @@
+<?php
+/**
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade Gally to newer versions in the future.
+ *
+ * @package   Gally
+ * @author    Gally Team <[email protected]>
+ * @copyright 2024-present Smile
+ * @license   Open Software License v. 3.0 (OSL-3.0)
+ */
+
+declare(strict_types=1);
+
+namespace Gally\OroPlugin\Entity;
+
+use Doctrine\ORM\Mapping as ORM;
+
+/**
+ * Entity that record the messages received for an index in order to know when to install it.
+ */
+#[ORM\Entity]
+#[ORM\Table(name: 'gally_index_batch_count')]
+class IndexBatchCount
+{
+    #[ORM\Id]
+    #[ORM\Column(name: 'index_name', type: 'string', length: 255)]
+    private string $indexName;
+
+    #[ORM\Column(name: 'message_count', type: 'integer')]
+    private int $messageCount = 1;
+
+    public function __construct(string $indexName)
+    {
+        $this->indexName = $indexName;
+    }
+
+    public function increment(): void
+    {
+        ++$this->messageCount;
+    }
+
+    public function getIndexName(): string
+    {
+        return $this->indexName;
+    }
+
+    public function setIndexName(string $indexName): void
+    {
+        $this->indexName = $indexName;
+    }
+
+    public function getMessageCount(): int
+    {
+        return $this->messageCount;
+    }
+
+    public function setMessageCount(int $messageCount): void
+    {
+        $this->messageCount = $messageCount;
+    }
+}

src/Indexer/Event/BeforeSaveIndexDataEvent.php

@@ -0,0 +1,43 @@
+<?php
+/**
+ * DISCLAIMER
+ *
+ * Do not edit or add to this file if you wish to upgrade Gally to newer versions in the future.
+ *
+ * @package   Gally
+ * @author    Gally Team <[email protected]>
+ * @copyright 2024-present Smile
+ * @license   Open Software License v. 3.0 (OSL-3.0)
+ */
+
+declare(strict_types=1);
+
+namespace Gally\OroPlugin\Indexer\Event;
+
+use Symfony\Contracts\EventDispatcher\Event;
+
+class BeforeSaveIndexDataEvent extends Event
+{
+    public const NAME = 'gally.indexer.before_save_index_data';
+
+    public function __construct(
+        private string $class,
+        private array $data
+    ) {
+    }
+
+    public function setData(array $data): void
+    {
+        $this->data = $data;
+    }
+
+    public function getData(): array
+    {
+        return $this->data;
+    }
+
+    public function getClass(): string
+    {
+        return $this->class;
+    }
+}

src/Indexer/IndexDataProvider.php

@@ -113,6 +113,10 @@ private function prepareIndexData(
                 );
             }
 
+            /**
+             * @var string $fieldName
+             * @var array  $values
+             */
             foreach ($this->toArray($fieldsValues) as $fieldName => $values) {
                 $singleValueFieldName = $this->cleanFieldName((string) $fieldName);
                 foreach ($this->toArray($values) as $value) {