Sync: review sync context handling

- Rename `claimFilterValue()` to `claimFilter()`
- Rename `getFilter()` to `getFilters()`
- Add `DeferredSyncEntityPolicy`
- Add `withDeferredSyncEntityPolicy()` and
  `getDeferredSyncEntityPolicy()` to `ISyncContext` / `SyncContext`
- Allow entities resolved by `SyncStore::resolveDeferredEntities()` to
  be scoped to entities deferred since a checkpoint returned by
  `SyncStore::getDeferredEntityCheckpoint()`
- Apply deferred sync entity policies in `SyncEntityProvider`

Author: lkrms

src/Facade/Sync.php

@@ -26,6 +26,7 @@
  * @method static SyncStore entity(int $providerId, class-string<ISyncEntity> $entityType, int|string $entityId, ISyncEntity $entity) Register a sync entity (see {@see SyncStore::entity()})
  * @method static SyncStore entityType(class-string<ISyncEntity> $entity) Register a sync entity type and set its ID (unless already registered) (see {@see SyncStore::entityType()})
  * @method static SyncStore error(SyncError|SyncErrorBuilder $error, bool $deduplicate = false, bool $toConsole = false) Report an error that occurred during a sync operation
+ * @method static int getDeferredEntityCheckpoint() Get a checkpoint to delineate between entities already in the deferred entity queue and any subsequently deferred sync entities (see {@see SyncStore::getDeferredEntityCheckpoint()})
  * @method static ISyncEntity|null getEntity(int $providerId, class-string<ISyncEntity> $entityType, int|string $entityId, bool|null $offline = null) Get a previously registered and/or stored sync entity (see {@see SyncStore::getEntity()})
  * @method static string|null getEntityTypeNamespace(class-string<ISyncEntity> $entity) Get the namespace of a sync entity type (see {@see SyncStore::getEntityTypeNamespace()})
  * @method static string|null getEntityTypeUri(class-string<ISyncEntity> $entity, bool $compact = true) Get the canonical URI of a sync entity type (see {@see SyncStore::getEntityTypeUri()})
@@ -39,7 +40,7 @@
  * @method static SyncStore namespace(string $prefix, string $uri, string $namespace, class-string<ISyncClassResolver>|null $resolver = null) Register a sync entity namespace (see {@see SyncStore::namespace()})
  * @method static SyncStore provider(ISyncProvider $provider) Register a sync provider and set its provider ID (see {@see SyncStore::provider()})
  * @method static SyncStore reportErrors(string $successText = 'No sync errors recorded') Report sync operation errors to the console (see {@see SyncStore::reportErrors()})
- * @method static ISyncEntity[] resolveDeferredEntities(?int $providerId = null, class-string<ISyncEntity> $entityType = null, bool|null $offline = null) Resolve deferred sync entities from their respective providers and/or the local entity store (see {@see SyncStore::resolveDeferredEntities()})
+ * @method static ISyncEntity[] resolveDeferredEntities(?int $fromCheckpoint = null, ?int $providerId = null, class-string<ISyncEntity> $entityType = null, bool|null $offline = null) Resolve deferred sync entities from their respective providers and/or the local entity store (see {@see SyncStore::resolveDeferredEntities()})
  *
  * @uses SyncStore
  *

src/Sync/Catalog/DeferredSyncEntityPolicy.php

@@ -0,0 +1,42 @@
+<?php declare(strict_types=1);
+
+namespace Lkrms\Sync\Catalog;
+
+use Lkrms\Concept\Enumeration;
+use Lkrms\Sync\Support\DeferredSyncEntity;
+use Lkrms\Sync\Support\SyncStore;
+
+/**
+ * Policies for sync entity deferral
+ *
+ * @extends Enumeration<int>
+ */
+final class DeferredSyncEntityPolicy extends Enumeration
+{
+    /**
+     * Do not resolve deferred entities
+     *
+     * If {@see SyncStore::resolveDeferredEntities()} is not called manually,
+     * unresolved {@see DeferredSyncEntity} instances may appear in object
+     * graphs returned by sync operations.
+     */
+    public const DO_NOT_RESOLVE = 0;
+
+    /**
+     * Resolve deferred entities immediately
+     *
+     * This is the least efficient policy because it produces the most round
+     * trips, but it does guarantee the return of fully resolved object graphs.
+     */
+    public const RESOLVE_EARLY = 1;
+
+    /**
+     * Resolve deferred entities after reaching the end of each stream of entity
+     * data
+     *
+     * This policy minimises the number of round trips to the backend, but
+     * unresolved {@see DeferredSyncEntity} instances may appear in object
+     * graphs until they have been fully traversed.
+     */
+    public const RESOLVE_LATE = 2;
+}

src/Sync/Command/GetSyncEntities.php

@@ -7,10 +7,8 @@
 use Lkrms\Cli\CliOption;
 use Lkrms\Facade\Console;
 use Lkrms\Facade\File;
-use Lkrms\Sync\Contract\ISyncContext;
 use Lkrms\Sync\Contract\ISyncEntity;
 use Lkrms\Sync\Contract\ISyncProvider;
-use Lkrms\Sync\Support\SyncContext;
 use Lkrms\Sync\Support\SyncIntrospector;
 use Lkrms\Sync\Support\SyncSerializeRules;
 use Lkrms\Utility\Convert;

src/Sync/Concept/SyncDefinition.php

@@ -33,7 +33,7 @@
  * @property-read TProvider $Provider The ISyncProvider servicing the entity
  * @property-read array<SyncOperation::*> $Operations A list of supported sync operations
  * @property-read ArrayKeyConformity::* $Conformity The conformity level of data returned by the provider for this entity
- * @property-read SyncFilterPolicy::* $FilterPolicy The action to take when filters are ignored by the provider
+ * @property-read SyncFilterPolicy::* $FilterPolicy The action to take when filters are unclaimed by the provider
  * @property-read array<SyncOperation::*,Closure(ISyncDefinition<TEntity,TProvider>, SyncOperation::*, ISyncContext, mixed...): mixed> $Overrides An array that maps sync operations to closures that override any other implementations
  * @property-read IPipeline<mixed[],TEntity,array{0:int,1:ISyncContext,2?:int|string|TEntity|TEntity[]|null,...}>|null $PipelineFromBackend A pipeline that maps data from the provider to entity-compatible associative arrays, or `null` if mapping is not required
  * @property-read IPipeline<TEntity,mixed[],array{0:int,1:ISyncContext,2?:int|string|TEntity|TEntity[]|null,...}>|null $PipelineToBackend A pipeline that maps serialized entities to data compatible with the provider, or `null` if mapping is not required
@@ -101,13 +101,13 @@ abstract protected function getClosure(int $operation): ?Closure;
     protected $Conformity;
 
     /**
-     * The action to take when filters are ignored by the provider
+     * The action to take when filters are unclaimed by the provider
      *
      * To prevent a request for entities that meet one or more criteria
      * inadvertently reaching the backend as a request for a larger set of
      * entities--if not all of them--the default policy if there are unclaimed
      * filters is {@see SyncFilterPolicy::THROW_EXCEPTION}. See
-     * {@see SyncFilterPolicy} for alternative policies or
+     * {@see SyncFilterPolicy} for alternative policies and
      * {@see ISyncContext::withArgs()} for more information about filters.
      *
      * @var SyncFilterPolicy::*
@@ -369,7 +369,7 @@ function (array $data, IPipeline $pipeline, $arg) use (&$ctx, &$closure) {
     }
 
     /**
-     * Enforce the ignored filter policy
+     * Enforce the unclaimed filter policy
      *
      * @param SyncOperation::* $operation
      * @param array{}|null $empty
@@ -381,7 +381,7 @@ final protected function applyFilterPolicy(int $operation, ISyncContext $ctx, ?b
         $returnEmpty = false;
 
         if (SyncFilterPolicy::IGNORE === $this->FilterPolicy ||
-                !($filter = $ctx->getFilter())) {
+                !($filter = $ctx->getFilters())) {
             return;
         }
 

src/Sync/Contract/ISyncContext.php

@@ -3,24 +3,25 @@
 namespace Lkrms\Sync\Contract;
 
 use Lkrms\Contract\IProviderContext;
+use Lkrms\Sync\Catalog\DeferredSyncEntityPolicy;
 use Lkrms\Sync\Catalog\SyncFilterPolicy;
 use Lkrms\Sync\Catalog\SyncOperation;
 use Lkrms\Sync\Exception\SyncInvalidFilterException;
 
 /**
- * The context within which a sync entity is instantiated
+ * The context within which a sync entity is instantiated by a provider
  *
  */
 interface ISyncContext extends IProviderContext
 {
     /**
-     * Convert non-mandatory sync operation arguments to a normalised filter and
-     * add it to the context
+     * Normalise non-mandatory sync operation arguments and add them to the
+     * context
      *
      * If, after removing the operation's mandatory arguments from `$args`, the
      * remaining values match one of the following signatures, they are mapped
-     * to an associative array and returned by {@see ISyncContext::getFilter()}
-     * until updated by another call to {@see ISyncContext::withArgs()}.
+     * to an associative array surfaced by {@see ISyncContext::getFilters()} and
+     * {@see ISyncContext::claimFilter()}:
      *
      * 1. One array argument (`fn(...$mandatoryArgs, array $filter)`)
      *    - Alphanumeric keys are converted to snake_case
@@ -41,9 +42,12 @@ interface ISyncContext extends IProviderContext
      * If `$args` doesn't match any of these, a
      * {@see SyncInvalidFilterException} is thrown.
      *
-     * Using {@see ISyncContext::claimFilterValue()} to claim values from the
-     * filter is recommended. Depending on the provider's
-     * {@see SyncFilterPolicy}, unclaimed values may cause requests to fail.
+     * Using {@see ISyncContext::claimFilter()} to claim filters is recommended.
+     * Depending on the provider's {@see SyncFilterPolicy}, unclaimed filters
+     * may cause requests to fail.
+     *
+     * When a filter is claimed, it is removed from the context.
+     * {@see ISyncContext::getFilters()} only returns unclaimed filters.
      *
      * {@see ISyncEntity} objects are replaced with the return value of
      * {@see ISyncEntity::id()} when `$args` contains an array or a list of
@@ -57,27 +61,50 @@ interface ISyncContext extends IProviderContext
     public function withArgs(int $operation, ...$args);
 
     /**
-     * Use a callback to enforce the provider's ignored filter policy
+     * Use a callback to enforce the provider's unclaimed filter policy
      *
      * Allows providers to enforce their {@see SyncFilterPolicy} by calling
      * {@see ISyncContext::maybeApplyFilterPolicy()} in scenarios where
      * enforcement before a sync operation starts isn't possible.
      *
+     * @see ISyncContext::maybeApplyFilterPolicy()
+     *
      * @param (callable(ISyncContext, ?bool &$returnEmpty, array{}|null &$empty): void)|null $callback
      * @return $this
      */
     public function withFilterPolicyCallback(?callable $callback);
 
     /**
-     * Run the ignored filter policy callback (if provided)
+     * Apply a deferred sync entity policy to the context
+     *
+     * @param DeferredSyncEntityPolicy::* $policy
+     * @return $this
+     */
+    public function withDeferredSyncEntityPolicy(int $policy);
+
+    /**
+     * Run the unclaimed filter policy callback
      *
      * Example:
      *
      * ```php
      * <?php
-     * $ctx->maybeApplyFilterPolicy($returnEmpty, $empty);
-     * if ($returnEmpty) {
-     *     return $empty;
+     * class Provider extends \Lkrms\Sync\Concept\HttpSyncProvider
+     * {
+     *     public function getList_Entity(\Lkrms\Sync\Contract\ISyncContext $ctx): iterable
+     *     {
+     *         if ($ctx->claimFilter('pending')) {
+     *             $entryTypes[] = 0;
+     *         }
+     *         if ($ctx->claimFilter('completed')) {
+     *             $entryTypes[] = 1;
+     *         }
+     *         $ctx->maybeApplyFilterPolicy($returnEmpty, $empty);
+     *         if ($returnEmpty) {
+     *             return $empty;
+     *         }
+     *         // ...
+     *     }
      * }
      * ```
      *
@@ -92,19 +119,26 @@ public function maybeApplyFilterPolicy(?bool &$returnEmpty, &$empty): void;
      *
      * @return array<string,mixed>
      */
-    public function getFilter(): array;
+    public function getFilters(): array;
 
     /**
      * Get a value from the filter most recently passed via optional sync
      * operation arguments
      *
      * Unlike other {@see ISyncContext} methods,
-     * {@see ISyncContext::claimFilterValue()} modifies the object it is called
+     * {@see ISyncContext::claimFilter()} modifies the object it is called
      * on instead of returning a modified clone.
      *
      * @return mixed `null` if the value has already been claimed or wasn't
      * passed to the operation.
      * @see ISyncContext::withArgs()
      */
-    public function claimFilterValue(string $key);
+    public function claimFilter(string $key);
+
+    /**
+     * Get the deferred sync entity policy applied to the context
+     *
+     * @return DeferredSyncEntityPolicy::*
+     */
+    public function getDeferredSyncEntityPolicy(): int;
 }

src/Sync/Support/DbSyncDefinitionBuilder.php

@@ -26,7 +26,7 @@
  * @method $this operations(array<SyncOperation::*> $value) A list of supported sync operations
  * @method $this table(?string $value) Set DbSyncDefinition::$Table
  * @method $this conformity(ArrayKeyConformity::* $value) The conformity level of data returned by the provider for this entity (see {@see SyncDefinition::$Conformity})
- * @method $this filterPolicy(SyncFilterPolicy::* $value) The action to take when filters are ignored by the provider (see {@see SyncDefinition::$FilterPolicy})
+ * @method $this filterPolicy(SyncFilterPolicy::* $value) The action to take when filters are unclaimed by the provider (see {@see SyncDefinition::$FilterPolicy})
  * @method $this overrides(array<SyncOperation::*,Closure(ISyncDefinition<TEntity,TProvider>, SyncOperation::*, ISyncContext, mixed...): mixed> $value) An array that maps sync operations to closures that override any other implementations (see {@see SyncDefinition::$Overrides})
  * @method $this pipelineFromBackend(IPipeline<mixed[],TEntity,array{0:int,1:ISyncContext,2?:int|string|TEntity|TEntity[]|null,...}>|null $value) A pipeline that maps data from the provider to entity-compatible associative arrays, or `null` if mapping is not required
  * @method $this pipelineToBackend(IPipeline<TEntity,mixed[],array{0:int,1:ISyncContext,2?:int|string|TEntity|TEntity[]|null,...}>|null $value) A pipeline that maps serialized entities to data compatible with the provider, or `null` if mapping is not required

src/Sync/Support/HttpSyncDefinitionBuilder.php

@@ -32,7 +32,7 @@
  * @method $this pager(?ICurlerPager $value) The pagination handler for the endpoint servicing the entity (see {@see HttpSyncDefinition::$Pager})
  * @method $this callback((callable(HttpSyncDefinition<TEntity,TProvider>, SyncOperation::*, ISyncContext, mixed...): HttpSyncDefinition<TEntity,TProvider>)|null $value) A callback applied to the definition before every sync operation (see {@see HttpSyncDefinition::$Callback})
  * @method $this conformity(ArrayKeyConformity::* $value) The conformity level of data returned by the provider for this entity (see {@see SyncDefinition::$Conformity})
- * @method $this filterPolicy(SyncFilterPolicy::* $value) The action to take when filters are ignored by the provider (see {@see SyncDefinition::$FilterPolicy})
+ * @method $this filterPolicy(SyncFilterPolicy::* $value) The action to take when filters are unclaimed by the provider (see {@see SyncDefinition::$FilterPolicy})
  * @method $this expiry(?int $value) The time, in seconds, before responses from the provider expire (see {@see HttpSyncDefinition::$Expiry})
  * @method $this methodMap(array<SyncOperation::*,string> $value) An array that maps sync operations to HTTP request methods (see {@see HttpSyncDefinition::$MethodMap})
  * @method $this syncOneEntityPerRequest(bool $value = true) If true, perform CREATE_LIST, UPDATE_LIST and DELETE_LIST operations on one entity per HTTP request (default: false)

src/Sync/Support/SyncContext.php

@@ -3,6 +3,7 @@
 namespace Lkrms\Sync\Support;
 
 use Lkrms\Support\ProviderContext;
+use Lkrms\Sync\Catalog\DeferredSyncEntityPolicy;
 use Lkrms\Sync\Catalog\SyncOperation;
 use Lkrms\Sync\Contract\ISyncContext;
 use Lkrms\Sync\Contract\ISyncEntity;
@@ -11,26 +12,37 @@
 use Lkrms\Utility\Test;
 
 /**
- * The context within which a sync entity is instantiated
+ * The context within which a sync entity is instantiated by a provider
  *
  */
 final class SyncContext extends ProviderContext implements ISyncContext
 {
     /**
      * @var array<string,mixed>
      */
-    protected array $Filter = [];
+    protected array $Filters = [];
 
     /**
      * @var (callable(ISyncContext, ?bool &$returnEmpty, mixed &$empty): void)|null
      */
     protected $FilterPolicyCallback;
 
+    /**
+     * @var DeferredSyncEntityPolicy::*
+     */
+    protected $DeferredSyncEntityPolicy = DeferredSyncEntityPolicy::RESOLVE_EARLY;
+
+    /**
+     * @inheritDoc
+     */
     public function withFilterPolicyCallback(?callable $callback)
     {
         return $this->withPropertyValue('FilterPolicyCallback', $callback);
     }
 
+    /**
+     * @inheritDoc
+     */
     public function maybeApplyFilterPolicy(?bool &$returnEmpty, &$empty): void
     {
         $returnEmpty = false;
@@ -40,6 +52,9 @@ public function maybeApplyFilterPolicy(?bool &$returnEmpty, &$empty): void
         }
     }
 
+    /**
+     * @inheritDoc
+     */
     public function withArgs(int $operation, ...$args)
     {
         // READ_LIST is the only operation with no mandatory argument after
@@ -49,11 +64,11 @@ public function withArgs(int $operation, ...$args)
         }
 
         if (empty($args)) {
-            return $this->withPropertyValue('Filter', []);
+            return $this->withPropertyValue('Filters', []);
         }
 
         if (is_array($args[0]) && count($args) === 1) {
-            return $this->withPropertyValue('Filter', array_combine(
+            return $this->withPropertyValue('Filters', array_combine(
                 array_map(
                     fn($key) =>
                         preg_match('/[^[:alnum:]_-]/', $key) ? $key : Convert::toSnakeCase($key),
@@ -68,7 +83,7 @@ public function withArgs(int $operation, ...$args)
         }
 
         if (Test::isArrayOfArrayKey($args)) {
-            return $this->withPropertyValue('Filter', ['id' => $args]);
+            return $this->withPropertyValue('Filters', ['id' => $args]);
         }
 
         if (Test::isArrayOf($args, ISyncEntity::class)) {
@@ -88,20 +103,42 @@ public function withArgs(int $operation, ...$args)
         throw new SyncInvalidFilterException(...$args);
     }
 
-    public function getFilter(): array
+    /**
+     * @inheritDoc
+     */
+    public function withDeferredSyncEntityPolicy(int $policy)
     {
-        return $this->Filter;
+        return $this->withPropertyValue('DeferredSyncEntityPolicy', $policy);
     }
 
-    public function claimFilterValue(string $key)
+    /**
+     * @inheritDoc
+     */
+    public function getFilters(): array
+    {
+        return $this->Filters;
+    }
+
+    /**
+     * @inheritDoc
+     */
+    public function claimFilter(string $key)
     {
-        if (array_key_exists($key, $this->Filter)) {
-            $value = $this->Filter[$key];
-            unset($this->Filter[$key]);
+        if (array_key_exists($key, $this->Filters)) {
+            $value = $this->Filters[$key];
+            unset($this->Filters[$key]);
 
             return $value;
         }
 
         return null;
     }
+
+    /**
+     * @inheritDoc
+     */
+    public function getDeferredSyncEntityPolicy(): int
+    {
+        return $this->DeferredSyncEntityPolicy;
+    }
 }