From 25bada2f9b8bcc13dbf6b903945e53d8af642b06 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Sun, 2 Aug 2026 01:38:28 -0100 Subject: [PATCH] trigger remote sync on local file modification Signed-off-by: Maxence Lange --- appinfo/routes.php | 1 + lib/AppInfo/Application.php | 8 ++ lib/BackgroundJobs/NotifyRemoteFile.php | 38 +++++++ lib/ConfigLexicon.php | 2 + lib/Controller/SlaveController.php | 30 +++++ lib/Db/FileRequest.php | 28 +++++ lib/Db/ShareRequest.php | 11 +- lib/Exceptions/RemoteIsLocalException.php | 15 +++ lib/Listeners/SharedFileRefresh.php | 59 ++++++++++ lib/Model/FederatedShare.php | 15 ++- lib/Service/GlobalShareService.php | 129 ++++++++++++++++++++-- 11 files changed, 321 insertions(+), 15 deletions(-) create mode 100644 lib/BackgroundJobs/NotifyRemoteFile.php create mode 100644 lib/Exceptions/RemoteIsLocalException.php create mode 100644 lib/Listeners/SharedFileRefresh.php diff --git a/appinfo/routes.php b/appinfo/routes.php index 6d1f187d..543fbd47 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -11,6 +11,7 @@ ['name' => 'Slave#createAppToken', 'url' => '/v1/createapptoken', 'verb' => 'GET'], ['name' => 'Slave#discovery', 'url' => '/discovery', 'verb' => 'GET'], ['name' => 'Slave#sharedFile', 'url' => '/sharedfile', 'verb' => 'GET'], + ['name' => 'Slave#refreshSharedFile', 'url' => '/refreshSharedFile', 'verb' => 'GET'], ], 'routes' => [ [ diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 0d070206..6e00807a 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -16,6 +16,7 @@ use OCA\GlobalSiteSelector\GlobalSiteSelector; use OCA\GlobalSiteSelector\Listeners\AddContentSecurityPolicyListener; use OCA\GlobalSiteSelector\Listeners\DeletingUser; +use OCA\GlobalSiteSelector\Listeners\SharedFileRefresh; use OCA\GlobalSiteSelector\Listeners\UserChanged; use OCA\GlobalSiteSelector\Listeners\UserCreated; use OCA\GlobalSiteSelector\Listeners\UserDeleted; @@ -30,6 +31,9 @@ use OCP\AppFramework\Bootstrap\IBootstrap; use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\EventDispatcher\IEventDispatcher; +use OCP\Files\Events\Node\NodeDeletedEvent; +use OCP\Files\Events\Node\NodeRenamedEvent; +use OCP\Files\Events\Node\NodeWrittenEvent; use OCP\IRequest; use OCP\IUser; use OCP\IUserManager; @@ -86,6 +90,10 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(UserLoggedOutEvent::class, UserLoggedOut::class); $context->registerEventListener(UserChangedEvent::class, UserChanged::class); + $context->registerEventListener(NodeWrittenEvent::class, SharedFileRefresh::class); + $context->registerEventListener(NodeRenamedEvent::class, SharedFileRefresh::class); + $context->registerEventListener(NodeDeletedEvent::class, SharedFileRefresh::class); + $context->registerSetupCheck(LongJwtKeySetupCheck::class); $context->registerConfigLexicon(ConfigLexicon::class); diff --git a/lib/BackgroundJobs/NotifyRemoteFile.php b/lib/BackgroundJobs/NotifyRemoteFile.php new file mode 100644 index 00000000..cc05e3f8 --- /dev/null +++ b/lib/BackgroundJobs/NotifyRemoteFile.php @@ -0,0 +1,38 @@ + $shares) { + $this->globalShareService->requestRemoteFileRefresh($instance, $shares); + } + } +} diff --git a/lib/ConfigLexicon.php b/lib/ConfigLexicon.php index c53034f5..92448e83 100644 --- a/lib/ConfigLexicon.php +++ b/lib/ConfigLexicon.php @@ -17,6 +17,7 @@ class ConfigLexicon implements ILexicon { public const GS_TOKENS = 'globalScaleTokens'; public const LOCAL_TOKEN = 'localToken'; public const REDIRECT_WEBDAV = 'redirectWebDAV'; + public const INSTANCE_MAIN_THREAD = 'requested_instance_main_thread'; #[\Override] public function getStrictness(): Strictness { @@ -32,6 +33,7 @@ public function getAppConfigs(): array { new Entry(key: self::GS_TOKENS, type: ValueType::ARRAY, defaultRaw: [], definition: 'list of token+host to navigate through GlobalScale', lazy: true), new Entry(key: self::LOCAL_TOKEN, type: ValueType::STRING, defaultRaw: '', definition: 'local token to id instance within GlobalScale', lazy: true), new Entry(key: self::REDIRECT_WEBDAV, type: ValueType::BOOL, defaultRaw: false, definition: 'redirect WebDAV request on Master to Slaves', lazy: false), + new Entry(key: self::INSTANCE_MAIN_THREAD, type: ValueType::INT, defaultRaw: 2, definition: 'when running event requests, maximum number of instances to reach before switching to background job', lazy: false), ]; } diff --git a/lib/Controller/SlaveController.php b/lib/Controller/SlaveController.php index dfd572ee..aafa45ba 100644 --- a/lib/Controller/SlaveController.php +++ b/lib/Controller/SlaveController.php @@ -15,6 +15,7 @@ use OCA\GlobalSiteSelector\Exceptions\MasterUrlException; use OCA\GlobalSiteSelector\Exceptions\SharedFileException; use OCA\GlobalSiteSelector\GlobalSiteSelector; +use OCA\GlobalSiteSelector\Model\FederatedShare; use OCA\GlobalSiteSelector\Model\LocalFile; use OCA\GlobalSiteSelector\Service\GlobalScaleService; use OCA\GlobalSiteSelector\Service\GlobalShareService; @@ -126,6 +127,35 @@ public function sharedFile(string $jwt): DataResponse { } + + /** + * initiate refresh on local versions of a remote federated file. + * request must contain encoded jwt. + */ + #[PublicPage] + #[NoCSRFRequired] + public function refreshSharedFile(string $jwt): DataResponse { + $key = $this->gss->getJwtKey(); + $decoded = (array)JWT::decode($jwt, new Key($key, Application::JWT_ALGORITHM)); + // JWT store data as stdClass, not array + $decoded = json_decode(json_encode($decoded), true); + $this->logger->debug('decoded request', ['data' => $decoded]); + $instance = $decoded['instance'] ?? ''; + + foreach ($decoded['shares'] as $entry) { + $federatedShare = new FederatedShare(); + $federatedShare->import($entry); + $this->globalShareService->refreshSharedTarget( + $instance, + $federatedShare->getId(), + $federatedShare->getShareToken(), + $federatedShare->getTarget() + ); + } + + return new DataResponse([]); + } + #[PublicPage] #[NoCSRFRequired] #[UseSession] diff --git a/lib/Db/FileRequest.php b/lib/Db/FileRequest.php index 367c4a75..d38ad6cc 100644 --- a/lib/Db/FileRequest.php +++ b/lib/Db/FileRequest.php @@ -185,6 +185,34 @@ public function getTeamStorages(FederatedShare $federatedShare, string $instance return $storage; } + /** + * returns an array containing user and mountpoint from an external share; based on remote + * instance that owns the file, the shareId on the remote instance and + * the share token. + * + * If not known, user and mountpoint are null in the returned array. + */ + public function getMountPointFromShare(string $instance, int $remoteId, string $shareToken): array { + $qb = $this->connection->getQueryBuilder(); + $qb->select('user', 'mountpoint') + ->from('share_external') + ->where( + $qb->expr()->andX( + $qb->expr()->like('remote', $qb->createNamedParameter('%://' . str_replace('%', '', $instance) . '/')), + $qb->expr()->eq('remote_id', $qb->createNamedParameter($remoteId, IQueryBuilder::PARAM_INT)), + $qb->expr()->eq('share_token', $qb->createNamedParameter($shareToken)), + ) + ); + + $result = $qb->executeQuery(); + $row = $result->fetch(); + if ($row === false) { + return [null, null]; + } + + return [$row['user'], $row['mountpoint']]; + } + /** * returns the mount using the id of a node, * userid can then be extracted and used to retrieve the file's root folder diff --git a/lib/Db/ShareRequest.php b/lib/Db/ShareRequest.php index 9ca88d34..c4ef6785 100644 --- a/lib/Db/ShareRequest.php +++ b/lib/Db/ShareRequest.php @@ -25,12 +25,14 @@ public function __construct( /** * returns list of existing federated shares providing access to a list * of files, in relation to the specified instance. + * if instance is NULL then all instances are returned * * @param LocalFile[] $files + * @param string|null $instance * * @return FederatedShare[] */ - public function getFederatedSharesRelatedToRemoteInstance(array $files, string $instance): array { + public function getFederatedSharesRelatedToRemoteInstance(array $files, ?string $instance = null): array { $indexedFiles = $ids = []; foreach ($files as $entry) { $indexedFiles[$entry->getId()] = $entry; @@ -38,7 +40,7 @@ public function getFederatedSharesRelatedToRemoteInstance(array $files, string $ } $qb = $this->connection->getQueryBuilder(); - $qb->select('s.id', 's.file_source', 's.share_type', 's.share_with', 's.permissions') + $qb->select('s.id', 's.file_source', 's.share_type', 's.share_with', 's.permissions', 's.token') ->from('share', 's') ->where( $qb->expr()->andX( @@ -46,7 +48,7 @@ public function getFederatedSharesRelatedToRemoteInstance(array $files, string $ $qb->expr()->orX( $qb->expr()->andX( $qb->expr()->in('share_type', $qb->createNamedParameter([IShare::TYPE_REMOTE, IShare::TYPE_REMOTE_GROUP], IQueryBuilder::PARAM_INT_ARRAY)), - $qb->expr()->like('share_with', $qb->createNamedParameter('%@' . $instance)), + $qb->expr()->like('share_with', $qb->createNamedParameter('%@' . ($instance ?? '%'))), ), $qb->expr()->in('share_type', $qb->createNamedParameter([IShare::TYPE_CIRCLE], IQueryBuilder::PARAM_INT_ARRAY)), ) @@ -57,7 +59,7 @@ public function getFederatedSharesRelatedToRemoteInstance(array $files, string $ $shares = []; while ($row = $result->fetch()) { $shareWith = $row['share_with']; - if (str_ends_with(strtolower($shareWith), '@' . strtolower($instance))) { + if ($instance !== null && str_ends_with(strtolower($shareWith), '@' . strtolower($instance))) { $shareWith = substr($shareWith, 0, -strlen('@' . $instance)); } @@ -65,6 +67,7 @@ public function getFederatedSharesRelatedToRemoteInstance(array $files, string $ $federatedShare->setId($row['id']) ->setFileId($row['file_source']) ->setShareType($row['share_type']) + ->setShareToken($row['token']) ->setShareWith($shareWith) ->setPermissions($row['permissions']) ->setTarget($indexedFiles[$row['file_source']]); diff --git a/lib/Exceptions/RemoteIsLocalException.php b/lib/Exceptions/RemoteIsLocalException.php new file mode 100644 index 00000000..fa3d4f53 --- /dev/null +++ b/lib/Exceptions/RemoteIsLocalException.php @@ -0,0 +1,15 @@ + + */ +class SharedFileRefresh implements IEventListener { + public function __construct( + private readonly GlobalShareService $globalShareService, + private readonly LoggerInterface $logger, + ) { + } + + /** + * @param Event $event + */ + #[\Override] + public function handle(Event $event): void { + switch (get_class($event)) { + case NodeWrittenEvent::class: + $fileId = $event->getNode()->getId(); + break; + + case NodeRenamedEvent::class: + $fileId = $event->getTarget()->getId(); + break; + + case NodeDeletedEvent::class: + $fileId = $event->getNode()->getParentId(); + break; + + default: + return; + } + + try { + // file is modified locally, broadcasting the event to other instances + $this->globalShareService->refreshFileAcrossGlobalScale($fileId); + } catch (Throwable $e) { + $this->logger->warning('issue while refreshing file across GS', ['exception' => $e]); + } + } +} diff --git a/lib/Model/FederatedShare.php b/lib/Model/FederatedShare.php index 6dec2dd9..61f30558 100644 --- a/lib/Model/FederatedShare.php +++ b/lib/Model/FederatedShare.php @@ -16,6 +16,7 @@ class FederatedShare implements JsonSerializable { private int $fileId = 0; private int $shareType = 0; private string $shareWith = ''; + private string $shareToken = ''; private int $permissions = 0; private bool $bounce = false; private string $remote = ''; @@ -62,6 +63,15 @@ public function getShareWith(): string { return $this->shareWith; } + public function setShareToken(string $shareToken): self { + $this->shareToken = $shareToken; + return $this; + } + + public function getShareToken(): string { + return $this->shareToken; + } + public function setPermissions(int $permissions): self { $this->permissions = $permissions; return $this; @@ -120,6 +130,7 @@ public function import(array $data): self { ->setFileId($data['fileId'] ?? 0) ->setShareType($data['shareType'] ?? 0) ->setShareWith($data['shareWith'] ?? '') + ->setShareToken($data['shareToken'] ?? '') ->setPermissions($data['permissions'] ?? 0); } @@ -133,7 +144,7 @@ public function import(array $data): self { } /** - * @return array{id: int, fileId: int, shareType: int, shareWith: string, permissions: int, target: array, remote: string, remoteId: int} + * @return array{id: int, fileId: int, shareType: int, shareWith: string, shareToken: string, permissions: int, target: array, remote: string, remoteId: int} */ #[\Override] public function jsonSerialize(): array { @@ -151,9 +162,9 @@ public function jsonSerialize(): array { 'fileId' => $this->getFileId(), 'shareType' => $this->getShareType(), 'shareWith' => $this->getShareWith(), + 'shareToken' => $this->getShareToken(), 'permissions' => $this->getPermissions(), 'target' => $this->getTarget(), ]; - } } diff --git a/lib/Service/GlobalShareService.php b/lib/Service/GlobalShareService.php index 175b192d..900b3c18 100644 --- a/lib/Service/GlobalShareService.php +++ b/lib/Service/GlobalShareService.php @@ -9,26 +9,33 @@ namespace OCA\GlobalSiteSelector\Service; use Exception; +use OC\User\NoUserException; use OCA\Circles\CirclesManager; use OCA\Circles\Model\Circle; use OCA\Files_Sharing\External\MountProvider; use OCA\GlobalSiteSelector\AppInfo\Application; +use OCA\GlobalSiteSelector\BackgroundJobs\NotifyRemoteFile; +use OCA\GlobalSiteSelector\ConfigLexicon; use OCA\GlobalSiteSelector\Db\FileRequest; use OCA\GlobalSiteSelector\Db\ShareRequest; use OCA\GlobalSiteSelector\Exceptions\LocalFederatedShareException; +use OCA\GlobalSiteSelector\Exceptions\RemoteIsLocalException; use OCA\GlobalSiteSelector\Exceptions\SharedFileException; use OCA\GlobalSiteSelector\GlobalSiteSelector; use OCA\GlobalSiteSelector\Model\FederatedShare; use OCA\GlobalSiteSelector\Model\LocalFile; use OCA\GlobalSiteSelector\Vendor\Firebase\JWT\JWT; use OCP\AppFramework\Http; +use OCP\BackgroundJob\IJobList; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; +use OCP\IAppConfig; use OCP\IGroupManager; use OCP\IUserManager; use OCP\IUserSession; use OCP\Share\IShare; +use OCP\User\Exceptions\UserNotFoundException; use Psr\Log\LoggerInterface; use UnhandledMatchError; @@ -37,10 +44,12 @@ class GlobalShareService { private array $currentGroups = []; private array $currentTeams = []; public function __construct( + private readonly IAppConfig $appConfig, private readonly IRootFolder $rootFolder, private readonly IUserSession $userSession, private readonly FileRequest $fileRequest, private readonly ShareRequest $shareRequest, + private readonly IJobList $jobList, private readonly GlobalSiteSelector $gss, private readonly GlobalScaleService $globalScaleService, private readonly IUserManager $userManager, @@ -142,6 +151,90 @@ public function getSharedFiles(int $fileId, int $shareId = 0, ?string $instance } + /** + * Rescan a part of a user root folder, based on a federated share and the + * path to reach a target. + */ + public function refreshSharedTarget(string $instance, int $shareId, string $shareToken, LocalFile $target): void { + $this->logger->debug('reaching details about federated share to refresh', ['shareId' => $shareId, 'shareToken' => $shareToken, 'target' => $target?->jsonSerialize()]); + [$userId, $mountPoint] = $this->fileRequest->getMountPointFromShare($instance, $shareId, $shareToken); + if ($userId === null || $mountPoint === null) { + return; + } + + $path = implode('/', array_reverse($target->getPath())); + $this->logger->debug('refreshing shared target', ['userId' => $userId, 'mountPoint' => $mountPoint, 'path' => $path]); + try { + $this->rootFolder->getUserFolder($userId) + ->get($mountPoint) + ->getStorage() + ->getScanner() + ->scan($path); + } catch (NotFoundException|NotPermittedException|NoUserException|UserNotFoundException $e) { + $this->logger->warning('could not refresh shared target', ['exception' => $e, 'mountPoint' => $mountPoint, 'path' => $path, 'userId' => $userId, 'shareToken' => $shareToken]); + } + } + + + /** + * Based on a file id, will retrieve federated share and send a notification to each remote instance + * This to be used when a local file is modified to keep remote instance in sync with local file. + * + * @throws SharedFileException + */ + public function refreshFileAcrossGlobalScale(int $fileId): void { + $files = $this->getRelatedFiles($fileId); + if (empty($files)) { + throw new SharedFileException('file not found'); + } + + // confirm mountPoint is local + $mountPoint = array_slice($files, -1)[0]; + if ($this->getFederatedShareFromTargetLocalFile($mountPoint) !== null) { + return; + } + + $federatedShares = $this->shareRequest->getFederatedSharesRelatedToRemoteInstance($files, null); + $instances = []; + + // regroup federated shares linked to the node by instances + foreach ($federatedShares as $federatedShare) { + $getShareWith = $federatedShare->getShareWith(); + $pos = strrpos($getShareWith, '@'); + if ($pos === false) { + continue; + } + $instance = substr($getShareWith, $pos + 1); + $federatedShare->setShareWith(substr($getShareWith, 0, $pos)); + if (!array_key_exists($instance, $instances)) { + $instances[$instance] = []; + } + $instances[$instance][] = $federatedShare->jsonSerialize(); + } + + if (count($instances) > $this->appConfig->getValueInt(Application::APP_ID, ConfigLexicon::INSTANCE_MAIN_THREAD)) { + $this->jobList->add(NotifyRemoteFile::class, ['instances' => $instances]); + return; + } + + foreach ($instances as $instance => $shares) { + $this->requestRemoteFileRefresh($instance, $shares); + } + } + + /** + * send notification to a remote instance to initiate a rescan of a federated + * shared in order to keep in sync. + */ + public function requestRemoteFileRefresh(string $remote, array $federatedShares): void { + $responseCode = 0; + try { + $this->requestRemoteInstance($remote, 'Slave.refreshSharedFile', ['shares' => $federatedShares], $responseCode); + } catch (RemoteIsLocalException) { + } + } + + /** * get details about a shared remote file based on the address of the remote * instance and the id of the file as stored on that remote instance @@ -239,6 +332,7 @@ private function getIdFromSharedTarget(int $shareId, LocalFile $target): int { return $this->getFinalFileId($fileOwner, $fileId, $target); } + /** * Return a file id based on a list of available shares. * A preferred share is selected based on permissions. @@ -339,13 +433,11 @@ private function getFinalFileId(string $user, int $nodeId, LocalFile $target): i } /** - * request remote instance to get the list of federated shares between both instances that would - * provide access to file id search can also be performed on the share id. + * GET request a remote instance of the GlobalScale using app route and a payload * - * @return FederatedShare[] - * @throws LocalFederatedShareException if the federated share is not remote + * @throws RemoteIsLocalException if remote is local */ - private function requestRemoteFederatedShares(string &$remote, array $search, bool $redirected = false): array { + private function requestRemoteInstance(string &$remote, string $route, array $payload, ?int &$responseCode = null): array { if (str_contains($remote, '://')) { $remote = parse_url($remote, PHP_URL_HOST); } @@ -353,17 +445,36 @@ private function requestRemoteFederatedShares(string &$remote, array $search, bo // this should not happen, but we keep a trace if ($this->globalScaleService->isLocalAddress($remote)) { $this->logger->warning('remote is local', ['exception' => new Exception(), 'remote' => $remote]); - return []; + throw new RemoteIsLocalException('remote is local'); } $responseCode = 0; $result = $this->globalScaleService->requestGssOcs( $remote, - 'Slave.sharedFile', - ['jwt' => JWT::encode(array_merge($search, ['instance' => $this->globalScaleService->getLocalAddress()]), $this->gss->getJwtKey(), Application::JWT_ALGORITHM)], + $route, + ['jwt' => JWT::encode(array_merge($payload, ['instance' => $this->globalScaleService->getLocalAddress()]), $this->gss->getJwtKey(), Application::JWT_ALGORITHM)], $responseCode); - $this->logger->warning('result from remote gss ocs', ['remote' => $remote, 'search' => $search, 'data' => $result, 'responseCode' => $responseCode]); + $this->logger->debug('result from remote gss ocs', ['remote' => $remote, 'payload' => $payload, 'data' => $result, 'responseCode' => $responseCode]); + + return $result; + } + + + /** + * request remote instance to get the list of federated shares between both instances that would + * provide access to file id search can also be performed on the share id. + * + * @return FederatedShare[] + * @throws LocalFederatedShareException if the federated share is not remote + */ + private function requestRemoteFederatedShares(string &$remote, array $search, bool $redirected = false): array { + $responseCode = 0; + try { + $result = $this->requestRemoteInstance($remote, 'Slave.sharedFile', $search, $responseCode); + } catch (RemoteIsLocalException) { + return []; + } // in case file is not on remote instance, we get a redirection if (!$redirected && $responseCode === Http::STATUS_MOVED_PERMANENTLY) {