From 03bc65b80d2fcba87e09d8b5964d90fb0e3b81de Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Thu, 17 Apr 2025 15:56:15 +0530 Subject: [PATCH 01/46] feat: add support for lct in library_context --- .../content_libraries/library_context.py | 137 ++++++++++++++---- .../core/djangoapps/content_staging/views.py | 30 ++-- .../learning_context/learning_context.py | 34 ++--- .../xblock/learning_context/manager.py | 6 +- setup.py | 1 + 5 files changed, 151 insertions(+), 57 deletions(-) diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 34a5a90269fe..41af54bd3e6f 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -8,8 +8,8 @@ from openedx_events.content_authoring.data import LibraryBlockData, LibraryContainerData from openedx_events.content_authoring.signals import LIBRARY_BLOCK_UPDATED, LIBRARY_CONTAINER_UPDATED -from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryUsageLocatorV2, LibraryLocatorV2 +from opaque_keys.edx.keys import UsageKeyV2, OpaqueKey +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryLocatorV2, LibraryUsageLocatorV2 from openedx_learning.api import authoring as authoring_api from openedx.core.djangoapps.content_libraries import api, permissions @@ -20,41 +20,50 @@ log = logging.getLogger(__name__) -class LibraryContextImpl(LearningContext): - """ - Implements content libraries as a learning context. - - This is the *new* content libraries based on Learning Core, not the old content - libraries based on modulestore. - """ +class LibraryContextPermissionBase: + locator_type = OpaqueKey def __init__(self, **kwargs): super().__init__(**kwargs) self.use_draft = kwargs.get('use_draft', None) - def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def get_library_key(self, opaque_key: OpaqueKey): """ - Assuming a block with the specified ID (usage_key) exists, does the + Get library key from given opaque_key. + """ + raise NotImplementedError() + + def can_edit_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: + """ + Assuming a block with the specified ID (opaque_key) exists, does the specified user have permission to edit it (make changes to the fields / authored data store)? May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) - return self._check_perm(user, usage_key.lib_key, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) + assert isinstance(opaque_key, self.locator_type) + return self._check_perm( + user, + self.get_library_key(opaque_key), + permissions.CAN_EDIT_THIS_CONTENT_LIBRARY + ) - def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_view_block_for_editing(self, user: UserType, opaque_key: OpaqueKey) -> bool: """ - Assuming a block with the specified ID (usage_key) exists, does the + Assuming a block with the specified ID (opaque_key) exists, does the specified user have permission to view its fields and OLX details (but not necessarily to make changes to it)? May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) - return self._check_perm(user, usage_key.lib_key, permissions.CAN_VIEW_THIS_CONTENT_LIBRARY) + assert isinstance(opaque_key, self.locator_type) + return self._check_perm( + user, + self.get_library_key(opaque_key), + permissions.CAN_VIEW_THIS_CONTENT_LIBRARY + ) - def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_view_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: """ Does the specified usage key exist in its context, and if so, does the specified user have permission to view it and interact with it (call @@ -62,8 +71,12 @@ def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) - return self._check_perm(user, usage_key.lib_key, permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY) + assert isinstance(opaque_key, self.locator_type) + return self._check_perm( + user, + self.get_library_key(opaque_key), + permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY + ) def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: """ Helper method to check a permission for the various can_ methods""" @@ -76,6 +89,23 @@ def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: # A 404 is probably what you want in this case, not a 500 error, so do that by default. raise NotFound(f"Content Library '{lib_key}' does not exist") from exc + +class LibraryContextImpl(LibraryContextPermissionBase, LearningContext): + """ + Implements content libraries as a learning context. + + This is the *new* content libraries based on Learning Core, not the old content + libraries based on modulestore. + """ + + locator_type = LibraryUsageLocatorV2 + + def get_library_key(self, opaque_key: OpaqueKey): + """ + Get library key from given opaque_key. + """ + return opaque_key.lib_key + def block_exists(self, usage_key: LibraryUsageLocatorV2): """ Does the block for this usage_key exist in this Library? @@ -103,25 +133,25 @@ def block_exists(self, usage_key: LibraryUsageLocatorV2): local_key=usage_key.block_id, ) - def send_block_updated_event(self, usage_key: UsageKeyV2): + def send_block_updated_event(self, opaque_key: OpaqueKey): """ Send a "block updated" event for the library block with the given usage_key. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) + assert isinstance(opaque_key, self.locator_type) LIBRARY_BLOCK_UPDATED.send_event( library_block=LibraryBlockData( - library_key=usage_key.lib_key, - usage_key=usage_key, + library_key=opaque_key.lib_key, + usage_key=opaque_key, ) ) - def send_container_updated_events(self, usage_key: UsageKeyV2): + def send_container_updated_events(self, opaque_key: OpaqueKey): """ Send "container updated" events for containers that contains the library block with the given usage_key. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) - affected_containers = api.get_containers_contains_component(usage_key) + assert isinstance(opaque_key, self.locator_type) + affected_containers = api.get_containers_contains_component(opaque_key) for container in affected_containers: LIBRARY_CONTAINER_UPDATED.send_event( library_container=LibraryContainerData( @@ -129,3 +159,56 @@ def send_container_updated_events(self, usage_key: UsageKeyV2): background=True, ) ) + + +class LibraryContextContainerImpl(LibraryContextPermissionBase, LearningContext): + """ + Implements content libraries as a learning context for containers in libraries. + + This is the *new* content libraries based on Learning Core, not the old content + libraries based on modulestore. + """ + + locator_type = LibraryContainerLocator + + def get_library_key(self, opaque_key: OpaqueKey): + """ + Get library key from given opaque_key. + """ + return opaque_key.library_key + + def container_exists(self, container_key: LibraryContainerLocator): + """ + Does the container for this key exist in this Library? + + Note that this applies to all versions, i.e. you can put a container key for + a piece of content that has been soft-deleted (removed from Drafts), and + it will still return True here. That's because for the purposes of + permission checking, we just want to know whether that block has ever + existed in this Library, because we could be looking at any older + version of it. + """ + try: + content_lib = ContentLibrary.objects.get_by_key(container_key.library_key) # type: ignore[attr-defined] + except ContentLibrary.DoesNotExist: + return False + + learning_package = content_lib.learning_package + if learning_package is None: + return False + + return authoring_api.container_exists_by_key( + learning_package.id, + container_key, + ) + + def send_block_updated_event(self, opaque_key: OpaqueKey): + """ + Send a "block updated" event for the library block with the given usage_key. + """ + assert isinstance(opaque_key, self.locator_type) + LIBRARY_CONTAINER_UPDATED.send_event( + library_container=LibraryContainerData( + container_key=opaque_key, + ) + ) diff --git a/openedx/core/djangoapps/content_staging/views.py b/openedx/core/djangoapps/content_staging/views.py index f0a20540ba8d..e90400e1dbc1 100644 --- a/openedx/core/djangoapps/content_staging/views.py +++ b/openedx/core/djangoapps/content_staging/views.py @@ -10,7 +10,7 @@ import edx_api_doc_tools as apidocs from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey -from opaque_keys.edx.locator import CourseLocator, LibraryLocatorV2 +from opaque_keys.edx.locator import CourseLocator, LibraryContainerLocator, LibraryLocatorV2 from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError from rest_framework.response import Response from rest_framework.views import APIView @@ -91,12 +91,18 @@ def post(self, request): # Check if the content exists and the user has permission to read it. # Parse the usage key: try: - usage_key = UsageKey.from_string(request.data["usage_key"]) - except (ValueError, InvalidKeyError): - raise ValidationError('Invalid usage key') # lint-amnesty, pylint: disable=raise-missing-from - if usage_key.block_type in ('course', 'chapter', 'sequential'): - raise ValidationError('Requested XBlock tree is too large') - course_key = usage_key.context_key + opaque_key = UsageKey.from_string(request.data["usage_key"]) + if opaque_key.block_type in ('course', 'chapter', 'sequential'): + raise ValidationError('Requested XBlock tree is too large') + except InvalidKeyError: + try: + # Check if valid library container + opaque_key = LibraryContainerLocator.from_string(request.data["usage_key"]) + if opaque_key.container_type != 'unit': + raise ValidationError('Requested XBlock tree is too large') + except (ValueError, InvalidKeyError): + raise ValidationError('Invalid usage key') # lint-amnesty, pylint: disable=raise-missing-from + course_key = getattr(opaque_key, 'context_key', None) or getattr(opaque_key, 'library_key', None) # Load the block and copy it to the user's clipboard try: @@ -106,7 +112,7 @@ def post(self, request): raise PermissionDenied( "You must be a member of the course team in Studio to export OLX using this API." ) - block = modulestore().get_item(usage_key) + block = modulestore().get_item(opaque_key) version_num = None elif isinstance(course_key, LibraryLocatorV2): @@ -115,8 +121,12 @@ def post(self, request): request.user, lib_api.permissions.CAN_VIEW_THIS_CONTENT_LIBRARY ) - block = xblock_api.load_block(usage_key, user=None) - version_num = lib_api.get_library_block(usage_key).draft_version_num + if isinstance(opaque_key, LibraryContainerLocator): + # TODO: load unit block data to staging content, probably need to convert unit from library to xml + raise NotImplementedError("Containers not supported yet") + else: + block = xblock_api.load_block(opaque_key, user=None) + version_num = lib_api.get_library_block(opaque_key).draft_version_num else: raise ValidationError("Invalid usage_key for the content.") diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index dc7a21f1c397..e0a754fe2128 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -3,7 +3,7 @@ of content where learning happens. """ from openedx.core.types import User as UserType -from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.keys import OpaqueKey class LearningContext: @@ -25,44 +25,44 @@ def __init__(self, **kwargs): parameters without changing the API. """ - def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument + def can_edit_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: # pylint: disable=unused-argument """ - Assuming a block with the specified ID (usage_key) exists, does the + Assuming a block with the specified ID (opaque_key) exists, does the specified user have permission to edit it (make changes to the fields / authored data store)? user: a Django User object (may be an AnonymousUser) - usage_key: the UsageKeyV2 subclass used for this learning context + opaque_key: the OpaqueKey subclass used for this learning context Must return a boolean. """ return False - def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_view_block_for_editing(self, user: UserType, opaque_key: OpaqueKey) -> bool: """ - Assuming a block with the specified ID (usage_key) exists, does the + Assuming a block with the specified ID (opaque_key) exists, does the specified user have permission to view its fields and OLX details (but not necessarily to make changes to it)? """ - return self.can_edit_block(user, usage_key) + return self.can_edit_block(user, opaque_key) - def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument + def can_view_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: # pylint: disable=unused-argument """ - Assuming a block with the specified ID (usage_key) exists, does the + Assuming a block with the specified ID (opaque_key) exists, does the specified user have permission to view it and interact with it (call handlers, save user state, etc.)? This is also sometimes called the "can_learn" permission. user: a Django User object (may be an AnonymousUser) - usage_key: the UsageKeyV2 subclass used for this learning context + opaque_key: the OpaqueKey subclass used for this learning context Must return a boolean. """ return False - def definition_for_usage(self, usage_key, **kwargs): + def definition_for_usage(self, opaque_key, **kwargs): """ Given a usage key in this context, return the key indicating the actual XBlock definition. @@ -70,17 +70,17 @@ def definition_for_usage(self, usage_key, **kwargs): """ raise NotImplementedError - def send_block_updated_event(self, usage_key): + def send_block_updated_event(self, opaque_key): """ - Send a "block updated" event for the block with the given usage_key in this context. + Send a "block updated" event for the block with the given opaque_key in this context. - usage_key: the UsageKeyV2 subclass used for this learning context + opaque_key: the OpaqueKey subclass used for this learning context """ - def send_container_updated_events(self, usage_key): + def send_container_updated_events(self, opaque_key): """ Send "container updated" events for containers that contains the block with - the given usage_key in this context. + the given opaque_key in this context. - usage_key: the UsageKeyV2 subclass used for this learning context + opaque_key: the OpaqueKey subclass used for this learning context """ diff --git a/openedx/core/djangoapps/xblock/learning_context/manager.py b/openedx/core/djangoapps/xblock/learning_context/manager.py index b7ed1c3c3426..dc390616969e 100644 --- a/openedx/core/djangoapps/xblock/learning_context/manager.py +++ b/openedx/core/djangoapps/xblock/learning_context/manager.py @@ -3,7 +3,7 @@ """ from edx_django_utils.plugins import PluginManager from opaque_keys import OpaqueKey -from opaque_keys.edx.keys import LearningContextKey, UsageKeyV2 +from opaque_keys.edx.keys import LearningContextKey, LibraryItemKey, UsageKeyV2 from openedx.core.djangoapps.xblock.apps import get_xblock_app_config @@ -33,8 +33,8 @@ def get_learning_context_impl(key): Raises PluginError if there is some misconfiguration causing the context implementation to not be installed. """ - if isinstance(key, LearningContextKey): - context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' + if isinstance(key, LearningContextKey) or isinstance(key, LibraryItemKey): + context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' or 'lct' elif isinstance(key, UsageKeyV2): context_type = key.context_key.CANONICAL_NAMESPACE elif isinstance(key, OpaqueKey): diff --git a/setup.py b/setup.py index 3b8f8c59498d..829bc5e08152 100644 --- a/setup.py +++ b/setup.py @@ -181,6 +181,7 @@ ], 'openedx.learning_context': [ 'lib = openedx.core.djangoapps.content_libraries.library_context:LibraryContextImpl', + 'lct = openedx.core.djangoapps.content_libraries.library_context:LibraryContextContainerImpl', ], 'openedx.dynamic_partition_generator': [ 'content_type_gating = openedx.features.content_type_gating.partitions:create_content_gating_partition', From 157f690753246320dbbe65a45d185bfe341a1d92 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Sat, 19 Apr 2025 22:30:16 +0530 Subject: [PATCH 02/46] feat: library unit sync --- cms/djangoapps/contentstore/helpers.py | 23 ++++ .../xblock_storage_handlers/view_handlers.py | 37 +++++-- cms/lib/xblock/upstream_sync.py | 100 ++++++++++++------ .../content_libraries/api/containers.py | 13 ++- openedx/core/djangoapps/xblock/api.py | 12 ++- .../xblock/runtime/learning_core_runtime.py | 75 +++++++------ 6 files changed, 185 insertions(+), 75 deletions(-) diff --git a/cms/djangoapps/contentstore/helpers.py b/cms/djangoapps/contentstore/helpers.py index 506ce766ff23..cebaef7dc2bd 100644 --- a/cms/djangoapps/contentstore/helpers.py +++ b/cms/djangoapps/contentstore/helpers.py @@ -790,3 +790,26 @@ def _get_usage_key_from_node(node, parent_id: str) -> UsageKey | None: ) return usage_key + + +def concat_static_file_notices(notices: list[StaticFileNotices]) -> StaticFileNotices: + """Combines multiple static file notices into a single object + + Args: + notices: list of StaticFileNotices + + Returns: + Single StaticFileNotices + """ + new_files = [] + conflicting_files = [] + error_files = [] + for notice in notices: + new_files.append(notice.new_files) + conflicting_files.append(notice.conflicting_files) + error_files.append(notice.error_files) + return StaticFileNotices( + new_files=new_files, + conflicting_files=conflicting_files, + error_files=error_files, + ) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 3506825ae357..8cbd86f6dc28 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -19,6 +19,7 @@ from django.http import HttpResponse, HttpResponseBadRequest from django.utils.translation import gettext as _ from edx_django_utils.plugins import pluggable_override +from openedx.core.djangoapps.content_libraries.api.containers import get_container_children from openedx.core.djangoapps.content_tagging.api import get_object_tag_counts from edx_proctoring.api import ( does_backend_support_onboarding, @@ -27,7 +28,7 @@ ) from edx_proctoring.exceptions import ProctoredExamNotFoundException from help_tokens.core import HelpUrlExpert -from opaque_keys.edx.locator import LibraryUsageLocator +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocator from pytz import UTC from xblock.core import XBlock from xblock.fields import Scope @@ -35,7 +36,7 @@ from cms.djangoapps.contentstore.config.waffle import SHOW_REVIEW_RULES_FLAG from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.lib.ai_aside_summary_config import AiAsideSummaryConfig -from cms.lib.xblock.upstream_sync import BadUpstream, sync_from_upstream +from cms.lib.xblock.upstream_sync import BadUpstream, get_upstream_key, sync_from_upstream from common.djangoapps.static_replace import replace_static_urls from common.djangoapps.student.auth import ( has_studio_read_access, @@ -77,6 +78,7 @@ from .create_xblock import create_xblock from .xblock_helpers import usage_key_with_run from ..helpers import ( + concat_static_file_notices, get_parent_xblock, import_staged_content_from_user_clipboard, import_static_assets_for_library_sync, @@ -522,6 +524,27 @@ def create_item(request): return _create_block(request) +def sync_library_content(created_block, request): + upstream_key = get_upstream_key(created_block) + lib_block = sync_from_upstream(downstream=created_block, user=request.user) + static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) + modulestore().update_item(created_block, request.user.id) + if isinstance(upstream_key, LibraryContainerLocator): + notices = [static_file_notices] + for child in get_container_children(upstream_key, published=True): + child_block = create_xblock( + parent_locator=created_block.location, + user=request.user, + category=child.usage_key.block_type, + display_name=child.display_name, + ) + child_block.upstream = child.usage_key + sync_library_content(child_block, request) + notices.append(sync_library_content(child, request)) + static_file_notices = concat_static_file_notices(notices) + return static_file_notices + + @login_required @expect_json def _create_block(request): @@ -578,7 +601,7 @@ def _create_block(request): created_block = create_xblock( parent_locator=parent_locator, user=request.user, - category=category, + category="vertical" if category == "unit" else category, display_name=request.json.get("display_name"), boilerplate=request.json.get("boilerplate"), ) @@ -590,10 +613,10 @@ def _create_block(request): # If it contains library_content_key, the block is being imported from a v2 library # so it needs to be synced with upstream block. if upstream_ref := request.json.get("library_content_key"): + # Set `created_block.upstream` and then sync this with the upstream (library) version. + created_block.upstream = upstream_ref try: - # Set `created_block.upstream` and then sync this with the upstream (library) version. - created_block.upstream = upstream_ref - lib_block = sync_from_upstream(downstream=created_block, user=request.user) + static_file_notices = sync_library_content(created_block, request) except BadUpstream as exc: _delete_item(created_block.location, request.user) log.exception( @@ -601,8 +624,6 @@ def _create_block(request): f"using provided library_content_key='{upstream_ref}'" ) return JsonResponse({"error": str(exc)}, status=400) - static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) - modulestore().update_item(created_block, request.user.id) response["upstreamRef"] = upstream_ref response["static_file_notices"] = asdict(static_file_notices) response["parent_locator"] = parent_locator diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 22cad3c6d36a..43d336c819c6 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -11,6 +11,7 @@ """ from __future__ import annotations +from functools import lru_cache import logging import typing as t from dataclasses import dataclass, asdict @@ -19,13 +20,14 @@ from django.core.exceptions import PermissionDenied from django.utils.translation import gettext_lazy as _ from rest_framework.exceptions import NotFound -from opaque_keys import InvalidKeyError +from opaque_keys import InvalidKeyError, OpaqueKey from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from xblock.exceptions import XBlockNotFoundError from xblock.fields import Scope, String, Integer from xblock.core import XBlockMixin, XBlock + if t.TYPE_CHECKING: from django.contrib.auth.models import User # pylint: disable=imported-auth-user @@ -146,37 +148,23 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: If link exists, is supported, and is followable, returns UpstreamLink. Otherwise, raises an UpstreamLinkException. """ - if not downstream.upstream: - raise NoUpstream() - if not isinstance(downstream.usage_key.context_key, CourseKey): - raise BadDownstream(_("Cannot update content because it does not belong to a course.")) - if downstream.has_children: - raise BadDownstream(_("Updating content with children is not yet supported.")) - try: - upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) - except InvalidKeyError as exc: - raise BadUpstream(_("Reference to linked library item is malformed")) from exc - downstream_type = downstream.usage_key.block_type - if upstream_key.block_type != downstream_type: - # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. - # It could be reasonable to relax this requirement in the future if there's product need for it. - # For example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. - raise BadUpstream( - _("Content type mismatch: {downstream_type} cannot be linked to {upstream_type}.").format( - downstream_type=downstream_type, upstream_type=upstream_key.block_type - ) - ) from TypeError( - f"downstream block '{downstream.usage_key}' is linked to " - f"upstream block of different type '{upstream_key}'" - ) + upstream_key = get_upstream_key(downstream) # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. from openedx.core.djangoapps.content_libraries.api import ( - get_library_block # pylint: disable=wrong-import-order + ContentLibraryContainerNotFound, # pylint: disable=wrong-import-order + get_container, # pylint: disable=wrong-import-order + get_library_block, # pylint: disable=wrong-import-order ) - try: - lib_meta = get_library_block(upstream_key) - except XBlockNotFoundError as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc + if isinstance(upstream_key, LibraryUsageLocatorV2): + try: + lib_meta = get_library_block(upstream_key) + except XBlockNotFoundError as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc + else: + try: + lib_meta = get_container(upstream_key) + except ContentLibraryContainerNotFound as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc return cls( upstream_ref=downstream.upstream, version_synced=downstream.upstream_version, @@ -215,6 +203,55 @@ def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBloc _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) +@lru_cache +def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryContainerLocator: + """ + Convert upstream key to proper type for given downstream block. + + Args: + downstream: XBlock + + Returns: + Parsed upstream key + + Raises: + NoUpstream: + BadDownstream: + BadUpstream: + """ + if not downstream.upstream: + raise NoUpstream() + if not isinstance(downstream.usage_key.context_key, CourseKey): + raise BadDownstream(_("Cannot update content because it does not belong to a course.")) + if downstream.has_children: + try: + upstream_key = LibraryContainerLocator.from_string(downstream.upstream) + upstream_type = upstream_key.container_type + except InvalidKeyError as exc: + raise BadUpstream(_("Reference to linked library item is malformed")) from exc + else: + try: + upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) + upstream_type = upstream_key.block_type + except InvalidKeyError as exc: + raise BadUpstream(_("Reference to linked library item is malformed")) from exc + downstream_type = downstream.usage_key.block_type + upstream_type = "vertical" if upstream_type == "unit" else upstream_type + if upstream_type != downstream_type: + # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. + # It could be reasonable to relax this requirement in the future if there's product need for it. + # For example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. + raise BadUpstream( + _("Content type mismatch: {downstream_type} cannot be linked to {upstream_type}.").format( + downstream_type=downstream_type, upstream_type=upstream_type + ) + ) from TypeError( + f"downstream block '{downstream.usage_key}' is linked to " + f"upstream block of different type '{upstream_key}'" + ) + return upstream_key + + def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[UpstreamLink, XBlock]: """ Load the upstream metadata and content for a downstream block. @@ -225,11 +262,12 @@ def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[Upstr If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. """ link = UpstreamLink.get_for_block(downstream) # can raise UpstreamLinkException + upstream_key = get_upstream_key(downstream) # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. from openedx.core.djangoapps.xblock.api import load_block, CheckPerm, LatestVersion # pylint: disable=wrong-import-order try: lib_block: XBlock = load_block( - LibraryUsageLocatorV2.from_string(downstream.upstream), + upstream_key, user, check_permission=CheckPerm.CAN_READ_AS_AUTHOR, version=LatestVersion.PUBLISHED, diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index 393755ed1c91..9c236cc6192f 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -8,6 +8,7 @@ from enum import Enum import logging from uuid import uuid4 +from lxml import etree from django.utils.text import slugify from opaque_keys.edx.keys import UsageKeyV2 @@ -27,7 +28,7 @@ LIBRARY_CONTAINER_UPDATED, ) from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import Container +from openedx_learning.api.authoring_models import Container, Unit from openedx.core.djangoapps.content_libraries.api.collections import library_collection_locator from openedx.core.djangoapps.xblock.api import get_component_from_usage_key @@ -54,6 +55,7 @@ "update_container_children", "get_containers_contains_component", "publish_container_changes", + "library_container_xml", ] log = logging.getLogger(__name__) @@ -372,7 +374,7 @@ def restore_container(container_key: LibraryContainerLocator) -> None: def get_container_children( container_key: LibraryContainerLocator, published=False, -) -> list[authoring_api.ContainerEntityListEntry]: +) -> list[LibraryXBlockMetadata | ContainerMetadata]: """ Get the entities contained in the given container (e.g. the components/xblocks in a unit) """ @@ -489,3 +491,10 @@ def publish_container_changes(container_key: LibraryContainerLocator, user_id: i f"PublishableEntity {record.entity.pk} / {record.entity.key} was modified during publish operation " "but is of unknown type." ) + + +def library_container_xml(container: ContainerMetadata): + """Converts given unit to xml without including children components""" + xml_object = etree.Element(container.container_type.value) + xml_object.set("display_name", container.display_name) + return xml_object diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index c806fefc87c5..d7484014a30e 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -19,12 +19,13 @@ from openedx_learning.api import authoring as authoring_api from openedx_learning.api.authoring_models import Component, ComponentVersion from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from rest_framework.exceptions import NotFound from xblock.core import XBlock from xblock.exceptions import NoSuchUsage, NoSuchViewError from xblock.plugin import PluginMissingError +from openedx.core.djangoapps.content_libraries.api.exceptions import ContentLibraryContainerNotFound from openedx.core.types import User as UserType from openedx.core.djangoapps.xblock.apps import get_xblock_app_config from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl @@ -70,7 +71,7 @@ def get_runtime(user: UserType): def load_block( - usage_key: UsageKeyV2, + usage_key: UsageKeyV2 | LibraryContainerLocator, user: UserType, *, check_permission: CheckPerm | None = CheckPerm.CAN_LEARN, @@ -114,11 +115,14 @@ def load_block( runtime = get_runtime(user=user) try: - return runtime.get_block(usage_key, version=version) + if isinstance(usage_key, UsageKeyV2): + return runtime.get_block(usage_key, version=version) + else: + return runtime.get_container_block(usage_key, version=version) except NoSuchUsage as exc: # Convert NoSuchUsage to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The component '{usage_key}' does not exist.") from exc - except ComponentVersion.DoesNotExist as exc: + except (ComponentVersion.DoesNotExist, ContentLibraryContainerNotFound) as exc: # Convert ComponentVersion.DoesNotExist to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The requested version of component '{usage_key}' does not exist.") from exc diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 5f9cba6c3a22..180e46b22663 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -167,36 +167,8 @@ class LearningCoreXBlockRuntime(XBlockRuntime): (eventually) asset storage. """ - def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion = LatestVersion.AUTO): - """ - Fetch an XBlock from Learning Core data models. - - This method will find the OLX for the content in Learning Core, parse it - into an XBlock (with mixins) instance, and properly initialize our - internal LearningCoreFieldData instance with the field values from the - parsed OLX. - """ - # We can do this more efficiently in a single query later, but for now - # just get it the easy way. - component = self._get_component_from_usage_key(usage_key) - - version = get_auto_latest_version(version) - if self.authored_data_mode == AuthoredDataMode.STRICTLY_PUBLISHED and version != LatestVersion.PUBLISHED: - raise ValidationError("This runtime only allows accessing the published version of components") - if version == LatestVersion.DRAFT: - component_version = component.versioning.draft - elif version == LatestVersion.PUBLISHED: - component_version = component.versioning.published - else: - assert isinstance(version, int) - component_version = component.versioning.version_num(version) - if component_version is None: - raise NoSuchUsage(usage_key) - - content = component_version.contents.get( - componentversioncontent__key="block.xml" - ) - xml_node = etree.fromstring(content.text) + def _initialize_block(self, content, usage_key, version: int | LatestVersion): + xml_node = etree.fromstring(content) block_type = usage_key.block_type keys = ScopeIds(self.user_id, block_type, None, usage_key) @@ -231,6 +203,49 @@ def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion return block + def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion = LatestVersion.AUTO): + """ + Fetch an XBlock from Learning Core data models. + + This method will find the OLX for the content in Learning Core, parse it + into an XBlock (with mixins) instance, and properly initialize our + internal LearningCoreFieldData instance with the field values from the + parsed OLX. + """ + # We can do this more efficiently in a single query later, but for now + # just get it the easy way. + component = self._get_component_from_usage_key(usage_key) + + version = get_auto_latest_version(version) + if self.authored_data_mode == AuthoredDataMode.STRICTLY_PUBLISHED and version != LatestVersion.PUBLISHED: + raise ValidationError("This runtime only allows accessing the published version of components") + if version == LatestVersion.DRAFT: + component_version = component.versioning.draft + elif version == LatestVersion.PUBLISHED: + component_version = component.versioning.published + else: + assert isinstance(version, int) + component_version = component.versioning.version_num(version) + if component_version is None: + raise NoSuchUsage(usage_key) + + content = component_version.contents.get( + componentversioncontent__key="block.xml" + ) + return self._initialize_block(content.text, usage_key, version) + + def get_container_block(self, container_key, *, version: int | LatestVersion = LatestVersion.AUTO): + """ + Fetch container from learning core data models. + + This method create a very basic olx for container and parse it into an XBlock instance. + """ + from openedx.core.djangoapps.content_libraries.api.containers import get_container, library_container_xml + container = get_container(container_key) + content = library_container_xml(container) + xml = etree.tostring(content) + return self._initialize_block(xml.decode(), container_key, version) + def get_block_assets(self, block, fetch_asset_data): """ Return a list of StaticFile entries. From 04adc267dd33ac2f19d3ad3235b4deb67cf535fc Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Sun, 20 Apr 2025 18:31:19 +0530 Subject: [PATCH 03/46] fixup! feat: library unit sync --- .../xblock_storage_handlers/view_handlers.py | 6 +++--- .../xblock/runtime/learning_core_runtime.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 8cbd86f6dc28..047a0445300f 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -533,14 +533,14 @@ def sync_library_content(created_block, request): notices = [static_file_notices] for child in get_container_children(upstream_key, published=True): child_block = create_xblock( - parent_locator=created_block.location, + parent_locator=str(created_block.location), user=request.user, category=child.usage_key.block_type, display_name=child.display_name, ) - child_block.upstream = child.usage_key + child_block.upstream = str(child.usage_key) sync_library_content(child_block, request) - notices.append(sync_library_content(child, request)) + notices.append(sync_library_content(child_block, request)) static_file_notices = concat_static_file_notices(notices) return static_file_notices diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 180e46b22663..4b849070a6f5 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -12,6 +12,7 @@ from django.db.transaction import atomic from django.urls import reverse +from opaque_keys.edx.keys import UsageKeyV2 from openedx_learning.api import authoring as authoring_api from lxml import etree @@ -167,9 +168,8 @@ class LearningCoreXBlockRuntime(XBlockRuntime): (eventually) asset storage. """ - def _initialize_block(self, content, usage_key, version: int | LatestVersion): + def _initialize_block(self, content, usage_key, block_type, version: int | LatestVersion): xml_node = etree.fromstring(content) - block_type = usage_key.block_type keys = ScopeIds(self.user_id, block_type, None, usage_key) if xml_node.get("url_name", None): @@ -232,7 +232,7 @@ def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion content = component_version.contents.get( componentversioncontent__key="block.xml" ) - return self._initialize_block(content.text, usage_key, version) + return self._initialize_block(content.text, usage_key, usage_key.block_type, version) def get_container_block(self, container_key, *, version: int | LatestVersion = LatestVersion.AUTO): """ @@ -244,7 +244,8 @@ def get_container_block(self, container_key, *, version: int | LatestVersion = L container = get_container(container_key) content = library_container_xml(container) xml = etree.tostring(content) - return self._initialize_block(xml.decode(), container_key, version) + block_type = "vertical" if container_key.container_type == "unit" else container_key.container_type + return self._initialize_block(xml.decode(), container_key, block_type, version) def get_block_assets(self, block, fetch_asset_data): """ @@ -261,6 +262,9 @@ def get_block_assets(self, block, fetch_asset_data): lookups one by one is going to get slow. At some point we're going to want something to look up a bunch of blocks at once. """ + if not isinstance(block.usage_key, UsageKeyV2): + # TODO: handle assets for containers if required. + return [] component_version = self._get_component_version_from_block(block) # cvc = the ComponentVersionContent through-table From ce0686e0698b3d623fdd7cdee573e0472470e514 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Sun, 20 Apr 2025 18:45:04 +0530 Subject: [PATCH 04/46] fixup! feat: library unit sync --- cms/djangoapps/contentstore/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cms/djangoapps/contentstore/helpers.py b/cms/djangoapps/contentstore/helpers.py index cebaef7dc2bd..0f7384af72ad 100644 --- a/cms/djangoapps/contentstore/helpers.py +++ b/cms/djangoapps/contentstore/helpers.py @@ -805,9 +805,9 @@ def concat_static_file_notices(notices: list[StaticFileNotices]) -> StaticFileNo conflicting_files = [] error_files = [] for notice in notices: - new_files.append(notice.new_files) - conflicting_files.append(notice.conflicting_files) - error_files.append(notice.error_files) + new_files.extend(notice.new_files) + conflicting_files.extend(notice.conflicting_files) + error_files.extend(notice.error_files) return StaticFileNotices( new_files=new_files, conflicting_files=conflicting_files, From 5549ee4f9b8867e9fb31b3de1b1265bd3c3b5d33 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Mon, 21 Apr 2025 16:22:11 +0530 Subject: [PATCH 05/46] fixup! feat: library unit sync --- .../xblock_storage_handlers/view_handlers.py | 28 ++++++++++--------- .../content_libraries/api/containers.py | 4 +-- openedx/core/djangoapps/xblock/api.py | 7 +++-- .../xblock/runtime/learning_core_runtime.py | 4 +-- 4 files changed, 23 insertions(+), 20 deletions(-) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 047a0445300f..7c66699605b8 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -528,20 +528,22 @@ def sync_library_content(created_block, request): upstream_key = get_upstream_key(created_block) lib_block = sync_from_upstream(downstream=created_block, user=request.user) static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) - modulestore().update_item(created_block, request.user.id) + store = modulestore() + store.update_item(created_block, request.user.id) if isinstance(upstream_key, LibraryContainerLocator): - notices = [static_file_notices] - for child in get_container_children(upstream_key, published=True): - child_block = create_xblock( - parent_locator=str(created_block.location), - user=request.user, - category=child.usage_key.block_type, - display_name=child.display_name, - ) - child_block.upstream = str(child.usage_key) - sync_library_content(child_block, request) - notices.append(sync_library_content(child_block, request)) - static_file_notices = concat_static_file_notices(notices) + with store.bulk_operations(created_block.location.course_key): + notices = [static_file_notices] + for child in get_container_children(upstream_key, published=True): + child_block = create_xblock( + parent_locator=str(created_block.location), + user=request.user, + category=child.usage_key.block_type, + display_name=child.display_name, + ) + child_block.upstream = str(child.usage_key) + sync_library_content(child_block, request) + notices.append(sync_library_content(child_block, request)) + static_file_notices = concat_static_file_notices(notices) return static_file_notices diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index 9c236cc6192f..4f92137ae4ea 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -493,8 +493,8 @@ def publish_container_changes(container_key: LibraryContainerLocator, user_id: i ) -def library_container_xml(container: ContainerMetadata): +def library_container_xml(container: ContainerMetadata, block_type: str | None = None): """Converts given unit to xml without including children components""" - xml_object = etree.Element(container.container_type.value) + xml_object = etree.Element(block_type or container.container_type.value) xml_object.set("display_name", container.display_name) return xml_object diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index d7484014a30e..ed5385103cca 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -97,14 +97,13 @@ def load_block( # Now, check if the block exists in this context and if the user has # permission to render this XBlock view: if check_permission and user is not None: + has_perm = False if check_permission == CheckPerm.CAN_EDIT: has_perm = context_impl.can_edit_block(user, usage_key) elif check_permission == CheckPerm.CAN_READ_AS_AUTHOR: has_perm = context_impl.can_view_block_for_editing(user, usage_key) elif check_permission == CheckPerm.CAN_LEARN: has_perm = context_impl.can_view_block(user, usage_key) - else: - has_perm = False if not has_perm: raise PermissionDenied(f"You don't have permission to access the component '{usage_key}'.") @@ -117,8 +116,10 @@ def load_block( try: if isinstance(usage_key, UsageKeyV2): return runtime.get_block(usage_key, version=version) - else: + elif isinstance(usage_key, LibraryContainerLocator): return runtime.get_container_block(usage_key, version=version) + else: + raise NotFound(f"The component '{usage_key}' does not exist.") except NoSuchUsage as exc: # Convert NoSuchUsage to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The component '{usage_key}' does not exist.") from exc diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 4b849070a6f5..a5221d2a6df4 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -242,9 +242,9 @@ def get_container_block(self, container_key, *, version: int | LatestVersion = L """ from openedx.core.djangoapps.content_libraries.api.containers import get_container, library_container_xml container = get_container(container_key) - content = library_container_xml(container) - xml = etree.tostring(content) block_type = "vertical" if container_key.container_type == "unit" else container_key.container_type + content = library_container_xml(container, block_type) + xml = etree.tostring(content) return self._initialize_block(xml.decode(), container_key, block_type, version) def get_block_assets(self, block, fetch_asset_data): From b87bc7bffd9f07ce08cbd81f900b82ab190532aa Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Mon, 21 Apr 2025 20:21:48 +0530 Subject: [PATCH 06/46] feat: unit sync using emulated library container usage locator key --- cms/djangoapps/contentstore/models.py | 4 +- cms/djangoapps/contentstore/utils.py | 4 +- .../xblock_storage_handlers/view_handlers.py | 8 ++-- cms/lib/xblock/upstream_sync.py | 27 ++++-------- .../content_libraries/api/__init__.py | 5 ++- .../content_libraries/api/containers.py | 8 ++-- .../content_libraries/api/generic.py | 44 +++++++++++++++++++ openedx/core/djangoapps/xblock/api.py | 9 +--- .../xblock/runtime/learning_core_runtime.py | 41 +++++++---------- 9 files changed, 87 insertions(+), 63 deletions(-) create mode 100644 openedx/core/djangoapps/content_libraries/api/generic.py diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index ca6171118c00..a8e59639a33a 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -14,7 +14,7 @@ from django.utils.translation import gettext_lazy as _ from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField from opaque_keys.edx.keys import CourseKey, UsageKey -from openedx_learning.api.authoring_models import Component, PublishableEntity +from openedx_learning.api.authoring_models import Component, Container, PublishableEntity from openedx_learning.lib.fields import ( immutable_uuid_field, key_field, @@ -144,7 +144,7 @@ class Meta: @classmethod def update_or_create( cls, - upstream_block: Component | None, + upstream_block: Component | Container | None, /, upstream_usage_key: UsageKey, upstream_context_key: str, diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 1ff309aa6e50..10dfd48c6638 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -86,6 +86,7 @@ from common.djangoapps.xblock_django.api import deprecated_xblocks from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService from openedx.core import toggles as core_toggles +from openedx.core.djangoapps.content_libraries.api import get_library_content from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_RESOURCES_MICROFRONTEND @@ -95,7 +96,6 @@ from openedx.core.djangoapps.models.course_details import CourseDetails from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.site_configuration.models import SiteConfiguration -from openedx.core.djangoapps.xblock.api import get_component_from_usage_key from openedx.core.lib.courses import course_image_url from openedx.core.lib.html_to_text import html_to_text from openedx.core.lib.teams_config import CONTENT_GROUPS_FOR_TEAMS, TEAM_SCHEME @@ -2380,7 +2380,7 @@ def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, c return None upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) try: - lib_component = get_component_from_usage_key(upstream_usage_key) + lib_component = get_library_content(upstream_usage_key) except ObjectDoesNotExist: log.error(f"Library component not found for {upstream_usage_key}") lib_component = None diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 7c66699605b8..72cddf82ea79 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -28,7 +28,7 @@ ) from edx_proctoring.exceptions import ProctoredExamNotFoundException from help_tokens.core import HelpUrlExpert -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocator +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryContainerUsageLocator, LibraryUsageLocator from pytz import UTC from xblock.core import XBlock from xblock.fields import Scope @@ -526,14 +526,16 @@ def create_item(request): def sync_library_content(created_block, request): upstream_key = get_upstream_key(created_block) + created_block.upstream = str(upstream_key) lib_block = sync_from_upstream(downstream=created_block, user=request.user) static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) store = modulestore() store.update_item(created_block, request.user.id) - if isinstance(upstream_key, LibraryContainerLocator): + if isinstance(upstream_key, LibraryContainerUsageLocator): + container_key = LibraryContainerLocator.from_usage_key(upstream_key) with store.bulk_operations(created_block.location.course_key): notices = [static_file_notices] - for child in get_container_children(upstream_key, published=True): + for child in get_container_children(container_key, published=True): child_block = create_xblock( parent_locator=str(created_block.location), user=request.user, diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 43d336c819c6..5ae09c6b0382 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -22,7 +22,7 @@ from rest_framework.exceptions import NotFound from opaque_keys import InvalidKeyError, OpaqueKey from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryContainerUsageLocator, LibraryUsageLocatorV2 from xblock.exceptions import XBlockNotFoundError from xblock.fields import Scope, String, Integer from xblock.core import XBlockMixin, XBlock @@ -152,19 +152,12 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. from openedx.core.djangoapps.content_libraries.api import ( ContentLibraryContainerNotFound, # pylint: disable=wrong-import-order - get_container, # pylint: disable=wrong-import-order - get_library_block, # pylint: disable=wrong-import-order + get_library_content_metadata, # pylint: disable=wrong-import-order ) - if isinstance(upstream_key, LibraryUsageLocatorV2): - try: - lib_meta = get_library_block(upstream_key) - except XBlockNotFoundError as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc - else: - try: - lib_meta = get_container(upstream_key) - except ContentLibraryContainerNotFound as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc + try: + lib_meta = get_library_content_metadata(upstream_key) + except (XBlockNotFoundError, ContentLibraryContainerNotFound) as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc return cls( upstream_ref=downstream.upstream, version_synced=downstream.upstream_version, @@ -204,7 +197,7 @@ def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBloc @lru_cache -def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryContainerLocator: +def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryContainerUsageLocator: """ Convert upstream key to proper type for given downstream block. @@ -225,18 +218,16 @@ def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryConta raise BadDownstream(_("Cannot update content because it does not belong to a course.")) if downstream.has_children: try: - upstream_key = LibraryContainerLocator.from_string(downstream.upstream) - upstream_type = upstream_key.container_type + upstream_key = LibraryContainerLocator.from_string(downstream.upstream).lib_usage_key except InvalidKeyError as exc: raise BadUpstream(_("Reference to linked library item is malformed")) from exc else: try: upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) - upstream_type = upstream_key.block_type except InvalidKeyError as exc: raise BadUpstream(_("Reference to linked library item is malformed")) from exc downstream_type = downstream.usage_key.block_type - upstream_type = "vertical" if upstream_type == "unit" else upstream_type + upstream_type = "vertical" if upstream_key.block_type == "unit" else upstream_key.block_type if upstream_type != downstream_type: # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. # It could be reasonable to relax this requirement in the future if there's product need for it. diff --git a/openedx/core/djangoapps/content_libraries/api/__init__.py b/openedx/core/djangoapps/content_libraries/api/__init__.py index 4e0b4fcce48e..7e47f6cee8b3 100644 --- a/openedx/core/djangoapps/content_libraries/api/__init__.py +++ b/openedx/core/djangoapps/content_libraries/api/__init__.py @@ -1,10 +1,11 @@ """ Python API for working with content libraries """ +from . import permissions +from .blocks import * from .collections import * from .containers import * from .courseware_import import * from .exceptions import * +from .generic import * from .libraries import * -from .blocks import * -from . import permissions diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index 4f92137ae4ea..4b08a5521a79 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -28,7 +28,7 @@ LIBRARY_CONTAINER_UPDATED, ) from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import Container, Unit +from openedx_learning.api.authoring_models import PublishableEntityVersion from openedx.core.djangoapps.content_libraries.api.collections import library_collection_locator from openedx.core.djangoapps.xblock.api import get_component_from_usage_key @@ -493,8 +493,8 @@ def publish_container_changes(container_key: LibraryContainerLocator, user_id: i ) -def library_container_xml(container: ContainerMetadata, block_type: str | None = None): +def library_container_xml(container: PublishableEntityVersion, block_type: str): """Converts given unit to xml without including children components""" - xml_object = etree.Element(block_type or container.container_type.value) - xml_object.set("display_name", container.display_name) + xml_object = etree.Element(block_type) + xml_object.set("display_name", container.title) return xml_object diff --git a/openedx/core/djangoapps/content_libraries/api/generic.py b/openedx/core/djangoapps/content_libraries/api/generic.py new file mode 100644 index 000000000000..83046060717d --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/api/generic.py @@ -0,0 +1,44 @@ +""" +Content libraries API methods to return blocks or containers based on given keys + +These methods don't enforce permissions (only the REST APIs do). +""" + +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryContainerUsageLocator, LibraryUsageLocatorV2 +from openedx_learning.api.authoring_models import Component, Container + +from openedx.core.djangoapps.xblock.api import get_component_from_usage_key + +from .blocks import LibraryXBlockMetadata, get_library_block +from .containers import ContainerMetadata, get_container, get_container_from_key + +__all__ = [ + "get_library_content_metadata", + "get_library_content", +] + + +def get_library_content_metadata( + usage_key: LibraryUsageLocatorV2 | LibraryContainerUsageLocator, + include_collections=False, +) -> LibraryXBlockMetadata | ContainerMetadata: + """ + Helper api method to return appropriate library content i.e. block metadata or container metadata based on + usage_key. + """ + if isinstance(usage_key, LibraryContainerUsageLocator): + container_key = LibraryContainerLocator.from_usage_key(usage_key) + return get_container(container_key, include_collections) + return get_library_block(usage_key, include_collections) + + +def get_library_content( + usage_key: LibraryUsageLocatorV2 | LibraryContainerUsageLocator, +) -> Container | Component: + """ + Helper api method to return appropriate library content i.e. block or container based on usage_key + """ + if isinstance(usage_key, LibraryContainerUsageLocator): + container_key = LibraryContainerLocator.from_usage_key(usage_key) + return get_container_from_key(container_key) + return get_component_from_usage_key(usage_key) diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index ed5385103cca..a29627afdd4c 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -71,7 +71,7 @@ def get_runtime(user: UserType): def load_block( - usage_key: UsageKeyV2 | LibraryContainerLocator, + usage_key: UsageKeyV2, user: UserType, *, check_permission: CheckPerm | None = CheckPerm.CAN_LEARN, @@ -114,12 +114,7 @@ def load_block( runtime = get_runtime(user=user) try: - if isinstance(usage_key, UsageKeyV2): - return runtime.get_block(usage_key, version=version) - elif isinstance(usage_key, LibraryContainerLocator): - return runtime.get_container_block(usage_key, version=version) - else: - raise NotFound(f"The component '{usage_key}' does not exist.") + return runtime.get_block(usage_key, version=version) except NoSuchUsage as exc: # Convert NoSuchUsage to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The component '{usage_key}' does not exist.") from exc diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index a5221d2a6df4..39c50b94f839 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -13,6 +13,7 @@ from django.urls import reverse from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.locator import LibraryContainerUsageLocator from openedx_learning.api import authoring as authoring_api from lxml import etree @@ -229,23 +230,17 @@ def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion if component_version is None: raise NoSuchUsage(usage_key) - content = component_version.contents.get( - componentversioncontent__key="block.xml" - ) - return self._initialize_block(content.text, usage_key, usage_key.block_type, version) - - def get_container_block(self, container_key, *, version: int | LatestVersion = LatestVersion.AUTO): - """ - Fetch container from learning core data models. - - This method create a very basic olx for container and parse it into an XBlock instance. - """ - from openedx.core.djangoapps.content_libraries.api.containers import get_container, library_container_xml - container = get_container(container_key) - block_type = "vertical" if container_key.container_type == "unit" else container_key.container_type - content = library_container_xml(container, block_type) - xml = etree.tostring(content) - return self._initialize_block(xml.decode(), container_key, block_type, version) + if isinstance(usage_key, LibraryContainerUsageLocator): + from openedx.core.djangoapps.content_libraries.api.containers import library_container_xml + block_type = "vertical" if usage_key.block_type == "unit" else usage_key.block_type + content = library_container_xml(component_version, block_type) + content = etree.tostring(content) + else: + content = component_version.contents.get( + componentversioncontent__key="block.xml" + ) + content = content.text + return self._initialize_block(content, usage_key, usage_key.block_type, version) def get_block_assets(self, block, fetch_asset_data): """ @@ -262,7 +257,7 @@ def get_block_assets(self, block, fetch_asset_data): lookups one by one is going to get slow. At some point we're going to want something to look up a bunch of blocks at once. """ - if not isinstance(block.usage_key, UsageKeyV2): + if isinstance(block.usage_key, LibraryContainerUsageLocator): # TODO: handle assets for containers if required. return [] component_version = self._get_component_version_from_block(block) @@ -341,6 +336,7 @@ def save_block(self, block): def _get_component_from_usage_key(self, usage_key): """ + Gets library block or container based on given usage_key. Note that Components aren't ever really truly deleted, so this will return a Component if this usage key has ever been used, even if it was later deleted. @@ -348,14 +344,9 @@ def _get_component_from_usage_key(self, usage_key): TODO: This is the third place where we're implementing this. Figure out where the definitive place should be and have everything else call that. """ - learning_package = authoring_api.get_learning_package_by_key(str(usage_key.lib_key)) + from openedx.core.djangoapps.content_libraries.api import get_library_content try: - component = authoring_api.get_component_by_key( - learning_package.id, - namespace='xblock.v1', - type_name=usage_key.block_type, - local_key=usage_key.block_id, - ) + component = get_library_content(usage_key) except ObjectDoesNotExist as exc: raise NoSuchUsage(usage_key) from exc From 87762b57d2625b1992b9b709bda3f956be655324 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Mon, 21 Apr 2025 20:24:18 +0530 Subject: [PATCH 07/46] Revert "feat: add support for lct in library_context" This reverts commit 72ba6ef2ae6ffb5c83670d522b75c200810fca17. --- .../content_libraries/library_context.py | 137 ++++-------------- .../core/djangoapps/content_staging/views.py | 30 ++-- .../learning_context/learning_context.py | 34 ++--- .../xblock/learning_context/manager.py | 6 +- setup.py | 1 - 5 files changed, 57 insertions(+), 151 deletions(-) diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 41af54bd3e6f..34a5a90269fe 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -8,8 +8,8 @@ from openedx_events.content_authoring.data import LibraryBlockData, LibraryContainerData from openedx_events.content_authoring.signals import LIBRARY_BLOCK_UPDATED, LIBRARY_CONTAINER_UPDATED -from opaque_keys.edx.keys import UsageKeyV2, OpaqueKey -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryLocatorV2, LibraryUsageLocatorV2 +from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.locator import LibraryUsageLocatorV2, LibraryLocatorV2 from openedx_learning.api import authoring as authoring_api from openedx.core.djangoapps.content_libraries import api, permissions @@ -20,50 +20,41 @@ log = logging.getLogger(__name__) -class LibraryContextPermissionBase: - locator_type = OpaqueKey +class LibraryContextImpl(LearningContext): + """ + Implements content libraries as a learning context. + + This is the *new* content libraries based on Learning Core, not the old content + libraries based on modulestore. + """ def __init__(self, **kwargs): super().__init__(**kwargs) self.use_draft = kwargs.get('use_draft', None) - def get_library_key(self, opaque_key: OpaqueKey): + def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ - Get library key from given opaque_key. - """ - raise NotImplementedError() - - def can_edit_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: - """ - Assuming a block with the specified ID (opaque_key) exists, does the + Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the fields / authored data store)? May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(opaque_key, self.locator_type) - return self._check_perm( - user, - self.get_library_key(opaque_key), - permissions.CAN_EDIT_THIS_CONTENT_LIBRARY - ) + assert isinstance(usage_key, LibraryUsageLocatorV2) + return self._check_perm(user, usage_key.lib_key, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) - def can_view_block_for_editing(self, user: UserType, opaque_key: OpaqueKey) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ - Assuming a block with the specified ID (opaque_key) exists, does the + Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but not necessarily to make changes to it)? May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(opaque_key, self.locator_type) - return self._check_perm( - user, - self.get_library_key(opaque_key), - permissions.CAN_VIEW_THIS_CONTENT_LIBRARY - ) + assert isinstance(usage_key, LibraryUsageLocatorV2) + return self._check_perm(user, usage_key.lib_key, permissions.CAN_VIEW_THIS_CONTENT_LIBRARY) - def can_view_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: + def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ Does the specified usage key exist in its context, and if so, does the specified user have permission to view it and interact with it (call @@ -71,12 +62,8 @@ def can_view_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(opaque_key, self.locator_type) - return self._check_perm( - user, - self.get_library_key(opaque_key), - permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY - ) + assert isinstance(usage_key, LibraryUsageLocatorV2) + return self._check_perm(user, usage_key.lib_key, permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY) def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: """ Helper method to check a permission for the various can_ methods""" @@ -89,23 +76,6 @@ def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: # A 404 is probably what you want in this case, not a 500 error, so do that by default. raise NotFound(f"Content Library '{lib_key}' does not exist") from exc - -class LibraryContextImpl(LibraryContextPermissionBase, LearningContext): - """ - Implements content libraries as a learning context. - - This is the *new* content libraries based on Learning Core, not the old content - libraries based on modulestore. - """ - - locator_type = LibraryUsageLocatorV2 - - def get_library_key(self, opaque_key: OpaqueKey): - """ - Get library key from given opaque_key. - """ - return opaque_key.lib_key - def block_exists(self, usage_key: LibraryUsageLocatorV2): """ Does the block for this usage_key exist in this Library? @@ -133,25 +103,25 @@ def block_exists(self, usage_key: LibraryUsageLocatorV2): local_key=usage_key.block_id, ) - def send_block_updated_event(self, opaque_key: OpaqueKey): + def send_block_updated_event(self, usage_key: UsageKeyV2): """ Send a "block updated" event for the library block with the given usage_key. """ - assert isinstance(opaque_key, self.locator_type) + assert isinstance(usage_key, LibraryUsageLocatorV2) LIBRARY_BLOCK_UPDATED.send_event( library_block=LibraryBlockData( - library_key=opaque_key.lib_key, - usage_key=opaque_key, + library_key=usage_key.lib_key, + usage_key=usage_key, ) ) - def send_container_updated_events(self, opaque_key: OpaqueKey): + def send_container_updated_events(self, usage_key: UsageKeyV2): """ Send "container updated" events for containers that contains the library block with the given usage_key. """ - assert isinstance(opaque_key, self.locator_type) - affected_containers = api.get_containers_contains_component(opaque_key) + assert isinstance(usage_key, LibraryUsageLocatorV2) + affected_containers = api.get_containers_contains_component(usage_key) for container in affected_containers: LIBRARY_CONTAINER_UPDATED.send_event( library_container=LibraryContainerData( @@ -159,56 +129,3 @@ def send_container_updated_events(self, opaque_key: OpaqueKey): background=True, ) ) - - -class LibraryContextContainerImpl(LibraryContextPermissionBase, LearningContext): - """ - Implements content libraries as a learning context for containers in libraries. - - This is the *new* content libraries based on Learning Core, not the old content - libraries based on modulestore. - """ - - locator_type = LibraryContainerLocator - - def get_library_key(self, opaque_key: OpaqueKey): - """ - Get library key from given opaque_key. - """ - return opaque_key.library_key - - def container_exists(self, container_key: LibraryContainerLocator): - """ - Does the container for this key exist in this Library? - - Note that this applies to all versions, i.e. you can put a container key for - a piece of content that has been soft-deleted (removed from Drafts), and - it will still return True here. That's because for the purposes of - permission checking, we just want to know whether that block has ever - existed in this Library, because we could be looking at any older - version of it. - """ - try: - content_lib = ContentLibrary.objects.get_by_key(container_key.library_key) # type: ignore[attr-defined] - except ContentLibrary.DoesNotExist: - return False - - learning_package = content_lib.learning_package - if learning_package is None: - return False - - return authoring_api.container_exists_by_key( - learning_package.id, - container_key, - ) - - def send_block_updated_event(self, opaque_key: OpaqueKey): - """ - Send a "block updated" event for the library block with the given usage_key. - """ - assert isinstance(opaque_key, self.locator_type) - LIBRARY_CONTAINER_UPDATED.send_event( - library_container=LibraryContainerData( - container_key=opaque_key, - ) - ) diff --git a/openedx/core/djangoapps/content_staging/views.py b/openedx/core/djangoapps/content_staging/views.py index e90400e1dbc1..f0a20540ba8d 100644 --- a/openedx/core/djangoapps/content_staging/views.py +++ b/openedx/core/djangoapps/content_staging/views.py @@ -10,7 +10,7 @@ import edx_api_doc_tools as apidocs from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey -from opaque_keys.edx.locator import CourseLocator, LibraryContainerLocator, LibraryLocatorV2 +from opaque_keys.edx.locator import CourseLocator, LibraryLocatorV2 from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError from rest_framework.response import Response from rest_framework.views import APIView @@ -91,18 +91,12 @@ def post(self, request): # Check if the content exists and the user has permission to read it. # Parse the usage key: try: - opaque_key = UsageKey.from_string(request.data["usage_key"]) - if opaque_key.block_type in ('course', 'chapter', 'sequential'): - raise ValidationError('Requested XBlock tree is too large') - except InvalidKeyError: - try: - # Check if valid library container - opaque_key = LibraryContainerLocator.from_string(request.data["usage_key"]) - if opaque_key.container_type != 'unit': - raise ValidationError('Requested XBlock tree is too large') - except (ValueError, InvalidKeyError): - raise ValidationError('Invalid usage key') # lint-amnesty, pylint: disable=raise-missing-from - course_key = getattr(opaque_key, 'context_key', None) or getattr(opaque_key, 'library_key', None) + usage_key = UsageKey.from_string(request.data["usage_key"]) + except (ValueError, InvalidKeyError): + raise ValidationError('Invalid usage key') # lint-amnesty, pylint: disable=raise-missing-from + if usage_key.block_type in ('course', 'chapter', 'sequential'): + raise ValidationError('Requested XBlock tree is too large') + course_key = usage_key.context_key # Load the block and copy it to the user's clipboard try: @@ -112,7 +106,7 @@ def post(self, request): raise PermissionDenied( "You must be a member of the course team in Studio to export OLX using this API." ) - block = modulestore().get_item(opaque_key) + block = modulestore().get_item(usage_key) version_num = None elif isinstance(course_key, LibraryLocatorV2): @@ -121,12 +115,8 @@ def post(self, request): request.user, lib_api.permissions.CAN_VIEW_THIS_CONTENT_LIBRARY ) - if isinstance(opaque_key, LibraryContainerLocator): - # TODO: load unit block data to staging content, probably need to convert unit from library to xml - raise NotImplementedError("Containers not supported yet") - else: - block = xblock_api.load_block(opaque_key, user=None) - version_num = lib_api.get_library_block(opaque_key).draft_version_num + block = xblock_api.load_block(usage_key, user=None) + version_num = lib_api.get_library_block(usage_key).draft_version_num else: raise ValidationError("Invalid usage_key for the content.") diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index e0a754fe2128..dc7a21f1c397 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -3,7 +3,7 @@ of content where learning happens. """ from openedx.core.types import User as UserType -from opaque_keys.edx.keys import OpaqueKey +from opaque_keys.edx.keys import UsageKeyV2 class LearningContext: @@ -25,44 +25,44 @@ def __init__(self, **kwargs): parameters without changing the API. """ - def can_edit_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: # pylint: disable=unused-argument + def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument """ - Assuming a block with the specified ID (opaque_key) exists, does the + Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the fields / authored data store)? user: a Django User object (may be an AnonymousUser) - opaque_key: the OpaqueKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context Must return a boolean. """ return False - def can_view_block_for_editing(self, user: UserType, opaque_key: OpaqueKey) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ - Assuming a block with the specified ID (opaque_key) exists, does the + Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but not necessarily to make changes to it)? """ - return self.can_edit_block(user, opaque_key) + return self.can_edit_block(user, usage_key) - def can_view_block(self, user: UserType, opaque_key: OpaqueKey) -> bool: # pylint: disable=unused-argument + def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument """ - Assuming a block with the specified ID (opaque_key) exists, does the + Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view it and interact with it (call handlers, save user state, etc.)? This is also sometimes called the "can_learn" permission. user: a Django User object (may be an AnonymousUser) - opaque_key: the OpaqueKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context Must return a boolean. """ return False - def definition_for_usage(self, opaque_key, **kwargs): + def definition_for_usage(self, usage_key, **kwargs): """ Given a usage key in this context, return the key indicating the actual XBlock definition. @@ -70,17 +70,17 @@ def definition_for_usage(self, opaque_key, **kwargs): """ raise NotImplementedError - def send_block_updated_event(self, opaque_key): + def send_block_updated_event(self, usage_key): """ - Send a "block updated" event for the block with the given opaque_key in this context. + Send a "block updated" event for the block with the given usage_key in this context. - opaque_key: the OpaqueKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context """ - def send_container_updated_events(self, opaque_key): + def send_container_updated_events(self, usage_key): """ Send "container updated" events for containers that contains the block with - the given opaque_key in this context. + the given usage_key in this context. - opaque_key: the OpaqueKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context """ diff --git a/openedx/core/djangoapps/xblock/learning_context/manager.py b/openedx/core/djangoapps/xblock/learning_context/manager.py index dc390616969e..b7ed1c3c3426 100644 --- a/openedx/core/djangoapps/xblock/learning_context/manager.py +++ b/openedx/core/djangoapps/xblock/learning_context/manager.py @@ -3,7 +3,7 @@ """ from edx_django_utils.plugins import PluginManager from opaque_keys import OpaqueKey -from opaque_keys.edx.keys import LearningContextKey, LibraryItemKey, UsageKeyV2 +from opaque_keys.edx.keys import LearningContextKey, UsageKeyV2 from openedx.core.djangoapps.xblock.apps import get_xblock_app_config @@ -33,8 +33,8 @@ def get_learning_context_impl(key): Raises PluginError if there is some misconfiguration causing the context implementation to not be installed. """ - if isinstance(key, LearningContextKey) or isinstance(key, LibraryItemKey): - context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' or 'lct' + if isinstance(key, LearningContextKey): + context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' elif isinstance(key, UsageKeyV2): context_type = key.context_key.CANONICAL_NAMESPACE elif isinstance(key, OpaqueKey): diff --git a/setup.py b/setup.py index 829bc5e08152..3b8f8c59498d 100644 --- a/setup.py +++ b/setup.py @@ -181,7 +181,6 @@ ], 'openedx.learning_context': [ 'lib = openedx.core.djangoapps.content_libraries.library_context:LibraryContextImpl', - 'lct = openedx.core.djangoapps.content_libraries.library_context:LibraryContextContainerImpl', ], 'openedx.dynamic_partition_generator': [ 'content_type_gating = openedx.features.content_type_gating.partitions:create_content_gating_partition', From fcfc0eeb6dacd9f76c75854f8dd733abf081ff1d Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Tue, 22 Apr 2025 10:18:04 +0530 Subject: [PATCH 08/46] Revert "feat: unit sync using emulated library container usage locator key" This reverts commit e8a8d58b10d182aa552cacfc2641db7bc791f619. --- cms/djangoapps/contentstore/models.py | 4 +- cms/djangoapps/contentstore/utils.py | 4 +- .../xblock_storage_handlers/view_handlers.py | 8 ++-- cms/lib/xblock/upstream_sync.py | 27 ++++++++---- .../content_libraries/api/__init__.py | 5 +-- .../content_libraries/api/containers.py | 8 ++-- .../content_libraries/api/generic.py | 44 ------------------- openedx/core/djangoapps/xblock/api.py | 9 +++- .../xblock/runtime/learning_core_runtime.py | 41 ++++++++++------- 9 files changed, 63 insertions(+), 87 deletions(-) delete mode 100644 openedx/core/djangoapps/content_libraries/api/generic.py diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index a8e59639a33a..ca6171118c00 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -14,7 +14,7 @@ from django.utils.translation import gettext_lazy as _ from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField from opaque_keys.edx.keys import CourseKey, UsageKey -from openedx_learning.api.authoring_models import Component, Container, PublishableEntity +from openedx_learning.api.authoring_models import Component, PublishableEntity from openedx_learning.lib.fields import ( immutable_uuid_field, key_field, @@ -144,7 +144,7 @@ class Meta: @classmethod def update_or_create( cls, - upstream_block: Component | Container | None, + upstream_block: Component | None, /, upstream_usage_key: UsageKey, upstream_context_key: str, diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 10dfd48c6638..1ff309aa6e50 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -86,7 +86,6 @@ from common.djangoapps.xblock_django.api import deprecated_xblocks from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService from openedx.core import toggles as core_toggles -from openedx.core.djangoapps.content_libraries.api import get_library_content from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_RESOURCES_MICROFRONTEND @@ -96,6 +95,7 @@ from openedx.core.djangoapps.models.course_details import CourseDetails from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers from openedx.core.djangoapps.site_configuration.models import SiteConfiguration +from openedx.core.djangoapps.xblock.api import get_component_from_usage_key from openedx.core.lib.courses import course_image_url from openedx.core.lib.html_to_text import html_to_text from openedx.core.lib.teams_config import CONTENT_GROUPS_FOR_TEAMS, TEAM_SCHEME @@ -2380,7 +2380,7 @@ def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, c return None upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) try: - lib_component = get_library_content(upstream_usage_key) + lib_component = get_component_from_usage_key(upstream_usage_key) except ObjectDoesNotExist: log.error(f"Library component not found for {upstream_usage_key}") lib_component = None diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 72cddf82ea79..7c66699605b8 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -28,7 +28,7 @@ ) from edx_proctoring.exceptions import ProctoredExamNotFoundException from help_tokens.core import HelpUrlExpert -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryContainerUsageLocator, LibraryUsageLocator +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocator from pytz import UTC from xblock.core import XBlock from xblock.fields import Scope @@ -526,16 +526,14 @@ def create_item(request): def sync_library_content(created_block, request): upstream_key = get_upstream_key(created_block) - created_block.upstream = str(upstream_key) lib_block = sync_from_upstream(downstream=created_block, user=request.user) static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) store = modulestore() store.update_item(created_block, request.user.id) - if isinstance(upstream_key, LibraryContainerUsageLocator): - container_key = LibraryContainerLocator.from_usage_key(upstream_key) + if isinstance(upstream_key, LibraryContainerLocator): with store.bulk_operations(created_block.location.course_key): notices = [static_file_notices] - for child in get_container_children(container_key, published=True): + for child in get_container_children(upstream_key, published=True): child_block = create_xblock( parent_locator=str(created_block.location), user=request.user, diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 5ae09c6b0382..43d336c819c6 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -22,7 +22,7 @@ from rest_framework.exceptions import NotFound from opaque_keys import InvalidKeyError, OpaqueKey from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryContainerUsageLocator, LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from xblock.exceptions import XBlockNotFoundError from xblock.fields import Scope, String, Integer from xblock.core import XBlockMixin, XBlock @@ -152,12 +152,19 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. from openedx.core.djangoapps.content_libraries.api import ( ContentLibraryContainerNotFound, # pylint: disable=wrong-import-order - get_library_content_metadata, # pylint: disable=wrong-import-order + get_container, # pylint: disable=wrong-import-order + get_library_block, # pylint: disable=wrong-import-order ) - try: - lib_meta = get_library_content_metadata(upstream_key) - except (XBlockNotFoundError, ContentLibraryContainerNotFound) as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc + if isinstance(upstream_key, LibraryUsageLocatorV2): + try: + lib_meta = get_library_block(upstream_key) + except XBlockNotFoundError as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc + else: + try: + lib_meta = get_container(upstream_key) + except ContentLibraryContainerNotFound as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc return cls( upstream_ref=downstream.upstream, version_synced=downstream.upstream_version, @@ -197,7 +204,7 @@ def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBloc @lru_cache -def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryContainerUsageLocator: +def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryContainerLocator: """ Convert upstream key to proper type for given downstream block. @@ -218,16 +225,18 @@ def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryConta raise BadDownstream(_("Cannot update content because it does not belong to a course.")) if downstream.has_children: try: - upstream_key = LibraryContainerLocator.from_string(downstream.upstream).lib_usage_key + upstream_key = LibraryContainerLocator.from_string(downstream.upstream) + upstream_type = upstream_key.container_type except InvalidKeyError as exc: raise BadUpstream(_("Reference to linked library item is malformed")) from exc else: try: upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) + upstream_type = upstream_key.block_type except InvalidKeyError as exc: raise BadUpstream(_("Reference to linked library item is malformed")) from exc downstream_type = downstream.usage_key.block_type - upstream_type = "vertical" if upstream_key.block_type == "unit" else upstream_key.block_type + upstream_type = "vertical" if upstream_type == "unit" else upstream_type if upstream_type != downstream_type: # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. # It could be reasonable to relax this requirement in the future if there's product need for it. diff --git a/openedx/core/djangoapps/content_libraries/api/__init__.py b/openedx/core/djangoapps/content_libraries/api/__init__.py index 7e47f6cee8b3..4e0b4fcce48e 100644 --- a/openedx/core/djangoapps/content_libraries/api/__init__.py +++ b/openedx/core/djangoapps/content_libraries/api/__init__.py @@ -1,11 +1,10 @@ """ Python API for working with content libraries """ -from . import permissions -from .blocks import * from .collections import * from .containers import * from .courseware_import import * from .exceptions import * -from .generic import * from .libraries import * +from .blocks import * +from . import permissions diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index 4b08a5521a79..4f92137ae4ea 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -28,7 +28,7 @@ LIBRARY_CONTAINER_UPDATED, ) from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import PublishableEntityVersion +from openedx_learning.api.authoring_models import Container, Unit from openedx.core.djangoapps.content_libraries.api.collections import library_collection_locator from openedx.core.djangoapps.xblock.api import get_component_from_usage_key @@ -493,8 +493,8 @@ def publish_container_changes(container_key: LibraryContainerLocator, user_id: i ) -def library_container_xml(container: PublishableEntityVersion, block_type: str): +def library_container_xml(container: ContainerMetadata, block_type: str | None = None): """Converts given unit to xml without including children components""" - xml_object = etree.Element(block_type) - xml_object.set("display_name", container.title) + xml_object = etree.Element(block_type or container.container_type.value) + xml_object.set("display_name", container.display_name) return xml_object diff --git a/openedx/core/djangoapps/content_libraries/api/generic.py b/openedx/core/djangoapps/content_libraries/api/generic.py deleted file mode 100644 index 83046060717d..000000000000 --- a/openedx/core/djangoapps/content_libraries/api/generic.py +++ /dev/null @@ -1,44 +0,0 @@ -""" -Content libraries API methods to return blocks or containers based on given keys - -These methods don't enforce permissions (only the REST APIs do). -""" - -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryContainerUsageLocator, LibraryUsageLocatorV2 -from openedx_learning.api.authoring_models import Component, Container - -from openedx.core.djangoapps.xblock.api import get_component_from_usage_key - -from .blocks import LibraryXBlockMetadata, get_library_block -from .containers import ContainerMetadata, get_container, get_container_from_key - -__all__ = [ - "get_library_content_metadata", - "get_library_content", -] - - -def get_library_content_metadata( - usage_key: LibraryUsageLocatorV2 | LibraryContainerUsageLocator, - include_collections=False, -) -> LibraryXBlockMetadata | ContainerMetadata: - """ - Helper api method to return appropriate library content i.e. block metadata or container metadata based on - usage_key. - """ - if isinstance(usage_key, LibraryContainerUsageLocator): - container_key = LibraryContainerLocator.from_usage_key(usage_key) - return get_container(container_key, include_collections) - return get_library_block(usage_key, include_collections) - - -def get_library_content( - usage_key: LibraryUsageLocatorV2 | LibraryContainerUsageLocator, -) -> Container | Component: - """ - Helper api method to return appropriate library content i.e. block or container based on usage_key - """ - if isinstance(usage_key, LibraryContainerUsageLocator): - container_key = LibraryContainerLocator.from_usage_key(usage_key) - return get_container_from_key(container_key) - return get_component_from_usage_key(usage_key) diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index a29627afdd4c..ed5385103cca 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -71,7 +71,7 @@ def get_runtime(user: UserType): def load_block( - usage_key: UsageKeyV2, + usage_key: UsageKeyV2 | LibraryContainerLocator, user: UserType, *, check_permission: CheckPerm | None = CheckPerm.CAN_LEARN, @@ -114,7 +114,12 @@ def load_block( runtime = get_runtime(user=user) try: - return runtime.get_block(usage_key, version=version) + if isinstance(usage_key, UsageKeyV2): + return runtime.get_block(usage_key, version=version) + elif isinstance(usage_key, LibraryContainerLocator): + return runtime.get_container_block(usage_key, version=version) + else: + raise NotFound(f"The component '{usage_key}' does not exist.") except NoSuchUsage as exc: # Convert NoSuchUsage to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The component '{usage_key}' does not exist.") from exc diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 39c50b94f839..a5221d2a6df4 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -13,7 +13,6 @@ from django.urls import reverse from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerUsageLocator from openedx_learning.api import authoring as authoring_api from lxml import etree @@ -230,17 +229,23 @@ def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion if component_version is None: raise NoSuchUsage(usage_key) - if isinstance(usage_key, LibraryContainerUsageLocator): - from openedx.core.djangoapps.content_libraries.api.containers import library_container_xml - block_type = "vertical" if usage_key.block_type == "unit" else usage_key.block_type - content = library_container_xml(component_version, block_type) - content = etree.tostring(content) - else: - content = component_version.contents.get( - componentversioncontent__key="block.xml" - ) - content = content.text - return self._initialize_block(content, usage_key, usage_key.block_type, version) + content = component_version.contents.get( + componentversioncontent__key="block.xml" + ) + return self._initialize_block(content.text, usage_key, usage_key.block_type, version) + + def get_container_block(self, container_key, *, version: int | LatestVersion = LatestVersion.AUTO): + """ + Fetch container from learning core data models. + + This method create a very basic olx for container and parse it into an XBlock instance. + """ + from openedx.core.djangoapps.content_libraries.api.containers import get_container, library_container_xml + container = get_container(container_key) + block_type = "vertical" if container_key.container_type == "unit" else container_key.container_type + content = library_container_xml(container, block_type) + xml = etree.tostring(content) + return self._initialize_block(xml.decode(), container_key, block_type, version) def get_block_assets(self, block, fetch_asset_data): """ @@ -257,7 +262,7 @@ def get_block_assets(self, block, fetch_asset_data): lookups one by one is going to get slow. At some point we're going to want something to look up a bunch of blocks at once. """ - if isinstance(block.usage_key, LibraryContainerUsageLocator): + if not isinstance(block.usage_key, UsageKeyV2): # TODO: handle assets for containers if required. return [] component_version = self._get_component_version_from_block(block) @@ -336,7 +341,6 @@ def save_block(self, block): def _get_component_from_usage_key(self, usage_key): """ - Gets library block or container based on given usage_key. Note that Components aren't ever really truly deleted, so this will return a Component if this usage key has ever been used, even if it was later deleted. @@ -344,9 +348,14 @@ def _get_component_from_usage_key(self, usage_key): TODO: This is the third place where we're implementing this. Figure out where the definitive place should be and have everything else call that. """ - from openedx.core.djangoapps.content_libraries.api import get_library_content + learning_package = authoring_api.get_learning_package_by_key(str(usage_key.lib_key)) try: - component = get_library_content(usage_key) + component = authoring_api.get_component_by_key( + learning_package.id, + namespace='xblock.v1', + type_name=usage_key.block_type, + local_key=usage_key.block_id, + ) except ObjectDoesNotExist as exc: raise NoSuchUsage(usage_key) from exc From 8f89cd68c85224438e7e35384daf42f195144cd3 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Tue, 22 Apr 2025 19:49:21 +0530 Subject: [PATCH 09/46] feat: separate managers for component and container upstream sync --- cms/djangoapps/contentstore/helpers.py | 5 +- .../rest_api/v2/views/downstreams.py | 5 +- .../xblock_storage_handlers/view_handlers.py | 46 +-- cms/lib/xblock/container_upstream_sync.py | 202 ++++++++++ cms/lib/xblock/test/test_upstream_sync.py | 6 +- cms/lib/xblock/upstream_sync.py | 375 ++++++++++-------- .../content_libraries/api/containers.py | 4 +- .../content_libraries/library_context.py | 20 +- .../xblock/learning_context/manager.py | 4 +- 9 files changed, 452 insertions(+), 215 deletions(-) create mode 100644 cms/lib/xblock/container_upstream_sync.py diff --git a/cms/djangoapps/contentstore/helpers.py b/cms/djangoapps/contentstore/helpers.py index 0f7384af72ad..375ec2194fa4 100644 --- a/cms/djangoapps/contentstore/helpers.py +++ b/cms/djangoapps/contentstore/helpers.py @@ -31,7 +31,7 @@ ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel -from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException, fetch_customizable_fields +from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException, ComponentUpstreamSyncManager from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers import openedx.core.djangoapps.content_staging.api as content_staging_api import openedx.core.djangoapps.content_tagging.api as content_tagging_api @@ -456,7 +456,8 @@ def _fetch_and_set_upstream_link( # later wants to restore it, it will restore to the value that the field had when the block was pasted. Of # course, if the author later syncs updates from a *future* published upstream version, then that will fetch # new values from the published upstream content. - fetch_customizable_fields(upstream=temp_xblock, downstream=temp_xblock, user=user) + manager = ComponentUpstreamSyncManager(downstream=temp_xblock, user=user, upstream=temp_xblock) + manager.update_customizable_fields(only_fetch=True) def _import_xml_node_to_parent( diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py index 29e6a3961dcc..3fd18340f38a 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py @@ -103,11 +103,11 @@ from cms.lib.xblock.upstream_sync import ( BadDownstream, BadUpstream, + ComponentUpstreamSyncManager, NoUpstream, UpstreamLink, UpstreamLinkException, decline_sync, - fetch_customizable_fields, sever_upstream_link, sync_from_upstream, ) @@ -257,7 +257,8 @@ def put(self, request: _AuthenticatedRequest, usage_key_string: str) -> Response # Even if we're not syncing (i.e., updating the downstream's values with the upstream's), we still need # to fetch the upstream's customizable values and store them as hidden fields on the downstream. This # ensures that downstream authors can restore defaults based on the upstream. - fetch_customizable_fields(downstream=downstream, user=request.user) + manager = ComponentUpstreamSyncManager(downstream=downstream, user=request.user) + manager.update_customizable_fields(only_fetch=True) except BadDownstream as exc: logger.exception( "'%s' is an invalid downstream; refusing to set its upstream to '%s'", diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 7c66699605b8..46e1c38b3f22 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -19,7 +19,6 @@ from django.http import HttpResponse, HttpResponseBadRequest from django.utils.translation import gettext as _ from edx_django_utils.plugins import pluggable_override -from openedx.core.djangoapps.content_libraries.api.containers import get_container_children from openedx.core.djangoapps.content_tagging.api import get_object_tag_counts from edx_proctoring.api import ( does_backend_support_onboarding, @@ -28,7 +27,7 @@ ) from edx_proctoring.exceptions import ProctoredExamNotFoundException from help_tokens.core import HelpUrlExpert -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocator +from opaque_keys.edx.locator import LibraryUsageLocator, LibraryUsageLocatorV2 from pytz import UTC from xblock.core import XBlock from xblock.fields import Scope @@ -36,7 +35,8 @@ from cms.djangoapps.contentstore.config.waffle import SHOW_REVIEW_RULES_FLAG from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.lib.ai_aside_summary_config import AiAsideSummaryConfig -from cms.lib.xblock.upstream_sync import BadUpstream, get_upstream_key, sync_from_upstream +from cms.lib.xblock.upstream_sync import BadUpstream, check_and_parse_upstream_key, sync_from_upstream +from cms.lib.xblock.container_upstream_sync import sync_from_upstream_container from common.djangoapps.static_replace import replace_static_urls from common.djangoapps.student.auth import ( has_studio_read_access, @@ -524,26 +524,23 @@ def create_item(request): return _create_block(request) -def sync_library_content(created_block, request): - upstream_key = get_upstream_key(created_block) - lib_block = sync_from_upstream(downstream=created_block, user=request.user) - static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) - store = modulestore() - store.update_item(created_block, request.user.id) - if isinstance(upstream_key, LibraryContainerLocator): - with store.bulk_operations(created_block.location.course_key): - notices = [static_file_notices] - for child in get_container_children(upstream_key, published=True): - child_block = create_xblock( - parent_locator=str(created_block.location), - user=request.user, - category=child.usage_key.block_type, - display_name=child.display_name, - ) - child_block.upstream = str(child.usage_key) - sync_library_content(child_block, request) - notices.append(sync_library_content(child_block, request)) - static_file_notices = concat_static_file_notices(notices) +def sync_library_content(created_block, request, store): + upstream_key = check_and_parse_upstream_key(created_block.upstream, created_block.usage_key) + if isinstance(upstream_key, LibraryUsageLocatorV2): + lib_block = sync_from_upstream(downstream=created_block, user=request.user) + static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) + store.update_item(created_block, request.user.id) + else: + lib_block, children_blocks = sync_from_upstream_container(downstream=created_block, user=request.user) + notices = [import_static_assets_for_library_sync(created_block, lib_block, request)] + with store.bulk_operations(created_block.location.context_key): + children_blocks_usage_keys = [] + for child_block in children_blocks: + notices.append(sync_library_content(child_block, request, store)) + children_blocks_usage_keys.append(child_block.usage_key) + created_block.children = children_blocks_usage_keys + store.update_item(created_block, request.user.id) + static_file_notices = concat_static_file_notices(notices) return static_file_notices @@ -618,7 +615,8 @@ def _create_block(request): # Set `created_block.upstream` and then sync this with the upstream (library) version. created_block.upstream = upstream_ref try: - static_file_notices = sync_library_content(created_block, request) + store = modulestore() + static_file_notices = sync_library_content(created_block, request, store) except BadUpstream as exc: _delete_item(created_block.location, request.user) log.exception( diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/container_upstream_sync.py new file mode 100644 index 000000000000..d1ae43302fc9 --- /dev/null +++ b/cms/lib/xblock/container_upstream_sync.py @@ -0,0 +1,202 @@ +import logging +import typing as t +from dataclasses import asdict, dataclass + +from django.conf import settings +from django.contrib.auth.models import User # pylint: disable=imported-auth-user +from django.core.exceptions import PermissionDenied +from django.utils.translation import gettext_lazy as _ +from opaque_keys import InvalidKeyError +from opaque_keys.edx.locator import LibraryContainerLocator +from rest_framework.exceptions import NotFound +from xblock.core import XBlock +from xblock.fields import Scope + +from cms.djangoapps.contentstore.xblock_storage_handlers.create_xblock import create_xblock +from openedx.core.djangoapps.content_libraries.api.containers import get_container_children + +from .upstream_sync import ( + BadUpstream, + BaseUpstreamLink, + BaseUpstreamSyncManager, + UpstreamLinkException, + check_and_parse_upstream_key, +) + +logger = logging.getLogger(__name__) + + +@dataclass(frozen=True) +class ContainerUpstreamLink(BaseUpstreamLink): + """ + Metadata about some downstream content's relationship with its linked upstream content. + """ + + @property + def upstream_link(self) -> str | None: + """ + Link to edit/view upstream block in library. + """ + if self.version_available is None or self.upstream_ref is None: + return None + try: + container_key = LibraryContainerLocator.from_string(self.upstream_ref) + except InvalidKeyError: + return None + return _get_library_container_url(container_key) + + def to_json(self) -> dict[str, t.Any]: + """ + Get an JSON-API-friendly representation of this upstream link. + """ + return { + **asdict(self), + "ready_to_sync": self.ready_to_sync, + "upstream_link": self.upstream_link, + } + + @classmethod + def get_for_container(cls, downstream: XBlock) -> t.Self: + """ + Get info on a container's relationship with its linked upstream content (without actually loading the content). + + Currently, the only supported upstreams are LC-backed Library Components. This may change in the future (see + module docstring). + + If link exists, is supported, and is followable, returns UpstreamLink. + Otherwise, raises an UpstreamLinkException. + """ + upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) + # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + from openedx.core.djangoapps.content_libraries.api import ( + ContentLibraryContainerNotFound, # pylint: disable=wrong-import-order + get_container, # pylint: disable=wrong-import-order + ) + if not isinstance(upstream_key, LibraryContainerLocator): + raise BadUpstream(_("Invalid upstream_key")) + try: + lib_meta = get_container(upstream_key) + except ContentLibraryContainerNotFound as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc + return cls( + upstream_ref=downstream.upstream, + version_synced=downstream.upstream_version, + version_available=(lib_meta.published_version_num if lib_meta else None), + version_declined=downstream.upstream_version_declined, + error_message=None, + ) + + @classmethod + def try_get_for_container(cls, downstream: XBlock) -> t.Self: + """ + Same as `get_for_container`, but upon failure, sets `.error_message` instead of raising an exception. + """ + try: + return cls.get_for_container(downstream) + except UpstreamLinkException as exc: + logger.exception( + "Tried to inspect an unsupported, broken, or missing downstream->upstream link: '%s'->'%s'", + downstream.usage_key, + downstream.upstream, + ) + return cls( + upstream_ref=downstream.upstream, + version_synced=downstream.upstream_version, + version_available=None, + version_declined=None, + error_message=str(exc), + ) + + +def _get_library_container_url(container_key: LibraryContainerLocator): + """ + Gets authoring url for given library_key. + """ + library_url = None + if mfe_base_url := settings.COURSE_AUTHORING_MICROFRONTEND_URL: # type: ignore + library_key = container_key.context_key + library_url = f'{mfe_base_url}/library/{library_key}/units?container_key={container_key}' + return library_url + + +class ContainerUpstreamSyncManager(BaseUpstreamSyncManager): + def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: + super().__init__(downstream, user, upstream) + if not isinstance(self.upstream_key, LibraryContainerLocator): + raise BadUpstream('Invalid upstream key') + self.link = ContainerUpstreamLink.get_for_container(downstream) + if not upstream: + self.upstream = self._load_upstream_link_and_container_block() + self.syncable_field_names: set[str] = self._get_synchronizable_fields() + self.new_children_blocks: list[XBlock] = [] + + def _get_synchronizable_fields(self) -> set[str]: + """ + The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. + """ + return set.intersection(*[ + set( + field_name + for (field_name, field) in block.__class__.fields.items() + if field.scope in [Scope.settings, Scope.content] + ) + for block in [self.upstream, self.downstream] + ]) + + def _load_upstream_link_and_container_block(self) -> XBlock: + """ + Load the upstream metadata and content for a downstream block. + + Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be + relaxed in the future (see module docstring). + + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + from openedx.core.djangoapps.xblock.api import ( # pylint: disable=wrong-import-order + CheckPerm, + LatestVersion, + load_block, + ) + try: + lib_block: XBlock = load_block( + self.upstream_key, + self.user, + check_permission=CheckPerm.CAN_READ_AS_AUTHOR, + version=LatestVersion.PUBLISHED, + ) + except (NotFound, PermissionDenied) as exc: + raise BadUpstream(_("Linked library item could not be loaded: {}").format(self.upstream_key)) from exc + return lib_block + + def sync_new_children_blocks(self): + for child in get_container_children(self.upstream_key, published=True): + child_block = create_xblock( + parent_locator=str(self.downstream.location), + user=self.user, + category=child.usage_key.block_type, + display_name=child.display_name, + ) + child_block.upstream = str(child.usage_key) + self.new_children_blocks.append(child_block) + return self.new_children_blocks + + def sync(self) -> None: + super().sync() + self.sync_new_children_blocks() + + +def sync_from_upstream_container(downstream: XBlock, user: User) -> tuple[XBlock, list[XBlock]]: + """ + Update `downstream` with content+settings from the latest available version of its linked upstream content. + + Preserves overrides to customizable fields; overwrites overrides to other fields. + Does not save `downstream` to the store. That is left up to the caller. + + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + manager = ContainerUpstreamSyncManager(downstream, user) + manager.sync() + downstream.upstream_version = manager.link.version_available + return manager.upstream, manager.new_children_blocks + diff --git a/cms/lib/xblock/test/test_upstream_sync.py b/cms/lib/xblock/test/test_upstream_sync.py index 71fa7d51bb9d..93c87349f680 100644 --- a/cms/lib/xblock/test/test_upstream_sync.py +++ b/cms/lib/xblock/test/test_upstream_sync.py @@ -9,8 +9,9 @@ from organizations.models import Organization from cms.lib.xblock.upstream_sync import ( + ComponentUpstreamSyncManager, UpstreamLink, - sync_from_upstream, decline_sync, fetch_customizable_fields, sever_upstream_link, + sync_from_upstream, decline_sync, sever_upstream_link, NoUpstream, BadUpstream, BadDownstream, ) from common.djangoapps.student.tests.factories import UserFactory @@ -415,7 +416,8 @@ def test_fetch_customizable_fields(self, initial_upstream_display_name): # fetch! upstream = xblock.load_block(self.upstream_key, self.user) - fetch_customizable_fields(upstream=upstream, downstream=downstream, user=self.user) + manager = ComponentUpstreamSyncManager(downstream=downstream, user=self.user, upstream=upstream) + manager.update_customizable_fields(only_fetch=True) # Ensure: fetching doesn't affect the upstream link (or lack thereof). assert not downstream.upstream diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 43d336c819c6..50d599f9256c 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -11,22 +11,21 @@ """ from __future__ import annotations -from functools import lru_cache import logging import typing as t -from dataclasses import dataclass, asdict +from dataclasses import asdict, dataclass +from functools import lru_cache from django.conf import settings from django.core.exceptions import PermissionDenied from django.utils.translation import gettext_lazy as _ -from rest_framework.exceptions import NotFound -from opaque_keys import InvalidKeyError, OpaqueKey -from opaque_keys.edx.keys import CourseKey +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey, UsageKeyV2 from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 +from rest_framework.exceptions import NotFound +from xblock.core import XBlock, XBlockMixin from xblock.exceptions import XBlockNotFoundError -from xblock.fields import Scope, String, Integer -from xblock.core import XBlockMixin, XBlock - +from xblock.fields import Integer, Scope, String if t.TYPE_CHECKING: from django.contrib.auth.models import User # pylint: disable=imported-auth-user @@ -71,9 +70,9 @@ def __init__(self): @dataclass(frozen=True) -class UpstreamLink: +class BaseUpstreamLink: """ - Metadata about some downstream content's relationship with its linked upstream content. + Base class to track metadata about some downstream content's relationship with its linked upstream content. """ upstream_ref: str | None # Reference to the upstream content, e.g., a serialized library block usage key. version_synced: int | None # Version of the upstream to which the downstream was last synced. @@ -93,6 +92,25 @@ def ready_to_sync(self) -> bool: self.version_available > (self.version_declined or 0) ) + +@dataclass(frozen=True) +class UpstreamLink(BaseUpstreamLink): + """ + Metadata about some downstream content's relationship with its linked upstream content. + """ + + @property + def ready_to_sync(self) -> bool: + """ + Should we invite the downstream's authors to sync the latest upstream updates? + """ + return bool( + self.upstream_ref and + self.version_available and + self.version_available > (self.version_synced or 0) and + self.version_available > (self.version_declined or 0) + ) + @property def upstream_link(self) -> str | None: """ @@ -148,23 +166,17 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: If link exists, is supported, and is followable, returns UpstreamLink. Otherwise, raises an UpstreamLinkException. """ - upstream_key = get_upstream_key(downstream) + upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. from openedx.core.djangoapps.content_libraries.api import ( - ContentLibraryContainerNotFound, # pylint: disable=wrong-import-order - get_container, # pylint: disable=wrong-import-order get_library_block, # pylint: disable=wrong-import-order ) - if isinstance(upstream_key, LibraryUsageLocatorV2): - try: - lib_meta = get_library_block(upstream_key) - except XBlockNotFoundError as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc - else: - try: - lib_meta = get_container(upstream_key) - except ContentLibraryContainerNotFound as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc + if not isinstance(upstream_key, LibraryUsageLocatorV2): + raise BadUpstream(_("Invalid upstream_key")) + try: + lib_meta = get_library_block(upstream_key) + except XBlockNotFoundError as exc: + raise BadUpstream(_("Linked library item was not found in the system")) from exc return cls( upstream_ref=downstream.upstream, version_synced=downstream.upstream_version, @@ -174,6 +186,156 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: ) +class BaseUpstreamSyncManager: + def __init__( + self, + downstream: XBlock, + user: User, + upstream: XBlock | None, + ) -> None: + self.downstream = downstream + self.user = user + self.upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) + self.link: BaseUpstreamLink + self.upstream = upstream + self.syncable_field_names: set[str] + + def update_customizable_fields(self, *, only_fetch: bool) -> None: + """ + For each customizable field: + * Save the upstream value to a hidden field on the downstream ("FETCH"). + * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: + * Update it the downstream field's value from the upstream field ("SYNC"). + + Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. + + * Say that the customizable fields are [display_name, max_attempts]. + + * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). + * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: + * Set `course_problem.display_name = lib_problem.display_name` ("sync"). + """ + for field_name, fetch_field_name in self.downstream.get_customizable_fields().items(): + + if field_name not in self.syncable_field_names: + continue + + # Downstream-only fields don't have an upstream fetch field + if fetch_field_name is None: + continue + + # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). + old_upstream_value = getattr(self.downstream, fetch_field_name) + new_upstream_value = getattr(self.upstream, field_name) + setattr(self.downstream, fetch_field_name, new_upstream_value) + + if only_fetch: + continue + + # Okay, now for the nuanced part... + # We need to update the downstream field *iff it has not been customized**. + # Determining whether a field has been customized will differ in Beta vs Future release. + # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) + + ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. + # if field_name in downstream.downstream_customized: + # continue + + ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. + downstream_value = getattr(self.downstream, field_name) + if old_upstream_value and downstream_value != old_upstream_value: + continue # Field has been customized. Don't touch it. Move on. + + # Field isn't customized -- SYNC it! + setattr(self.downstream, field_name, new_upstream_value) + + def update_non_customizable_fields(self) -> None: + """ + For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. + """ + customizable_fields = set(self.downstream.get_customizable_fields().keys()) + isVideoBlock = self.downstream.usage_key.block_type == "video" + for field_name in self.syncable_field_names - customizable_fields: + if isVideoBlock and field_name == 'edx_video_id': + # Avoid overwriting edx_video_id between blocks + continue + new_upstream_value = getattr(self.upstream, field_name) + setattr(self.downstream, field_name, new_upstream_value) + + def update_tags(self) -> None: + """ + Update tags from `upstream` to `downstream` + """ + from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only + # For any block synced with an upstream, copy the tags as read_only + # This keeps tags added locally. + copy_tags_as_read_only( + str(self.upstream.location), + str(self.downstream.location), + ) + + def sync(self) -> None: + """ + Update `downstream` with content+settings from the latest available version of its linked upstream content. + + Preserves overrides to customizable fields; overwrites overrides to other fields. + Does not save `downstream` to the store. That is left up to the caller. + + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + self.update_customizable_fields(only_fetch=False) + self.update_non_customizable_fields() + self.update_tags() + + +class ComponentUpstreamSyncManager(BaseUpstreamSyncManager): + def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: + super().__init__(downstream, user, upstream) + self.link = UpstreamLink.get_for_block(downstream) + if not upstream: + self.upstream = self._load_upstream_link_and_block() + self.syncable_field_names: set[str] = self._get_synchronizable_fields() + + def _get_synchronizable_fields(self) -> set[str]: + """ + The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. + """ + return set.intersection(*[ + set( + field_name + for (field_name, field) in block.__class__.fields.items() + if field.scope in [Scope.settings, Scope.content] + ) + for block in [self.upstream, self.downstream] + ]) + + def _load_upstream_link_and_block(self) -> XBlock: + """ + Load the upstream metadata and content for a downstream block. + + Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be + relaxed in the future (see module docstring). + + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + from openedx.core.djangoapps.xblock.api import ( + CheckPerm, + LatestVersion, + load_block, + ) + try: + lib_block: XBlock = load_block( + self.upstream_key, + self.user, + check_permission=CheckPerm.CAN_READ_AS_AUTHOR, + version=LatestVersion.PUBLISHED, + ) + except (NotFound, PermissionDenied) as exc: + raise BadUpstream(_("Linked library item could not be loaded: {}").format(self.upstream_key)) from exc + return lib_block + + def sync_from_upstream(downstream: XBlock, user: User) -> XBlock: """ Update `downstream` with content+settings from the latest available version of its linked upstream content. @@ -183,28 +345,17 @@ def sync_from_upstream(downstream: XBlock, user: User) -> XBlock: If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. """ - link, upstream = _load_upstream_link_and_block(downstream, user) - _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=False) - _update_non_customizable_fields(upstream=upstream, downstream=downstream) - _update_tags(upstream=upstream, downstream=downstream) - downstream.upstream_version = link.version_available - return upstream - - -def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: - """ - Fetch upstream-defined value of customizable fields and save them on the downstream. - - If `upstream` is provided, use that block as the upstream. - Otherwise, load the block specified by `downstream.upstream`, which may raise an UpstreamLinkException. - """ - if not upstream: - _link, upstream = _load_upstream_link_and_block(downstream, user) - _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) + manager = ComponentUpstreamSyncManager(downstream, user) + manager.sync() + downstream.upstream_version = manager.link.version_available + return manager.upstream @lru_cache -def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryContainerLocator: +def check_and_parse_upstream_key( + upstream: str | None, + downstream_usage_key: UsageKeyV2, +) -> LibraryUsageLocatorV2 | LibraryContainerLocator: """ Convert upstream key to proper type for given downstream block. @@ -219,23 +370,20 @@ def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryConta BadDownstream: BadUpstream: """ - if not downstream.upstream: + if not upstream: raise NoUpstream() - if not isinstance(downstream.usage_key.context_key, CourseKey): + if not isinstance(downstream_usage_key.context_key, CourseKey): raise BadDownstream(_("Cannot update content because it does not belong to a course.")) - if downstream.has_children: + try: + upstream_key = LibraryUsageLocatorV2.from_string(upstream) + upstream_type = upstream_key.block_type + except InvalidKeyError: try: - upstream_key = LibraryContainerLocator.from_string(downstream.upstream) + upstream_key = LibraryContainerLocator.from_string(upstream) upstream_type = upstream_key.container_type except InvalidKeyError as exc: raise BadUpstream(_("Reference to linked library item is malformed")) from exc - else: - try: - upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) - upstream_type = upstream_key.block_type - except InvalidKeyError as exc: - raise BadUpstream(_("Reference to linked library item is malformed")) from exc - downstream_type = downstream.usage_key.block_type + downstream_type = downstream_usage_key.block_type upstream_type = "vertical" if upstream_type == "unit" else upstream_type if upstream_type != downstream_type: # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. @@ -246,131 +394,12 @@ def get_upstream_key(downstream: XBlock) -> LibraryUsageLocatorV2 | LibraryConta downstream_type=downstream_type, upstream_type=upstream_type ) ) from TypeError( - f"downstream block '{downstream.usage_key}' is linked to " + f"downstream block '{downstream_usage_key}' is linked to " f"upstream block of different type '{upstream_key}'" ) return upstream_key -def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[UpstreamLink, XBlock]: - """ - Load the upstream metadata and content for a downstream block. - - Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be - relaxed in the future (see module docstring). - - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. - """ - link = UpstreamLink.get_for_block(downstream) # can raise UpstreamLinkException - upstream_key = get_upstream_key(downstream) - # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. - from openedx.core.djangoapps.xblock.api import load_block, CheckPerm, LatestVersion # pylint: disable=wrong-import-order - try: - lib_block: XBlock = load_block( - upstream_key, - user, - check_permission=CheckPerm.CAN_READ_AS_AUTHOR, - version=LatestVersion.PUBLISHED, - ) - except (NotFound, PermissionDenied) as exc: - raise BadUpstream(_("Linked library item could not be loaded: {}").format(downstream.upstream)) from exc - return link, lib_block - - -def _update_customizable_fields(*, upstream: XBlock, downstream: XBlock, only_fetch: bool) -> None: - """ - For each customizable field: - * Save the upstream value to a hidden field on the downstream ("FETCH"). - * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: - * Update it the downstream field's value from the upstream field ("SYNC"). - - Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. - - * Say that the customizable fields are [display_name, max_attempts]. - - * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). - * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: - * Set `course_problem.display_name = lib_problem.display_name` ("sync"). - """ - syncable_field_names = _get_synchronizable_fields(upstream, downstream) - - for field_name, fetch_field_name in downstream.get_customizable_fields().items(): - - if field_name not in syncable_field_names: - continue - - # Downstream-only fields don't have an upstream fetch field - if fetch_field_name is None: - continue - - # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). - old_upstream_value = getattr(downstream, fetch_field_name) - new_upstream_value = getattr(upstream, field_name) - setattr(downstream, fetch_field_name, new_upstream_value) - - if only_fetch: - continue - - # Okay, now for the nuanced part... - # We need to update the downstream field *iff it has not been customized**. - # Determining whether a field has been customized will differ in Beta vs Future release. - # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) - - ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. - # if field_name in downstream.downstream_customized: - # continue - - ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. - downstream_value = getattr(downstream, field_name) - if old_upstream_value and downstream_value != old_upstream_value: - continue # Field has been customized. Don't touch it. Move on. - - # Field isn't customized -- SYNC it! - setattr(downstream, field_name, new_upstream_value) - - -def _update_non_customizable_fields(*, upstream: XBlock, downstream: XBlock) -> None: - """ - For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. - """ - syncable_fields = _get_synchronizable_fields(upstream, downstream) - customizable_fields = set(downstream.get_customizable_fields().keys()) - isVideoBlock = downstream.usage_key.block_type == "video" - for field_name in syncable_fields - customizable_fields: - if isVideoBlock and field_name == 'edx_video_id': - # Avoid overwriting edx_video_id between blocks - continue - new_upstream_value = getattr(upstream, field_name) - setattr(downstream, field_name, new_upstream_value) - - -def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: - """ - Update tags from `upstream` to `downstream` - """ - from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only - # For any block synced with an upstream, copy the tags as read_only - # This keeps tags added locally. - copy_tags_as_read_only( - str(upstream.location), - str(downstream.location), - ) - - -def _get_synchronizable_fields(upstream: XBlock, downstream: XBlock) -> set[str]: - """ - The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. - """ - return set.intersection(*[ - set( - field_name - for (field_name, field) in block.__class__.fields.items() - if field.scope in [Scope.settings, Scope.content] - ) - for block in [upstream, downstream] - ]) - - def decline_sync(downstream: XBlock) -> None: """ Given an XBlock that is linked to upstream content, mark the latest available update as 'declined' so that its diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index 4f92137ae4ea..caf4d04c68c2 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -382,13 +382,13 @@ def get_container_children( if container_key.container_type == ContainerType.Unit.value: child_components = authoring_api.get_components_in_unit(container.unit, published=published) return [LibraryXBlockMetadata.from_component( - container_key.library_key, + container_key.lib_key, entry.component ) for entry in child_components] else: child_entities = authoring_api.get_entities_in_container(container, published=published) return [ContainerMetadata.from_container( - container_key.library_key, + container_key.lib_key, entry.entity ) for entry in child_entities] diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 34a5a90269fe..1cb4a4adcfbd 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -4,12 +4,13 @@ import logging from django.core.exceptions import PermissionDenied +from opaque_keys import OpaqueKey from rest_framework.exceptions import NotFound from openedx_events.content_authoring.data import LibraryBlockData, LibraryContainerData from openedx_events.content_authoring.signals import LIBRARY_BLOCK_UPDATED, LIBRARY_CONTAINER_UPDATED from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryUsageLocatorV2, LibraryLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2, LibraryLocatorV2 from openedx_learning.api import authoring as authoring_api from openedx.core.djangoapps.content_libraries import api, permissions @@ -32,7 +33,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.use_draft = kwargs.get('use_draft', None) - def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_edit_block(self, user: UserType, usage_key: OpaqueKey) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the @@ -40,10 +41,10 @@ def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) + self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) - def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: OpaqueKey) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but @@ -51,10 +52,10 @@ def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> b May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) + self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_VIEW_THIS_CONTENT_LIBRARY) - def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_view_block(self, user: UserType, usage_key: OpaqueKey) -> bool: """ Does the specified usage key exist in its context, and if so, does the specified user have permission to view it and interact with it (call @@ -62,9 +63,12 @@ def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: May raise ContentLibraryNotFound if the library does not exist. """ - assert isinstance(usage_key, LibraryUsageLocatorV2) + self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY) + def _assert_key_instance(self, usage_key: OpaqueKey): + assert isinstance(usage_key, LibraryUsageLocatorV2) or isinstance(usage_key, LibraryContainerLocator) + def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: """ Helper method to check a permission for the various can_ methods""" try: @@ -110,7 +114,7 @@ def send_block_updated_event(self, usage_key: UsageKeyV2): assert isinstance(usage_key, LibraryUsageLocatorV2) LIBRARY_BLOCK_UPDATED.send_event( library_block=LibraryBlockData( - library_key=usage_key.lib_key, + library_key=usage_key.context_key, usage_key=usage_key, ) ) diff --git a/openedx/core/djangoapps/xblock/learning_context/manager.py b/openedx/core/djangoapps/xblock/learning_context/manager.py index b7ed1c3c3426..3d4ab5eabf4e 100644 --- a/openedx/core/djangoapps/xblock/learning_context/manager.py +++ b/openedx/core/djangoapps/xblock/learning_context/manager.py @@ -3,7 +3,7 @@ """ from edx_django_utils.plugins import PluginManager from opaque_keys import OpaqueKey -from opaque_keys.edx.keys import LearningContextKey, UsageKeyV2 +from opaque_keys.edx.keys import LearningContextKey, LibraryItemKey, UsageKeyV2 from openedx.core.djangoapps.xblock.apps import get_xblock_app_config @@ -35,7 +35,7 @@ def get_learning_context_impl(key): """ if isinstance(key, LearningContextKey): context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' - elif isinstance(key, UsageKeyV2): + elif isinstance(key, UsageKeyV2) or isinstance(key, LibraryItemKey): context_type = key.context_key.CANONICAL_NAMESPACE elif isinstance(key, OpaqueKey): # Maybe this is an older modulestore key etc. From b3c0fc308b19021aa3d99d767b8c416c7c4cbc55 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Tue, 22 Apr 2025 20:06:56 +0530 Subject: [PATCH 10/46] fixup! feat: separate managers for component and container upstream sync --- cms/djangoapps/contentstore/helpers.py | 6 +++--- .../contentstore/xblock_storage_handlers/view_handlers.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/helpers.py b/cms/djangoapps/contentstore/helpers.py index 375ec2194fa4..82c4b3856f40 100644 --- a/cms/djangoapps/contentstore/helpers.py +++ b/cms/djangoapps/contentstore/helpers.py @@ -810,7 +810,7 @@ def concat_static_file_notices(notices: list[StaticFileNotices]) -> StaticFileNo conflicting_files.extend(notice.conflicting_files) error_files.extend(notice.error_files) return StaticFileNotices( - new_files=new_files, - conflicting_files=conflicting_files, - error_files=error_files, + new_files=list(set(new_files)), + conflicting_files=list(set(conflicting_files)), + error_files=list(set(error_files)), ) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 46e1c38b3f22..157b9179ac79 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -524,7 +524,7 @@ def create_item(request): return _create_block(request) -def sync_library_content(created_block, request, store): +def sync_library_content(created_block: XBlock, request, store): upstream_key = check_and_parse_upstream_key(created_block.upstream, created_block.usage_key) if isinstance(upstream_key, LibraryUsageLocatorV2): lib_block = sync_from_upstream(downstream=created_block, user=request.user) From 95395cda0ef0b3c73341cc2b254976088a175643 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Tue, 22 Apr 2025 20:22:06 +0530 Subject: [PATCH 11/46] feat: remove upstream links from unit child components on sync --- .../contentstore/xblock_storage_handlers/view_handlers.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 157b9179ac79..1143fc70eb98 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -524,10 +524,13 @@ def create_item(request): return _create_block(request) -def sync_library_content(created_block: XBlock, request, store): +def sync_library_content(created_block: XBlock, request, store, remove_upstream_link: bool = False): upstream_key = check_and_parse_upstream_key(created_block.upstream, created_block.usage_key) if isinstance(upstream_key, LibraryUsageLocatorV2): lib_block = sync_from_upstream(downstream=created_block, user=request.user) + if remove_upstream_link: + # Removing upstream link from child components + created_block.upstream = None static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) store.update_item(created_block, request.user.id) else: @@ -536,7 +539,7 @@ def sync_library_content(created_block: XBlock, request, store): with store.bulk_operations(created_block.location.context_key): children_blocks_usage_keys = [] for child_block in children_blocks: - notices.append(sync_library_content(child_block, request, store)) + notices.append(sync_library_content(child_block, request, store, remove_upstream_link=True)) children_blocks_usage_keys.append(child_block.usage_key) created_block.children = children_blocks_usage_keys store.update_item(created_block, request.user.id) From 2d5e63ed97041fadac0399dbb954acfa34f25fe4 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Tue, 22 Apr 2025 20:40:28 +0530 Subject: [PATCH 12/46] refactor!: rename publishable entity links table and update upstream_block field --- cms/djangoapps/contentstore/admin.py | 8 ++-- .../commands/recreate_upstream_links.py | 4 +- ...shableentitylink_componentlink_and_more.py | 28 +++++++++++++ cms/djangoapps/contentstore/models.py | 40 +++++++++---------- .../rest_api/v2/serializers/__init__.py | 8 ++-- .../rest_api/v2/serializers/downstreams.py | 8 ++-- .../rest_api/v2/views/downstreams.py | 16 ++++---- .../contentstore/signals/handlers.py | 6 +-- cms/djangoapps/contentstore/tasks.py | 6 +-- .../tests/test_upstream_downstream_links.py | 22 +++++----- cms/djangoapps/contentstore/utils.py | 4 +- .../tests/test_mixed_modulestore.py | 2 +- 12 files changed, 90 insertions(+), 62 deletions(-) create mode 100644 cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index 0b01abe05073..1da5652a16da 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -14,7 +14,7 @@ BackfillCourseTabsConfig, CleanStaleCertificateAvailabilityDatesConfig, LearningContextLinksStatus, - PublishableEntityLink, + ComponentLink, VideoUploadConfig ) from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate @@ -88,10 +88,10 @@ class CleanStaleCertificateAvailabilityDatesConfigAdmin(ConfigurationModelAdmin) pass -@admin.register(PublishableEntityLink) -class PublishableEntityLinkAdmin(admin.ModelAdmin): +@admin.register(ComponentLink) +class ComponentLinkAdmin(admin.ModelAdmin): """ - PublishableEntityLink admin. + ComponentLink admin. """ fields = ( "uuid", diff --git a/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py b/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py index c1a8454cd56c..9e86df4d26a9 100644 --- a/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py +++ b/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py @@ -1,5 +1,5 @@ """ -Management command to recreate upstream-dowstream links in PublishableEntityLink for course(s). +Management command to recreate upstream-dowstream links in ComponentLink for course(s). This command can be run for all the courses or for given list of courses. """ @@ -23,7 +23,7 @@ class Command(BaseCommand): """ - Recreate links for course(s) in PublishableEntityLink table. + Recreate links for course(s) in ComponentLink table. Examples: # Recreate upstream links for two courses. diff --git a/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py b/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py new file mode 100644 index 000000000000..6764aab48cd2 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.20 on 2025-04-22 15:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'), + ('contentstore', '0009_learningcontextlinksstatus_publishableentitylink'), + ] + + operations = [ + migrations.RenameModel( + old_name='PublishableEntityLink', + new_name='ComponentLink', + ), + migrations.AlterModelOptions( + name='componentlink', + options={'verbose_name': 'Component Link', 'verbose_name_plural': 'Component Links'}, + ), + migrations.AlterField( + model_name='componentlink', + name='upstream_block', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='links', to='oel_components.component'), + ), + ] diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index ca6171118c00..2c06e07ab9e6 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -14,7 +14,7 @@ from django.utils.translation import gettext_lazy as _ from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField from opaque_keys.edx.keys import CourseKey, UsageKey -from openedx_learning.api.authoring_models import Component, PublishableEntity +from openedx_learning.api.authoring_models import Component from openedx_learning.lib.fields import ( immutable_uuid_field, key_field, @@ -80,14 +80,14 @@ class Meta: ) -class PublishableEntityLink(models.Model): +class ComponentLink(models.Model): """ This represents link between any two publishable entities or link between publishable entity and a course xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses. """ uuid = immutable_uuid_field() upstream_block = models.ForeignKey( - PublishableEntity, + Component, on_delete=models.SET_NULL, related_name="links", null=True, @@ -124,10 +124,10 @@ def upstream_version(self) -> int | None: Returns upstream block version number if available. """ version_num = None - if hasattr(self.upstream_block, 'published'): - if hasattr(self.upstream_block.published, 'version'): - if hasattr(self.upstream_block.published.version, 'version_num'): - version_num = self.upstream_block.published.version.version_num + if hasattr(self.upstream_block.publishable_entity, 'published'): + if hasattr(self.upstream_block.publishable_entity.published, 'version'): + if hasattr(self.upstream_block.publishable_entity.published.version, 'version_num'): + version_num = self.upstream_block.publishable_entity.published.version.version_num return version_num @property @@ -135,11 +135,11 @@ def upstream_context_title(self) -> str: """ Returns upstream context title. """ - return self.upstream_block.learning_package.title + return self.upstream_block.publishable_entity.learning_package.title class Meta: - verbose_name = _("Publishable Entity Link") - verbose_name_plural = _("Publishable Entity Links") + verbose_name = _("Component Link") + verbose_name_plural = _("Component Links") @classmethod def update_or_create( @@ -153,7 +153,7 @@ def update_or_create( version_synced: int, version_declined: int | None = None, created: datetime | None = None, - ) -> "PublishableEntityLink": + ) -> "ComponentLink": """ Update or create entity link. This will only update `updated` field if something has changed. """ @@ -170,7 +170,7 @@ def update_or_create( if upstream_block: new_values.update( { - 'upstream_block': upstream_block.publishable_entity, + 'upstream_block': upstream_block, } ) try: @@ -201,21 +201,21 @@ def update_or_create( def filter_links( cls, **link_filter, - ) -> QuerySet["PublishableEntityLink"]: + ) -> QuerySet["ComponentLink"]: """ Get all links along with sync flag, upstream context title and version, with optional filtering. """ ready_to_sync = link_filter.pop('ready_to_sync', None) result = cls.objects.filter(**link_filter).select_related( - "upstream_block__published__version", - "upstream_block__learning_package" + "upstream_block__publishable_entity__published__version", + "upstream_block__publishable_entity__learning_package" ).annotate( ready_to_sync=( GreaterThan( - Coalesce("upstream_block__published__version__version_num", 0), + Coalesce("upstream_block__publishable_entity__published__version__version_num", 0), Coalesce("version_synced", 0) ) & GreaterThan( - Coalesce("upstream_block__published__version__version_num", 0), + Coalesce("upstream_block__publishable_entity__published__version__version_num", 0), Coalesce("version_declined", 0) ) ) @@ -225,7 +225,7 @@ def filter_links( return result @classmethod - def get_by_upstream_usage_key(cls, upstream_usage_key: UsageKey) -> QuerySet["PublishableEntityLink"]: + def get_by_upstream_usage_key(cls, upstream_usage_key: UsageKey) -> QuerySet["ComponentLink"]: """ Get all downstream context keys for given upstream usage key """ @@ -255,7 +255,7 @@ def summarize_by_downstream_context(cls, downstream_context_key: CourseKey) -> Q """ result = cls.filter_links(downstream_context_key=downstream_context_key).values( "upstream_context_key", - upstream_context_title=F("upstream_block__learning_package__title"), + upstream_context_title=F("upstream_block__publishable_entity__learning_package__title"), ).annotate( ready_to_sync_count=Count("id", Q(ready_to_sync=True)), total_count=Count('id') @@ -275,7 +275,7 @@ class LearningContextLinksStatusChoices(models.TextChoices): class LearningContextLinksStatus(models.Model): """ - This table stores current processing status of upstream-downstream links in PublishableEntityLink table for a + This table stores current processing status of upstream-downstream links in ComponentLink table for a course or a learning context. """ context_key = CourseKeyField( diff --git a/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py b/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py index 1815b4435d25..739960c95eb8 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py +++ b/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py @@ -1,13 +1,13 @@ """Module for v2 serializers.""" from cms.djangoapps.contentstore.rest_api.v2.serializers.downstreams import ( - PublishableEntityLinksSerializer, - PublishableEntityLinksSummarySerializer, + ComponentLinksSerializer, + ComponentLinksSummarySerializer, ) from cms.djangoapps.contentstore.rest_api.v2.serializers.home import CourseHomeTabSerializerV2 __all__ = [ 'CourseHomeTabSerializerV2', - 'PublishableEntityLinksSerializer', - 'PublishableEntityLinksSummarySerializer', + 'ComponentLinksSerializer', + 'ComponentLinksSummarySerializer', ] diff --git a/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py index 1a2139d4cf91..44a31e8e94e9 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py @@ -4,10 +4,10 @@ from rest_framework import serializers -from cms.djangoapps.contentstore.models import PublishableEntityLink +from cms.djangoapps.contentstore.models import ComponentLink -class PublishableEntityLinksSerializer(serializers.ModelSerializer): +class ComponentLinksSerializer(serializers.ModelSerializer): """ Serializer for publishable entity links. """ @@ -16,11 +16,11 @@ class PublishableEntityLinksSerializer(serializers.ModelSerializer): ready_to_sync = serializers.BooleanField() class Meta: - model = PublishableEntityLink + model = ComponentLink exclude = ['upstream_block', 'uuid'] -class PublishableEntityLinksSummarySerializer(serializers.Serializer): +class ComponentLinksSummarySerializer(serializers.Serializer): """ Serializer for summary for publishable entity links """ diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py index 3fd18340f38a..b59c980f7884 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py @@ -48,7 +48,7 @@ /api/contentstore/v2/downstreams /api/contentstore/v2/downstreams?course_id=course-v1:A+B+C&ready_to_sync=true GET: List downstream blocks that can be synced, filterable by course or sync-readiness. - 200: A paginated list of applicable & accessible downstream blocks. Entries are PublishableEntityLinks. + 200: A paginated list of applicable & accessible downstream blocks. Entries are ComponentLinks. /api/contentstore/v2/downstreams//summary GET: List summary of links by course key @@ -95,10 +95,10 @@ from xblock.core import XBlock from cms.djangoapps.contentstore.helpers import import_static_assets_for_library_sync -from cms.djangoapps.contentstore.models import PublishableEntityLink +from cms.djangoapps.contentstore.models import ComponentLink from cms.djangoapps.contentstore.rest_api.v2.serializers import ( - PublishableEntityLinksSerializer, - PublishableEntityLinksSummarySerializer, + ComponentLinksSerializer, + ComponentLinksSummarySerializer, ) from cms.lib.xblock.upstream_sync import ( BadDownstream, @@ -183,9 +183,9 @@ def get(self, request: _AuthenticatedRequest): link_filter["upstream_usage_key"] = UsageKey.from_string(upstream_usage_key) except InvalidKeyError as exc: raise ValidationError(detail=f"Malformed usage key: {upstream_usage_key}") from exc - links = PublishableEntityLink.filter_links(**link_filter) + links = ComponentLink.filter_links(**link_filter) paginated_links = paginator.paginate_queryset(links, self.request, view=self) - serializer = PublishableEntityLinksSerializer(paginated_links, many=True) + serializer = ComponentLinksSerializer(paginated_links, many=True) return paginator.get_paginated_response(serializer.data, self.request) @@ -217,8 +217,8 @@ def get(self, request: _AuthenticatedRequest, course_key_string: str): course_key = CourseKey.from_string(course_key_string) except InvalidKeyError as exc: raise ValidationError(detail=f"Malformed course key: {course_key_string}") from exc - links = PublishableEntityLink.summarize_by_downstream_context(downstream_context_key=course_key) - serializer = PublishableEntityLinksSummarySerializer(links, many=True) + links = ComponentLink.summarize_by_downstream_context(downstream_context_key=course_key) + serializer = ComponentLinksSummarySerializer(links, many=True) return Response(serializer.data) diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index 5635d4655622..83b10c997489 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -44,7 +44,7 @@ from xmodule.modulestore.django import SignalHandler, modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from ..models import PublishableEntityLink +from ..models import ComponentLink from ..tasks import ( create_or_update_upstream_links, handle_create_or_update_xblock_upstream_link, @@ -230,7 +230,7 @@ def handle_item_deleted(**kwargs): gating_api.set_required_content(course_key, block.location, None, None, None) id_list.add(block.location) - PublishableEntityLink.objects.filter(downstream_usage_key__in=id_list).delete() + ComponentLink.objects.filter(downstream_usage_key__in=id_list).delete() @receiver(GRADING_POLICY_CHANGED) @@ -278,7 +278,7 @@ def delete_upstream_downstream_link_handler(**kwargs): log.error("Received null or incorrect data for event") return - PublishableEntityLink.objects.filter( + ComponentLink.objects.filter( downstream_usage_key=xblock_info.usage_key ).delete() diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index e36a8edae8a2..289712e58b06 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -84,7 +84,7 @@ from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml from xmodule.modulestore.xml_importer import CourseImportException, import_course_from_xml, import_library_from_xml -from .models import LearningContextLinksStatus, LearningContextLinksStatusChoices, PublishableEntityLink +from .models import LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink from .outlines import update_outline_from_modulestore from .outlines_regenerate import CourseOutlineRegenerate from .toggles import bypass_olx_failure_enabled @@ -1475,7 +1475,7 @@ def create_or_update_upstream_links( updated=created, ) if replace: - PublishableEntityLink.objects.filter(downstream_context_key=course_key).delete() + ComponentLink.objects.filter(downstream_context_key=course_key).delete() try: xblocks = store.get_items(course_key, settings={"upstream": lambda x: x is not None}) except ItemNotFoundError: @@ -1501,7 +1501,7 @@ def handle_unlink_upstream_block(upstream_usage_key_string: str) -> None: LOGGER.exception(f'Invalid upstream usage_key: {upstream_usage_key_string}') return - for link in PublishableEntityLink.objects.filter( + for link in ComponentLink.objects.filter( upstream_usage_key=upstream_usage_key, ): make_copied_tags_editable(str(link.downstream_usage_key)) diff --git a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py index 3f0703d8b31e..0e684b871d9d 100644 --- a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py +++ b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py @@ -17,7 +17,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory -from ..models import LearningContextLinksStatus, LearningContextLinksStatusChoices, PublishableEntityLink +from ..models import LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink class BaseUpstreamLinksHelpers(TestCase): @@ -66,7 +66,7 @@ def _compare_links(self, course_key, expected): """ Compares links for given course with passed expected list of dicts. """ - links = list(PublishableEntityLink.objects.filter(downstream_context_key=course_key).values( + links = list(ComponentLink.objects.filter(downstream_context_key=course_key).values( 'upstream_block', 'upstream_usage_key', 'upstream_context_key', @@ -132,7 +132,7 @@ def test_call_for_single_course(self): """ # Pre-checks assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists() - assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_1).exists() + assert not ComponentLink.objects.filter(downstream_context_key=self.course_key_1).exists() # Run command self.call_command('--course', str(self.course_key_1)) # Post verfication @@ -147,9 +147,9 @@ def test_call_for_multiple_course(self): """ # Pre-checks assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists() - assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_2).exists() + assert not ComponentLink.objects.filter(downstream_context_key=self.course_key_2).exists() assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists() - assert not PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_3).exists() + assert not ComponentLink.objects.filter(downstream_context_key=self.course_key_3).exists() # Run command self.call_command('--course', str(self.course_key_2), '--course', str(self.course_key_3)) @@ -170,7 +170,7 @@ def test_call_for_all_courses(self): """ # Delete all links and status just to make sure --all option works LearningContextLinksStatus.objects.all().delete() - PublishableEntityLink.objects.all().delete() + ComponentLink.objects.all().delete() # Pre-checks assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists() assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists() @@ -257,9 +257,9 @@ def test_create_or_update_events(self): assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_1)).exists() assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_2)).exists() assert not LearningContextLinksStatus.objects.filter(context_key=str(self.course_key_3)).exists() - assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_1).count() == 3 - assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_2).count() == 3 - assert PublishableEntityLink.objects.filter(downstream_context_key=self.course_key_3).count() == 3 + assert ComponentLink.objects.filter(downstream_context_key=self.course_key_1).count() == 3 + assert ComponentLink.objects.filter(downstream_context_key=self.course_key_2).count() == 3 + assert ComponentLink.objects.filter(downstream_context_key=self.course_key_3).count() == 3 self._compare_links(self.course_key_1, self.expected_links_1) self._compare_links(self.course_key_2, self.expected_links_2) self._compare_links(self.course_key_3, self.expected_links_3) @@ -269,6 +269,6 @@ def test_delete_handler(self): Test whether links are deleted on deletion of xblock. """ usage_key = self.expected_links_1[0]["downstream_usage_key"] - assert PublishableEntityLink.objects.filter(downstream_usage_key=usage_key).exists() + assert ComponentLink.objects.filter(downstream_usage_key=usage_key).exists() self.store.delete_item(usage_key, self.user.id) - assert not PublishableEntityLink.objects.filter(downstream_usage_key=usage_key).exists() + assert not ComponentLink.objects.filter(downstream_usage_key=usage_key).exists() diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 1ff309aa6e50..dbce2ab48511 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -113,7 +113,7 @@ ) from xmodule.services import ConfigurationService, SettingsService, TeamsConfigurationService -from .models import PublishableEntityLink +from .models import ComponentLink IMPORTABLE_FILE_TYPES = ('.tar.gz', '.zip') log = logging.getLogger(__name__) @@ -2384,7 +2384,7 @@ def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, c except ObjectDoesNotExist: log.error(f"Library component not found for {upstream_usage_key}") lib_component = None - PublishableEntityLink.update_or_create( + ComponentLink.update_or_create( lib_component, upstream_usage_key=upstream_usage_key, upstream_context_key=str(upstream_usage_key.context_key), diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index bbc91f832f96..ecf20578ef91 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -173,7 +173,7 @@ def setUp(self): create_or_update_xblock_upstream_link_patch.start() self.addCleanup(create_or_update_xblock_upstream_link_patch.stop) publishableEntityLinkPatch = patch( - 'cms.djangoapps.contentstore.signals.handlers.PublishableEntityLink' + 'cms.djangoapps.contentstore.signals.handlers.ComponentLink' ) publishableEntityLinkPatch.start() self.addCleanup(publishableEntityLinkPatch.stop) From 7763b9bde8717a50a2b753b2dc4c2a61642889bb Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Tue, 22 Apr 2025 20:55:03 +0530 Subject: [PATCH 13/46] feat: create component link only for component xblocks --- cms/djangoapps/contentstore/utils.py | 34 ++++++++++++++++------------ 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index dbce2ab48511..22e1bda256b3 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -24,6 +24,7 @@ from help_tokens.core import HelpUrlExpert from lti_consumer.models import CourseAllowPIISharingInLTIFlag from milestones import api as milestones_api +from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey, UsageKey, UsageKeyV2 from opaque_keys.edx.locator import LibraryLocator from openedx_events.content_authoring.data import DuplicatedXBlockData @@ -2378,19 +2379,22 @@ def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, c """ if not xblock.upstream: return None - upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) try: - lib_component = get_component_from_usage_key(upstream_usage_key) - except ObjectDoesNotExist: - log.error(f"Library component not found for {upstream_usage_key}") - lib_component = None - ComponentLink.update_or_create( - lib_component, - upstream_usage_key=upstream_usage_key, - upstream_context_key=str(upstream_usage_key.context_key), - downstream_context_key=course_key, - downstream_usage_key=xblock.usage_key, - version_synced=xblock.upstream_version, - version_declined=xblock.upstream_version_declined, - created=created, - ) + upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) + try: + lib_component = get_component_from_usage_key(upstream_usage_key) + except ObjectDoesNotExist: + log.error(f"Library component not found for {upstream_usage_key}") + lib_component = None + ComponentLink.update_or_create( + lib_component, + upstream_usage_key=upstream_usage_key, + upstream_context_key=str(upstream_usage_key.context_key), + downstream_context_key=course_key, + downstream_usage_key=xblock.usage_key, + version_synced=xblock.upstream_version, + version_declined=xblock.upstream_version_declined, + created=created, + ) + except InvalidKeyError: + pass From bcb88fe12b7e85ce22ab3efe8ebbbda7eb159e1a Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 12:28:27 +0530 Subject: [PATCH 14/46] feat: container link model --- cms/djangoapps/contentstore/admin.py | 47 +++- ...ter_componentlink_options_containerlink.py | 65 +++++ cms/djangoapps/contentstore/models.py | 252 ++++++++++++------ .../rest_api/v2/serializers/__init__.py | 4 +- .../rest_api/v2/serializers/downstreams.py | 2 +- .../rest_api/v2/views/downstreams.py | 4 +- .../contentstore/signals/handlers.py | 7 +- cms/djangoapps/contentstore/tasks.py | 9 +- .../tests/test_upstream_downstream_links.py | 77 +++++- cms/djangoapps/contentstore/utils.py | 70 +++-- 10 files changed, 404 insertions(+), 133 deletions(-) create mode 100644 cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index 1da5652a16da..2e9f5b19b353 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -13,15 +13,15 @@ from cms.djangoapps.contentstore.models import ( BackfillCourseTabsConfig, CleanStaleCertificateAvailabilityDatesConfig, - LearningContextLinksStatus, ComponentLink, - VideoUploadConfig + ContainerLink, + LearningContextLinksStatus, + VideoUploadConfig, ) from cms.djangoapps.contentstore.outlines_regenerate import CourseOutlineRegenerate from openedx.core.djangoapps.content.learning_sequences.api import key_supports_outlines -from .tasks import update_outline_from_modulestore_task, update_all_outlines_from_modulestore_task - +from .tasks import update_all_outlines_from_modulestore_task, update_outline_from_modulestore_task log = logging.getLogger(__name__) @@ -127,6 +127,45 @@ def has_change_permission(self, request, obj=None): return False +@admin.register(ContainerLink) +class ContainerLinkAdmin(admin.ModelAdmin): + """ + ContainerLink admin. + """ + fields = ( + "uuid", + "upstream_block", + "upstream_container_key", + "upstream_context_key", + "downstream_usage_key", + "downstream_context_key", + "version_synced", + "version_declined", + "created", + "updated", + ) + readonly_fields = fields + list_display = [ + "upstream_block", + "upstream_container_key", + "downstream_usage_key", + "version_synced", + "updated", + ] + search_fields = [ + "upstream_container_key", + "upstream_context_key", + "downstream_usage_key", + "downstream_context_key", + ] + + def has_add_permission(self, request): + return False + + def has_change_permission(self, request, obj=None): + return False + + @admin.register(LearningContextLinksStatus) class LearningContextLinksStatusAdmin(admin.ModelAdmin): """ diff --git a/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py b/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py new file mode 100644 index 000000000000..c2d0f8433837 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py @@ -0,0 +1,65 @@ +# Generated by Django 4.2.20 on 2025-04-23 05:47 + +import uuid + +import django.db.models.deletion +import opaque_keys.edx.django.models +import openedx_learning.lib.fields +import openedx_learning.lib.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'), + ('contentstore', '0010_rename_publishableentitylink_componentlink_and_more'), + ] + + operations = [ + migrations.AlterModelOptions( + name='componentlink', + options={}, + ), + migrations.CreateModel( + name='ContainerLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ( + 'upstream_context_key', + openedx_learning.lib.fields.MultiCollationCharField( + db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, + db_index=True, + help_text='Upstream context key i.e., learning_package/library key', + max_length=500, + ), + ), + ('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)), + ('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)), + ('version_synced', models.IntegerField()), + ('version_declined', models.IntegerField(blank=True, null=True)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ( + 'upstream_container_key', + opaque_keys.edx.django.models.LibraryItemField( + help_text='Upstream block usage key, this value cannot be null and useful to track upstream library blocks that do not exist yet', + max_length=255, + ), + ), + ( + 'upstream_block', + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='links', + to='oel_publishing.container', + ), + ), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index 2c06e07ab9e6..1b2c2f96bd96 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -4,6 +4,7 @@ from datetime import datetime, timezone +from pyexpat import model from config_models.models import ConfigurationModel from django.db import models @@ -12,9 +13,11 @@ from django.db.models.functions import Coalesce from django.db.models.lookups import GreaterThan from django.utils.translation import gettext_lazy as _ -from opaque_keys.edx.django.models import CourseKeyField, UsageKeyField +from opaque_keys.edx.django.models import CourseKeyField, LibraryItemField, UsageKeyField from opaque_keys.edx.keys import CourseKey, UsageKey -from openedx_learning.api.authoring_models import Component +from opaque_keys.edx.locator import LibraryContainerLocator +from openedx_learning.api.authoring_models import Component, Container +from openedx_learning.apps.authoring.publishing.api import get_published_version from openedx_learning.lib.fields import ( immutable_uuid_field, key_field, @@ -80,26 +83,12 @@ class Meta: ) -class ComponentLink(models.Model): +class EntityLinkBase(models.Model): """ - This represents link between any two publishable entities or link between publishable entity and a course - xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses. + Abstract base class that defines fields and functions for storing link between two publishable entities + or links between publishable entity and a course xblock. """ uuid = immutable_uuid_field() - upstream_block = models.ForeignKey( - Component, - on_delete=models.SET_NULL, - related_name="links", - null=True, - blank=True, - ) - upstream_usage_key = UsageKeyField( - max_length=255, - help_text=_( - "Upstream block usage key, this value cannot be null" - " and useful to track upstream library blocks that do not exist yet" - ) - ) # Search by library/upstream context key upstream_context_key = key_field( help_text=_("Upstream context key i.e., learning_package/library key"), @@ -115,20 +104,16 @@ class ComponentLink(models.Model): created = manual_date_time_field() updated = manual_date_time_field() - def __str__(self): - return f"{self.upstream_usage_key}->{self.downstream_usage_key}" + class Meta: + abstract = True @property def upstream_version(self) -> int | None: """ Returns upstream block version number if available. """ - version_num = None - if hasattr(self.upstream_block.publishable_entity, 'published'): - if hasattr(self.upstream_block.publishable_entity.published, 'version'): - if hasattr(self.upstream_block.publishable_entity.published.version, 'version_num'): - version_num = self.upstream_block.publishable_entity.published.version.version_num - return version_num + published_version = get_published_version(self.upstream_block.publishable_entity.id) + return published_version.version_num if published_version else None @property def upstream_context_title(self) -> str: @@ -137,9 +122,85 @@ def upstream_context_title(self) -> str: """ return self.upstream_block.publishable_entity.learning_package.title - class Meta: - verbose_name = _("Component Link") - verbose_name_plural = _("Component Links") + @classmethod + def filter_links( + cls, + **link_filter, + ) -> QuerySet["EntityLinkBase"]: + """ + Get all links along with sync flag, upstream context title and version, with optional filtering. + """ + ready_to_sync = link_filter.pop('ready_to_sync', None) + result = cls.objects.filter(**link_filter).select_related( + "upstream_block__publishable_entity__published__version", + "upstream_block__publishable_entity__learning_package" + ).annotate( + ready_to_sync=( + GreaterThan( + Coalesce("upstream_block__publishable_entity__published__version__version_num", 0), + Coalesce("version_synced", 0) + ) & GreaterThan( + Coalesce("upstream_block__publishable_entity__published__version__version_num", 0), + Coalesce("version_declined", 0) + ) + ) + ) + if ready_to_sync is not None: + result = result.filter(ready_to_sync=ready_to_sync) + return result + + @classmethod + def summarize_by_downstream_context(cls, downstream_context_key: CourseKey) -> QuerySet: + """ + Returns a summary of links by upstream context for given downstream_context_key. + Example: + [ + { + "upstream_context_title": "CS problems 3", + "upstream_context_key": "lib:OpenedX:CSPROB3", + "ready_to_sync_count": 11, + "total_count": 14 + }, + { + "upstream_context_title": "CS problems 2", + "upstream_context_key": "lib:OpenedX:CSPROB2", + "ready_to_sync_count": 15, + "total_count": 24 + }, + ] + """ + result = cls.filter_links(downstream_context_key=downstream_context_key).values( + "upstream_context_key", + upstream_context_title=F("upstream_block__publishable_entity__learning_package__title"), + ).annotate( + ready_to_sync_count=Count("id", Q(ready_to_sync=True)), + total_count=Count('id') + ) + return result + + +class ComponentLink(EntityLinkBase): + """ + This represents link between any two publishable entities or link between publishable entity and a course + xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses. + """ + upstream_block = models.ForeignKey( + Component, + on_delete=models.SET_NULL, + related_name="links", + null=True, + blank=True, + ) + upstream_usage_key = UsageKeyField( + max_length=255, + help_text=_( + "Upstream block usage key, this value cannot be null" + " and useful to track upstream library blocks that do not exist yet" + ) + ) + + def __str__(self): + return f"ComponentLink<{self.upstream_usage_key}->{self.downstream_usage_key}>" @classmethod def update_or_create( @@ -197,70 +258,85 @@ def update_or_create( link.save() return link - @classmethod - def filter_links( - cls, - **link_filter, - ) -> QuerySet["ComponentLink"]: - """ - Get all links along with sync flag, upstream context title and version, with optional filtering. - """ - ready_to_sync = link_filter.pop('ready_to_sync', None) - result = cls.objects.filter(**link_filter).select_related( - "upstream_block__publishable_entity__published__version", - "upstream_block__publishable_entity__learning_package" - ).annotate( - ready_to_sync=( - GreaterThan( - Coalesce("upstream_block__publishable_entity__published__version__version_num", 0), - Coalesce("version_synced", 0) - ) & GreaterThan( - Coalesce("upstream_block__publishable_entity__published__version__version_num", 0), - Coalesce("version_declined", 0) - ) - ) - ) - if ready_to_sync is not None: - result = result.filter(ready_to_sync=ready_to_sync) - return result - @classmethod - def get_by_upstream_usage_key(cls, upstream_usage_key: UsageKey) -> QuerySet["ComponentLink"]: - """ - Get all downstream context keys for given upstream usage key - """ - return cls.objects.filter( - upstream_usage_key=upstream_usage_key, +class ContainerLink(EntityLinkBase): + """ + This represents link between any two publishable entities or link between publishable entity and a course + xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses. + """ + upstream_block = models.ForeignKey( + Container, + on_delete=models.SET_NULL, + related_name="links", + null=True, + blank=True, + ) + upstream_container_key = LibraryItemField( + max_length=255, + help_text=_( + "Upstream block usage key, this value cannot be null" + " and useful to track upstream library blocks that do not exist yet" ) + ) + + def __str__(self): + return f"ContainerLink<{self.upstream_container_key}->{self.downstream_usage_key}>" @classmethod - def summarize_by_downstream_context(cls, downstream_context_key: CourseKey) -> QuerySet: + def update_or_create( + cls, + upstream_block: Container | None, + /, + upstream_container_key: LibraryContainerLocator, + upstream_context_key: str, + downstream_usage_key: UsageKey, + downstream_context_key: CourseKey, + version_synced: int, + version_declined: int | None = None, + created: datetime | None = None, + ) -> "ContainerLink": """ - Returns a summary of links by upstream context for given downstream_context_key. - Example: - [ - { - "upstream_context_title": "CS problems 3", - "upstream_context_key": "lib:OpenedX:CSPROB3", - "ready_to_sync_count": 11, - "total_count": 14 - }, - { - "upstream_context_title": "CS problems 2", - "upstream_context_key": "lib:OpenedX:CSPROB2", - "ready_to_sync_count": 15, - "total_count": 24 - }, - ] + Update or create entity link. This will only update `updated` field if something has changed. """ - result = cls.filter_links(downstream_context_key=downstream_context_key).values( - "upstream_context_key", - upstream_context_title=F("upstream_block__publishable_entity__learning_package__title"), - ).annotate( - ready_to_sync_count=Count("id", Q(ready_to_sync=True)), - total_count=Count('id') - ) - return result + if not created: + created = datetime.now(tz=timezone.utc) + new_values = { + 'upstream_container_key': upstream_container_key, + 'upstream_context_key': upstream_context_key, + 'downstream_usage_key': downstream_usage_key, + 'downstream_context_key': downstream_context_key, + 'version_synced': version_synced, + 'version_declined': version_declined, + } + if upstream_block: + new_values.update( + { + 'upstream_block': upstream_block, + } + ) + try: + link = cls.objects.get(downstream_usage_key=downstream_usage_key) + # TODO: until we save modified datetime for course xblocks in index, the modified time for links are updated + # everytime a downstream/course block is updated. This allows us to order links[1] based on recently + # modified downstream version. + # pylint: disable=line-too-long + # 1. https://github.com/open-craft/frontend-app-course-authoring/blob/0443d88824095f6f65a3a64b77244af590d4edff/src/course-libraries/ReviewTabContent.tsx#L222-L233 + has_changes = True # change to false once above condition is met. + for key, value in new_values.items(): + prev = getattr(link, key) + # None != None is True, so we need to check for it specially + if prev != value and ~(prev is None and value is None): + has_changes = True + setattr(link, key, value) + if has_changes: + link.updated = created + link.save() + except cls.DoesNotExist: + link = cls(**new_values) + link.created = created + link.updated = created + link.save() + return link class LearningContextLinksStatusChoices(models.TextChoices): diff --git a/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py b/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py index 739960c95eb8..4c275a1976b0 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py +++ b/cms/djangoapps/contentstore/rest_api/v2/serializers/__init__.py @@ -2,12 +2,12 @@ from cms.djangoapps.contentstore.rest_api.v2.serializers.downstreams import ( ComponentLinksSerializer, - ComponentLinksSummarySerializer, + PublishableEntityLinksSummarySerializer, ) from cms.djangoapps.contentstore.rest_api.v2.serializers.home import CourseHomeTabSerializerV2 __all__ = [ 'CourseHomeTabSerializerV2', 'ComponentLinksSerializer', - 'ComponentLinksSummarySerializer', + 'PublishableEntityLinksSummarySerializer', ] diff --git a/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py index 44a31e8e94e9..8fc6bb1991e1 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/serializers/downstreams.py @@ -20,7 +20,7 @@ class Meta: exclude = ['upstream_block', 'uuid'] -class ComponentLinksSummarySerializer(serializers.Serializer): +class PublishableEntityLinksSummarySerializer(serializers.Serializer): """ Serializer for summary for publishable entity links """ diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py index b59c980f7884..49d85152f4fb 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py @@ -98,7 +98,7 @@ from cms.djangoapps.contentstore.models import ComponentLink from cms.djangoapps.contentstore.rest_api.v2.serializers import ( ComponentLinksSerializer, - ComponentLinksSummarySerializer, + PublishableEntityLinksSummarySerializer, ) from cms.lib.xblock.upstream_sync import ( BadDownstream, @@ -218,7 +218,7 @@ def get(self, request: _AuthenticatedRequest, course_key_string: str): except InvalidKeyError as exc: raise ValidationError(detail=f"Malformed course key: {course_key_string}") from exc links = ComponentLink.summarize_by_downstream_context(downstream_context_key=course_key) - serializer = ComponentLinksSummarySerializer(links, many=True) + serializer = PublishableEntityLinksSummarySerializer(links, many=True) return Response(serializer.data) diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index 83b10c997489..dbf75c85e2f8 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -44,7 +44,7 @@ from xmodule.modulestore.django import SignalHandler, modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from ..models import ComponentLink +from ..models import ComponentLink, ContainerLink from ..tasks import ( create_or_update_upstream_links, handle_create_or_update_xblock_upstream_link, @@ -281,6 +281,11 @@ def delete_upstream_downstream_link_handler(**kwargs): ComponentLink.objects.filter( downstream_usage_key=xblock_info.usage_key ).delete() + ContainerLink.objects.filter( + downstream_usage_key=xblock_info.usage_key + ).delete() + + @receiver(COURSE_IMPORT_COMPLETED) diff --git a/cms/djangoapps/contentstore/tasks.py b/cms/djangoapps/contentstore/tasks.py index 289712e58b06..b6cb2af53d56 100644 --- a/cms/djangoapps/contentstore/tasks.py +++ b/cms/djangoapps/contentstore/tasks.py @@ -84,7 +84,7 @@ from xmodule.modulestore.xml_exporter import export_course_to_xml, export_library_to_xml from xmodule.modulestore.xml_importer import CourseImportException, import_course_from_xml, import_library_from_xml -from .models import LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink +from .models import ContainerLink, LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink from .outlines import update_outline_from_modulestore from .outlines_regenerate import CourseOutlineRegenerate from .toggles import bypass_olx_failure_enabled @@ -1476,6 +1476,7 @@ def create_or_update_upstream_links( ) if replace: ComponentLink.objects.filter(downstream_context_key=course_key).delete() + ContainerLink.objects.filter(downstream_context_key=course_key).delete() try: xblocks = store.get_items(course_key, settings={"upstream": lambda x: x is not None}) except ItemNotFoundError: @@ -1483,7 +1484,7 @@ def create_or_update_upstream_links( course_status.update_status(LearningContextLinksStatusChoices.FAILED) return for xblock in xblocks: - create_or_update_xblock_upstream_link(xblock, course_key_str, created) + create_or_update_xblock_upstream_link(xblock, course_key, created) course_status.update_status(LearningContextLinksStatusChoices.COMPLETED) @@ -1505,3 +1506,7 @@ def handle_unlink_upstream_block(upstream_usage_key_string: str) -> None: upstream_usage_key=upstream_usage_key, ): make_copied_tags_editable(str(link.downstream_usage_key)) + for link in ContainerLink.objects.filter( + upstream_usage_key=upstream_usage_key, + ): + make_copied_tags_editable(str(link.downstream_usage_key)) diff --git a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py index 0e684b871d9d..a36540d3646e 100644 --- a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py +++ b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py @@ -9,7 +9,7 @@ from django.core.management.base import CommandError from django.test import TestCase from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from openedx_events.tests.utils import OpenEdxEventsTestMixin from common.djangoapps.student.tests.factories import UserFactory @@ -17,7 +17,7 @@ from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory -from ..models import LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink +from ..models import ContainerLink, LearningContextLinksStatus, LearningContextLinksStatusChoices, ComponentLink class BaseUpstreamLinksHelpers(TestCase): @@ -44,6 +44,36 @@ def _create_block(self, num: int, category="html"): upstream_version=num, ) + def _create_unit(self, num: int): + """ + Create xblock with random upstream key and version number. + """ + random_upstream = LibraryContainerLocator.from_string( + f"lct:OpenedX:CSPROB2:unit:{uuid4()}" + ) + return random_upstream, BlockFactory.create( + parent=self.sequence, # pylint: disable=attribute-defined-outside-init + category='vertical', + display_name=f"An unit Block - {num}", + upstream=str(random_upstream), + upstream_version=num, + ) + + def _create_unit_and_expected_container_link(self, course_key: str | CourseKey, num_blocks: int = 3): + data = [] + for i in range(num_blocks): + upstream, block = self._create_unit(i + 1) + data.append({ + "upstream_block": None, + "downstream_context_key": course_key, + "downstream_usage_key": block.usage_key, + "upstream_container_key": upstream, + "upstream_context_key": str(upstream.context_key), + "version_synced": i + 1, + "version_declined": None, + }) + return data + def _create_block_and_expected_links_data(self, course_key: str | CourseKey, num_blocks: int = 3): """ Creates xblocks and its expected links data for given course_key @@ -62,7 +92,7 @@ def _create_block_and_expected_links_data(self, course_key: str | CourseKey, num }) return data - def _compare_links(self, course_key, expected): + def _compare_links(self, course_key, expected_component_links, expected_container_links): """ Compares links for given course with passed expected list of dicts. """ @@ -75,7 +105,17 @@ def _compare_links(self, course_key, expected): 'version_synced', 'version_declined', )) - self.assertListEqual(links, expected) + self.assertListEqual(links, expected_component_links) + container_links = list(ContainerLink.objects.filter(downstream_context_key=course_key).values( + 'upstream_block', + 'upstream_container_key', + 'upstream_context_key', + 'downstream_usage_key', + 'downstream_context_key', + 'version_synced', + 'version_declined', + )) + self.assertListEqual(container_links, expected_container_links) @skip_unless_cms @@ -95,16 +135,19 @@ def setUp(self): with self.store.bulk_operations(course_key_1): self._set_course_data(course_1) self.expected_links_1 = self._create_block_and_expected_links_data(course_key_1) + self.expected_container_links_1 = self._create_unit_and_expected_container_link(course_key_1) self.course_2 = course_2 = CourseFactory.create(emit_signals=True) self.course_key_2 = course_key_2 = self.course_2.id with self.store.bulk_operations(course_key_2): self._set_course_data(course_2) self.expected_links_2 = self._create_block_and_expected_links_data(course_key_2) + self.expected_container_links_2 = self._create_unit_and_expected_container_link(course_key_2) self.course_3 = course_3 = CourseFactory.create(emit_signals=True) self.course_key_3 = course_key_3 = self.course_3.id with self.store.bulk_operations(course_key_3): self._set_course_data(course_3) self.expected_links_3 = self._create_block_and_expected_links_data(course_key_3) + self.expected_container_links_3 = self._create_unit_and_expected_container_link(course_key_3) def call_command(self, *args, **kwargs): """ @@ -139,7 +182,7 @@ def test_call_for_single_course(self): assert LearningContextLinksStatus.objects.filter( context_key=str(self.course_key_1) ).first().status == LearningContextLinksStatusChoices.COMPLETED - self._compare_links(self.course_key_1, self.expected_links_1) + self._compare_links(self.course_key_1, self.expected_links_1, self.expected_container_links_1) def test_call_for_multiple_course(self): """ @@ -161,8 +204,8 @@ def test_call_for_multiple_course(self): assert LearningContextLinksStatus.objects.filter( context_key=str(self.course_key_3) ).first().status == LearningContextLinksStatusChoices.COMPLETED - self._compare_links(self.course_key_2, self.expected_links_2) - self._compare_links(self.course_key_3, self.expected_links_3) + self._compare_links(self.course_key_2, self.expected_links_2, self.expected_container_links_2) + self._compare_links(self.course_key_3, self.expected_links_3, self.expected_container_links_3) def test_call_for_all_courses(self): """ @@ -189,9 +232,9 @@ def test_call_for_all_courses(self): assert LearningContextLinksStatus.objects.filter( context_key=str(self.course_key_3) ).first().status == LearningContextLinksStatusChoices.COMPLETED - self._compare_links(self.course_key_1, self.expected_links_1) - self._compare_links(self.course_key_2, self.expected_links_2) - self._compare_links(self.course_key_3, self.expected_links_3) + self._compare_links(self.course_key_1, self.expected_links_1, self.expected_container_links_1) + self._compare_links(self.course_key_2, self.expected_links_2, self.expected_container_links_2) + self._compare_links(self.course_key_3, self.expected_links_3, self.expected_container_links_3) def test_call_for_invalid_course(self): """ @@ -239,16 +282,19 @@ def setUp(self): with self.store.bulk_operations(course_key_1): self._set_course_data(course_1) self.expected_links_1 = self._create_block_and_expected_links_data(course_key_1) + self.expected_container_links_1 = self._create_unit_and_expected_container_link(course_key_1) self.course_2 = course_2 = CourseFactory.create(emit_signals=True) self.course_key_2 = course_key_2 = self.course_2.id with self.store.bulk_operations(course_key_2): self._set_course_data(course_2) self.expected_links_2 = self._create_block_and_expected_links_data(course_key_2) + self.expected_container_links_2 = self._create_unit_and_expected_container_link(course_key_2) self.course_3 = course_3 = CourseFactory.create(emit_signals=True) self.course_key_3 = course_key_3 = self.course_3.id with self.store.bulk_operations(course_key_3): self._set_course_data(course_3) self.expected_links_3 = self._create_block_and_expected_links_data(course_key_3) + self.expected_container_links_3 = self._create_unit_and_expected_container_link(course_key_3) def test_create_or_update_events(self): """ @@ -260,9 +306,9 @@ def test_create_or_update_events(self): assert ComponentLink.objects.filter(downstream_context_key=self.course_key_1).count() == 3 assert ComponentLink.objects.filter(downstream_context_key=self.course_key_2).count() == 3 assert ComponentLink.objects.filter(downstream_context_key=self.course_key_3).count() == 3 - self._compare_links(self.course_key_1, self.expected_links_1) - self._compare_links(self.course_key_2, self.expected_links_2) - self._compare_links(self.course_key_3, self.expected_links_3) + self._compare_links(self.course_key_1, self.expected_links_1, self.expected_container_links_1) + self._compare_links(self.course_key_2, self.expected_links_2, self.expected_container_links_2) + self._compare_links(self.course_key_3, self.expected_links_3, self.expected_container_links_3) def test_delete_handler(self): """ @@ -272,3 +318,8 @@ def test_delete_handler(self): assert ComponentLink.objects.filter(downstream_usage_key=usage_key).exists() self.store.delete_item(usage_key, self.user.id) assert not ComponentLink.objects.filter(downstream_usage_key=usage_key).exists() + + usage_key = self.expected_container_links_1[0]["downstream_usage_key"] + assert ContainerLink.objects.filter(downstream_usage_key=usage_key).exists() + self.store.delete_item(usage_key, self.user.id) + assert not ContainerLink.objects.filter(downstream_usage_key=usage_key).exists() diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 22e1bda256b3..e3e7c3b985fd 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -26,7 +26,7 @@ from milestones import api as milestones_api from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey, UsageKey, UsageKeyV2 -from opaque_keys.edx.locator import LibraryLocator +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryLocator from openedx_events.content_authoring.data import DuplicatedXBlockData from openedx_events.content_authoring.signals import XBLOCK_DUPLICATED from openedx_events.learning.data import CourseNotificationData @@ -87,6 +87,7 @@ from common.djangoapps.xblock_django.api import deprecated_xblocks from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService from openedx.core import toggles as core_toggles +from openedx.core.djangoapps.content_libraries.api.containers import get_container_from_key from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_RESOURCES_MICROFRONTEND @@ -114,7 +115,7 @@ ) from xmodule.services import ConfigurationService, SettingsService, TeamsConfigurationService -from .models import ComponentLink +from .models import ComponentLink, ContainerLink IMPORTABLE_FILE_TYPES = ('.tar.gz', '.zip') log = logging.getLogger(__name__) @@ -2373,28 +2374,57 @@ def get_xblock_render_context(request, block): return "" -def create_or_update_xblock_upstream_link(xblock, course_key: str | CourseKey, created: datetime | None = None) -> None: +def _create_or_update_component_link(course_key: CourseKey, created: datetime | None, xblock): + upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) + try: + lib_component = get_component_from_usage_key(upstream_usage_key) + except ObjectDoesNotExist: + log.error(f"Library component not found for {upstream_usage_key}") + lib_component = None + ComponentLink.update_or_create( + lib_component, + upstream_usage_key=upstream_usage_key, + upstream_context_key=str(upstream_usage_key.context_key), + downstream_context_key=course_key, + downstream_usage_key=xblock.usage_key, + version_synced=xblock.upstream_version, + version_declined=xblock.upstream_version_declined, + created=created, + ) + + +def _create_or_update_container_link(course_key: CourseKey, created: datetime | None, xblock): + upstream_container_key = LibraryContainerLocator.from_string(xblock.upstream) + try: + lib_component = get_container_from_key(upstream_container_key) + except ObjectDoesNotExist: + log.error(f"Library component not found for {upstream_container_key}") + lib_component = None + ContainerLink.update_or_create( + lib_component, + upstream_container_key=upstream_container_key, + upstream_context_key=str(upstream_container_key.context_key), + downstream_context_key=course_key, + downstream_usage_key=xblock.usage_key, + version_synced=xblock.upstream_version, + version_declined=xblock.upstream_version_declined, + created=created, + ) + + +def create_or_update_xblock_upstream_link(xblock, course_key: CourseKey, created: datetime | None = None) -> None: """ Create or update upstream->downstream link in database for given xblock. """ if not xblock.upstream: return None try: - upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) - try: - lib_component = get_component_from_usage_key(upstream_usage_key) - except ObjectDoesNotExist: - log.error(f"Library component not found for {upstream_usage_key}") - lib_component = None - ComponentLink.update_or_create( - lib_component, - upstream_usage_key=upstream_usage_key, - upstream_context_key=str(upstream_usage_key.context_key), - downstream_context_key=course_key, - downstream_usage_key=xblock.usage_key, - version_synced=xblock.upstream_version, - version_declined=xblock.upstream_version_declined, - created=created, - ) + # Try to create component link + _create_or_update_component_link(course_key, created, xblock) except InvalidKeyError: - pass + # It is possible that the upstream is a container and UsageKeyV2 parse failed + # Create upstream container link + try: + _create_or_update_container_link(course_key, created, xblock) + except InvalidKeyError: + log.error(f"Invalid key: {xblock.upstream}") From 4442dac0860319e7d9865e6e0c4322621b56e657 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 12:46:28 +0530 Subject: [PATCH 15/46] chore: fix lint issues --- .../management/commands/recreate_upstream_links.py | 2 +- ...me_publishableentitylink_componentlink_and_more.py | 11 ++++++++--- .../0011_alter_componentlink_options_containerlink.py | 5 ++++- cms/djangoapps/contentstore/models.py | 1 - cms/djangoapps/contentstore/signals/handlers.py | 2 -- cms/lib/xblock/container_upstream_sync.py | 1 - 6 files changed, 13 insertions(+), 9 deletions(-) diff --git a/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py b/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py index 9e86df4d26a9..a0f04bb279e9 100644 --- a/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py +++ b/cms/djangoapps/contentstore/management/commands/recreate_upstream_links.py @@ -23,7 +23,7 @@ class Command(BaseCommand): """ - Recreate links for course(s) in ComponentLink table. + Recreate upstream links for course(s) in ComponentLink and ContainerLink tables. Examples: # Recreate upstream links for two courses. diff --git a/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py b/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py index 6764aab48cd2..8068f4f024d2 100644 --- a/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py +++ b/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py @@ -1,11 +1,10 @@ # Generated by Django 4.2.20 on 2025-04-22 15:08 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'), ('contentstore', '0009_learningcontextlinksstatus_publishableentitylink'), @@ -23,6 +22,12 @@ class Migration(migrations.Migration): migrations.AlterField( model_name='componentlink', name='upstream_block', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='links', to='oel_components.component'), + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='links', + to='oel_components.component', + ), ), ] diff --git a/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py b/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py index c2d0f8433837..f47db143929e 100644 --- a/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py +++ b/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py @@ -43,7 +43,10 @@ class Migration(migrations.Migration): ( 'upstream_container_key', opaque_keys.edx.django.models.LibraryItemField( - help_text='Upstream block usage key, this value cannot be null and useful to track upstream library blocks that do not exist yet', + help_text=( + 'Upstream block usage key, this value cannot be null and useful to track upstream' + 'library blocks that do not exist yet' + ), max_length=255, ), ), diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index 1b2c2f96bd96..369d49517787 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -4,7 +4,6 @@ from datetime import datetime, timezone -from pyexpat import model from config_models.models import ConfigurationModel from django.db import models diff --git a/cms/djangoapps/contentstore/signals/handlers.py b/cms/djangoapps/contentstore/signals/handlers.py index dbf75c85e2f8..b8be6ee84f8b 100644 --- a/cms/djangoapps/contentstore/signals/handlers.py +++ b/cms/djangoapps/contentstore/signals/handlers.py @@ -286,8 +286,6 @@ def delete_upstream_downstream_link_handler(**kwargs): ).delete() - - @receiver(COURSE_IMPORT_COMPLETED) def handle_new_course_import(**kwargs): """ diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/container_upstream_sync.py index d1ae43302fc9..185ab6ca8888 100644 --- a/cms/lib/xblock/container_upstream_sync.py +++ b/cms/lib/xblock/container_upstream_sync.py @@ -199,4 +199,3 @@ def sync_from_upstream_container(downstream: XBlock, user: User) -> tuple[XBlock manager.sync() downstream.upstream_version = manager.link.version_available return manager.upstream, manager.new_children_blocks - From ce1247a63e405fe055e0a839ef121c659f40833d Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 12:57:50 +0530 Subject: [PATCH 16/46] temp: point opaque keys to dev branch --- requirements/edx/base.txt | 2 +- requirements/edx/development.txt | 2 +- requirements/edx/doc.txt | 2 +- requirements/edx/kernel.in | 2 +- requirements/edx/testing.txt | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 1ec89fa01fa5..c62b1256c903 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -477,7 +477,7 @@ edx-milestones==0.6.0 # via -r requirements/edx/kernel.in edx-name-affirmation==3.0.1 # via -r requirements/edx/kernel.in -edx-opaque-keys[django]==2.12.0 +edx-opaque-keys[django] @ git+https://github.com/open-craft/opaque-keys@navin/fal-4077/fix-new-locators # via # -r requirements/edx/kernel.in # edx-bulk-grades diff --git a/requirements/edx/development.txt b/requirements/edx/development.txt index 8453feba1be8..959fd947e020 100644 --- a/requirements/edx/development.txt +++ b/requirements/edx/development.txt @@ -766,7 +766,7 @@ edx-name-affirmation==3.0.1 # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt -edx-opaque-keys[django]==2.12.0 +edx-opaque-keys[django] @ git+https://github.com/open-craft/opaque-keys@navin/fal-4077/fix-new-locators # via # -r requirements/edx/doc.txt # -r requirements/edx/testing.txt diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index ebd21ff42cf9..7f26e70db273 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -561,7 +561,7 @@ edx-milestones==0.6.0 # via -r requirements/edx/base.txt edx-name-affirmation==3.0.1 # via -r requirements/edx/base.txt -edx-opaque-keys[django]==2.12.0 +edx-opaque-keys[django] @ git+https://github.com/open-craft/opaque-keys@navin/fal-4077/fix-new-locators # via # -r requirements/edx/base.txt # edx-bulk-grades diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 55fa2f29082d..cb77ed9adb2d 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -75,7 +75,7 @@ edx-event-bus-kafka>=5.6.0 # Kafka implementation of event bus edx-event-bus-redis edx-milestones edx-name-affirmation -edx-opaque-keys>=2.12.0 +edx-opaque-keys @ git+https://github.com/open-craft/opaque-keys@navin/fal-4077/fix-new-locators edx-organizations edx-proctoring>=2.0.1 # using hash to support django42 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index eebd02b46c5c..ffab32dcfb94 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -588,7 +588,7 @@ edx-milestones==0.6.0 # via -r requirements/edx/base.txt edx-name-affirmation==3.0.1 # via -r requirements/edx/base.txt -edx-opaque-keys[django]==2.12.0 +edx-opaque-keys[django] @ git+https://github.com/open-craft/opaque-keys@navin/fal-4077/fix-new-locators # via # -r requirements/edx/base.txt # edx-bulk-grades From 16ae3b4c5851bec2b1392739ea7b5fc7251fd93f Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 14:54:12 +0530 Subject: [PATCH 17/46] fix: mypy errors --- cms/lib/xblock/upstream_sync.py | 3 +++ .../content_libraries/library_context.py | 16 +++++++--------- .../xblock/learning_context/learning_context.py | 15 ++++++++------- 3 files changed, 18 insertions(+), 16 deletions(-) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 50d599f9256c..5a862c91cdb8 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -266,6 +266,8 @@ def update_tags(self) -> None: """ Update tags from `upstream` to `downstream` """ + if not self.upstream: + return from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only # For any block synced with an upstream, copy the tags as read_only # This keeps tags added locally. @@ -370,6 +372,7 @@ def check_and_parse_upstream_key( BadDownstream: BadUpstream: """ + upstream_key: LibraryContainerLocator | LibraryUsageLocatorV2 if not upstream: raise NoUpstream() if not isinstance(downstream_usage_key.context_key, CourseKey): diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 1cb4a4adcfbd..7fd37761086f 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -4,14 +4,12 @@ import logging from django.core.exceptions import PermissionDenied -from opaque_keys import OpaqueKey -from rest_framework.exceptions import NotFound - +from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryLocatorV2, LibraryUsageLocatorV2 from openedx_events.content_authoring.data import LibraryBlockData, LibraryContainerData from openedx_events.content_authoring.signals import LIBRARY_BLOCK_UPDATED, LIBRARY_CONTAINER_UPDATED -from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2, LibraryLocatorV2 from openedx_learning.api import authoring as authoring_api +from rest_framework.exceptions import NotFound from openedx.core.djangoapps.content_libraries import api, permissions from openedx.core.djangoapps.content_libraries.models import ContentLibrary @@ -33,7 +31,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.use_draft = kwargs.get('use_draft', None) - def can_edit_block(self, user: UserType, usage_key: OpaqueKey) -> bool: + def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the @@ -44,7 +42,7 @@ def can_edit_block(self, user: UserType, usage_key: OpaqueKey) -> bool: self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) - def can_view_block_for_editing(self, user: UserType, usage_key: OpaqueKey) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but @@ -55,7 +53,7 @@ def can_view_block_for_editing(self, user: UserType, usage_key: OpaqueKey) -> bo self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_VIEW_THIS_CONTENT_LIBRARY) - def can_view_block(self, user: UserType, usage_key: OpaqueKey) -> bool: + def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: """ Does the specified usage key exist in its context, and if so, does the specified user have permission to view it and interact with it (call @@ -66,7 +64,7 @@ def can_view_block(self, user: UserType, usage_key: OpaqueKey) -> bool: self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY) - def _assert_key_instance(self, usage_key: OpaqueKey): + def _assert_key_instance(self, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator): assert isinstance(usage_key, LibraryUsageLocatorV2) or isinstance(usage_key, LibraryContainerLocator) def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index dc7a21f1c397..e1d10f054904 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -2,8 +2,9 @@ A "Learning Context" is a course, a library, a program, or some other collection of content where learning happens. """ +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 + from openedx.core.types import User as UserType -from opaque_keys.edx.keys import UsageKeyV2 class LearningContext: @@ -25,7 +26,7 @@ def __init__(self, **kwargs): parameters without changing the API. """ - def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument + def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: # pylint: disable=unused-argument """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the @@ -33,13 +34,13 @@ def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pyli user: a Django User object (may be an AnonymousUser) - usage_key: the UsageKeyV2 subclass used for this learning context + usage_key: the UsageKeyV2 | LibraryItemKey subclass used for this learning context Must return a boolean. """ return False - def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but @@ -47,7 +48,7 @@ def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> b """ return self.can_edit_block(user, usage_key) - def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument + def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: # pylint: disable=unused-argument """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view it and interact with it (call @@ -56,7 +57,7 @@ def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pyli user: a Django User object (may be an AnonymousUser) - usage_key: the UsageKeyV2 subclass used for this learning context + usage_key: the UsageKeyV2 | LibraryItemKey subclass used for this learning context Must return a boolean. """ @@ -74,7 +75,7 @@ def send_block_updated_event(self, usage_key): """ Send a "block updated" event for the block with the given usage_key in this context. - usage_key: the UsageKeyV2 subclass used for this learning context + usage_key: the UsageKeyV2 | LibraryItemKey subclass used for this learning context """ def send_container_updated_events(self, usage_key): From b8eb17158b1e8a04275190444f99b81411b04d4b Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 16:15:38 +0530 Subject: [PATCH 18/46] fix: import lint errors --- cms/djangoapps/contentstore/models.py | 2 +- cms/djangoapps/contentstore/utils.py | 2 +- cms/lib/xblock/container_upstream_sync.py | 2 +- .../content_libraries/api/containers.py | 11 +----- openedx/core/djangoapps/xblock/api.py | 22 +++++------ .../xblock/runtime/learning_core_runtime.py | 37 ++++++++++++++----- openedx/core/djangoapps/xblock/utils.py | 13 +++++++ 7 files changed, 53 insertions(+), 36 deletions(-) diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index 369d49517787..77cb5186579f 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -15,8 +15,8 @@ from opaque_keys.edx.django.models import CourseKeyField, LibraryItemField, UsageKeyField from opaque_keys.edx.keys import CourseKey, UsageKey from opaque_keys.edx.locator import LibraryContainerLocator +from openedx_learning.api.authoring import get_published_version from openedx_learning.api.authoring_models import Component, Container -from openedx_learning.apps.authoring.publishing.api import get_published_version from openedx_learning.lib.fields import ( immutable_uuid_field, key_field, diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index e3e7c3b985fd..8e8da422ea31 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -87,7 +87,7 @@ from common.djangoapps.xblock_django.api import deprecated_xblocks from common.djangoapps.xblock_django.user_service import DjangoXBlockUserService from openedx.core import toggles as core_toggles -from openedx.core.djangoapps.content_libraries.api.containers import get_container_from_key +from openedx.core.djangoapps.content_libraries.api import get_container_from_key from openedx.core.djangoapps.content_tagging.toggles import is_tagging_feature_disabled from openedx.core.djangoapps.credit.api import get_credit_requirements, is_credit_course from openedx.core.djangoapps.discussions.config.waffle import ENABLE_PAGES_AND_RESOURCES_MICROFRONTEND diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/container_upstream_sync.py index 185ab6ca8888..a0a49bda12b3 100644 --- a/cms/lib/xblock/container_upstream_sync.py +++ b/cms/lib/xblock/container_upstream_sync.py @@ -13,7 +13,7 @@ from xblock.fields import Scope from cms.djangoapps.contentstore.xblock_storage_handlers.create_xblock import create_xblock -from openedx.core.djangoapps.content_libraries.api.containers import get_container_children +from openedx.core.djangoapps.content_libraries.api import get_container_children from .upstream_sync import ( BadUpstream, diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index caf4d04c68c2..32a4229aeda5 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -8,7 +8,6 @@ from enum import Enum import logging from uuid import uuid4 -from lxml import etree from django.utils.text import slugify from opaque_keys.edx.keys import UsageKeyV2 @@ -28,7 +27,7 @@ LIBRARY_CONTAINER_UPDATED, ) from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import Container, Unit +from openedx_learning.api.authoring_models import Container from openedx.core.djangoapps.content_libraries.api.collections import library_collection_locator from openedx.core.djangoapps.xblock.api import get_component_from_usage_key @@ -55,7 +54,6 @@ "update_container_children", "get_containers_contains_component", "publish_container_changes", - "library_container_xml", ] log = logging.getLogger(__name__) @@ -491,10 +489,3 @@ def publish_container_changes(container_key: LibraryContainerLocator, user_id: i f"PublishableEntity {record.entity.pk} / {record.entity.key} was modified during publish operation " "but is of unknown type." ) - - -def library_container_xml(container: ContainerMetadata, block_type: str | None = None): - """Converts given unit to xml without including children components""" - xml_object = etree.Element(block_type or container.container_type.value) - xml_object.set("display_name", container.display_name) - return xml_object diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index ed5385103cca..008fcb94f5aa 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -8,41 +8,37 @@ Studio APIs cover use cases like adding/deleting/editing blocks. """ # pylint: disable=unused-import -from enum import Enum -from datetime import datetime import logging -import threading from django.core.exceptions import PermissionDenied from django.urls import reverse from django.utils.translation import gettext as _ -from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import Component, ComponentVersion from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator +from openedx_learning.api import authoring as authoring_api +from openedx_learning.api.authoring_models import Component, ComponentVersion, Container from rest_framework.exceptions import NotFound from xblock.core import XBlock from xblock.exceptions import NoSuchUsage, NoSuchViewError from xblock.plugin import PluginMissingError -from openedx.core.djangoapps.content_libraries.api.exceptions import ContentLibraryContainerNotFound -from openedx.core.types import User as UserType from openedx.core.djangoapps.xblock.apps import get_xblock_app_config + +# Made available as part of this package's public API: from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl from openedx.core.djangoapps.xblock.runtime.learning_core_runtime import ( LearningCoreFieldData, LearningCoreXBlockRuntime, ) +from openedx.core.types import User as UserType + from .data import CheckPerm, LatestVersion -from .rest_api.url_converters import VersionConverter from .utils import ( + get_auto_latest_version, get_secure_token_for_xblock_handler, get_xblock_id_for_anonymous_user, - get_auto_latest_version, ) -from .runtime.learning_core_runtime import LearningCoreXBlockRuntime - # Made available as part of this package's public API: from openedx.core.djangoapps.xblock.learning_context import LearningContext @@ -123,7 +119,7 @@ def load_block( except NoSuchUsage as exc: # Convert NoSuchUsage to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The component '{usage_key}' does not exist.") from exc - except (ComponentVersion.DoesNotExist, ContentLibraryContainerNotFound) as exc: + except (ComponentVersion.DoesNotExist, Container.DoesNotExist) as exc: # Convert ComponentVersion.DoesNotExist to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The requested version of component '{usage_key}' does not exist.") from exc diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index a5221d2a6df4..22a0079ad74d 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -11,26 +11,24 @@ from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db.transaction import atomic from django.urls import reverse - +from lxml import etree from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.locator import LibraryContainerLocator from openedx_learning.api import authoring as authoring_api - -from lxml import etree - from xblock.core import XBlock from xblock.exceptions import NoSuchUsage -from xblock.fields import Field, Scope, ScopeIds from xblock.field_data import FieldData +from xblock.fields import Field, Scope, ScopeIds from openedx.core.djangoapps.xblock.api import get_xblock_app_config from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from openedx.core.lib.xblock_serializer.data import StaticFile + from ..data import AuthoredDataMode, LatestVersion -from ..utils import get_auto_latest_version from ..learning_context.manager import get_learning_context_impl +from ..utils import get_auto_latest_version, library_container_xml from .runtime import XBlockRuntime - log = logging.getLogger(__name__) @@ -240,10 +238,13 @@ def get_container_block(self, container_key, *, version: int | LatestVersion = L This method create a very basic olx for container and parse it into an XBlock instance. """ - from openedx.core.djangoapps.content_libraries.api.containers import get_container, library_container_xml - container = get_container(container_key) + container = self._get_container_from_key(container_key) block_type = "vertical" if container_key.container_type == "unit" else container_key.container_type - content = library_container_xml(container, block_type) + content = library_container_xml( + container_key, + display_name=container.versioning.draft.title if container.versioning.draft else None, + block_type=block_type + ) xml = etree.tostring(content) return self._initialize_block(xml.decode(), container_key, block_type, version) @@ -339,6 +340,22 @@ def save_block(self, block): learning_context.send_block_updated_event(usage_key) learning_context.send_container_updated_events(usage_key) + def _get_container_from_key(self, container_key: LibraryContainerLocator): + """ + TODO: This is the third place where we're implementing this. Figure out + where the definitive place should be and have everything else call that. + """ + learning_package = authoring_api.get_learning_package_by_key(str(container_key.lib_key)) + try: + component = authoring_api.get_container_by_key( + learning_package.id, + key=container_key.container_id, + ) + except ObjectDoesNotExist as exc: + raise NoSuchUsage(container_key) from exc + + return component + def _get_component_from_usage_key(self, usage_key): """ Note that Components aren't ever really truly deleted, so this will diff --git a/openedx/core/djangoapps/xblock/utils.py b/openedx/core/djangoapps/xblock/utils.py index b4ae054cf498..248f503c66df 100644 --- a/openedx/core/djangoapps/xblock/utils.py +++ b/openedx/core/djangoapps/xblock/utils.py @@ -10,6 +10,8 @@ import crum from django.conf import settings +from lxml import etree +from opaque_keys.edx.locator import LibraryContainerLocator from openedx.core.djangoapps.xblock.apps import get_xblock_app_config @@ -186,3 +188,14 @@ def get_auto_latest_version(version: int | LatestVersion) -> int | LatestVersion else LatestVersion.PUBLISHED ) return version + + +def library_container_xml( + container_key: LibraryContainerLocator, + display_name: str | None = None, + block_type: str | None = None, +): + """Converts given unit to xml without including children components""" + xml_object = etree.Element(block_type or container_key.container_type) + xml_object.set("display_name", display_name) + return xml_object From e0c8662451ee165a7e33136b95363d07be249ae5 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 16:33:45 +0530 Subject: [PATCH 19/46] fix: failing tests --- cms/lib/xblock/container_upstream_sync.py | 2 +- cms/lib/xblock/test/test_upstream_sync.py | 18 +++++++++++------- cms/lib/xblock/upstream_sync.py | 8 ++++++-- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/container_upstream_sync.py index a0a49bda12b3..2082db678645 100644 --- a/cms/lib/xblock/container_upstream_sync.py +++ b/cms/lib/xblock/container_upstream_sync.py @@ -124,8 +124,8 @@ def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = Non super().__init__(downstream, user, upstream) if not isinstance(self.upstream_key, LibraryContainerLocator): raise BadUpstream('Invalid upstream key') - self.link = ContainerUpstreamLink.get_for_container(downstream) if not upstream: + self.link = ContainerUpstreamLink.get_for_container(downstream) self.upstream = self._load_upstream_link_and_container_block() self.syncable_field_names: set[str] = self._get_synchronizable_fields() self.new_children_blocks: list[XBlock] = [] diff --git a/cms/lib/xblock/test/test_upstream_sync.py b/cms/lib/xblock/test/test_upstream_sync.py index 93c87349f680..703f84c9c6c1 100644 --- a/cms/lib/xblock/test/test_upstream_sync.py +++ b/cms/lib/xblock/test/test_upstream_sync.py @@ -2,24 +2,28 @@ Test CMS's upstream->downstream syncing system """ import datetime -import ddt -from pytz import utc +import ddt from organizations.api import ensure_organization from organizations.models import Organization +from pytz import utc from cms.lib.xblock.upstream_sync import ( + BadDownstream, + BadUpstream, ComponentUpstreamSyncManager, + NoUpstream, UpstreamLink, - sync_from_upstream, decline_sync, sever_upstream_link, - NoUpstream, BadUpstream, BadDownstream, + decline_sync, + sever_upstream_link, + sync_from_upstream, ) from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as libs from openedx.core.djangoapps.content_tagging import api as tagging_api from openedx.core.djangoapps.xblock import api as xblock from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import CourseFactory, BlockFactory +from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory @ddt.ddt @@ -399,7 +403,7 @@ def test_sync_updates_to_modified_content(self): # assert downstream.upstream_display_name == downstream.display_name == "Title V5" @ddt.data(None, "Title From Some Other Upstream Version") - def test_fetch_customizable_fields(self, initial_upstream_display_name): + def test_update_customizable_fields(self, initial_upstream_display_name): """ Can we fetch a block's upstream field values without syncing it? @@ -410,7 +414,7 @@ def test_fetch_customizable_fields(self, initial_upstream_display_name): downstream.display_name = "Some Title" downstream.data = "Some content" - # Note that we're not linked to any upstream. fetch_customizable_fields shouldn't care. + # Note that we're not linked to any upstream. ComponentUpstreamSyncManager shouldn't care if upstream is passed. assert not downstream.upstream assert not downstream.upstream_version diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 5a862c91cdb8..65743e7a560e 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -195,7 +195,9 @@ def __init__( ) -> None: self.downstream = downstream self.user = user - self.upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) + if not upstream: + # Only parse upstream_key if upstream block is not passed else don't care about downstream.upstream + self.upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) self.link: BaseUpstreamLink self.upstream = upstream self.syncable_field_names: set[str] @@ -293,8 +295,10 @@ def sync(self) -> None: class ComponentUpstreamSyncManager(BaseUpstreamSyncManager): def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: super().__init__(downstream, user, upstream) - self.link = UpstreamLink.get_for_block(downstream) if not upstream: + # Only parse upstream_key if upstream block is not passed else don't care about + # downstream.upstream and upstream link + self.link = UpstreamLink.get_for_block(downstream) self.upstream = self._load_upstream_link_and_block() self.syncable_field_names: set[str] = self._get_synchronizable_fields() From 78bbb412b967cefcd5c769ee5e790434040cd956 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 16:51:05 +0530 Subject: [PATCH 20/46] fix: doc lint issues --- .../tests/test_upstream_downstream_links.py | 3 +++ cms/djangoapps/contentstore/utils.py | 6 ++++++ .../xblock_storage_handlers/view_handlers.py | 4 ++++ cms/lib/xblock/container_upstream_sync.py | 20 ++++++++++++++++++- cms/lib/xblock/upstream_sync.py | 12 +++++++++-- .../content_libraries/library_context.py | 8 ++++++-- 6 files changed, 48 insertions(+), 5 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py index a36540d3646e..a5ad9a2a8002 100644 --- a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py +++ b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py @@ -60,6 +60,9 @@ def _create_unit(self, num: int): ) def _create_unit_and_expected_container_link(self, course_key: str | CourseKey, num_blocks: int = 3): + """ + Create unit xblock with random upstream key and version number. + """ data = [] for i in range(num_blocks): upstream, block = self._create_unit(i + 1) diff --git a/cms/djangoapps/contentstore/utils.py b/cms/djangoapps/contentstore/utils.py index 8e8da422ea31..1a670e8fecc7 100644 --- a/cms/djangoapps/contentstore/utils.py +++ b/cms/djangoapps/contentstore/utils.py @@ -2375,6 +2375,9 @@ def get_xblock_render_context(request, block): def _create_or_update_component_link(course_key: CourseKey, created: datetime | None, xblock): + """ + Create or update upstream->downstream link for components in database for given xblock. + """ upstream_usage_key = UsageKeyV2.from_string(xblock.upstream) try: lib_component = get_component_from_usage_key(upstream_usage_key) @@ -2394,6 +2397,9 @@ def _create_or_update_component_link(course_key: CourseKey, created: datetime | def _create_or_update_container_link(course_key: CourseKey, created: datetime | None, xblock): + """ + Create or update upstream->downstream link for containers in database for given xblock. + """ upstream_container_key = LibraryContainerLocator.from_string(xblock.upstream) try: lib_component = get_container_from_key(upstream_container_key) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 1143fc70eb98..82ca84a5b319 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -525,6 +525,10 @@ def create_item(request): def sync_library_content(created_block: XBlock, request, store, remove_upstream_link: bool = False): + """ + Handle syncing library content for given xblock depending on its upstream type. + It can sync unit containers and lower level xblocks. + """ upstream_key = check_and_parse_upstream_key(created_block.upstream, created_block.usage_key) if isinstance(upstream_key, LibraryUsageLocatorV2): lib_block = sync_from_upstream(downstream=created_block, user=request.user) diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/container_upstream_sync.py index 2082db678645..ae40f8e8f64d 100644 --- a/cms/lib/xblock/container_upstream_sync.py +++ b/cms/lib/xblock/container_upstream_sync.py @@ -1,3 +1,14 @@ +""" +Synchronize content and settings from upstream containers to their downstream usages. + +* The upstream is a Container from a Learning Core-backed Content Library. +* The downstream is a block of matching type in a SplitModuleStore-backed Course. +* They are both on the same Open edX instance. + +HOWEVER, those assumptions may loosen in the future. So, we consider these to be INTERNAL ASSUMPIONS that should not be +exposed through this module's public Python interface. +""" + import logging import typing as t from dataclasses import asdict, dataclass @@ -120,6 +131,9 @@ def _get_library_container_url(container_key: LibraryContainerLocator): class ContainerUpstreamSyncManager(BaseUpstreamSyncManager): + """ + Manages sync process of downstream containers like unit with upstream containers. + """ def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: super().__init__(downstream, user, upstream) if not isinstance(self.upstream_key, LibraryContainerLocator): @@ -152,7 +166,8 @@ def _load_upstream_link_and_container_block(self) -> XBlock: If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. """ - # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, + # which loads before the djangoapps are ready. from openedx.core.djangoapps.xblock.api import ( # pylint: disable=wrong-import-order CheckPerm, LatestVersion, @@ -170,6 +185,9 @@ def _load_upstream_link_and_container_block(self) -> XBlock: return lib_block def sync_new_children_blocks(self): + """ + Creates children xblocks in course based on library container children. + """ for child in get_container_children(self.upstream_key, published=True): child_block = create_xblock( parent_locator=str(self.downstream.location), diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 65743e7a560e..7571c93ab17e 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -11,6 +11,7 @@ """ from __future__ import annotations +from abc import ABC import logging import typing as t from dataclasses import asdict, dataclass @@ -186,7 +187,10 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: ) -class BaseUpstreamSyncManager: +class BaseUpstreamSyncManager(ABC): + """ + Base manager class for managing upstream link sync process. + """ def __init__( self, downstream: XBlock, @@ -293,6 +297,9 @@ def sync(self) -> None: class ComponentUpstreamSyncManager(BaseUpstreamSyncManager): + """ + Manages sync process of downstream component with upstream components. + """ def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: super().__init__(downstream, user, upstream) if not upstream: @@ -324,7 +331,8 @@ def _load_upstream_link_and_block(self) -> XBlock: If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. """ - # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, + # which loads before the djangoapps are ready. from openedx.core.djangoapps.xblock.api import ( CheckPerm, LatestVersion, diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 7fd37761086f..536d8a496706 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -42,7 +42,11 @@ def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr self._assert_key_instance(usage_key) return self._check_perm(user, usage_key.lib_key, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) - def can_view_block_for_editing(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: + def can_view_block_for_editing( + self, + user: UserType, + usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator, + ) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but @@ -65,7 +69,7 @@ def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr return self._check_perm(user, usage_key.lib_key, permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY) def _assert_key_instance(self, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator): - assert isinstance(usage_key, LibraryUsageLocatorV2) or isinstance(usage_key, LibraryContainerLocator) + assert isinstance(usage_key, (LibraryUsageLocatorV2, LibraryContainerLocator)) def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: """ Helper method to check a permission for the various can_ methods""" From b1ba9e8785149acd0d14bbd0803f50f4d7f394bb Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 17:06:42 +0530 Subject: [PATCH 21/46] fix: lint issues --- .../rest_api/v2/views/tests/test_downstreams.py | 12 ++++++++---- .../xblock/learning_context/learning_context.py | 6 +++++- .../djangoapps/xblock/learning_context/manager.py | 2 +- .../xblock/runtime/learning_core_runtime.py | 3 +++ 4 files changed, 17 insertions(+), 6 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py index ca590273677f..e0f222d5eb68 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py @@ -11,7 +11,7 @@ from organizations.models import Organization from cms.djangoapps.contentstore.helpers import StaticFileNotices -from cms.lib.xblock.upstream_sync import BadUpstream, UpstreamLink +from cms.lib.xblock.upstream_sync import BadUpstream, ComponentUpstreamSyncManager, UpstreamLink from cms.djangoapps.contentstore.tests.utils import CourseTestCase from opaque_keys.edx.keys import UsageKey from common.djangoapps.student.tests.factories import UserFactory @@ -235,7 +235,7 @@ def call_api(self, usage_key_string, sync: str | None = None): content_type="application/json", ) - @patch.object(downstreams_views, "fetch_customizable_fields") + @patch.object(ComponentUpstreamSyncManager, "update_customizable_fields") @patch.object(downstreams_views, "sync_from_upstream") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) def test_200_with_sync(self, mock_sync, mock_fetch): @@ -250,7 +250,7 @@ def test_200_with_sync(self, mock_sync, mock_fetch): assert mock_fetch.call_count == 0 assert video_after.upstream == self.video_lib_id - @patch.object(downstreams_views, "fetch_customizable_fields") + @patch.object(ComponentUpstreamSyncManager, "update_customizable_fields") @patch.object(downstreams_views, "sync_from_upstream") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) def test_200_no_sync(self, mock_sync, mock_fetch): @@ -265,7 +265,11 @@ def test_200_no_sync(self, mock_sync, mock_fetch): assert mock_fetch.call_count == 1 assert video_after.upstream == self.video_lib_id - @patch.object(downstreams_views, "fetch_customizable_fields", side_effect=BadUpstream(MOCK_UPSTREAM_ERROR)) + @patch.object( + ComponentUpstreamSyncManager, + "update_customizable_fields", + side_effect=BadUpstream(MOCK_UPSTREAM_ERROR) + ) def test_400(self, sync: str): """ Do we raise a 400 if the provided upstream reference is malformed or not accessible? diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index e1d10f054904..db9030894b59 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -40,7 +40,11 @@ def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr """ return False - def can_view_block_for_editing(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: + def can_view_block_for_editing( + self, + user: UserType, + usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator, + ) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but diff --git a/openedx/core/djangoapps/xblock/learning_context/manager.py b/openedx/core/djangoapps/xblock/learning_context/manager.py index 3d4ab5eabf4e..6a3279a553cc 100644 --- a/openedx/core/djangoapps/xblock/learning_context/manager.py +++ b/openedx/core/djangoapps/xblock/learning_context/manager.py @@ -35,7 +35,7 @@ def get_learning_context_impl(key): """ if isinstance(key, LearningContextKey): context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' - elif isinstance(key, UsageKeyV2) or isinstance(key, LibraryItemKey): + elif isinstance(key, (UsageKeyV2, LibraryItemKey)): context_type = key.context_key.CANONICAL_NAMESPACE elif isinstance(key, OpaqueKey): # Maybe this is an older modulestore key etc. diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 22a0079ad74d..6890ad9c675e 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -167,6 +167,9 @@ class LearningCoreXBlockRuntime(XBlockRuntime): """ def _initialize_block(self, content, usage_key, block_type, version: int | LatestVersion): + """ + Creates new xblock from given content, usage_key and block_type + """ xml_node = etree.fromstring(content) keys = ScopeIds(self.user_id, block_type, None, usage_key) From fe316a18eca75ec6815523ae27e5c7130e6fe006 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 19:49:50 +0530 Subject: [PATCH 22/46] fix: failing tests --- xmodule/modulestore/tests/test_mixed_modulestore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index ecf20578ef91..6e15fab918df 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -1112,7 +1112,7 @@ def test_has_changes_missing_child(self, default_ms, default_branch): # check CONTENT_TAGGING_AUTO CourseWaffleFlag # Find: active_versions, 2 structures (published & draft), definition (unnecessary) # Sends: updated draft and published structures and active_versions - @ddt.data((ModuleStoreEnum.Type.split, 5, 2, 3)) + @ddt.data((ModuleStoreEnum.Type.split, 6, 2, 3)) @ddt.unpack def test_delete_item(self, default_ms, num_mysql, max_find, max_send): """ @@ -1135,7 +1135,7 @@ def test_delete_item(self, default_ms, num_mysql, max_find, max_send): # check CONTENT_TAGGING_AUTO CourseWaffleFlag # find: draft and published structures, definition (unnecessary) # sends: update published (why?), draft, and active_versions - @ddt.data((ModuleStoreEnum.Type.split, 5, 3, 3)) + @ddt.data((ModuleStoreEnum.Type.split, 6, 3, 3)) @ddt.unpack def test_delete_private_vertical(self, default_ms, num_mysql, max_find, max_send): """ @@ -1185,7 +1185,7 @@ def test_delete_private_vertical(self, default_ms, num_mysql, max_find, max_send # check CONTENT_TAGGING_AUTO CourseWaffleFlag # find: structure (cached) # send: update structure and active_versions - @ddt.data((ModuleStoreEnum.Type.split, 5, 1, 2)) + @ddt.data((ModuleStoreEnum.Type.split, 6, 1, 2)) @ddt.unpack def test_delete_draft_vertical(self, default_ms, num_mysql, max_find, max_send): """ From 3745db71a46acc8ec6027aa6a5fd72d07c5408c1 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 20:20:52 +0530 Subject: [PATCH 23/46] Revert "fix: failing tests" This reverts commit fe316a18eca75ec6815523ae27e5c7130e6fe006. --- xmodule/modulestore/tests/test_mixed_modulestore.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index 6e15fab918df..ecf20578ef91 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -1112,7 +1112,7 @@ def test_has_changes_missing_child(self, default_ms, default_branch): # check CONTENT_TAGGING_AUTO CourseWaffleFlag # Find: active_versions, 2 structures (published & draft), definition (unnecessary) # Sends: updated draft and published structures and active_versions - @ddt.data((ModuleStoreEnum.Type.split, 6, 2, 3)) + @ddt.data((ModuleStoreEnum.Type.split, 5, 2, 3)) @ddt.unpack def test_delete_item(self, default_ms, num_mysql, max_find, max_send): """ @@ -1135,7 +1135,7 @@ def test_delete_item(self, default_ms, num_mysql, max_find, max_send): # check CONTENT_TAGGING_AUTO CourseWaffleFlag # find: draft and published structures, definition (unnecessary) # sends: update published (why?), draft, and active_versions - @ddt.data((ModuleStoreEnum.Type.split, 6, 3, 3)) + @ddt.data((ModuleStoreEnum.Type.split, 5, 3, 3)) @ddt.unpack def test_delete_private_vertical(self, default_ms, num_mysql, max_find, max_send): """ @@ -1185,7 +1185,7 @@ def test_delete_private_vertical(self, default_ms, num_mysql, max_find, max_send # check CONTENT_TAGGING_AUTO CourseWaffleFlag # find: structure (cached) # send: update structure and active_versions - @ddt.data((ModuleStoreEnum.Type.split, 6, 1, 2)) + @ddt.data((ModuleStoreEnum.Type.split, 5, 1, 2)) @ddt.unpack def test_delete_draft_vertical(self, default_ms, num_mysql, max_find, max_send): """ From 18ee10f8a50ce246e5ab4ae21f45b38ce9533bbd Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 20:24:36 +0530 Subject: [PATCH 24/46] feat: update downstream api views --- .../rest_api/v2/views/downstreams.py | 17 +++++++++----- .../v2/views/tests/test_downstreams.py | 10 ++++----- .../xblock_storage_handlers/view_handlers.py | 22 +++++++++---------- .../tests/test_mixed_modulestore.py | 11 +++++++--- 4 files changed, 34 insertions(+), 26 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py index 49d85152f4fb..30bf93079ef1 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py @@ -94,12 +94,12 @@ from rest_framework.views import APIView from xblock.core import XBlock -from cms.djangoapps.contentstore.helpers import import_static_assets_for_library_sync from cms.djangoapps.contentstore.models import ComponentLink from cms.djangoapps.contentstore.rest_api.v2.serializers import ( ComponentLinksSerializer, PublishableEntityLinksSummarySerializer, ) +from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import sync_library_content from cms.lib.xblock.upstream_sync import ( BadDownstream, BadUpstream, @@ -109,7 +109,6 @@ UpstreamLinkException, decline_sync, sever_upstream_link, - sync_from_upstream, ) from common.djangoapps.student.auth import has_studio_read_access, has_studio_write_access from openedx.core.lib.api.view_utils import ( @@ -252,7 +251,11 @@ def put(self, request: _AuthenticatedRequest, usage_key_string: str) -> Response raise ValidationError({"sync": "must be 'true' or 'false'"}) try: if sync_param == "true" or sync_param is True: - sync_from_upstream(downstream=downstream, user=request.user) + sync_library_content( + downstream=downstream, + request=request, + store=modulestore() + ) else: # Even if we're not syncing (i.e., updating the downstream's values with the upstream's), we still need # to fetch the upstream's customizable values and store them as hidden fields on the downstream. This @@ -313,8 +316,11 @@ def post(self, request: _AuthenticatedRequest, usage_key_string: str) -> Respons if downstream.usage_key.block_type == "video": # Delete all transcripts so we can copy new ones from upstream clear_transcripts(downstream) - upstream = sync_from_upstream(downstream, request.user) - static_file_notices = import_static_assets_for_library_sync(downstream, upstream, request) + static_file_notices = sync_library_content( + downstream=downstream, + request=request, + store=modulestore() + ) except UpstreamLinkException as exc: logger.exception( "Could not sync from upstream '%s' to downstream '%s'", @@ -322,7 +328,6 @@ def post(self, request: _AuthenticatedRequest, usage_key_string: str) -> Respons usage_key_string, ) raise ValidationError(detail=str(exc)) from exc - modulestore().update_item(downstream, request.user.id) # Note: We call `get_for_block` (rather than `try_get_for_block`) because if anything is wrong with the # upstream at this point, then that is completely unexpected, so it's appropriate to let the 500 happen. response = UpstreamLink.get_for_block(downstream).to_json() diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py index e0f222d5eb68..c2428c6db97f 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py @@ -236,7 +236,7 @@ def call_api(self, usage_key_string, sync: str | None = None): ) @patch.object(ComponentUpstreamSyncManager, "update_customizable_fields") - @patch.object(downstreams_views, "sync_from_upstream") + @patch.object(downstreams_views, "sync_library_content") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) def test_200_with_sync(self, mock_sync, mock_fetch): """ @@ -251,7 +251,7 @@ def test_200_with_sync(self, mock_sync, mock_fetch): assert video_after.upstream == self.video_lib_id @patch.object(ComponentUpstreamSyncManager, "update_customizable_fields") - @patch.object(downstreams_views, "sync_from_upstream") + @patch.object(downstreams_views, "sync_library_content") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) def test_200_no_sync(self, mock_sync, mock_fetch): """ @@ -398,10 +398,9 @@ def call_api(self, usage_key_string): return self.client.post(f"/api/contentstore/v2/downstreams/{usage_key_string}/sync") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) - @patch.object(downstreams_views, "sync_from_upstream") - @patch.object(downstreams_views, "import_static_assets_for_library_sync", return_value=StaticFileNotices()) + @patch.object(downstreams_views, "sync_library_content", return_value=StaticFileNotices()) @patch.object(downstreams_views, "clear_transcripts") - def test_200(self, mock_sync_from_upstream, mock_import_staged_content, mock_clear_transcripts): + def test_200(self, mock_sync_from_upstream, mock_clear_transcripts): """ Does the happy path work? """ @@ -409,7 +408,6 @@ def test_200(self, mock_sync_from_upstream, mock_import_staged_content, mock_cle response = self.call_api(self.downstream_video_key) assert response.status_code == 200 assert mock_sync_from_upstream.call_count == 1 - assert mock_import_staged_content.call_count == 1 assert mock_clear_transcripts.call_count == 1 diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 82ca84a5b319..4ce591af40a7 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -524,29 +524,29 @@ def create_item(request): return _create_block(request) -def sync_library_content(created_block: XBlock, request, store, remove_upstream_link: bool = False): +def sync_library_content(downstream: XBlock, request, store, remove_upstream_link: bool = False): """ Handle syncing library content for given xblock depending on its upstream type. It can sync unit containers and lower level xblocks. """ - upstream_key = check_and_parse_upstream_key(created_block.upstream, created_block.usage_key) + upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) if isinstance(upstream_key, LibraryUsageLocatorV2): - lib_block = sync_from_upstream(downstream=created_block, user=request.user) + lib_block = sync_from_upstream(downstream=downstream, user=request.user) if remove_upstream_link: # Removing upstream link from child components - created_block.upstream = None - static_file_notices = import_static_assets_for_library_sync(created_block, lib_block, request) - store.update_item(created_block, request.user.id) + downstream.upstream = None + static_file_notices = import_static_assets_for_library_sync(downstream, lib_block, request) + store.update_item(downstream, request.user.id) else: - lib_block, children_blocks = sync_from_upstream_container(downstream=created_block, user=request.user) - notices = [import_static_assets_for_library_sync(created_block, lib_block, request)] - with store.bulk_operations(created_block.location.context_key): + lib_block, children_blocks = sync_from_upstream_container(downstream=downstream, user=request.user) + notices = [import_static_assets_for_library_sync(downstream, lib_block, request)] + with store.bulk_operations(downstream.location.context_key): children_blocks_usage_keys = [] for child_block in children_blocks: notices.append(sync_library_content(child_block, request, store, remove_upstream_link=True)) children_blocks_usage_keys.append(child_block.usage_key) - created_block.children = children_blocks_usage_keys - store.update_item(created_block, request.user.id) + downstream.children = children_blocks_usage_keys + store.update_item(downstream, request.user.id) static_file_notices = concat_static_file_notices(notices) return static_file_notices diff --git a/xmodule/modulestore/tests/test_mixed_modulestore.py b/xmodule/modulestore/tests/test_mixed_modulestore.py index ecf20578ef91..c7770beb0f3c 100644 --- a/xmodule/modulestore/tests/test_mixed_modulestore.py +++ b/xmodule/modulestore/tests/test_mixed_modulestore.py @@ -172,11 +172,16 @@ def setUp(self): ) create_or_update_xblock_upstream_link_patch.start() self.addCleanup(create_or_update_xblock_upstream_link_patch.stop) - publishableEntityLinkPatch = patch( + component_link_patch = patch( 'cms.djangoapps.contentstore.signals.handlers.ComponentLink' ) - publishableEntityLinkPatch.start() - self.addCleanup(publishableEntityLinkPatch.stop) + component_link_patch.start() + self.addCleanup(component_link_patch.stop) + container_link_patch = patch( + 'cms.djangoapps.contentstore.signals.handlers.ContainerLink' + ) + container_link_patch.start() + self.addCleanup(container_link_patch.stop) def _check_connection(self): """ From 0051e184a527e4d0f829d58b2e872f693b6a4724 Mon Sep 17 00:00:00 2001 From: Navin Karkera Date: Wed, 23 Apr 2025 20:48:21 +0530 Subject: [PATCH 25/46] feat: delete extra components in container on sync (not working) --- .../contentstore/views/entrance_exam.py | 2 +- .../xblock_storage_handlers/view_handlers.py | 8 ++++---- cms/lib/xblock/container_upstream_sync.py | 17 +++++++++++++++++ 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/cms/djangoapps/contentstore/views/entrance_exam.py b/cms/djangoapps/contentstore/views/entrance_exam.py index bbefb0e9e876..addf0e4386c4 100644 --- a/cms/djangoapps/contentstore/views/entrance_exam.py +++ b/cms/djangoapps/contentstore/views/entrance_exam.py @@ -278,5 +278,5 @@ def remove_entrance_exam_milestone_reference(request, course_key): ) for course_child in course_children: if course_child.is_entrance_exam: - delete_item(request, course_child.scope_ids.usage_id) + delete_item(request.user, course_child.scope_ids.usage_id) milestones_helpers.remove_content_references(str(course_child.scope_ids.usage_id)) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 4ce591af40a7..c086cce62283 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -538,9 +538,9 @@ def sync_library_content(downstream: XBlock, request, store, remove_upstream_lin static_file_notices = import_static_assets_for_library_sync(downstream, lib_block, request) store.update_item(downstream, request.user.id) else: - lib_block, children_blocks = sync_from_upstream_container(downstream=downstream, user=request.user) - notices = [import_static_assets_for_library_sync(downstream, lib_block, request)] with store.bulk_operations(downstream.location.context_key): + lib_block, children_blocks = sync_from_upstream_container(downstream=downstream, user=request.user) + notices = [import_static_assets_for_library_sync(downstream, lib_block, request)] children_blocks_usage_keys = [] for child_block in children_blocks: notices.append(sync_library_content(child_block, request, store, remove_upstream_link=True)) @@ -790,11 +790,11 @@ def _move_item(source_usage_key, target_parent_usage_key, user, target_index=Non @login_required -def delete_item(request, usage_key): +def delete_item(user, usage_key): """ Exposes internal helper method without breaking existing bindings/dependencies """ - _delete_item(usage_key, request.user) + _delete_item(usage_key, user) def _delete_item(usage_key, user): diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/container_upstream_sync.py index ae40f8e8f64d..141b6e4266f1 100644 --- a/cms/lib/xblock/container_upstream_sync.py +++ b/cms/lib/xblock/container_upstream_sync.py @@ -199,8 +199,25 @@ def sync_new_children_blocks(self): self.new_children_blocks.append(child_block) return self.new_children_blocks + def delete_extra_blocks(self): + """ + Deletes extra child blocks under the container that are not present in new version of library container. + """ + # TODO: Importing here to avoid circular imports, should be fixed later + from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import delete_item + current = self.downstream.children + latest = [str(child.usage_key) for child in get_container_children(self.upstream_key, published=True)] + for child in current: + # TODO: doesn't work for two reasons + # 1. child is not an XBlock but usage_key, so we need to load the child to get its upstream + # 2. Even if we get child, it won't have upstream set as we are not setting upstream for child components + # See: cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py:535 + if child.upstream not in latest: + delete_item(self.user, child.usage_key) + def sync(self) -> None: super().sync() + # self.delete_extra_blocks() self.sync_new_children_blocks() From 391e3af2bb7d9083920dea6903291c0bbbe9d2a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=B4mulo=20Penido?= Date: Mon, 21 Apr 2025 11:51:10 -0300 Subject: [PATCH 26/46] feat: disallow edits to units in courses that are sourced from a library --- cms/djangoapps/contentstore/views/preview.py | 13 +++++++++++-- .../xblock_storage_handlers/view_handlers.py | 5 +++++ cms/templates/studio_xblock_wrapper.html | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/cms/djangoapps/contentstore/views/preview.py b/cms/djangoapps/contentstore/views/preview.py index aa7421bb87b3..c7b7bc511d57 100644 --- a/cms/djangoapps/contentstore/views/preview.py +++ b/cms/djangoapps/contentstore/views/preview.py @@ -299,8 +299,16 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): if selected_groups_label: selected_groups_label = _('Access restricted to: {list_of_groups}').format(list_of_groups=selected_groups_label) # lint-amnesty, pylint: disable=line-too-long course = modulestore().get_course(xblock.location.course_key) - can_edit = context.get('can_edit', True) - can_add = context.get('can_add', True) + + if root_xblock.upstream and str(root_xblock.upstream).startswith('lct:'): + can_edit = False + can_add = False + parent_has_upstream = True + else: + can_edit = context.get('can_edit', True) + can_add = context.get('can_add', True) + parent_has_upstream = False + # Is this a course or a library? is_course = xblock.context_key.is_course tags_count_map = context.get('tags_count_map') @@ -329,6 +337,7 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): 'language': getattr(course, 'language', None), 'is_course': is_course, 'tags_count': tags_count, + 'parent_has_upstream': parent_has_upstream, } add_webpack_js_to_fragment(frag, "js/factories/xblock_validation") diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index c086cce62283..21086afb3e30 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -1316,6 +1316,11 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements if is_xblock_unit and summary_configuration.is_enabled(): xblock_info["summary_configuration_enabled"] = summary_configuration.is_summary_enabled(xblock_info['id']) + if xblock.upstream: + xblock_info["upstream"] = str(xblock.upstream) + else: + xblock_info["upstream"] = None + return xblock_info diff --git a/cms/templates/studio_xblock_wrapper.html b/cms/templates/studio_xblock_wrapper.html index 8f4090588613..1dc58cbb6f61 100644 --- a/cms/templates/studio_xblock_wrapper.html +++ b/cms/templates/studio_xblock_wrapper.html @@ -211,7 +211,7 @@ % endif - % elif not show_inline: + % elif not show_inline and not parent_has_upstream:
  • ${_("Details")} From c9ebcebfb910ae4a6da5f6f004c8e1f539b43079 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 15:48:24 -0700 Subject: [PATCH 27/46] revert: changes to xblock runtime --- .../content_libraries/library_context.py | 28 ++--- openedx/core/djangoapps/xblock/api.py | 33 +++--- .../learning_context/learning_context.py | 19 ++- .../xblock/learning_context/manager.py | 4 +- .../xblock/runtime/learning_core_runtime.py | 111 ++++++------------ openedx/core/djangoapps/xblock/utils.py | 13 -- requirements/edx/kernel.in | 2 +- 7 files changed, 73 insertions(+), 137 deletions(-) diff --git a/openedx/core/djangoapps/content_libraries/library_context.py b/openedx/core/djangoapps/content_libraries/library_context.py index 536d8a496706..34a5a90269fe 100644 --- a/openedx/core/djangoapps/content_libraries/library_context.py +++ b/openedx/core/djangoapps/content_libraries/library_context.py @@ -4,12 +4,13 @@ import logging from django.core.exceptions import PermissionDenied -from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryLocatorV2, LibraryUsageLocatorV2 +from rest_framework.exceptions import NotFound + from openedx_events.content_authoring.data import LibraryBlockData, LibraryContainerData from openedx_events.content_authoring.signals import LIBRARY_BLOCK_UPDATED, LIBRARY_CONTAINER_UPDATED +from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.locator import LibraryUsageLocatorV2, LibraryLocatorV2 from openedx_learning.api import authoring as authoring_api -from rest_framework.exceptions import NotFound from openedx.core.djangoapps.content_libraries import api, permissions from openedx.core.djangoapps.content_libraries.models import ContentLibrary @@ -31,7 +32,7 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.use_draft = kwargs.get('use_draft', None) - def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: + def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the @@ -39,14 +40,10 @@ def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr May raise ContentLibraryNotFound if the library does not exist. """ - self._assert_key_instance(usage_key) + assert isinstance(usage_key, LibraryUsageLocatorV2) return self._check_perm(user, usage_key.lib_key, permissions.CAN_EDIT_THIS_CONTENT_LIBRARY) - def can_view_block_for_editing( - self, - user: UserType, - usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator, - ) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but @@ -54,10 +51,10 @@ def can_view_block_for_editing( May raise ContentLibraryNotFound if the library does not exist. """ - self._assert_key_instance(usage_key) + assert isinstance(usage_key, LibraryUsageLocatorV2) return self._check_perm(user, usage_key.lib_key, permissions.CAN_VIEW_THIS_CONTENT_LIBRARY) - def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: + def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ Does the specified usage key exist in its context, and if so, does the specified user have permission to view it and interact with it (call @@ -65,12 +62,9 @@ def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr May raise ContentLibraryNotFound if the library does not exist. """ - self._assert_key_instance(usage_key) + assert isinstance(usage_key, LibraryUsageLocatorV2) return self._check_perm(user, usage_key.lib_key, permissions.CAN_LEARN_FROM_THIS_CONTENT_LIBRARY) - def _assert_key_instance(self, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator): - assert isinstance(usage_key, (LibraryUsageLocatorV2, LibraryContainerLocator)) - def _check_perm(self, user: UserType, lib_key: LibraryLocatorV2, perm) -> bool: """ Helper method to check a permission for the various can_ methods""" try: @@ -116,7 +110,7 @@ def send_block_updated_event(self, usage_key: UsageKeyV2): assert isinstance(usage_key, LibraryUsageLocatorV2) LIBRARY_BLOCK_UPDATED.send_event( library_block=LibraryBlockData( - library_key=usage_key.context_key, + library_key=usage_key.lib_key, usage_key=usage_key, ) ) diff --git a/openedx/core/djangoapps/xblock/api.py b/openedx/core/djangoapps/xblock/api.py index 008fcb94f5aa..c806fefc87c5 100644 --- a/openedx/core/djangoapps/xblock/api.py +++ b/openedx/core/djangoapps/xblock/api.py @@ -8,37 +8,40 @@ Studio APIs cover use cases like adding/deleting/editing blocks. """ # pylint: disable=unused-import +from enum import Enum +from datetime import datetime import logging +import threading from django.core.exceptions import PermissionDenied from django.urls import reverse from django.utils.translation import gettext as _ -from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerLocator from openedx_learning.api import authoring as authoring_api -from openedx_learning.api.authoring_models import Component, ComponentVersion, Container +from openedx_learning.api.authoring_models import Component, ComponentVersion +from opaque_keys.edx.keys import UsageKeyV2 +from opaque_keys.edx.locator import LibraryUsageLocatorV2 from rest_framework.exceptions import NotFound from xblock.core import XBlock from xblock.exceptions import NoSuchUsage, NoSuchViewError from xblock.plugin import PluginMissingError +from openedx.core.types import User as UserType from openedx.core.djangoapps.xblock.apps import get_xblock_app_config - -# Made available as part of this package's public API: from openedx.core.djangoapps.xblock.learning_context.manager import get_learning_context_impl from openedx.core.djangoapps.xblock.runtime.learning_core_runtime import ( LearningCoreFieldData, LearningCoreXBlockRuntime, ) -from openedx.core.types import User as UserType - from .data import CheckPerm, LatestVersion +from .rest_api.url_converters import VersionConverter from .utils import ( - get_auto_latest_version, get_secure_token_for_xblock_handler, get_xblock_id_for_anonymous_user, + get_auto_latest_version, ) +from .runtime.learning_core_runtime import LearningCoreXBlockRuntime + # Made available as part of this package's public API: from openedx.core.djangoapps.xblock.learning_context import LearningContext @@ -67,7 +70,7 @@ def get_runtime(user: UserType): def load_block( - usage_key: UsageKeyV2 | LibraryContainerLocator, + usage_key: UsageKeyV2, user: UserType, *, check_permission: CheckPerm | None = CheckPerm.CAN_LEARN, @@ -93,13 +96,14 @@ def load_block( # Now, check if the block exists in this context and if the user has # permission to render this XBlock view: if check_permission and user is not None: - has_perm = False if check_permission == CheckPerm.CAN_EDIT: has_perm = context_impl.can_edit_block(user, usage_key) elif check_permission == CheckPerm.CAN_READ_AS_AUTHOR: has_perm = context_impl.can_view_block_for_editing(user, usage_key) elif check_permission == CheckPerm.CAN_LEARN: has_perm = context_impl.can_view_block(user, usage_key) + else: + has_perm = False if not has_perm: raise PermissionDenied(f"You don't have permission to access the component '{usage_key}'.") @@ -110,16 +114,11 @@ def load_block( runtime = get_runtime(user=user) try: - if isinstance(usage_key, UsageKeyV2): - return runtime.get_block(usage_key, version=version) - elif isinstance(usage_key, LibraryContainerLocator): - return runtime.get_container_block(usage_key, version=version) - else: - raise NotFound(f"The component '{usage_key}' does not exist.") + return runtime.get_block(usage_key, version=version) except NoSuchUsage as exc: # Convert NoSuchUsage to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The component '{usage_key}' does not exist.") from exc - except (ComponentVersion.DoesNotExist, Container.DoesNotExist) as exc: + except ComponentVersion.DoesNotExist as exc: # Convert ComponentVersion.DoesNotExist to NotFound so we do the right thing (404 not 500) by default. raise NotFound(f"The requested version of component '{usage_key}' does not exist.") from exc diff --git a/openedx/core/djangoapps/xblock/learning_context/learning_context.py b/openedx/core/djangoapps/xblock/learning_context/learning_context.py index db9030894b59..dc7a21f1c397 100644 --- a/openedx/core/djangoapps/xblock/learning_context/learning_context.py +++ b/openedx/core/djangoapps/xblock/learning_context/learning_context.py @@ -2,9 +2,8 @@ A "Learning Context" is a course, a library, a program, or some other collection of content where learning happens. """ -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 - from openedx.core.types import User as UserType +from opaque_keys.edx.keys import UsageKeyV2 class LearningContext: @@ -26,7 +25,7 @@ def __init__(self, **kwargs): parameters without changing the API. """ - def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: # pylint: disable=unused-argument + def can_edit_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to edit it (make changes to the @@ -34,17 +33,13 @@ def can_edit_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr user: a Django User object (may be an AnonymousUser) - usage_key: the UsageKeyV2 | LibraryItemKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context Must return a boolean. """ return False - def can_view_block_for_editing( - self, - user: UserType, - usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator, - ) -> bool: + def can_view_block_for_editing(self, user: UserType, usage_key: UsageKeyV2) -> bool: """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view its fields and OLX details (but @@ -52,7 +47,7 @@ def can_view_block_for_editing( """ return self.can_edit_block(user, usage_key) - def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | LibraryContainerLocator) -> bool: # pylint: disable=unused-argument + def can_view_block(self, user: UserType, usage_key: UsageKeyV2) -> bool: # pylint: disable=unused-argument """ Assuming a block with the specified ID (usage_key) exists, does the specified user have permission to view it and interact with it (call @@ -61,7 +56,7 @@ def can_view_block(self, user: UserType, usage_key: LibraryUsageLocatorV2 | Libr user: a Django User object (may be an AnonymousUser) - usage_key: the UsageKeyV2 | LibraryItemKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context Must return a boolean. """ @@ -79,7 +74,7 @@ def send_block_updated_event(self, usage_key): """ Send a "block updated" event for the block with the given usage_key in this context. - usage_key: the UsageKeyV2 | LibraryItemKey subclass used for this learning context + usage_key: the UsageKeyV2 subclass used for this learning context """ def send_container_updated_events(self, usage_key): diff --git a/openedx/core/djangoapps/xblock/learning_context/manager.py b/openedx/core/djangoapps/xblock/learning_context/manager.py index 6a3279a553cc..b7ed1c3c3426 100644 --- a/openedx/core/djangoapps/xblock/learning_context/manager.py +++ b/openedx/core/djangoapps/xblock/learning_context/manager.py @@ -3,7 +3,7 @@ """ from edx_django_utils.plugins import PluginManager from opaque_keys import OpaqueKey -from opaque_keys.edx.keys import LearningContextKey, LibraryItemKey, UsageKeyV2 +from opaque_keys.edx.keys import LearningContextKey, UsageKeyV2 from openedx.core.djangoapps.xblock.apps import get_xblock_app_config @@ -35,7 +35,7 @@ def get_learning_context_impl(key): """ if isinstance(key, LearningContextKey): context_type = key.CANONICAL_NAMESPACE # e.g. 'lib' - elif isinstance(key, (UsageKeyV2, LibraryItemKey)): + elif isinstance(key, UsageKeyV2): context_type = key.context_key.CANONICAL_NAMESPACE elif isinstance(key, OpaqueKey): # Maybe this is an older modulestore key etc. diff --git a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py index 6890ad9c675e..5f9cba6c3a22 100644 --- a/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py +++ b/openedx/core/djangoapps/xblock/runtime/learning_core_runtime.py @@ -11,24 +11,25 @@ from django.core.exceptions import ObjectDoesNotExist, ValidationError from django.db.transaction import atomic from django.urls import reverse -from lxml import etree -from opaque_keys.edx.keys import UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerLocator + from openedx_learning.api import authoring as authoring_api + +from lxml import etree + from xblock.core import XBlock from xblock.exceptions import NoSuchUsage -from xblock.field_data import FieldData from xblock.fields import Field, Scope, ScopeIds +from xblock.field_data import FieldData from openedx.core.djangoapps.xblock.api import get_xblock_app_config from openedx.core.lib.xblock_serializer.api import serialize_modulestore_block_for_learning_core from openedx.core.lib.xblock_serializer.data import StaticFile - from ..data import AuthoredDataMode, LatestVersion +from ..utils import get_auto_latest_version from ..learning_context.manager import get_learning_context_impl -from ..utils import get_auto_latest_version, library_container_xml from .runtime import XBlockRuntime + log = logging.getLogger(__name__) @@ -166,11 +167,37 @@ class LearningCoreXBlockRuntime(XBlockRuntime): (eventually) asset storage. """ - def _initialize_block(self, content, usage_key, block_type, version: int | LatestVersion): + def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion = LatestVersion.AUTO): """ - Creates new xblock from given content, usage_key and block_type + Fetch an XBlock from Learning Core data models. + + This method will find the OLX for the content in Learning Core, parse it + into an XBlock (with mixins) instance, and properly initialize our + internal LearningCoreFieldData instance with the field values from the + parsed OLX. """ - xml_node = etree.fromstring(content) + # We can do this more efficiently in a single query later, but for now + # just get it the easy way. + component = self._get_component_from_usage_key(usage_key) + + version = get_auto_latest_version(version) + if self.authored_data_mode == AuthoredDataMode.STRICTLY_PUBLISHED and version != LatestVersion.PUBLISHED: + raise ValidationError("This runtime only allows accessing the published version of components") + if version == LatestVersion.DRAFT: + component_version = component.versioning.draft + elif version == LatestVersion.PUBLISHED: + component_version = component.versioning.published + else: + assert isinstance(version, int) + component_version = component.versioning.version_num(version) + if component_version is None: + raise NoSuchUsage(usage_key) + + content = component_version.contents.get( + componentversioncontent__key="block.xml" + ) + xml_node = etree.fromstring(content.text) + block_type = usage_key.block_type keys = ScopeIds(self.user_id, block_type, None, usage_key) if xml_node.get("url_name", None): @@ -204,53 +231,6 @@ def _initialize_block(self, content, usage_key, block_type, version: int | Lates return block - def get_block(self, usage_key, for_parent=None, *, version: int | LatestVersion = LatestVersion.AUTO): - """ - Fetch an XBlock from Learning Core data models. - - This method will find the OLX for the content in Learning Core, parse it - into an XBlock (with mixins) instance, and properly initialize our - internal LearningCoreFieldData instance with the field values from the - parsed OLX. - """ - # We can do this more efficiently in a single query later, but for now - # just get it the easy way. - component = self._get_component_from_usage_key(usage_key) - - version = get_auto_latest_version(version) - if self.authored_data_mode == AuthoredDataMode.STRICTLY_PUBLISHED and version != LatestVersion.PUBLISHED: - raise ValidationError("This runtime only allows accessing the published version of components") - if version == LatestVersion.DRAFT: - component_version = component.versioning.draft - elif version == LatestVersion.PUBLISHED: - component_version = component.versioning.published - else: - assert isinstance(version, int) - component_version = component.versioning.version_num(version) - if component_version is None: - raise NoSuchUsage(usage_key) - - content = component_version.contents.get( - componentversioncontent__key="block.xml" - ) - return self._initialize_block(content.text, usage_key, usage_key.block_type, version) - - def get_container_block(self, container_key, *, version: int | LatestVersion = LatestVersion.AUTO): - """ - Fetch container from learning core data models. - - This method create a very basic olx for container and parse it into an XBlock instance. - """ - container = self._get_container_from_key(container_key) - block_type = "vertical" if container_key.container_type == "unit" else container_key.container_type - content = library_container_xml( - container_key, - display_name=container.versioning.draft.title if container.versioning.draft else None, - block_type=block_type - ) - xml = etree.tostring(content) - return self._initialize_block(xml.decode(), container_key, block_type, version) - def get_block_assets(self, block, fetch_asset_data): """ Return a list of StaticFile entries. @@ -266,9 +246,6 @@ def get_block_assets(self, block, fetch_asset_data): lookups one by one is going to get slow. At some point we're going to want something to look up a bunch of blocks at once. """ - if not isinstance(block.usage_key, UsageKeyV2): - # TODO: handle assets for containers if required. - return [] component_version = self._get_component_version_from_block(block) # cvc = the ComponentVersionContent through-table @@ -343,22 +320,6 @@ def save_block(self, block): learning_context.send_block_updated_event(usage_key) learning_context.send_container_updated_events(usage_key) - def _get_container_from_key(self, container_key: LibraryContainerLocator): - """ - TODO: This is the third place where we're implementing this. Figure out - where the definitive place should be and have everything else call that. - """ - learning_package = authoring_api.get_learning_package_by_key(str(container_key.lib_key)) - try: - component = authoring_api.get_container_by_key( - learning_package.id, - key=container_key.container_id, - ) - except ObjectDoesNotExist as exc: - raise NoSuchUsage(container_key) from exc - - return component - def _get_component_from_usage_key(self, usage_key): """ Note that Components aren't ever really truly deleted, so this will diff --git a/openedx/core/djangoapps/xblock/utils.py b/openedx/core/djangoapps/xblock/utils.py index 248f503c66df..b4ae054cf498 100644 --- a/openedx/core/djangoapps/xblock/utils.py +++ b/openedx/core/djangoapps/xblock/utils.py @@ -10,8 +10,6 @@ import crum from django.conf import settings -from lxml import etree -from opaque_keys.edx.locator import LibraryContainerLocator from openedx.core.djangoapps.xblock.apps import get_xblock_app_config @@ -188,14 +186,3 @@ def get_auto_latest_version(version: int | LatestVersion) -> int | LatestVersion else LatestVersion.PUBLISHED ) return version - - -def library_container_xml( - container_key: LibraryContainerLocator, - display_name: str | None = None, - block_type: str | None = None, -): - """Converts given unit to xml without including children components""" - xml_object = etree.Element(block_type or container_key.container_type) - xml_object.set("display_name", display_name) - return xml_object diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index cb77ed9adb2d..55fa2f29082d 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -75,7 +75,7 @@ edx-event-bus-kafka>=5.6.0 # Kafka implementation of event bus edx-event-bus-redis edx-milestones edx-name-affirmation -edx-opaque-keys @ git+https://github.com/open-craft/opaque-keys@navin/fal-4077/fix-new-locators +edx-opaque-keys>=2.12.0 edx-organizations edx-proctoring>=2.0.1 # using hash to support django42 From 9439310677e9f7821ee6b63dc27ebc7a8519ce2a Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 16:11:20 -0700 Subject: [PATCH 28/46] chore: cleanups --- cms/djangoapps/contentstore/admin.py | 4 +- .../migrations/0010_container_link_models.py | 58 ++++++++++++++++ ...shableentitylink_componentlink_and_more.py | 33 --------- ...ter_componentlink_options_containerlink.py | 68 ------------------- cms/djangoapps/contentstore/models.py | 25 ++++--- .../xblock_storage_handlers/view_handlers.py | 2 +- cms/lib/xblock/upstream_sync.py | 2 + ...eam_sync.py => upstream_sync_container.py} | 0 mypy.ini | 1 + 9 files changed, 81 insertions(+), 112 deletions(-) create mode 100644 cms/djangoapps/contentstore/migrations/0010_container_link_models.py delete mode 100644 cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py delete mode 100644 cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py rename cms/lib/xblock/{container_upstream_sync.py => upstream_sync_container.py} (100%) diff --git a/cms/djangoapps/contentstore/admin.py b/cms/djangoapps/contentstore/admin.py index 2e9f5b19b353..67bb39b7a32a 100644 --- a/cms/djangoapps/contentstore/admin.py +++ b/cms/djangoapps/contentstore/admin.py @@ -134,7 +134,7 @@ class ContainerLinkAdmin(admin.ModelAdmin): """ fields = ( "uuid", - "upstream_block", + "upstream_container", "upstream_container_key", "upstream_context_key", "downstream_usage_key", @@ -146,7 +146,7 @@ class ContainerLinkAdmin(admin.ModelAdmin): ) readonly_fields = fields list_display = [ - "upstream_block", + "upstream_container", "upstream_container_key", "downstream_usage_key", "version_synced", diff --git a/cms/djangoapps/contentstore/migrations/0010_container_link_models.py b/cms/djangoapps/contentstore/migrations/0010_container_link_models.py new file mode 100644 index 000000000000..7a02a1c74df1 --- /dev/null +++ b/cms/djangoapps/contentstore/migrations/0010_container_link_models.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2.20 on 2025-04-22 15:08 +import uuid + +import django.db.models.deletion +import opaque_keys.edx.django.models +import openedx_learning.lib.fields +import openedx_learning.lib.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'), + ('contentstore', '0009_learningcontextlinksstatus_publishableentitylink'), + ] + + operations = [ + migrations.RenameModel( + old_name='PublishableEntityLink', + new_name='ComponentLink', + ), + migrations.AlterModelOptions( + name='componentlink', + options={'verbose_name': 'Component Link', 'verbose_name_plural': 'Component Links'}, + ), + migrations.AlterField( + model_name='componentlink', + name='upstream_block', + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name='links', + to='oel_components.component', + ), + ), + migrations.CreateModel( + name='ContainerLink', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), + ('upstream_context_key', openedx_learning.lib.fields.MultiCollationCharField(db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, db_index=True, help_text='Upstream context key i.e., learning_package/library key', max_length=500)), + ('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)), + ('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)), + ('version_synced', models.IntegerField()), + ('version_declined', models.IntegerField(blank=True, null=True)), + ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), + ('upstream_container_key', opaque_keys.edx.django.models.ContainerKeyField(help_text='Upstream block key (e.g. lct:...), this value cannot be null and is useful to track upstream library blocks that do not exist yet or were deleted.', max_length=255)), + ('upstream_container', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='links', to='oel_publishing.container')), + ], + options={ + 'abstract': False, + 'verbose_name': 'Container Link', + 'verbose_name_plural': 'Container Links', + }, + ), + ] diff --git a/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py b/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py deleted file mode 100644 index 8068f4f024d2..000000000000 --- a/cms/djangoapps/contentstore/migrations/0010_rename_publishableentitylink_componentlink_and_more.py +++ /dev/null @@ -1,33 +0,0 @@ -# Generated by Django 4.2.20 on 2025-04-22 15:08 - -import django.db.models.deletion -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ('oel_components', '0003_remove_componentversioncontent_learner_downloadable'), - ('contentstore', '0009_learningcontextlinksstatus_publishableentitylink'), - ] - - operations = [ - migrations.RenameModel( - old_name='PublishableEntityLink', - new_name='ComponentLink', - ), - migrations.AlterModelOptions( - name='componentlink', - options={'verbose_name': 'Component Link', 'verbose_name_plural': 'Component Links'}, - ), - migrations.AlterField( - model_name='componentlink', - name='upstream_block', - field=models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='links', - to='oel_components.component', - ), - ), - ] diff --git a/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py b/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py deleted file mode 100644 index f47db143929e..000000000000 --- a/cms/djangoapps/contentstore/migrations/0011_alter_componentlink_options_containerlink.py +++ /dev/null @@ -1,68 +0,0 @@ -# Generated by Django 4.2.20 on 2025-04-23 05:47 - -import uuid - -import django.db.models.deletion -import opaque_keys.edx.django.models -import openedx_learning.lib.fields -import openedx_learning.lib.validators -from django.db import migrations, models - - -class Migration(migrations.Migration): - dependencies = [ - ('oel_publishing', '0008_alter_draftchangelogrecord_options_and_more'), - ('contentstore', '0010_rename_publishableentitylink_componentlink_and_more'), - ] - - operations = [ - migrations.AlterModelOptions( - name='componentlink', - options={}, - ), - migrations.CreateModel( - name='ContainerLink', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True, verbose_name='UUID')), - ( - 'upstream_context_key', - openedx_learning.lib.fields.MultiCollationCharField( - db_collations={'mysql': 'utf8mb4_bin', 'sqlite': 'BINARY'}, - db_index=True, - help_text='Upstream context key i.e., learning_package/library key', - max_length=500, - ), - ), - ('downstream_usage_key', opaque_keys.edx.django.models.UsageKeyField(max_length=255, unique=True)), - ('downstream_context_key', opaque_keys.edx.django.models.CourseKeyField(db_index=True, max_length=255)), - ('version_synced', models.IntegerField()), - ('version_declined', models.IntegerField(blank=True, null=True)), - ('created', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), - ('updated', models.DateTimeField(validators=[openedx_learning.lib.validators.validate_utc_datetime])), - ( - 'upstream_container_key', - opaque_keys.edx.django.models.LibraryItemField( - help_text=( - 'Upstream block usage key, this value cannot be null and useful to track upstream' - 'library blocks that do not exist yet' - ), - max_length=255, - ), - ), - ( - 'upstream_block', - models.ForeignKey( - blank=True, - null=True, - on_delete=django.db.models.deletion.SET_NULL, - related_name='links', - to='oel_publishing.container', - ), - ), - ], - options={ - 'abstract': False, - }, - ), - ] diff --git a/cms/djangoapps/contentstore/models.py b/cms/djangoapps/contentstore/models.py index 77cb5186579f..565a63670f68 100644 --- a/cms/djangoapps/contentstore/models.py +++ b/cms/djangoapps/contentstore/models.py @@ -12,7 +12,7 @@ from django.db.models.functions import Coalesce from django.db.models.lookups import GreaterThan from django.utils.translation import gettext_lazy as _ -from opaque_keys.edx.django.models import CourseKeyField, LibraryItemField, UsageKeyField +from opaque_keys.edx.django.models import CourseKeyField, ContainerKeyField, UsageKeyField from opaque_keys.edx.keys import CourseKey, UsageKey from opaque_keys.edx.locator import LibraryContainerLocator from openedx_learning.api.authoring import get_published_version @@ -198,6 +198,10 @@ class ComponentLink(EntityLinkBase): ) ) + class Meta: + verbose_name = _("Component Link") + verbose_name_plural = _("Component Links") + def __str__(self): return f"ComponentLink<{self.upstream_usage_key}->{self.downstream_usage_key}>" @@ -263,28 +267,33 @@ class ContainerLink(EntityLinkBase): This represents link between any two publishable entities or link between publishable entity and a course xblock. It helps in tracking relationship between xblocks imported from libraries and used in different courses. """ - upstream_block = models.ForeignKey( + upstream_container = models.ForeignKey( Container, on_delete=models.SET_NULL, related_name="links", null=True, blank=True, ) - upstream_container_key = LibraryItemField( + upstream_container_key = ContainerKeyField( max_length=255, help_text=_( - "Upstream block usage key, this value cannot be null" - " and useful to track upstream library blocks that do not exist yet" + "Upstream block key (e.g. lct:...), this value cannot be null " + "and is useful to track upstream library blocks that do not exist yet " + "or were deleted." ) ) + class Meta: + verbose_name = _("Container Link") + verbose_name_plural = _("Container Links") + def __str__(self): return f"ContainerLink<{self.upstream_container_key}->{self.downstream_usage_key}>" @classmethod def update_or_create( cls, - upstream_block: Container | None, + upstream_container: Container | None, /, upstream_container_key: LibraryContainerLocator, upstream_context_key: str, @@ -307,10 +316,10 @@ def update_or_create( 'version_synced': version_synced, 'version_declined': version_declined, } - if upstream_block: + if upstream_container: new_values.update( { - 'upstream_block': upstream_block, + 'upstream_container': upstream_container, } ) try: diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index c086cce62283..3a4170d033fc 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -36,7 +36,7 @@ from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.lib.ai_aside_summary_config import AiAsideSummaryConfig from cms.lib.xblock.upstream_sync import BadUpstream, check_and_parse_upstream_key, sync_from_upstream -from cms.lib.xblock.container_upstream_sync import sync_from_upstream_container +from cms.lib.xblock.upstream_sync_container import sync_from_upstream_container from common.djangoapps.static_replace import replace_static_urls from common.djangoapps.student.auth import ( has_studio_read_access, diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 7571c93ab17e..442d31a3b203 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -338,6 +338,8 @@ def _load_upstream_link_and_block(self) -> XBlock: LatestVersion, load_block, ) + if not isinstance(self.upstream_key, LibraryUsageLocatorV2): + raise BadUpstream(_("Invalid upstream_key")) try: lib_block: XBlock = load_block( self.upstream_key, diff --git a/cms/lib/xblock/container_upstream_sync.py b/cms/lib/xblock/upstream_sync_container.py similarity index 100% rename from cms/lib/xblock/container_upstream_sync.py rename to cms/lib/xblock/upstream_sync_container.py diff --git a/mypy.ini b/mypy.ini index 83255c1a5af1..f0267364ffb0 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,6 +7,7 @@ plugins = mypy_drf_plugin.main files = cms/lib/xblock/upstream_sync.py, + cms/lib/xblock/upstream_sync_container.py, cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py, cms/djangoapps/import_from_modulestore, openedx/core/djangoapps/content/learning_sequences, From 6ac9f6f69a37ac97f87006cb36a3dbcf8a55501c Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 16:13:11 -0700 Subject: [PATCH 29/46] revert: changes to upstream_sync.py --- cms/lib/xblock/upstream_sync.py | 370 ++++++++++++-------------------- 1 file changed, 143 insertions(+), 227 deletions(-) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 442d31a3b203..22cad3c6d36a 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -11,22 +11,20 @@ """ from __future__ import annotations -from abc import ABC import logging import typing as t -from dataclasses import asdict, dataclass -from functools import lru_cache +from dataclasses import dataclass, asdict from django.conf import settings from django.core.exceptions import PermissionDenied from django.utils.translation import gettext_lazy as _ -from opaque_keys import InvalidKeyError -from opaque_keys.edx.keys import CourseKey, UsageKeyV2 -from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from rest_framework.exceptions import NotFound -from xblock.core import XBlock, XBlockMixin +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import CourseKey +from opaque_keys.edx.locator import LibraryUsageLocatorV2 from xblock.exceptions import XBlockNotFoundError -from xblock.fields import Integer, Scope, String +from xblock.fields import Scope, String, Integer +from xblock.core import XBlockMixin, XBlock if t.TYPE_CHECKING: from django.contrib.auth.models import User # pylint: disable=imported-auth-user @@ -71,9 +69,9 @@ def __init__(self): @dataclass(frozen=True) -class BaseUpstreamLink: +class UpstreamLink: """ - Base class to track metadata about some downstream content's relationship with its linked upstream content. + Metadata about some downstream content's relationship with its linked upstream content. """ upstream_ref: str | None # Reference to the upstream content, e.g., a serialized library block usage key. version_synced: int | None # Version of the upstream to which the downstream was last synced. @@ -93,25 +91,6 @@ def ready_to_sync(self) -> bool: self.version_available > (self.version_declined or 0) ) - -@dataclass(frozen=True) -class UpstreamLink(BaseUpstreamLink): - """ - Metadata about some downstream content's relationship with its linked upstream content. - """ - - @property - def ready_to_sync(self) -> bool: - """ - Should we invite the downstream's authors to sync the latest upstream updates? - """ - return bool( - self.upstream_ref and - self.version_available and - self.version_available > (self.version_synced or 0) and - self.version_available > (self.version_declined or 0) - ) - @property def upstream_link(self) -> str | None: """ @@ -167,13 +146,33 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: If link exists, is supported, and is followable, returns UpstreamLink. Otherwise, raises an UpstreamLinkException. """ - upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) + if not downstream.upstream: + raise NoUpstream() + if not isinstance(downstream.usage_key.context_key, CourseKey): + raise BadDownstream(_("Cannot update content because it does not belong to a course.")) + if downstream.has_children: + raise BadDownstream(_("Updating content with children is not yet supported.")) + try: + upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) + except InvalidKeyError as exc: + raise BadUpstream(_("Reference to linked library item is malformed")) from exc + downstream_type = downstream.usage_key.block_type + if upstream_key.block_type != downstream_type: + # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. + # It could be reasonable to relax this requirement in the future if there's product need for it. + # For example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. + raise BadUpstream( + _("Content type mismatch: {downstream_type} cannot be linked to {upstream_type}.").format( + downstream_type=downstream_type, upstream_type=upstream_key.block_type + ) + ) from TypeError( + f"downstream block '{downstream.usage_key}' is linked to " + f"upstream block of different type '{upstream_key}'" + ) # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. from openedx.core.djangoapps.content_libraries.api import ( - get_library_block, # pylint: disable=wrong-import-order + get_library_block # pylint: disable=wrong-import-order ) - if not isinstance(upstream_key, LibraryUsageLocatorV2): - raise BadUpstream(_("Invalid upstream_key")) try: lib_meta = get_library_block(upstream_key) except XBlockNotFoundError as exc: @@ -187,234 +186,151 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: ) -class BaseUpstreamSyncManager(ABC): - """ - Base manager class for managing upstream link sync process. +def sync_from_upstream(downstream: XBlock, user: User) -> XBlock: """ - def __init__( - self, - downstream: XBlock, - user: User, - upstream: XBlock | None, - ) -> None: - self.downstream = downstream - self.user = user - if not upstream: - # Only parse upstream_key if upstream block is not passed else don't care about downstream.upstream - self.upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) - self.link: BaseUpstreamLink - self.upstream = upstream - self.syncable_field_names: set[str] - - def update_customizable_fields(self, *, only_fetch: bool) -> None: - """ - For each customizable field: - * Save the upstream value to a hidden field on the downstream ("FETCH"). - * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: - * Update it the downstream field's value from the upstream field ("SYNC"). + Update `downstream` with content+settings from the latest available version of its linked upstream content. - Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. + Preserves overrides to customizable fields; overwrites overrides to other fields. + Does not save `downstream` to the store. That is left up to the caller. - * Say that the customizable fields are [display_name, max_attempts]. + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + link, upstream = _load_upstream_link_and_block(downstream, user) + _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=False) + _update_non_customizable_fields(upstream=upstream, downstream=downstream) + _update_tags(upstream=upstream, downstream=downstream) + downstream.upstream_version = link.version_available + return upstream - * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). - * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: - * Set `course_problem.display_name = lib_problem.display_name` ("sync"). - """ - for field_name, fetch_field_name in self.downstream.get_customizable_fields().items(): - if field_name not in self.syncable_field_names: - continue +def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: + """ + Fetch upstream-defined value of customizable fields and save them on the downstream. - # Downstream-only fields don't have an upstream fetch field - if fetch_field_name is None: - continue + If `upstream` is provided, use that block as the upstream. + Otherwise, load the block specified by `downstream.upstream`, which may raise an UpstreamLinkException. + """ + if not upstream: + _link, upstream = _load_upstream_link_and_block(downstream, user) + _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) - # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). - old_upstream_value = getattr(self.downstream, fetch_field_name) - new_upstream_value = getattr(self.upstream, field_name) - setattr(self.downstream, fetch_field_name, new_upstream_value) - if only_fetch: - continue +def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[UpstreamLink, XBlock]: + """ + Load the upstream metadata and content for a downstream block. - # Okay, now for the nuanced part... - # We need to update the downstream field *iff it has not been customized**. - # Determining whether a field has been customized will differ in Beta vs Future release. - # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) + Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be + relaxed in the future (see module docstring). - ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. - # if field_name in downstream.downstream_customized: - # continue + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + link = UpstreamLink.get_for_block(downstream) # can raise UpstreamLinkException + # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + from openedx.core.djangoapps.xblock.api import load_block, CheckPerm, LatestVersion # pylint: disable=wrong-import-order + try: + lib_block: XBlock = load_block( + LibraryUsageLocatorV2.from_string(downstream.upstream), + user, + check_permission=CheckPerm.CAN_READ_AS_AUTHOR, + version=LatestVersion.PUBLISHED, + ) + except (NotFound, PermissionDenied) as exc: + raise BadUpstream(_("Linked library item could not be loaded: {}").format(downstream.upstream)) from exc + return link, lib_block - ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. - downstream_value = getattr(self.downstream, field_name) - if old_upstream_value and downstream_value != old_upstream_value: - continue # Field has been customized. Don't touch it. Move on. - # Field isn't customized -- SYNC it! - setattr(self.downstream, field_name, new_upstream_value) +def _update_customizable_fields(*, upstream: XBlock, downstream: XBlock, only_fetch: bool) -> None: + """ + For each customizable field: + * Save the upstream value to a hidden field on the downstream ("FETCH"). + * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: + * Update it the downstream field's value from the upstream field ("SYNC"). - def update_non_customizable_fields(self) -> None: - """ - For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. - """ - customizable_fields = set(self.downstream.get_customizable_fields().keys()) - isVideoBlock = self.downstream.usage_key.block_type == "video" - for field_name in self.syncable_field_names - customizable_fields: - if isVideoBlock and field_name == 'edx_video_id': - # Avoid overwriting edx_video_id between blocks - continue - new_upstream_value = getattr(self.upstream, field_name) - setattr(self.downstream, field_name, new_upstream_value) - - def update_tags(self) -> None: - """ - Update tags from `upstream` to `downstream` - """ - if not self.upstream: - return - from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only - # For any block synced with an upstream, copy the tags as read_only - # This keeps tags added locally. - copy_tags_as_read_only( - str(self.upstream.location), - str(self.downstream.location), - ) + Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. - def sync(self) -> None: - """ - Update `downstream` with content+settings from the latest available version of its linked upstream content. + * Say that the customizable fields are [display_name, max_attempts]. - Preserves overrides to customizable fields; overwrites overrides to other fields. - Does not save `downstream` to the store. That is left up to the caller. + * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). + * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: + * Set `course_problem.display_name = lib_problem.display_name` ("sync"). + """ + syncable_field_names = _get_synchronizable_fields(upstream, downstream) - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. - """ - self.update_customizable_fields(only_fetch=False) - self.update_non_customizable_fields() - self.update_tags() + for field_name, fetch_field_name in downstream.get_customizable_fields().items(): + if field_name not in syncable_field_names: + continue -class ComponentUpstreamSyncManager(BaseUpstreamSyncManager): - """ - Manages sync process of downstream component with upstream components. - """ - def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: - super().__init__(downstream, user, upstream) - if not upstream: - # Only parse upstream_key if upstream block is not passed else don't care about - # downstream.upstream and upstream link - self.link = UpstreamLink.get_for_block(downstream) - self.upstream = self._load_upstream_link_and_block() - self.syncable_field_names: set[str] = self._get_synchronizable_fields() + # Downstream-only fields don't have an upstream fetch field + if fetch_field_name is None: + continue - def _get_synchronizable_fields(self) -> set[str]: - """ - The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. - """ - return set.intersection(*[ - set( - field_name - for (field_name, field) in block.__class__.fields.items() - if field.scope in [Scope.settings, Scope.content] - ) - for block in [self.upstream, self.downstream] - ]) + # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). + old_upstream_value = getattr(downstream, fetch_field_name) + new_upstream_value = getattr(upstream, field_name) + setattr(downstream, fetch_field_name, new_upstream_value) - def _load_upstream_link_and_block(self) -> XBlock: - """ - Load the upstream metadata and content for a downstream block. + if only_fetch: + continue - Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be - relaxed in the future (see module docstring). + # Okay, now for the nuanced part... + # We need to update the downstream field *iff it has not been customized**. + # Determining whether a field has been customized will differ in Beta vs Future release. + # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. - """ - # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, - # which loads before the djangoapps are ready. - from openedx.core.djangoapps.xblock.api import ( - CheckPerm, - LatestVersion, - load_block, - ) - if not isinstance(self.upstream_key, LibraryUsageLocatorV2): - raise BadUpstream(_("Invalid upstream_key")) - try: - lib_block: XBlock = load_block( - self.upstream_key, - self.user, - check_permission=CheckPerm.CAN_READ_AS_AUTHOR, - version=LatestVersion.PUBLISHED, - ) - except (NotFound, PermissionDenied) as exc: - raise BadUpstream(_("Linked library item could not be loaded: {}").format(self.upstream_key)) from exc - return lib_block + ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. + # if field_name in downstream.downstream_customized: + # continue + ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. + downstream_value = getattr(downstream, field_name) + if old_upstream_value and downstream_value != old_upstream_value: + continue # Field has been customized. Don't touch it. Move on. -def sync_from_upstream(downstream: XBlock, user: User) -> XBlock: - """ - Update `downstream` with content+settings from the latest available version of its linked upstream content. + # Field isn't customized -- SYNC it! + setattr(downstream, field_name, new_upstream_value) - Preserves overrides to customizable fields; overwrites overrides to other fields. - Does not save `downstream` to the store. That is left up to the caller. - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. +def _update_non_customizable_fields(*, upstream: XBlock, downstream: XBlock) -> None: + """ + For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. """ - manager = ComponentUpstreamSyncManager(downstream, user) - manager.sync() - downstream.upstream_version = manager.link.version_available - return manager.upstream + syncable_fields = _get_synchronizable_fields(upstream, downstream) + customizable_fields = set(downstream.get_customizable_fields().keys()) + isVideoBlock = downstream.usage_key.block_type == "video" + for field_name in syncable_fields - customizable_fields: + if isVideoBlock and field_name == 'edx_video_id': + # Avoid overwriting edx_video_id between blocks + continue + new_upstream_value = getattr(upstream, field_name) + setattr(downstream, field_name, new_upstream_value) -@lru_cache -def check_and_parse_upstream_key( - upstream: str | None, - downstream_usage_key: UsageKeyV2, -) -> LibraryUsageLocatorV2 | LibraryContainerLocator: +def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: """ - Convert upstream key to proper type for given downstream block. - - Args: - downstream: XBlock + Update tags from `upstream` to `downstream` + """ + from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only + # For any block synced with an upstream, copy the tags as read_only + # This keeps tags added locally. + copy_tags_as_read_only( + str(upstream.location), + str(downstream.location), + ) - Returns: - Parsed upstream key - Raises: - NoUpstream: - BadDownstream: - BadUpstream: +def _get_synchronizable_fields(upstream: XBlock, downstream: XBlock) -> set[str]: """ - upstream_key: LibraryContainerLocator | LibraryUsageLocatorV2 - if not upstream: - raise NoUpstream() - if not isinstance(downstream_usage_key.context_key, CourseKey): - raise BadDownstream(_("Cannot update content because it does not belong to a course.")) - try: - upstream_key = LibraryUsageLocatorV2.from_string(upstream) - upstream_type = upstream_key.block_type - except InvalidKeyError: - try: - upstream_key = LibraryContainerLocator.from_string(upstream) - upstream_type = upstream_key.container_type - except InvalidKeyError as exc: - raise BadUpstream(_("Reference to linked library item is malformed")) from exc - downstream_type = downstream_usage_key.block_type - upstream_type = "vertical" if upstream_type == "unit" else upstream_type - if upstream_type != downstream_type: - # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. - # It could be reasonable to relax this requirement in the future if there's product need for it. - # For example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. - raise BadUpstream( - _("Content type mismatch: {downstream_type} cannot be linked to {upstream_type}.").format( - downstream_type=downstream_type, upstream_type=upstream_type - ) - ) from TypeError( - f"downstream block '{downstream_usage_key}' is linked to " - f"upstream block of different type '{upstream_key}'" + The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. + """ + return set.intersection(*[ + set( + field_name + for (field_name, field) in block.__class__.fields.items() + if field.scope in [Scope.settings, Scope.content] ) - return upstream_key + for block in [upstream, downstream] + ]) def decline_sync(downstream: XBlock) -> None: From 29f8f7c0cdb5099b15ca787423298a42b04813cc Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 17:40:58 -0700 Subject: [PATCH 30/46] feat: WIP toward syncing --- cms/djangoapps/contentstore/helpers.py | 11 +- .../rest_api/v2/views/downstreams.py | 12 +- .../xblock_storage_handlers/view_handlers.py | 17 +- cms/lib/xblock/upstream_sync.py | 254 ++++---------- cms/lib/xblock/upstream_sync_block.py | 174 ++++++++++ cms/lib/xblock/upstream_sync_container.py | 321 ++++++------------ 6 files changed, 379 insertions(+), 410 deletions(-) create mode 100644 cms/lib/xblock/upstream_sync_block.py diff --git a/cms/djangoapps/contentstore/helpers.py b/cms/djangoapps/contentstore/helpers.py index 82c4b3856f40..f80e304fb791 100644 --- a/cms/djangoapps/contentstore/helpers.py +++ b/cms/djangoapps/contentstore/helpers.py @@ -31,7 +31,8 @@ ) from cms.djangoapps.models.settings.course_grading import CourseGradingModel -from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException, ComponentUpstreamSyncManager +from cms.lib.xblock.upstream_sync import UpstreamLink, UpstreamLinkException +from cms.lib.xblock.upstream_sync_block import fetch_customizable_fields_from_block from openedx.core.djangoapps.site_configuration import helpers as configuration_helpers import openedx.core.djangoapps.content_staging.api as content_staging_api import openedx.core.djangoapps.content_tagging.api as content_tagging_api @@ -416,7 +417,7 @@ def _fetch_and_set_upstream_link( user: User ): """ - Fetch and set upstream link for the given xblock. This function handles following cases: + Fetch and set upstream link for the given xblock which is being pasted. This function handles following cases: * the xblock is copied from a v2 library; the library block is set as upstream. * the xblock is copied from a course; no upstream is set, only copied_from_block is set. * the xblock is copied from a course where the source block was imported from a library; the original libary block @@ -425,7 +426,7 @@ def _fetch_and_set_upstream_link( # Try to link the pasted block (downstream) to the copied block (upstream). temp_xblock.upstream = copied_from_block try: - UpstreamLink.get_for_block(temp_xblock) + upstream_link = UpstreamLink.get_for_block(temp_xblock) except UpstreamLinkException: # Usually this will fail. For example, if the copied block is a modulestore course block, it can't be an # upstream. That's fine! Instead, we store a reference to where this block was copied from, in the @@ -456,8 +457,8 @@ def _fetch_and_set_upstream_link( # later wants to restore it, it will restore to the value that the field had when the block was pasted. Of # course, if the author later syncs updates from a *future* published upstream version, then that will fetch # new values from the published upstream content. - manager = ComponentUpstreamSyncManager(downstream=temp_xblock, user=user, upstream=temp_xblock) - manager.update_customizable_fields(only_fetch=True) + if isinstance(upstream_link.upstream_key, UsageKey): # only if upstream is a block, not a container + fetch_customizable_fields_from_block(downstream=temp_xblock, user=user, upstream=temp_xblock) def _import_xml_node_to_parent( diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py index 30bf93079ef1..39c2649118f5 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/downstreams.py @@ -87,6 +87,7 @@ from edx_rest_framework_extensions.paginators import DefaultPagination from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey, UsageKey +from opaque_keys.edx.locator import LibraryUsageLocatorV2, LibraryContainerLocator from rest_framework.exceptions import NotFound, ValidationError from rest_framework.fields import BooleanField from rest_framework.request import Request @@ -103,13 +104,14 @@ from cms.lib.xblock.upstream_sync import ( BadDownstream, BadUpstream, - ComponentUpstreamSyncManager, NoUpstream, UpstreamLink, UpstreamLinkException, decline_sync, sever_upstream_link, ) +from cms.lib.xblock.upstream_sync_block import fetch_customizable_fields_from_block +from cms.lib.xblock.upstream_sync_container import fetch_customizable_fields_from_container from common.djangoapps.student.auth import has_studio_read_access, has_studio_write_access from openedx.core.lib.api.view_utils import ( DeveloperErrorViewMixin, @@ -260,8 +262,12 @@ def put(self, request: _AuthenticatedRequest, usage_key_string: str) -> Response # Even if we're not syncing (i.e., updating the downstream's values with the upstream's), we still need # to fetch the upstream's customizable values and store them as hidden fields on the downstream. This # ensures that downstream authors can restore defaults based on the upstream. - manager = ComponentUpstreamSyncManager(downstream=downstream, user=request.user) - manager.update_customizable_fields(only_fetch=True) + link = UpstreamLink.get_for_block(downstream) + if isinstance(link.upstream_key, LibraryUsageLocatorV2): + fetch_customizable_fields_from_block(downstream=downstream, user=request.user) + else: + assert isinstance(link.upstream_key, LibraryContainerLocator) + fetch_customizable_fields_from_container(downstream=downstream, user=request.user) except BadDownstream as exc: logger.exception( "'%s' is an invalid downstream; refusing to set its upstream to '%s'", diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 3a4170d033fc..0df775b19ab7 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -33,9 +33,11 @@ from xblock.fields import Scope from cms.djangoapps.contentstore.config.waffle import SHOW_REVIEW_RULES_FLAG +from cms.djangoapps.contentstore.helpers import StaticFileNotices from cms.djangoapps.models.settings.course_grading import CourseGradingModel from cms.lib.ai_aside_summary_config import AiAsideSummaryConfig -from cms.lib.xblock.upstream_sync import BadUpstream, check_and_parse_upstream_key, sync_from_upstream +from cms.lib.xblock.upstream_sync import BadUpstream, UpstreamLink +from cms.lib.xblock.upstream_sync_block import sync_from_upstream_block from cms.lib.xblock.upstream_sync_container import sync_from_upstream_container from common.djangoapps.static_replace import replace_static_urls from common.djangoapps.student.auth import ( @@ -524,23 +526,24 @@ def create_item(request): return _create_block(request) -def sync_library_content(downstream: XBlock, request, store, remove_upstream_link: bool = False): +def sync_library_content(downstream: XBlock, request, store, remove_upstream_link: bool = False) -> StaticFileNotices: """ Handle syncing library content for given xblock depending on its upstream type. It can sync unit containers and lower level xblocks. """ - upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) + link = UpstreamLink.get_for_block(downstream) + upstream_key = link.upstream_key if isinstance(upstream_key, LibraryUsageLocatorV2): - lib_block = sync_from_upstream(downstream=downstream, user=request.user) + lib_block = sync_from_upstream_block(downstream=downstream, user=request.user) if remove_upstream_link: # Removing upstream link from child components downstream.upstream = None static_file_notices = import_static_assets_for_library_sync(downstream, lib_block, request) store.update_item(downstream, request.user.id) else: - with store.bulk_operations(downstream.location.context_key): - lib_block, children_blocks = sync_from_upstream_container(downstream=downstream, user=request.user) - notices = [import_static_assets_for_library_sync(downstream, lib_block, request)] + with store.bulk_operations(downstream.usage_key.context_key): + children_blocks = sync_from_upstream_container(downstream=downstream, user=request.user) + notices = [] children_blocks_usage_keys = [] for child_block in children_blocks: notices.append(sync_library_content(child_block, request, store, remove_upstream_link=True)) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 22cad3c6d36a..70e9b9e9658b 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -1,13 +1,17 @@ """ -Synchronize content and settings from upstream blocks to their downstream usages. +Synchronize content and settings from upstream content to their downstream +usages. At the time of writing, we assume that for any upstream-downstream linkage: -* The upstream is a Component from a Learning Core-backed Content Library. -* The downstream is a block of matching type in a SplitModuleStore-backed Course. +* The upstream is a Component or Container from a Learning Core-backed Content + Library. +* The downstream is a block of compatible type in a SplitModuleStore-backed + Course. * They are both on the same Open edX instance. -HOWEVER, those assumptions may loosen in the future. So, we consider these to be INTERNAL ASSUMPIONS that should not be -exposed through this module's public Python interface. +HOWEVER, those assumptions may loosen in the future. So, we consider these to be +INTERNAL ASSUMPIONS that should not be exposed through this module's public +Python interface. """ from __future__ import annotations @@ -16,12 +20,10 @@ from dataclasses import dataclass, asdict from django.conf import settings -from django.core.exceptions import PermissionDenied from django.utils.translation import gettext_lazy as _ -from rest_framework.exceptions import NotFound from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import CourseKey -from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from opaque_keys.edx.locator import LibraryContainerLocator, LibraryUsageLocatorV2 from xblock.exceptions import XBlockNotFoundError from xblock.fields import Scope, String, Integer from xblock.core import XBlockMixin, XBlock @@ -74,6 +76,7 @@ class UpstreamLink: Metadata about some downstream content's relationship with its linked upstream content. """ upstream_ref: str | None # Reference to the upstream content, e.g., a serialized library block usage key. + upstream_key: LibraryUsageLocatorV2 | LibraryContainerLocator | None # parsed opaque key version of upstream_ref version_synced: int | None # Version of the upstream to which the downstream was last synced. version_available: int | None # Latest version of the upstream that's available, or None if it couldn't be loaded. version_declined: int | None # Latest version which the user has declined to sync with, if any. @@ -129,6 +132,7 @@ def try_get_for_block(cls, downstream: XBlock) -> t.Self: ) return cls( upstream_ref=downstream.upstream, + upstream_key=None, version_synced=downstream.upstream_version, version_available=None, version_declined=None, @@ -138,201 +142,78 @@ def try_get_for_block(cls, downstream: XBlock) -> t.Self: @classmethod def get_for_block(cls, downstream: XBlock) -> t.Self: """ - Get info on a block's relationship with its linked upstream content (without actually loading the content). + Get info on a downstream block's relationship with its linked upstream + content (without actually loading the content). - Currently, the only supported upstreams are LC-backed Library Components. This may change in the future (see - module docstring). + Currently, the only supported upstreams are LC-backed Library Components + (XBlocks) or Containers. This may change in the future (see module + docstring). If link exists, is supported, and is followable, returns UpstreamLink. Otherwise, raises an UpstreamLinkException. """ + # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + from openedx.core.djangoapps.content_libraries import api as lib_api + if not downstream.upstream: raise NoUpstream() if not isinstance(downstream.usage_key.context_key, CourseKey): raise BadDownstream(_("Cannot update content because it does not belong to a course.")) - if downstream.has_children: - raise BadDownstream(_("Updating content with children is not yet supported.")) + downstream_type = downstream.usage_key.block_type + try: upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) - except InvalidKeyError as exc: - raise BadUpstream(_("Reference to linked library item is malformed")) from exc - downstream_type = downstream.usage_key.block_type - if upstream_key.block_type != downstream_type: - # Note: Currently, we strictly enforce that the downstream and upstream block_types must exactly match. - # It could be reasonable to relax this requirement in the future if there's product need for it. - # For example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. + except InvalidKeyError: + try: + upstream_key = LibraryContainerLocator.from_string(downstream.upstream) + except InvalidKeyError as exc: + raise BadUpstream(_("Reference to linked library item is malformed")) from exc + + if isinstance(upstream_key, LibraryUsageLocatorV2): + # The upstream is an XBlock + if downstream.has_children: + raise BadDownstream( + _("Updating content with children is not yet supported unless the upstream is a container."), + ) + expected_downstream_block_type = upstream_key.block_type + try: + block_meta = lib_api.get_library_block(upstream_key) + except XBlockNotFoundError as exc: + raise BadUpstream(_("Linked upstream library block was not found in the system")) from exc + version_available = block_meta.published_version_num + else: + # The upstream is a Container: + assert isinstance(upstream_key, LibraryContainerLocator) + try: + container_meta = lib_api.get_container(upstream_key) + except lib_api.ContentLibraryContainerNotFound as exc: + raise BadUpstream(_("Linked upstream library container was not found in the system")) from exc + expected_downstream_block_type = container_meta.container_type.olx_tag + version_available = container_meta.published_version_num + + if downstream_type != expected_downstream_block_type: + # Note: generally the upstream and downstream types must match, except that upstream containers + # may have e.g. container_type=unit while the downstream block has the equivalent block_type=vertical. + # It could be reasonable to relax this requirement in the future if there's product need for it. + # for example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. raise BadUpstream( - _("Content type mismatch: {downstream_type} cannot be linked to {upstream_type}.").format( - downstream_type=downstream_type, upstream_type=upstream_key.block_type + _("Content type mismatch: {downstream_id} ({downstream_type}) cannot be linked to {upstream_id}.").format( + downstream_id=downstream.usage_key, + downstream_type=downstream_type, + upstream_id=str(upstream_key), ) - ) from TypeError( - f"downstream block '{downstream.usage_key}' is linked to " - f"upstream block of different type '{upstream_key}'" ) - # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. - from openedx.core.djangoapps.content_libraries.api import ( - get_library_block # pylint: disable=wrong-import-order - ) - try: - lib_meta = get_library_block(upstream_key) - except XBlockNotFoundError as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc + return cls( upstream_ref=downstream.upstream, + upstream_key=upstream_key, version_synced=downstream.upstream_version, - version_available=(lib_meta.published_version_num if lib_meta else None), + version_available=version_available, version_declined=downstream.upstream_version_declined, error_message=None, ) -def sync_from_upstream(downstream: XBlock, user: User) -> XBlock: - """ - Update `downstream` with content+settings from the latest available version of its linked upstream content. - - Preserves overrides to customizable fields; overwrites overrides to other fields. - Does not save `downstream` to the store. That is left up to the caller. - - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. - """ - link, upstream = _load_upstream_link_and_block(downstream, user) - _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=False) - _update_non_customizable_fields(upstream=upstream, downstream=downstream) - _update_tags(upstream=upstream, downstream=downstream) - downstream.upstream_version = link.version_available - return upstream - - -def fetch_customizable_fields(*, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: - """ - Fetch upstream-defined value of customizable fields and save them on the downstream. - - If `upstream` is provided, use that block as the upstream. - Otherwise, load the block specified by `downstream.upstream`, which may raise an UpstreamLinkException. - """ - if not upstream: - _link, upstream = _load_upstream_link_and_block(downstream, user) - _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) - - -def _load_upstream_link_and_block(downstream: XBlock, user: User) -> tuple[UpstreamLink, XBlock]: - """ - Load the upstream metadata and content for a downstream block. - - Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be - relaxed in the future (see module docstring). - - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. - """ - link = UpstreamLink.get_for_block(downstream) # can raise UpstreamLinkException - # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. - from openedx.core.djangoapps.xblock.api import load_block, CheckPerm, LatestVersion # pylint: disable=wrong-import-order - try: - lib_block: XBlock = load_block( - LibraryUsageLocatorV2.from_string(downstream.upstream), - user, - check_permission=CheckPerm.CAN_READ_AS_AUTHOR, - version=LatestVersion.PUBLISHED, - ) - except (NotFound, PermissionDenied) as exc: - raise BadUpstream(_("Linked library item could not be loaded: {}").format(downstream.upstream)) from exc - return link, lib_block - - -def _update_customizable_fields(*, upstream: XBlock, downstream: XBlock, only_fetch: bool) -> None: - """ - For each customizable field: - * Save the upstream value to a hidden field on the downstream ("FETCH"). - * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: - * Update it the downstream field's value from the upstream field ("SYNC"). - - Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. - - * Say that the customizable fields are [display_name, max_attempts]. - - * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). - * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: - * Set `course_problem.display_name = lib_problem.display_name` ("sync"). - """ - syncable_field_names = _get_synchronizable_fields(upstream, downstream) - - for field_name, fetch_field_name in downstream.get_customizable_fields().items(): - - if field_name not in syncable_field_names: - continue - - # Downstream-only fields don't have an upstream fetch field - if fetch_field_name is None: - continue - - # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). - old_upstream_value = getattr(downstream, fetch_field_name) - new_upstream_value = getattr(upstream, field_name) - setattr(downstream, fetch_field_name, new_upstream_value) - - if only_fetch: - continue - - # Okay, now for the nuanced part... - # We need to update the downstream field *iff it has not been customized**. - # Determining whether a field has been customized will differ in Beta vs Future release. - # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) - - ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. - # if field_name in downstream.downstream_customized: - # continue - - ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. - downstream_value = getattr(downstream, field_name) - if old_upstream_value and downstream_value != old_upstream_value: - continue # Field has been customized. Don't touch it. Move on. - - # Field isn't customized -- SYNC it! - setattr(downstream, field_name, new_upstream_value) - - -def _update_non_customizable_fields(*, upstream: XBlock, downstream: XBlock) -> None: - """ - For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. - """ - syncable_fields = _get_synchronizable_fields(upstream, downstream) - customizable_fields = set(downstream.get_customizable_fields().keys()) - isVideoBlock = downstream.usage_key.block_type == "video" - for field_name in syncable_fields - customizable_fields: - if isVideoBlock and field_name == 'edx_video_id': - # Avoid overwriting edx_video_id between blocks - continue - new_upstream_value = getattr(upstream, field_name) - setattr(downstream, field_name, new_upstream_value) - - -def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: - """ - Update tags from `upstream` to `downstream` - """ - from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only - # For any block synced with an upstream, copy the tags as read_only - # This keeps tags added locally. - copy_tags_as_read_only( - str(upstream.location), - str(downstream.location), - ) - - -def _get_synchronizable_fields(upstream: XBlock, downstream: XBlock) -> set[str]: - """ - The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. - """ - return set.intersection(*[ - set( - field_name - for (field_name, field) in block.__class__.fields.items() - if field.scope in [Scope.settings, Scope.content] - ) - for block in [upstream, downstream] - ]) - - def decline_sync(downstream: XBlock) -> None: """ Given an XBlock that is linked to upstream content, mark the latest available update as 'declined' so that its @@ -393,10 +274,11 @@ class UpstreamSyncMixin(XBlockMixin): # Upstream synchronization metadata fields upstream = String( help=( - "The usage key of a block (generally within a content library) which serves as a source of upstream " - "updates for this block, or None if there is no such upstream. Please note: It is valid for this " - "field to hold a usage key for an upstream block that does not exist (or does not *yet* exist) on " - "this instance, particularly if this downstream block was imported from a different instance." + "The usage key or container key of the source block/container (generally within a content library) " + "which serves as a source of upstream updates for this block, or None if there is no such upstream. " + "Please note: It is valid for this field to hold a key for an upstream block/container that does not " + "exist (or does not *yet* exist) on this instance, particularly if this downstream block was imported " + "from a different instance." ), default=None, scope=Scope.settings, hidden=True, enforce_type=True ) @@ -419,7 +301,7 @@ class UpstreamSyncMixin(XBlockMixin): # Store the fetched upstream values for customizable fields. upstream_display_name = String( - help=("The value of display_name on the linked upstream block."), + help=("The value of display_name on the linked upstream block/container."), default=None, scope=Scope.settings, hidden=True, enforce_type=True, ) diff --git a/cms/lib/xblock/upstream_sync_block.py b/cms/lib/xblock/upstream_sync_block.py new file mode 100644 index 000000000000..c7ec740dbf11 --- /dev/null +++ b/cms/lib/xblock/upstream_sync_block.py @@ -0,0 +1,174 @@ +""" +Methods related to syncing a downstream XBlock with an upstream XBlock. + +See upstream_sync.py for general upstream sync code that applies even when the +upstream is a container, not an XBlock. +""" +from __future__ import annotations + +import typing as t + +from django.core.exceptions import PermissionDenied +from django.utils.translation import gettext_lazy as _ +from rest_framework.exceptions import NotFound +from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from xblock.fields import Scope +from xblock.core import XBlock + +from .upstream_sync import UpstreamLink, BadUpstream + +if t.TYPE_CHECKING: + from django.contrib.auth.models import User # pylint: disable=imported-auth-user + + +def sync_from_upstream_block(downstream: XBlock, user: User) -> XBlock: + """ + Update `downstream` with content+settings from the latest available version of its linked upstream content. + + Preserves overrides to customizable fields; overwrites overrides to other fields. + Does not save `downstream` to the store. That is left up to the caller. + + ⭐️ Does not save changes to modulestore nor handle static assets. The caller + will have to take care of that. + + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + link = UpstreamLink.get_for_block(downstream) # can raise UpstreamLinkException + if not isinstance(link.upstream_key, LibraryUsageLocatorV2): + raise TypeError("sync_from_upstream_block() only supports XBlock upstreams, not containers") + # Upstream is a library block: + upstream = _load_upstream_block(downstream, user) + _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=False) + _update_non_customizable_fields(upstream=upstream, downstream=downstream) + _update_tags(upstream=upstream, downstream=downstream) + downstream.upstream_version = link.version_available + return upstream + + +def fetch_customizable_fields_from_block(*, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: + """ + Fetch upstream-defined value of customizable fields and save them on the downstream. + + If `upstream` is provided, use that block as the upstream. + Otherwise, load the block specified by `downstream.upstream`, which may raise an UpstreamLinkException. + """ + if not upstream: + upstream = _load_upstream_block(downstream, user) + _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) + + +def _load_upstream_block(downstream: XBlock, user: User) -> XBlock: + """ + Load the upstream metadata and content for a downstream block. + + Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be + relaxed in the future (see module docstring). + + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. + """ + # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. + from openedx.core.djangoapps.xblock.api import load_block, CheckPerm, LatestVersion # pylint: disable=wrong-import-order + try: + lib_block: XBlock = load_block( + LibraryUsageLocatorV2.from_string(downstream.upstream), + user, + check_permission=CheckPerm.CAN_READ_AS_AUTHOR, + version=LatestVersion.PUBLISHED, + ) + except (NotFound, PermissionDenied) as exc: + raise BadUpstream(_("Linked library item could not be loaded: {}").format(downstream.upstream)) from exc + return lib_block + + +def _update_customizable_fields(*, upstream: XBlock, downstream: XBlock, only_fetch: bool) -> None: + """ + For each customizable field: + * Save the upstream value to a hidden field on the downstream ("FETCH"). + * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: + * Update it the downstream field's value from the upstream field ("SYNC"). + + Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. + + * Say that the customizable fields are [display_name, max_attempts]. + + * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). + * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: + * Set `course_problem.display_name = lib_problem.display_name` ("sync"). + """ + syncable_field_names = _get_synchronizable_fields(upstream, downstream) + + for field_name, fetch_field_name in downstream.get_customizable_fields().items(): + + if field_name not in syncable_field_names: + continue + + # Downstream-only fields don't have an upstream fetch field + if fetch_field_name is None: + continue + + # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). + old_upstream_value = getattr(downstream, fetch_field_name) + new_upstream_value = getattr(upstream, field_name) + setattr(downstream, fetch_field_name, new_upstream_value) + + if only_fetch: + continue + + # Okay, now for the nuanced part... + # We need to update the downstream field *iff it has not been customized**. + # Determining whether a field has been customized will differ in Beta vs Future release. + # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) + + ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. + # if field_name in downstream.downstream_customized: + # continue + + ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. + downstream_value = getattr(downstream, field_name) + if old_upstream_value and downstream_value != old_upstream_value: + continue # Field has been customized. Don't touch it. Move on. + + # Field isn't customized -- SYNC it! + setattr(downstream, field_name, new_upstream_value) + + +def _update_non_customizable_fields(*, upstream: XBlock, downstream: XBlock) -> None: + """ + For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. + """ + syncable_fields = _get_synchronizable_fields(upstream, downstream) + customizable_fields = set(downstream.get_customizable_fields().keys()) + isVideoBlock = downstream.usage_key.block_type == "video" + for field_name in syncable_fields - customizable_fields: + if isVideoBlock and field_name == 'edx_video_id': + # Avoid overwriting edx_video_id between blocks + continue + new_upstream_value = getattr(upstream, field_name) + setattr(downstream, field_name, new_upstream_value) + + +def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: + """ + Update tags from `upstream` to `downstream` + """ + from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only + # For any block synced with an upstream, copy the tags as read_only + # This keeps tags added locally. + copy_tags_as_read_only( + str(upstream.location), + str(downstream.location), + ) + + +def _get_synchronizable_fields(upstream: XBlock, downstream: XBlock) -> set[str]: + """ + The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. + """ + return set.intersection(*[ + set( + field_name + for (field_name, field) in block.__class__.fields.items() + if field.scope in [Scope.settings, Scope.content] + ) + for block in [upstream, downstream] + ]) diff --git a/cms/lib/xblock/upstream_sync_container.py b/cms/lib/xblock/upstream_sync_container.py index 141b6e4266f1..f3305c1881d5 100644 --- a/cms/lib/xblock/upstream_sync_container.py +++ b/cms/lib/xblock/upstream_sync_container.py @@ -1,236 +1,139 @@ """ -Synchronize content and settings from upstream containers to their downstream usages. +Methods related to syncing a downstream XBlock with an upstream Container. -* The upstream is a Container from a Learning Core-backed Content Library. -* The downstream is a block of matching type in a SplitModuleStore-backed Course. -* They are both on the same Open edX instance. - -HOWEVER, those assumptions may loosen in the future. So, we consider these to be INTERNAL ASSUMPIONS that should not be -exposed through this module's public Python interface. +See upstream_sync.py for general upstream sync code that applies even when the +upstream is a container, not an XBlock. """ +from __future__ import annotations -import logging import typing as t -from dataclasses import asdict, dataclass -from django.conf import settings -from django.contrib.auth.models import User # pylint: disable=imported-auth-user -from django.core.exceptions import PermissionDenied from django.utils.translation import gettext_lazy as _ -from opaque_keys import InvalidKeyError from opaque_keys.edx.locator import LibraryContainerLocator -from rest_framework.exceptions import NotFound from xblock.core import XBlock -from xblock.fields import Scope -from cms.djangoapps.contentstore.xblock_storage_handlers.create_xblock import create_xblock -from openedx.core.djangoapps.content_libraries.api import get_container_children +from openedx.core.djangoapps.content_libraries import api as lib_api +from .upstream_sync import UpstreamLink + +if t.TYPE_CHECKING: + from django.contrib.auth.models import User # pylint: disable=imported-auth-user -from .upstream_sync import ( - BadUpstream, - BaseUpstreamLink, - BaseUpstreamSyncManager, - UpstreamLinkException, - check_and_parse_upstream_key, -) -logger = logging.getLogger(__name__) +def sync_from_upstream_container( + downstream: XBlock, + user: User, +) -> list[lib_api.LibraryXBlockMetadata | lib_api.ContainerMetadata]: + """ + Update `downstream` with content+settings from the latest available version of its linked upstream content. + + Preserves overrides to customizable fields; overwrites overrides to other fields. + Does not save `downstream` to the store. That is left up to the caller. + If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. -@dataclass(frozen=True) -class ContainerUpstreamLink(BaseUpstreamLink): + ⭐️ Does not directly sync static assets (containers don't have them) nor + children. Returns a list of the upstream children so the caller can do that. + + Should children be handled in here? Maybe if sync_from_upstream_block + were updated to handle static assets and also save changes to modulestore. """ - Metadata about some downstream content's relationship with its linked upstream content. + link = UpstreamLink.get_for_block(downstream) # can raise UpstreamLinkException + if not isinstance(link.upstream_key, LibraryContainerLocator): + raise TypeError("sync_from_upstream_container() only supports Container upstreams, not containers") + lib_api.require_permission_for_library_key( # TODO: should permissions be checked at this low level? + link.upstream_key.lib_key, + user, + permission=lib_api.permissions.CAN_VIEW_THIS_CONTENT_LIBRARY, + ) + upstream_meta = lib_api.get_container(link.upstream_key, user) + upstream_children = lib_api.get_container_children(link.upstream_key, published=True) + _update_customizable_fields(upstream=upstream_meta, downstream=downstream, only_fetch=False) + _update_non_customizable_fields(upstream=upstream_meta, downstream=downstream) + _update_tags(upstream=upstream_meta, downstream=downstream) + downstream.upstream_version = link.version_available + return upstream_children + + +def fetch_customizable_fields_from_container(*, downstream: XBlock, user: User) -> None: """ + Fetch upstream-defined value of customizable fields and save them on the downstream. + + The container version only retrieves values from *published* containers. - @property - def upstream_link(self) -> str | None: - """ - Link to edit/view upstream block in library. - """ - if self.version_available is None or self.upstream_ref is None: - return None - try: - container_key = LibraryContainerLocator.from_string(self.upstream_ref) - except InvalidKeyError: - return None - return _get_library_container_url(container_key) - - def to_json(self) -> dict[str, t.Any]: - """ - Get an JSON-API-friendly representation of this upstream link. - """ - return { - **asdict(self), - "ready_to_sync": self.ready_to_sync, - "upstream_link": self.upstream_link, - } - - @classmethod - def get_for_container(cls, downstream: XBlock) -> t.Self: - """ - Get info on a container's relationship with its linked upstream content (without actually loading the content). - - Currently, the only supported upstreams are LC-backed Library Components. This may change in the future (see - module docstring). - - If link exists, is supported, and is followable, returns UpstreamLink. - Otherwise, raises an UpstreamLinkException. - """ - upstream_key = check_and_parse_upstream_key(downstream.upstream, downstream.usage_key) - # We import this here b/c UpstreamSyncMixin is used by cms/envs, which loads before the djangoapps are ready. - from openedx.core.djangoapps.content_libraries.api import ( - ContentLibraryContainerNotFound, # pylint: disable=wrong-import-order - get_container, # pylint: disable=wrong-import-order - ) - if not isinstance(upstream_key, LibraryContainerLocator): - raise BadUpstream(_("Invalid upstream_key")) - try: - lib_meta = get_container(upstream_key) - except ContentLibraryContainerNotFound as exc: - raise BadUpstream(_("Linked library item was not found in the system")) from exc - return cls( - upstream_ref=downstream.upstream, - version_synced=downstream.upstream_version, - version_available=(lib_meta.published_version_num if lib_meta else None), - version_declined=downstream.upstream_version_declined, - error_message=None, - ) - - @classmethod - def try_get_for_container(cls, downstream: XBlock) -> t.Self: - """ - Same as `get_for_container`, but upon failure, sets `.error_message` instead of raising an exception. - """ - try: - return cls.get_for_container(downstream) - except UpstreamLinkException as exc: - logger.exception( - "Tried to inspect an unsupported, broken, or missing downstream->upstream link: '%s'->'%s'", - downstream.usage_key, - downstream.upstream, - ) - return cls( - upstream_ref=downstream.upstream, - version_synced=downstream.upstream_version, - version_available=None, - version_declined=None, - error_message=str(exc), - ) - - -def _get_library_container_url(container_key: LibraryContainerLocator): + Basically, this sets the value of "upstream_display_name" on the downstream block. """ - Gets authoring url for given library_key. + upstream = lib_api.get_container(LibraryContainerLocator.from_string(downstream.upstream), user) + _update_customizable_fields(upstream=upstream, downstream=downstream, only_fetch=True) + + +def _update_customizable_fields(*, upstream: lib_api.ContainerMetadata, downstream: XBlock, only_fetch: bool) -> None: """ - library_url = None - if mfe_base_url := settings.COURSE_AUTHORING_MICROFRONTEND_URL: # type: ignore - library_key = container_key.context_key - library_url = f'{mfe_base_url}/library/{library_key}/units?container_key={container_key}' - return library_url + For each customizable field: + * Save the upstream value to a hidden field on the downstream ("FETCH"). + * If `not only_fetch`, and if the field *isn't* customized on the downstream, then: + * Update it the downstream field's value from the upstream field ("SYNC"). + Concrete example: Imagine `lib_problem` is our upstream and `course_problem` is our downstream. -class ContainerUpstreamSyncManager(BaseUpstreamSyncManager): + * Say that the customizable fields are [display_name, max_attempts]. + + * Set `course_problem.upstream_display_name = lib_problem.display_name` ("fetch"). + * If `not only_fetch`, and `course_problem.display_name` wasn't customized, then: + * Set `course_problem.display_name = lib_problem.display_name` ("sync"). """ - Manages sync process of downstream containers like unit with upstream containers. + # For now, the only supported container "field" is display_name + syncable_field_names = ["display_name"] + + for field_name, fetch_field_name in downstream.get_customizable_fields().items(): + + if field_name not in syncable_field_names: + continue + + # Downstream-only fields don't have an upstream fetch field + if fetch_field_name is None: + continue + + # FETCH the upstream's value and save it on the downstream (ie, `downstream.upstream_$FIELD`). + old_upstream_value = getattr(downstream, fetch_field_name) + new_upstream_value = getattr(upstream, f"published_{field_name}") + setattr(downstream, fetch_field_name, new_upstream_value) + + if only_fetch: + continue + + # Okay, now for the nuanced part... + # We need to update the downstream field *iff it has not been customized**. + # Determining whether a field has been customized will differ in Beta vs Future release. + # (See "PRESERVING DOWNSTREAM CUSTOMIZATIONS" comment below for details.) + + ## FUTURE BEHAVIOR: field is "customized" iff we have noticed that the user edited it. + # if field_name in downstream.downstream_customized: + # continue + + ## BETA BEHAVIOR: field is "customized" iff we have the prev upstream value, but field doesn't match it. + downstream_value = getattr(downstream, field_name) + if old_upstream_value and downstream_value != old_upstream_value: + continue # Field has been customized. Don't touch it. Move on. + + # Field isn't customized -- SYNC it! + setattr(downstream, field_name, new_upstream_value) + + +def _update_non_customizable_fields(*, upstream: lib_api.ContainerMetadata, downstream: XBlock) -> None: """ - def __init__(self, downstream: XBlock, user: User, upstream: XBlock | None = None) -> None: - super().__init__(downstream, user, upstream) - if not isinstance(self.upstream_key, LibraryContainerLocator): - raise BadUpstream('Invalid upstream key') - if not upstream: - self.link = ContainerUpstreamLink.get_for_container(downstream) - self.upstream = self._load_upstream_link_and_container_block() - self.syncable_field_names: set[str] = self._get_synchronizable_fields() - self.new_children_blocks: list[XBlock] = [] - - def _get_synchronizable_fields(self) -> set[str]: - """ - The syncable fields are the ones which are content- or settings-scoped AND are defined on both (up,down)stream. - """ - return set.intersection(*[ - set( - field_name - for (field_name, field) in block.__class__.fields.items() - if field.scope in [Scope.settings, Scope.content] - ) - for block in [self.upstream, self.downstream] - ]) - - def _load_upstream_link_and_container_block(self) -> XBlock: - """ - Load the upstream metadata and content for a downstream block. - - Assumes that the upstream content is an XBlock in an LC-backed content libraries. This assumption may need to be - relaxed in the future (see module docstring). - - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. - """ - # We import load_block here b/c UpstreamSyncMixin is used by cms/envs, - # which loads before the djangoapps are ready. - from openedx.core.djangoapps.xblock.api import ( # pylint: disable=wrong-import-order - CheckPerm, - LatestVersion, - load_block, - ) - try: - lib_block: XBlock = load_block( - self.upstream_key, - self.user, - check_permission=CheckPerm.CAN_READ_AS_AUTHOR, - version=LatestVersion.PUBLISHED, - ) - except (NotFound, PermissionDenied) as exc: - raise BadUpstream(_("Linked library item could not be loaded: {}").format(self.upstream_key)) from exc - return lib_block - - def sync_new_children_blocks(self): - """ - Creates children xblocks in course based on library container children. - """ - for child in get_container_children(self.upstream_key, published=True): - child_block = create_xblock( - parent_locator=str(self.downstream.location), - user=self.user, - category=child.usage_key.block_type, - display_name=child.display_name, - ) - child_block.upstream = str(child.usage_key) - self.new_children_blocks.append(child_block) - return self.new_children_blocks - - def delete_extra_blocks(self): - """ - Deletes extra child blocks under the container that are not present in new version of library container. - """ - # TODO: Importing here to avoid circular imports, should be fixed later - from cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers import delete_item - current = self.downstream.children - latest = [str(child.usage_key) for child in get_container_children(self.upstream_key, published=True)] - for child in current: - # TODO: doesn't work for two reasons - # 1. child is not an XBlock but usage_key, so we need to load the child to get its upstream - # 2. Even if we get child, it won't have upstream set as we are not setting upstream for child components - # See: cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py:535 - if child.upstream not in latest: - delete_item(self.user, child.usage_key) - - def sync(self) -> None: - super().sync() - # self.delete_extra_blocks() - self.sync_new_children_blocks() - - -def sync_from_upstream_container(downstream: XBlock, user: User) -> tuple[XBlock, list[XBlock]]: + For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. """ - Update `downstream` with content+settings from the latest available version of its linked upstream content. + pass - Preserves overrides to customizable fields; overwrites overrides to other fields. - Does not save `downstream` to the store. That is left up to the caller. - If `downstream` lacks a valid+supported upstream link, this raises an UpstreamLinkException. +def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: + """ + Update tags from `upstream` to `downstream` """ - manager = ContainerUpstreamSyncManager(downstream, user) - manager.sync() - downstream.upstream_version = manager.link.version_available - return manager.upstream, manager.new_children_blocks + from openedx.core.djangoapps.content_tagging.api import copy_tags_as_read_only + # For any block synced with an upstream, copy the tags as read_only + # This keeps tags added locally. + copy_tags_as_read_only( + str(upstream.location), + str(downstream.location), + ) From eb818344ef3766f2961d48c4970f89423f87b319 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 17:55:23 -0700 Subject: [PATCH 31/46] fix: duplicate definitions of LibraryXBlockMetadata --- .../content_libraries/api/__init__.py | 1 + .../content_libraries/api/block_metadata.py | 84 +++++++++++++++++++ .../content_libraries/api/blocks.py | 62 +------------- .../content_libraries/api/containers.py | 3 +- .../content_libraries/api/libraries.py | 48 ----------- 5 files changed, 88 insertions(+), 110 deletions(-) create mode 100644 openedx/core/djangoapps/content_libraries/api/block_metadata.py diff --git a/openedx/core/djangoapps/content_libraries/api/__init__.py b/openedx/core/djangoapps/content_libraries/api/__init__.py index 4e0b4fcce48e..6c5cbce2a2fa 100644 --- a/openedx/core/djangoapps/content_libraries/api/__init__.py +++ b/openedx/core/djangoapps/content_libraries/api/__init__.py @@ -1,6 +1,7 @@ """ Python API for working with content libraries """ +from .block_metadata import * from .collections import * from .containers import * from .courseware_import import * diff --git a/openedx/core/djangoapps/content_libraries/api/block_metadata.py b/openedx/core/djangoapps/content_libraries/api/block_metadata.py new file mode 100644 index 000000000000..37752255a7ea --- /dev/null +++ b/openedx/core/djangoapps/content_libraries/api/block_metadata.py @@ -0,0 +1,84 @@ +""" +Content libraries API methods related to XBlocks/Components. + +These methods don't enforce permissions (only the REST APIs do). +""" +from __future__ import annotations +from dataclasses import dataclass + +from django.utils.translation import gettext as _ +from opaque_keys.edx.locator import LibraryUsageLocatorV2 +from .libraries import ( + library_component_usage_key, + PublishableItem, +) + +# The public API is only the following symbols: +__all__ = [ + "LibraryXBlockMetadata", + "LibraryXBlockStaticFile", +] + + +@dataclass(frozen=True, kw_only=True) +class LibraryXBlockMetadata(PublishableItem): + """ + Class that represents the metadata about an XBlock in a content library. + """ + usage_key: LibraryUsageLocatorV2 + # TODO: move tags_count to LibraryItem as all objects under a library can be tagged. + tags_count: int = 0 + + @classmethod + def from_component(cls, library_key, component, associated_collections=None): + """ + Construct a LibraryXBlockMetadata from a Component object. + """ + # Import content_tagging.api here to avoid circular imports + from openedx.core.djangoapps.content_tagging.api import get_object_tag_counts + last_publish_log = component.versioning.last_publish_log + + published_by = None + if last_publish_log and last_publish_log.published_by: + published_by = last_publish_log.published_by.username + + draft = component.versioning.draft + published = component.versioning.published + last_draft_created = draft.created if draft else None + last_draft_created_by = draft.publishable_entity_version.created_by if draft else None + usage_key = library_component_usage_key(library_key, component) + tags = get_object_tag_counts(str(usage_key), count_implicit=True) + + return cls( + usage_key=library_component_usage_key( + library_key, + component, + ), + display_name=draft.title, + created=component.created, + modified=draft.created, + draft_version_num=draft.version_num, + published_version_num=published.version_num if published else None, + last_published=None if last_publish_log is None else last_publish_log.published_at, + published_by=published_by, + last_draft_created=last_draft_created, + last_draft_created_by=last_draft_created_by, + has_unpublished_changes=component.versioning.has_unpublished_changes, + collections=associated_collections or [], + tags_count=tags.get(str(usage_key), 0), + ) + + +@dataclass(frozen=True) +class LibraryXBlockStaticFile: + """ + Class that represents a static file in a content library, associated with + a particular XBlock. + """ + # File path e.g. "diagram.png" + # In some rare cases it might contain a folder part, e.g. "en/track1.srt" + path: str + # Publicly accessible URL where the file can be downloaded + url: str + # Size in bytes + size: int diff --git a/openedx/core/djangoapps/content_libraries/api/blocks.py b/openedx/core/djangoapps/content_libraries/api/blocks.py index 1bfb054f4239..7ec465836a50 100644 --- a/openedx/core/djangoapps/content_libraries/api/blocks.py +++ b/openedx/core/djangoapps/content_libraries/api/blocks.py @@ -55,6 +55,7 @@ InvalidNameError, LibraryBlockAlreadyExists, ) +from .block_metadata import LibraryXBlockMetadata, LibraryXBlockStaticFile from .containers import ( create_container, get_container, @@ -79,9 +80,6 @@ # The public API is only the following symbols: __all__ = [ - # Models - "LibraryXBlockMetadata", - "LibraryXBlockStaticFile", # API methods "get_library_components", "get_library_block", @@ -99,64 +97,6 @@ "publish_component_changes", ] - -@dataclass(frozen=True, kw_only=True) -class LibraryXBlockMetadata(PublishableItem): - """ - Class that represents the metadata about an XBlock in a content library. - """ - usage_key: LibraryUsageLocatorV2 - - @classmethod - def from_component(cls, library_key, component, associated_collections=None): - """ - Construct a LibraryXBlockMetadata from a Component object. - """ - last_publish_log = component.versioning.last_publish_log - - published_by = None - if last_publish_log and last_publish_log.published_by: - published_by = last_publish_log.published_by.username - - draft = component.versioning.draft - published = component.versioning.published - last_draft_created = draft.created if draft else None - last_draft_created_by = draft.publishable_entity_version.created_by if draft else None - - return cls( - usage_key=library_component_usage_key( - library_key, - component, - ), - display_name=draft.title, - created=component.created, - modified=draft.created, - draft_version_num=draft.version_num, - published_version_num=published.version_num if published else None, - last_published=None if last_publish_log is None else last_publish_log.published_at, - published_by=published_by, - last_draft_created=last_draft_created, - last_draft_created_by=last_draft_created_by, - has_unpublished_changes=component.versioning.has_unpublished_changes, - collections=associated_collections or [], - ) - - -@dataclass(frozen=True) -class LibraryXBlockStaticFile: - """ - Class that represents a static file in a content library, associated with - a particular XBlock. - """ - # File path e.g. "diagram.png" - # In some rare cases it might contain a folder part, e.g. "en/track1.srt" - path: str - # Publicly accessible URL where the file can be downloaded - url: str - # Size in bytes - size: int - - def get_library_components( library_key: LibraryLocatorV2, text_search: str | None = None, diff --git a/openedx/core/djangoapps/content_libraries/api/containers.py b/openedx/core/djangoapps/content_libraries/api/containers.py index d1d1cccfc394..31f971ea3c0c 100644 --- a/openedx/core/djangoapps/content_libraries/api/containers.py +++ b/openedx/core/djangoapps/content_libraries/api/containers.py @@ -34,7 +34,8 @@ from ..models import ContentLibrary from .exceptions import ContentLibraryContainerNotFound -from .libraries import LibraryXBlockMetadata, PublishableItem, library_component_usage_key +from .libraries import PublishableItem, library_component_usage_key +from .block_metadata import LibraryXBlockMetadata # The public API is only the following symbols: __all__ = [ diff --git a/openedx/core/djangoapps/content_libraries/api/libraries.py b/openedx/core/djangoapps/content_libraries/api/libraries.py index d214dcc3fd86..8e238f278096 100644 --- a/openedx/core/djangoapps/content_libraries/api/libraries.py +++ b/openedx/core/djangoapps/content_libraries/api/libraries.py @@ -205,54 +205,6 @@ class PublishableItem(LibraryItem): can_stand_alone: bool = True -@dataclass(frozen=True, kw_only=True) -class LibraryXBlockMetadata(PublishableItem): - """ - Class that represents the metadata about an XBlock in a content library. - """ - usage_key: LibraryUsageLocatorV2 - # TODO: move tags_count to LibraryItem as all objects under a library can be tagged. - tags_count: int = 0 - - @classmethod - def from_component(cls, library_key, component, associated_collections=None): - """ - Construct a LibraryXBlockMetadata from a Component object. - """ - # Import content_tagging.api here to avoid circular imports - from openedx.core.djangoapps.content_tagging.api import get_object_tag_counts - - last_publish_log = component.versioning.last_publish_log - - published_by = None - if last_publish_log and last_publish_log.published_by: - published_by = last_publish_log.published_by.username - - draft = component.versioning.draft - published = component.versioning.published - last_draft_created = draft.created if draft else None - last_draft_created_by = draft.publishable_entity_version.created_by if draft else None - usage_key = library_component_usage_key(library_key, component) - tags = get_object_tag_counts(str(usage_key), count_implicit=True) - - return cls( - usage_key=usage_key, - display_name=draft.title, - created=component.created, - modified=draft.created, - draft_version_num=draft.version_num, - published_version_num=published.version_num if published else None, - last_published=None if last_publish_log is None else last_publish_log.published_at, - published_by=published_by, - last_draft_created=last_draft_created, - last_draft_created_by=last_draft_created_by, - has_unpublished_changes=component.versioning.has_unpublished_changes, - collections=associated_collections or [], - can_stand_alone=component.publishable_entity.can_stand_alone, - tags_count=tags.get(str(usage_key), 0), - ) - - @dataclass(frozen=True) class LibraryXBlockStaticFile: """ From 2859bc13285d73953b336e7a7b3534a003737a24 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 17:55:32 -0700 Subject: [PATCH 32/46] fix: type issues --- cms/lib/xblock/upstream_sync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 70e9b9e9658b..caa8d6b99a52 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -161,6 +161,7 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: raise BadDownstream(_("Cannot update content because it does not belong to a course.")) downstream_type = downstream.usage_key.block_type + upstream_key: LibraryUsageLocatorV2 | LibraryContainerLocator try: upstream_key = LibraryUsageLocatorV2.from_string(downstream.upstream) except InvalidKeyError: From 3d116918d4d48cf55cf4c519271281f69c9accb8 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 18:28:33 -0700 Subject: [PATCH 33/46] test: fix tests --- .../tests/test_upstream_downstream_links.py | 4 +- cms/lib/xblock/test/test_upstream_sync.py | 68 +++++++++++-------- 2 files changed, 43 insertions(+), 29 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py index a5ad9a2a8002..90fec8471651 100644 --- a/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py +++ b/cms/djangoapps/contentstore/tests/test_upstream_downstream_links.py @@ -67,7 +67,7 @@ def _create_unit_and_expected_container_link(self, course_key: str | CourseKey, for i in range(num_blocks): upstream, block = self._create_unit(i + 1) data.append({ - "upstream_block": None, + "upstream_container": None, "downstream_context_key": course_key, "downstream_usage_key": block.usage_key, "upstream_container_key": upstream, @@ -110,7 +110,7 @@ def _compare_links(self, course_key, expected_component_links, expected_containe )) self.assertListEqual(links, expected_component_links) container_links = list(ContainerLink.objects.filter(downstream_context_key=course_key).values( - 'upstream_block', + 'upstream_container', 'upstream_container_key', 'upstream_context_key', 'downstream_usage_key', diff --git a/cms/lib/xblock/test/test_upstream_sync.py b/cms/lib/xblock/test/test_upstream_sync.py index 703f84c9c6c1..8902d67c8cb1 100644 --- a/cms/lib/xblock/test/test_upstream_sync.py +++ b/cms/lib/xblock/test/test_upstream_sync.py @@ -11,13 +11,12 @@ from cms.lib.xblock.upstream_sync import ( BadDownstream, BadUpstream, - ComponentUpstreamSyncManager, NoUpstream, UpstreamLink, decline_sync, sever_upstream_link, - sync_from_upstream, ) +from cms.lib.xblock.upstream_sync_block import sync_from_upstream_block, fetch_customizable_fields_from_block from common.djangoapps.student.tests.factories import UserFactory from openedx.core.djangoapps.content_libraries import api as libs from openedx.core.djangoapps.content_tagging import api as tagging_api @@ -118,7 +117,7 @@ def test_sync_bad_downstream(self): downstream_lib_block.save() with self.assertRaises(BadDownstream): - sync_from_upstream(downstream_lib_block, self.user) + sync_from_upstream_block(downstream_lib_block, self.user) assert downstream_lib_block.display_name == "Another lib block" assert downstream_lib_block.data == "another lib block" @@ -132,7 +131,7 @@ def test_sync_no_upstream(self): block.data = "Block content" with self.assertRaises(NoUpstream): - sync_from_upstream(block, self.user) + sync_from_upstream_block(block, self.user) assert block.display_name == "Block Title" assert block.data == "Block content" @@ -143,7 +142,6 @@ def test_sync_no_upstream(self): ("course-v1:Oops+ItsA+CourseKey", ".*is malformed.*"), ("block-v1:The+Wrong+KindOfUsageKey+type@html+block@nope", ".*is malformed.*"), ("lb:TestX:NoSuchLib:html:block-id", ".*not found in the system.*"), - ("lb:TestX:TestLib:video:should-be-html-but-is-a-video", ".*type mismatch.*"), ("lb:TestX:TestLib:html:no-such-html", ".*not found in the system.*"), ) @ddt.unpack @@ -156,12 +154,29 @@ def test_sync_bad_upstream(self, upstream, message_regex): block.data = "Block content" with self.assertRaisesRegex(BadUpstream, message_regex): - sync_from_upstream(block, self.user) + sync_from_upstream_block(block, self.user) assert block.display_name == "Block Title" assert block.data == "Block content" assert not block.upstream_display_name + def test_sync_incompatible_upstream(self): + """ + Syncing with a bad upstream raises BadUpstream, but doesn't affect the block + """ + downstream_block = BlockFactory.create( + category='html', parent=self.unit, upstream=str(self.upstream_problem_key), + ) + downstream_block.display_name = "Block Title" + downstream_block.data = "Block content" + + with self.assertRaisesRegex(BadUpstream, "Content type mismatch.*"): + sync_from_upstream_block(downstream_block, self.user) + + assert downstream_block.display_name == "Block Title" + assert downstream_block.data == "Block content" + assert not downstream_block.upstream_display_name + def test_sync_not_accessible(self): """ Syncing with an block that exists, but is inaccessible, raises BadUpstream @@ -169,7 +184,7 @@ def test_sync_not_accessible(self): downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(self.upstream_key)) user_who_cannot_read_upstream = UserFactory.create(username="rando", is_staff=False, is_superuser=False) with self.assertRaisesRegex(BadUpstream, ".*could not be loaded.*") as exc: - sync_from_upstream(downstream, user_who_cannot_read_upstream) + sync_from_upstream_block(downstream, user_who_cannot_read_upstream) def test_sync_updates_happy_path(self): """ @@ -178,7 +193,7 @@ def test_sync_updates_happy_path(self): downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(self.upstream_key)) # Initial sync - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) assert downstream.upstream_version == 2 # Library blocks start at version 2 (v1 is the empty new block) assert downstream.upstream_display_name == "Upstream Title V2" assert downstream.display_name == "Upstream Title V2" @@ -199,7 +214,7 @@ def test_sync_updates_happy_path(self): tagging_api.tag_object(str(self.upstream_key), self.taxonomy_all_org, new_upstream_tags) # Assert that un-published updates are not yet pulled into downstream - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) assert downstream.upstream_version == 2 # Library blocks start at version 2 (v1 is the empty new block) assert downstream.upstream_display_name == "Upstream Title V2" assert downstream.display_name == "Upstream Title V2" @@ -209,7 +224,7 @@ def test_sync_updates_happy_path(self): libs.publish_changes(self.library.key, self.user.id) # Follow-up sync. Assert that updates are pulled into downstream. - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) assert downstream.upstream_version == 3 assert downstream.upstream_display_name == "Upstream Title V3" assert downstream.display_name == "Upstream Title V3" @@ -229,7 +244,7 @@ def test_sync_updates_to_downstream_only_fields(self): downstream = BlockFactory.create(category='problem', parent=self.unit, upstream=str(self.upstream_problem_key)) # Initial sync - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) # These fields are copied from upstream assert downstream.upstream_display_name == "Upstream Problem Title V2" @@ -293,7 +308,7 @@ def test_sync_updates_to_downstream_only_fields(self): downstream.save() # Follow-up sync. - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) # "unsafe" customizations are overridden by upstream assert downstream.upstream_display_name == "Upstream Problem Title V3" @@ -322,7 +337,7 @@ def test_sync_updates_to_modified_content(self): downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(self.upstream_key)) # Initial sync - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) assert downstream.upstream_display_name == "Upstream Title V2" assert downstream.display_name == "Upstream Title V2" assert downstream.data == "Upstream content V2" @@ -340,7 +355,7 @@ def test_sync_updates_to_modified_content(self): downstream.save() # Follow-up sync. Assert that updates are pulled into downstream, but customizations are saved. - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) assert downstream.upstream_display_name == "Upstream Title V3" assert downstream.display_name == "Downstream Title Override" # "safe" customization survives assert downstream.data == "Upstream content V3" # "unsafe" override is gone @@ -357,7 +372,7 @@ def test_sync_updates_to_modified_content(self): # """ # # Start with an uncustomized downstream block. # downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(self.upstream_key)) - # sync_from_upstream(downstream, self.user) + # sync_from_upstream_block(downstream, self.user) # assert downstream.downstream_customized == [] # assert downstream.display_name == downstream.upstream_display_name == "Upstream Title V2" # @@ -367,7 +382,7 @@ def test_sync_updates_to_modified_content(self): # assert downstream.downstream_customized == ["display_name"] # # # Syncing should retain the customization. - # sync_from_upstream(downstream, self.user) + # sync_from_upstream_block(downstream, self.user) # assert downstream.upstream_version == 2 # assert downstream.upstream_display_name == "Upstream Title V2" # assert downstream.display_name == "Title V3" @@ -378,7 +393,7 @@ def test_sync_updates_to_modified_content(self): # upstream.save() # # # ...which is reflected when we sync. - # sync_from_upstream(downstream, self.user) + # sync_from_upstream_block(downstream, self.user) # assert downstream.upstream_version == 3 # assert downstream.upstream_display_name == downstream.display_name == "Title V3" # @@ -389,7 +404,7 @@ def test_sync_updates_to_modified_content(self): # upstream.save() # # # ...then the downstream title should remain put. - # sync_from_upstream(downstream, self.user) + # sync_from_upstream_block(downstream, self.user) # assert downstream.upstream_version == 4 # assert downstream.upstream_display_name == "Title V4" # assert downstream.display_name == "Title V3" @@ -398,7 +413,7 @@ def test_sync_updates_to_modified_content(self): # downstream.downstream_customized = [] # upstream.display_name = "Title V5" # upstream.save() - # sync_from_upstream(downstream, self.user) + # sync_from_upstream_block(downstream, self.user) # assert downstream.upstream_version == 5 # assert downstream.upstream_display_name == downstream.display_name == "Title V5" @@ -414,14 +429,13 @@ def test_update_customizable_fields(self, initial_upstream_display_name): downstream.display_name = "Some Title" downstream.data = "Some content" - # Note that we're not linked to any upstream. ComponentUpstreamSyncManager shouldn't care if upstream is passed. + # Note that we're not linked to any upstream. fetch_customizable_fields_from_block shouldn't care. assert not downstream.upstream assert not downstream.upstream_version # fetch! upstream = xblock.load_block(self.upstream_key, self.user) - manager = ComponentUpstreamSyncManager(downstream=downstream, user=self.user, upstream=upstream) - manager.update_customizable_fields(only_fetch=True) + fetch_customizable_fields_from_block(upstream=upstream, downstream=downstream, user=self.user) # Ensure: fetching doesn't affect the upstream link (or lack thereof). assert not downstream.upstream @@ -447,7 +461,7 @@ def test_prompt_and_decline_sync(self): assert link.ready_to_sync is True # Initial sync to V2 - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) link = UpstreamLink.get_for_block(downstream) assert link.version_synced == 2 assert link.version_declined is None @@ -497,7 +511,7 @@ def test_sever_upstream_link(self): """ # Start with a course block that is linked+synced to a content library block. downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(self.upstream_key)) - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) # (sanity checks) assert downstream.upstream == str(self.upstream_key) @@ -537,7 +551,7 @@ def test_sync_library_block_tags(self): downstream = BlockFactory.create(category='html', parent=self.unit, upstream=str(upstream_lib_block_key)) # Initial sync - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) # Verify tags object_tags = tagging_api.get_object_tags(str(downstream.location)) @@ -553,7 +567,7 @@ def test_sync_library_block_tags(self): tagging_api.tag_object(str(upstream_lib_block_key), self.taxonomy_all_org, new_upstream_tags) # Follow-up sync. - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) #Verify tags object_tags = tagging_api.get_object_tags(str(downstream.location)) @@ -566,7 +580,7 @@ def test_sync_video_block(self): downstream.edx_video_id = "test_video_id" # Sync - sync_from_upstream(downstream, self.user) + sync_from_upstream_block(downstream, self.user) assert downstream.upstream_version == 2 assert downstream.upstream_display_name == "Video Test" assert downstream.display_name == "Video Test" From 406ce79a677b5f7cf8f8e8f94ac60adbe033e9cd Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 18:43:20 -0700 Subject: [PATCH 34/46] fix: duplicate definitions of LibraryXBlockMetadata (fixup) --- openedx/core/djangoapps/content_libraries/api/blocks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openedx/core/djangoapps/content_libraries/api/blocks.py b/openedx/core/djangoapps/content_libraries/api/blocks.py index 7ec465836a50..d440055448f2 100644 --- a/openedx/core/djangoapps/content_libraries/api/blocks.py +++ b/openedx/core/djangoapps/content_libraries/api/blocks.py @@ -6,7 +6,6 @@ from __future__ import annotations import logging import mimetypes -from dataclasses import dataclass from datetime import datetime, timezone from typing import TYPE_CHECKING from uuid import uuid4 @@ -66,7 +65,6 @@ ) from .libraries import ( library_collection_locator, - library_component_usage_key, PublishableItem, ) @@ -97,6 +95,7 @@ "publish_component_changes", ] + def get_library_components( library_key: LibraryLocatorV2, text_search: str | None = None, From 84d048542632e5455dcd6827b82dcc25dde7a662 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 19:00:11 -0700 Subject: [PATCH 35/46] chore: quality issues --- cms/lib/xblock/upstream_sync.py | 4 +++- cms/lib/xblock/upstream_sync_container.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index caa8d6b99a52..401d32258f6d 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -198,7 +198,9 @@ def get_for_block(cls, downstream: XBlock) -> t.Self: # It could be reasonable to relax this requirement in the future if there's product need for it. # for example, there's no reason that a StaticTabBlock couldn't take updates from an HtmlBlock. raise BadUpstream( - _("Content type mismatch: {downstream_id} ({downstream_type}) cannot be linked to {upstream_id}.").format( + _( + "Content type mismatch: {downstream_id} ({downstream_type}) cannot be linked to {upstream_id}." + ).format( downstream_id=downstream.usage_key, downstream_type=downstream_type, upstream_id=str(upstream_key), diff --git a/cms/lib/xblock/upstream_sync_container.py b/cms/lib/xblock/upstream_sync_container.py index f3305c1881d5..3837288d6d6c 100644 --- a/cms/lib/xblock/upstream_sync_container.py +++ b/cms/lib/xblock/upstream_sync_container.py @@ -123,7 +123,7 @@ def _update_non_customizable_fields(*, upstream: lib_api.ContainerMetadata, down """ For each field `downstream.blah` that isn't customizable: set it to `upstream.blah`. """ - pass + # For now, there's nothing to do here - containers don't have any non-customizable fields. def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: From b2737966369f125453c053b29e38660a936f7d91 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 19:17:40 -0700 Subject: [PATCH 36/46] test: update tests --- .../v2/views/tests/test_downstreams.py | 20 +++++++++---------- cms/lib/xblock/upstream_sync.py | 4 +++- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py index c2428c6db97f..426b49dc53e7 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstreams.py @@ -11,8 +11,9 @@ from organizations.models import Organization from cms.djangoapps.contentstore.helpers import StaticFileNotices -from cms.lib.xblock.upstream_sync import BadUpstream, ComponentUpstreamSyncManager, UpstreamLink +from cms.lib.xblock.upstream_sync import BadUpstream, UpstreamLink from cms.djangoapps.contentstore.tests.utils import CourseTestCase +from cms.djangoapps.contentstore.xblock_storage_handlers import view_handlers as xblock_view_handlers from opaque_keys.edx.keys import UsageKey from common.djangoapps.student.tests.factories import UserFactory from xmodule.modulestore.django import modulestore @@ -32,6 +33,7 @@ def _get_upstream_link_good_and_syncable(downstream): return UpstreamLink( upstream_ref=downstream.upstream, + upstream_key=UsageKey.from_string(downstream.upstream), version_synced=downstream.upstream_version, version_available=(downstream.upstream_version or 0) + 1, version_declined=downstream.upstream_version_declined, @@ -235,7 +237,7 @@ def call_api(self, usage_key_string, sync: str | None = None): content_type="application/json", ) - @patch.object(ComponentUpstreamSyncManager, "update_customizable_fields") + @patch.object(downstreams_views, "fetch_customizable_fields_from_block") @patch.object(downstreams_views, "sync_library_content") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) def test_200_with_sync(self, mock_sync, mock_fetch): @@ -250,7 +252,7 @@ def test_200_with_sync(self, mock_sync, mock_fetch): assert mock_fetch.call_count == 0 assert video_after.upstream == self.video_lib_id - @patch.object(ComponentUpstreamSyncManager, "update_customizable_fields") + @patch.object(downstreams_views, "fetch_customizable_fields_from_block") @patch.object(downstreams_views, "sync_library_content") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) def test_200_no_sync(self, mock_sync, mock_fetch): @@ -266,9 +268,7 @@ def test_200_no_sync(self, mock_sync, mock_fetch): assert video_after.upstream == self.video_lib_id @patch.object( - ComponentUpstreamSyncManager, - "update_customizable_fields", - side_effect=BadUpstream(MOCK_UPSTREAM_ERROR) + downstreams_views, "fetch_customizable_fields_from_block", side_effect=BadUpstream(MOCK_UPSTREAM_ERROR), ) def test_400(self, sync: str): """ @@ -370,7 +370,7 @@ def test_200(self): @patch("cms.djangoapps.contentstore.helpers._insert_static_files_into_downstream_xblock") @patch("cms.djangoapps.contentstore.helpers.content_staging_api.stage_xblock_temporarily") - @patch("cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers.sync_from_upstream") + @patch("cms.djangoapps.contentstore.xblock_storage_handlers.view_handlers.sync_from_upstream_block") def test_200_video(self, mock_sync, mock_stage, mock_insert): mock_lib_block = MagicMock() mock_lib_block.runtime.get_block_assets.return_value = ['mocked_asset'] @@ -398,16 +398,16 @@ def call_api(self, usage_key_string): return self.client.post(f"/api/contentstore/v2/downstreams/{usage_key_string}/sync") @patch.object(UpstreamLink, "get_for_block", _get_upstream_link_good_and_syncable) - @patch.object(downstreams_views, "sync_library_content", return_value=StaticFileNotices()) + @patch.object(xblock_view_handlers, "import_static_assets_for_library_sync", return_value=StaticFileNotices()) @patch.object(downstreams_views, "clear_transcripts") - def test_200(self, mock_sync_from_upstream, mock_clear_transcripts): + def test_200(self, mock_import_staged_content, mock_clear_transcripts): """ Does the happy path work? """ self.client.login(username="superuser", password="password") response = self.call_api(self.downstream_video_key) assert response.status_code == 200 - assert mock_sync_from_upstream.call_count == 1 + assert mock_import_staged_content.call_count == 1 assert mock_clear_transcripts.call_count == 1 diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 401d32258f6d..3708f803e880 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -111,11 +111,13 @@ def to_json(self) -> dict[str, t.Any]: """ Get an JSON-API-friendly representation of this upstream link. """ - return { + data = { **asdict(self), "ready_to_sync": self.ready_to_sync, "upstream_link": self.upstream_link, } + del data["upstream_key"] # As JSON (string), this would be redundant with upstream_ref + return data @classmethod def try_get_for_block(cls, downstream: XBlock) -> t.Self: From f613566475ca6f9ffd76b71de1373d29a1d41153 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 20:33:58 -0700 Subject: [PATCH 37/46] test: add a new integration test suite for syncing --- .../tests/test_downstream_sync_integration.py | 136 ++++++++++++++++++ .../xblock_storage_handlers/view_handlers.py | 2 +- .../content_libraries/tests/base.py | 4 + 3 files changed, 141 insertions(+), 1 deletion(-) create mode 100644 cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py new file mode 100644 index 000000000000..473e56b0bc74 --- /dev/null +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -0,0 +1,136 @@ +""" +Unit and integration tests to ensure that syncing content from libraries to +courses is working. +""" +from xml.etree import ElementTree + +import ddt +from opaque_keys.edx.locator import LibraryContainerLocator + +from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase +from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory, ToyCourseFactory +from common.djangoapps.student.tests.factories import UserFactory +from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest + + +@ddt.ddt +class CourseToLibraryTestCase(ContentLibrariesRestApiTest, ModuleStoreTestCase): + """ + Tests that involve syncing content from libraries to courses. + """ + maxDiff = None # Necessary for debugging OLX differences + + def setUp(self): + super().setUp() + # self.user is set up by ContentLibrariesRestApiTest + + # The source library (contains the upstreams): + self.library = self._create_library(slug="testlib", title="Upstream Library") + lib_id = self.library["id"] # the library ID as a string + self.upstream_problem1 = self._add_block_to_library(lib_id, "problem", "prob1", can_stand_alone=True) + self._set_library_block_olx( + self.upstream_problem1["id"], + 'multiple choice...' + ) + self.upstream_problem2 = self._add_block_to_library(lib_id, "problem", "prob2", can_stand_alone=True) + self._set_library_block_olx( + self.upstream_problem2["id"], + 'multi select...' + ) + self.upstream_html1 = self._add_block_to_library(lib_id, "html", "html1", can_stand_alone=False) + self._set_library_block_olx( + self.upstream_html1["id"], + 'This is the HTML.' + ) + self._commit_library_changes(lib_id) # publish everything + + # The destination course: + self.course = CourseFactory.create() + self.course_section = BlockFactory.create(category='chapter', parent=self.course) + self.course_subsection = BlockFactory.create(category='sequential', parent=self.course_section) + self.course_unit = BlockFactory.create(category='vertical', parent=self.course_subsection) + + def _get_sync_status(self, usage_key: str): + return self._api('get', f"/api/contentstore/v2/downstreams/{usage_key}", {}, expect_response=200) + + def _sync_downstream(self, usage_key: str): + return self._api('post', f"/api/contentstore/v2/downstreams/{usage_key}/sync", {}, expect_response=200) + + def _get_course_block_olx(self, usage_key: str): + data = self._api('get', f'/api/olx-export/v1/xblock/{usage_key}/', {}, expect_response=200) + return data["blocks"][data["root_block_id"]]["olx"] + + def _create_block_from_upstream( + self, + block_category: str, + parent_usage_key: str, + upstream_key: str, + expect_response: int = 200, + ): + """ + Call the CMS API for inserting an XBlock that's cloned from a library + item. i.e. copy a *published* library block into a course, and create an + upstream link. + """ + return self._api('post', "/xblock/", { + "category": block_category, + "parent_locator": parent_usage_key, + "library_content_key": upstream_key, + }, expect_response=expect_response) + + def assertXmlEqual(self, xml_str_a: str, xml_str_b: str) -> bool: + """ Assert that the given XML strings are equal, ignoring attribute order and some whitespace variations. """ + self.assertEqual( + ElementTree.canonicalize(xml_str_a, strip_text=True), + ElementTree.canonicalize(xml_str_b, strip_text=True), + ) + + # OLX attributes that will appear on capa problems when saved/exported. Excludes "markdown" + standard_capa_attributes = """ + markdown_edited="false" + matlab_api_key="null" + name="null" + rerandomize="never" + source_code="null" + tags="[]" + use_latex_compiler="false" + """ + + #################################################################################################################### + + def test_problem_sync(self): + """ + Test that we can sync a problem from a library into a course. + """ + # First, create the problem in the course, using the upstream problem as a template: + downstream_problem1 = self._create_block_from_upstream( + block_category="problem", + parent_usage_key=str(self.course_subsection.usage_key), + upstream_key=self.upstream_problem1["id"], + ) + status = self._get_sync_status(downstream_problem1["locator"]) + self.assertDictContainsEntries(status, { + 'upstream_ref': self.upstream_problem1["id"], # e.g. 'lb:CL-TEST:testlib:problem:prob1' + 'version_available': 2, + 'version_synced': 2, + 'version_declined': None, + 'ready_to_sync': False, + 'error_message': None, + # 'upstream_link': 'http://course-authoring-mfe/library/lib:CL-TEST:testlib/components?usageKey=...' + }) + assert status["upstream_link"].startswith("http://course-authoring-mfe/library/") + assert status["upstream_link"].endswith(f"/components?usageKey={self.upstream_problem1['id']}") + + downstream_olx = self._get_course_block_olx(downstream_problem1["locator"]) + # Notice that per UpstreamSyncMixin.get_customizable_fields(), some fields like + # weight and max_attempts are DROPPED entirely from the upstream version when creating the downstream: + self.assertXmlEqual(downstream_olx, f""" + multiple choice... + """) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 0df775b19ab7..620b32af3c04 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -610,7 +610,7 @@ def _create_block(request): created_block = create_xblock( parent_locator=parent_locator, user=request.user, - category="vertical" if category == "unit" else category, + category=category, display_name=request.json.get("display_name"), boilerplate=request.json.get("boilerplate"), ) diff --git a/openedx/core/djangoapps/content_libraries/tests/base.py b/openedx/core/djangoapps/content_libraries/tests/base.py index b81382ab001e..6068d9c20e72 100644 --- a/openedx/core/djangoapps/content_libraries/tests/base.py +++ b/openedx/core/djangoapps/content_libraries/tests/base.py @@ -2,6 +2,7 @@ Tests for Learning-Core-based Content Libraries """ from contextlib import contextmanager +import json from io import BytesIO from urllib.parse import urlencode @@ -9,6 +10,7 @@ from rest_framework.test import APITransactionTestCase, APIClient from common.djangoapps.student.tests.factories import UserFactory +from common.djangoapps.util.json_request import JsonResponse as SpecialJsonResponse from openedx.core.djangoapps.content_libraries.constants import ALL_RIGHTS_RESERVED from openedx.core.djangolib.testing.utils import skip_unless_cms @@ -113,6 +115,8 @@ def _api(self, method, url, data, expect_response): response = getattr(self.client, method)(url, data, format="json") assert response.status_code == expect_response,\ 'Unexpected response code {}:\n{}'.format(response.status_code, getattr(response, 'data', '(no data)')) + if isinstance(response, SpecialJsonResponse): # Required for some old APIs in the CMS that aren't using DRF + return json.loads(response.content) return response.data @contextmanager From b71f11acca533eec2ebbbf1562f09bbc69967f1d Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 20:52:23 -0700 Subject: [PATCH 38/46] test: continue building out integration tests --- .../tests/test_downstream_sync_integration.py | 77 +++++++++++++++++-- 1 file changed, 72 insertions(+), 5 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index 473e56b0bc74..6e1b59b63b1b 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -2,6 +2,7 @@ Unit and integration tests to ensure that syncing content from libraries to courses is working. """ +from typing import Any from xml.etree import ElementTree import ddt @@ -77,6 +78,12 @@ def _create_block_from_upstream( "parent_locator": parent_usage_key, "library_content_key": upstream_key, }, expect_response=expect_response) + + def _update_course_block_fields(self, usage_key: str, fields: dict[str, Any] = None): + """ Update fields of an XBlock """ + return self._api('patch', f"/xblock/{usage_key}", { + "metadata": fields, + }, expect_response=200) def assertXmlEqual(self, xml_str_a: str, xml_str_b: str) -> bool: """ Assert that the given XML strings are equal, ignoring attribute order and some whitespace variations. """ @@ -102,7 +109,7 @@ def test_problem_sync(self): """ Test that we can sync a problem from a library into a course. """ - # First, create the problem in the course, using the upstream problem as a template: + # 1️⃣ First, create the problem in the course, using the upstream problem as a template: downstream_problem1 = self._create_block_from_upstream( block_category="problem", parent_usage_key=str(self.course_subsection.usage_key), @@ -121,10 +128,11 @@ def test_problem_sync(self): assert status["upstream_link"].startswith("http://course-authoring-mfe/library/") assert status["upstream_link"].endswith(f"/components?usageKey={self.upstream_problem1['id']}") - downstream_olx = self._get_course_block_olx(downstream_problem1["locator"]) - # Notice that per UpstreamSyncMixin.get_customizable_fields(), some fields like - # weight and max_attempts are DROPPED entirely from the upstream version when creating the downstream: - self.assertXmlEqual(downstream_olx, f""" + # Check the OLX of the downstream block. Notice that: + # (1) fields display_name and markdown, as well as the 'data' (content/body of the ) are synced. + # (2) per UpstreamSyncMixin.get_customizable_fields(), some fields like weight and max_attempts are + # DROPPED entirely from the upstream version when creating the downstream: + self.assertXmlEqual(self._get_course_block_olx(downstream_problem1["locator"]), f""" multiple choice... """) + + # 2️⃣ Now, lets modify the upstream problem AND the downstream problem: + + self._update_course_block_fields(downstream_problem1["locator"], { + "display_name": "Custom Display Name", + "max_attempts": 3, + "markdown": "blow me away, scotty!", # This change will be lost + }) + + self._set_library_block_olx( + self.upstream_problem1["id"], + 'multiple choice v2...' + ) + self._publish_library_block(self.upstream_problem1["id"]) + + # Here's how the downstream OLX looks now, before we sync: + self.assertXmlEqual(self._get_course_block_olx(downstream_problem1["locator"]), f""" + multiple choice... + """) + + status = self._get_sync_status(downstream_problem1["locator"]) + self.assertDictContainsEntries(status, { + 'upstream_ref': self.upstream_problem1["id"], # e.g. 'lb:CL-TEST:testlib:problem:prob1' + 'version_available': 3, # <--- updated + 'version_synced': 2, + 'version_declined': None, + 'ready_to_sync': True, # <--- updated + 'error_message': None, + }) + + # 3️⃣ Now, sync and check the resulting OLX of the downstream + + self._sync_downstream(downstream_problem1["locator"]) + + # Here's how the downstream OLX looks now, after we synced it. + # Notice: + # (1) content like "markdown" and the body XML content are synced + # (2) the "display_name" is left alone (customized downstream), but + # (3) "upstream_display_name" is updated. + # (4) The customized "max_attempts" is also still present. + self.assertXmlEqual(self._get_course_block_olx(downstream_problem1["locator"]), f""" + multiple choice v2... + """) From f0e5556f56f471a6ba050925547bf953dda8d222 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 21:29:41 -0700 Subject: [PATCH 39/46] feat: partially implement container+child syncing --- .../tests/test_downstream_sync_integration.py | 35 ++++++++++++-- .../xblock_storage_handlers/view_handlers.py | 48 +++++++++++++++---- cms/lib/xblock/upstream_sync_container.py | 6 +-- 3 files changed, 72 insertions(+), 17 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index 6e1b59b63b1b..73501bc268b1 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -6,11 +6,9 @@ from xml.etree import ElementTree import ddt -from opaque_keys.edx.locator import LibraryContainerLocator from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase -from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory, ToyCourseFactory -from common.djangoapps.student.tests.factories import UserFactory +from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest @@ -43,6 +41,12 @@ def setUp(self): self.upstream_html1["id"], 'This is the HTML.' ) + self.upstream_unit = self._create_container(lib_id, "unit", slug="u1", display_name="Unit 1 Title") + self._add_container_components(self.upstream_unit["id"], [ + self.upstream_html1["id"], + self.upstream_problem1["id"], + self.upstream_problem2["id"], + ]) self._commit_library_changes(lib_id) # publish everything # The destination course: @@ -102,7 +106,7 @@ def assertXmlEqual(self, xml_str_a: str, xml_str_b: str) -> bool: tags="[]" use_latex_compiler="false" """ - + #################################################################################################################### def test_problem_sync(self): @@ -201,3 +205,26 @@ def test_problem_sync(self): {self.standard_capa_attributes} >multiple choice v2... """) + + def test_unit_sync(self): + """ + Test that we can sync a unit from the library into the course + """ + # Create a "vertical" block based on a "unit" container: + downstream_unit = self._create_block_from_upstream( + block_category="vertical", + parent_usage_key=str(self.course_subsection.usage_key), + upstream_key=self.upstream_unit["id"], + ) + status = self._get_sync_status(downstream_unit["locator"]) + self.assertDictContainsEntries(status, { + 'upstream_ref': self.upstream_unit["id"], # e.g. 'lct:CL-TEST:testlib:unit:u1' + 'version_available': 2, + 'version_synced': 2, + 'version_declined': None, + 'ready_to_sync': False, + 'error_message': None, + 'upstream_link': 'http://course-authoring-mfe/library/lib:CL-TEST:testlib/units' + }) + assert status["upstream_link"].startswith("http://course-authoring-mfe/library/") + assert status["upstream_link"].endswith(f"/units/{self.upstream_unit['id']}") diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 620b32af3c04..b57dbb2da365 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -10,6 +10,7 @@ """ import logging from datetime import datetime +from uuid import uuid4 from attrs import asdict from django.conf import settings @@ -19,6 +20,7 @@ from django.http import HttpResponse, HttpResponseBadRequest from django.utils.translation import gettext as _ from edx_django_utils.plugins import pluggable_override +from openedx.core.djangoapps.content_libraries.api import LibraryXBlockMetadata from openedx.core.djangoapps.content_tagging.api import get_object_tag_counts from edx_proctoring.api import ( does_backend_support_onboarding, @@ -526,7 +528,7 @@ def create_item(request): return _create_block(request) -def sync_library_content(downstream: XBlock, request, store, remove_upstream_link: bool = False) -> StaticFileNotices: +def sync_library_content(downstream: XBlock, request, store) -> StaticFileNotices: """ Handle syncing library content for given xblock depending on its upstream type. It can sync unit containers and lower level xblocks. @@ -535,20 +537,46 @@ def sync_library_content(downstream: XBlock, request, store, remove_upstream_lin upstream_key = link.upstream_key if isinstance(upstream_key, LibraryUsageLocatorV2): lib_block = sync_from_upstream_block(downstream=downstream, user=request.user) - if remove_upstream_link: - # Removing upstream link from child components - downstream.upstream = None static_file_notices = import_static_assets_for_library_sync(downstream, lib_block, request) store.update_item(downstream, request.user.id) else: with store.bulk_operations(downstream.usage_key.context_key): - children_blocks = sync_from_upstream_container(downstream=downstream, user=request.user) + upstream_children = sync_from_upstream_container(downstream=downstream, user=request.user) + downstream_children = downstream.get_children() + # For now in this BETA version of syncing, we do really dumb 1:1 child matching + # that will blow away any changes if the downstream container has had blocks added, + # re-ordered, or deleted. + # In the future, we need to do something more sophisticated. + + # Delete any "extra" children in the downstream + while len(downstream_children) > len(upstream_children): + del downstream_children[-1] + # Sync the children: notices = [] - children_blocks_usage_keys = [] - for child_block in children_blocks: - notices.append(sync_library_content(child_block, request, store, remove_upstream_link=True)) - children_blocks_usage_keys.append(child_block.usage_key) - downstream.children = children_blocks_usage_keys + for i in range(len(upstream_children)): + upstream_child = upstream_children[i] + assert isinstance(upstream_child, LibraryXBlockMetadata) # for now we only support units + downstream_child = downstream_children[i] if i < len(downstream_children) else None + + if downstream_child and downstream_child.upstream != upstream_child.usage_key: + # This downstream block was added, or re-ordered, or no longer aligns with an upstream block. + store.delete_item(downstream_child.usage_key, user_id=request.user.id) + downstream_child = None + + if not downstream_child: + downstream_child = store.create_child( + parent_usage_key=downstream.usage_key, + position=i, + user_id=request.user.id, + block_type=upstream_child.usage_key.block_type, + # TODO: Can we generate a unique but friendly block_id, perhaps using upstream block_id + block_id=f"{upstream_child.usage_key.block_type}{uuid4().hex[:8]}", + fields={ + "upstream": str(upstream_child.usage_key), + }, + ) + result = sync_library_content(downstream=downstream_child, request=request, store=store) + notices.append(result) store.update_item(downstream, request.user.id) static_file_notices = concat_static_file_notices(notices) return static_file_notices diff --git a/cms/lib/xblock/upstream_sync_container.py b/cms/lib/xblock/upstream_sync_container.py index 3837288d6d6c..44e3b429ec05 100644 --- a/cms/lib/xblock/upstream_sync_container.py +++ b/cms/lib/xblock/upstream_sync_container.py @@ -126,7 +126,7 @@ def _update_non_customizable_fields(*, upstream: lib_api.ContainerMetadata, down # For now, there's nothing to do here - containers don't have any non-customizable fields. -def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: +def _update_tags(*, upstream: lib_api.ContainerMetadata, downstream: XBlock) -> None: """ Update tags from `upstream` to `downstream` """ @@ -134,6 +134,6 @@ def _update_tags(*, upstream: XBlock, downstream: XBlock) -> None: # For any block synced with an upstream, copy the tags as read_only # This keeps tags added locally. copy_tags_as_read_only( - str(upstream.location), - str(downstream.location), + str(upstream.container_key), + str(downstream.usage_key), ) From 3dfcc26edf24fcaa2455b30eeb0d53a11d41f27b Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 22:01:34 -0700 Subject: [PATCH 40/46] test: continue building out integration tests --- .../tests/test_downstream_sync_integration.py | 8 +++---- cms/lib/xblock/upstream_sync.py | 24 ++++++++++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index 73501bc268b1..08a6289a8a9c 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -60,7 +60,7 @@ def _get_sync_status(self, usage_key: str): def _sync_downstream(self, usage_key: str): return self._api('post', f"/api/contentstore/v2/downstreams/{usage_key}/sync", {}, expect_response=200) - + def _get_course_block_olx(self, usage_key: str): data = self._api('get', f'/api/olx-export/v1/xblock/{usage_key}/', {}, expect_response=200) return data["blocks"][data["root_block_id"]]["olx"] @@ -82,7 +82,7 @@ def _create_block_from_upstream( "parent_locator": parent_usage_key, "library_content_key": upstream_key, }, expect_response=expect_response) - + def _update_course_block_fields(self, usage_key: str, fields: dict[str, Any] = None): """ Update fields of an XBlock """ return self._api('patch', f"/xblock/{usage_key}", { @@ -210,7 +210,7 @@ def test_unit_sync(self): """ Test that we can sync a unit from the library into the course """ - # Create a "vertical" block based on a "unit" container: + # 1️⃣ Create a "vertical" block in the course based on a "unit" container: downstream_unit = self._create_block_from_upstream( block_category="vertical", parent_usage_key=str(self.course_subsection.usage_key), @@ -224,7 +224,7 @@ def test_unit_sync(self): 'version_declined': None, 'ready_to_sync': False, 'error_message': None, - 'upstream_link': 'http://course-authoring-mfe/library/lib:CL-TEST:testlib/units' + # 'upstream_link': 'http://course-authoring-mfe/library/lib:CL-TEST:testlib/units/...' }) assert status["upstream_link"].startswith("http://course-authoring-mfe/library/") assert status["upstream_link"].endswith(f"/units/{self.upstream_unit['id']}") diff --git a/cms/lib/xblock/upstream_sync.py b/cms/lib/xblock/upstream_sync.py index 3708f803e880..ab0eef333641 100644 --- a/cms/lib/xblock/upstream_sync.py +++ b/cms/lib/xblock/upstream_sync.py @@ -99,13 +99,13 @@ def upstream_link(self) -> str | None: """ Link to edit/view upstream block in library. """ - if self.version_available is None or self.upstream_ref is None: + if self.version_available is None or self.upstream_key is None: return None - try: - usage_key = LibraryUsageLocatorV2.from_string(self.upstream_ref) - except InvalidKeyError: - return None - return _get_library_xblock_url(usage_key) + if isinstance(self.upstream_key, LibraryUsageLocatorV2): + return _get_library_xblock_url(self.upstream_key) + if isinstance(self.upstream_key, LibraryContainerLocator): + return _get_library_container_url(self.upstream_key) + return None def to_json(self) -> dict[str, t.Any]: """ @@ -269,6 +269,18 @@ def _get_library_xblock_url(usage_key: LibraryUsageLocatorV2): return library_url +def _get_library_container_url(container_key: LibraryContainerLocator): + """ + Gets authoring url for given container_key. + """ + library_url = None + if mfe_base_url := settings.COURSE_AUTHORING_MICROFRONTEND_URL: # type: ignore + library_key = container_key.lib_key + if container_key.container_type == "unit": + library_url = f'{mfe_base_url}/library/{library_key}/units/{container_key}' + return library_url + + class UpstreamSyncMixin(XBlockMixin): """ Allows an XBlock in the CMS to be associated & synced with an upstream. From 29324b9b09ec62949fcd50d27de70db3f2745723 Mon Sep 17 00:00:00 2001 From: Jillian Vogel Date: Thu, 24 Apr 2025 13:57:34 +0930 Subject: [PATCH 41/46] feat: use existing can_edit flag and remove added parent_has_upstream Also fixes cause of some test failures. --- cms/djangoapps/contentstore/views/preview.py | 7 ++----- .../xblock_storage_handlers/view_handlers.py | 5 +---- cms/templates/studio_xblock_wrapper.html | 13 +++++++------ 3 files changed, 10 insertions(+), 15 deletions(-) diff --git a/cms/djangoapps/contentstore/views/preview.py b/cms/djangoapps/contentstore/views/preview.py index c7b7bc511d57..a5e3b575e3f8 100644 --- a/cms/djangoapps/contentstore/views/preview.py +++ b/cms/djangoapps/contentstore/views/preview.py @@ -300,14 +300,12 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): selected_groups_label = _('Access restricted to: {list_of_groups}').format(list_of_groups=selected_groups_label) # lint-amnesty, pylint: disable=line-too-long course = modulestore().get_course(xblock.location.course_key) - if root_xblock.upstream and str(root_xblock.upstream).startswith('lct:'): + if root_xblock and root_xblock.upstream and str(root_xblock.upstream).startswith('lct:'): can_edit = False can_add = False - parent_has_upstream = True else: can_edit = context.get('can_edit', True) can_add = context.get('can_add', True) - parent_has_upstream = False # Is this a course or a library? is_course = xblock.context_key.is_course @@ -323,7 +321,7 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): 'is_root': is_root, 'is_reorderable': is_reorderable, 'can_edit': can_edit, - 'can_edit_visibility': context.get('can_edit_visibility', is_course), + 'can_edit_visibility': can_edit and context.get('can_edit_visibility', is_course), 'course_authoring_url': settings.COURSE_AUTHORING_MICROFRONTEND_URL, 'is_loading': context.get('is_loading', False), 'is_selected': context.get('is_selected', False), @@ -337,7 +335,6 @@ def _studio_wrap_xblock(xblock, view, frag, context, display_name_only=False): 'language': getattr(course, 'language', None), 'is_course': is_course, 'tags_count': tags_count, - 'parent_has_upstream': parent_has_upstream, } add_webpack_js_to_fragment(frag, "js/factories/xblock_validation") diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index 21086afb3e30..a65e9debaa6d 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -1316,10 +1316,7 @@ def create_xblock_info( # lint-amnesty, pylint: disable=too-many-statements if is_xblock_unit and summary_configuration.is_enabled(): xblock_info["summary_configuration_enabled"] = summary_configuration.is_summary_enabled(xblock_info['id']) - if xblock.upstream: - xblock_info["upstream"] = str(xblock.upstream) - else: - xblock_info["upstream"] = None + xblock_info["upstream"] = str(xblock.upstream) if xblock.upstream else None return xblock_info diff --git a/cms/templates/studio_xblock_wrapper.html b/cms/templates/studio_xblock_wrapper.html index 1dc58cbb6f61..c705c2e6adf5 100644 --- a/cms/templates/studio_xblock_wrapper.html +++ b/cms/templates/studio_xblock_wrapper.html @@ -211,12 +211,13 @@
  • % endif - % elif not show_inline and not parent_has_upstream: -
  • - - ${_("Details")} - -
  • + % if not show_inline: +
  • + + ${_("Details")} + +
  • + % endif % endif % endif From 7ac5109bbe33b06becb9dacfb1cd5ba800c64ef7 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 22:32:16 -0700 Subject: [PATCH 42/46] test: continue building out integration tests --- .../tests/test_downstream_sync_integration.py | 43 +++++++++++++++++++ .../xblock_storage_handlers/view_handlers.py | 2 +- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index 08a6289a8a9c..b6601e90307c 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -65,6 +65,12 @@ def _get_course_block_olx(self, usage_key: str): data = self._api('get', f'/api/olx-export/v1/xblock/{usage_key}/', {}, expect_response=200) return data["blocks"][data["root_block_id"]]["olx"] + # def _get_course_block_fields(self, usage_key: str): + # return self._api('get', f'/xblock/{usage_key}', {}, expect_response=200) + + # def _get_course_block_children(self, usage_key: str): + # return self._api('get', f'/xblock/container/{usage_key}', {}, expect_response=200) + def _create_block_from_upstream( self, block_category: str, @@ -228,3 +234,40 @@ def test_unit_sync(self): }) assert status["upstream_link"].startswith("http://course-authoring-mfe/library/") assert status["upstream_link"].endswith(f"/units/{self.upstream_unit['id']}") + + # Check that the downstream container matches our expectations. + # Note that: + # (1) Every XBlock has an "upstream" field + # (2) some "downstream only" fields like weight and max_attempts are omitted. + self.assertXmlEqual(self._get_course_block_olx(downstream_unit["locator"]), f""" + + This is the HTML. + multiple choice... + multi select... + + """) diff --git a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py index b57dbb2da365..1269e11d1be4 100644 --- a/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py +++ b/cms/djangoapps/contentstore/xblock_storage_handlers/view_handlers.py @@ -542,6 +542,7 @@ def sync_library_content(downstream: XBlock, request, store) -> StaticFileNotice else: with store.bulk_operations(downstream.usage_key.context_key): upstream_children = sync_from_upstream_container(downstream=downstream, user=request.user) + store.update_item(downstream, request.user.id) downstream_children = downstream.get_children() # For now in this BETA version of syncing, we do really dumb 1:1 child matching # that will blow away any changes if the downstream container has had blocks added, @@ -577,7 +578,6 @@ def sync_library_content(downstream: XBlock, request, store) -> StaticFileNotice ) result = sync_library_content(downstream=downstream_child, request=request, store=store) notices.append(result) - store.update_item(downstream, request.user.id) static_file_notices = concat_static_file_notices(notices) return static_file_notices From bf39a3ebdb3cad28cf534656b43b4875f39733a7 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 22:32:30 -0700 Subject: [PATCH 43/46] fix: blockserializer wasn't always serializing all HTML block fields --- openedx/core/lib/xblock_serializer/block_serializer.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index 53be26937d4a..b8eabfcb4f11 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -152,6 +152,9 @@ def _serialize_html_block(self, block) -> etree.Element: olx_node.attrib["editor"] = block.editor if block.use_latex_compiler: olx_node.attrib["use_latex_compiler"] = "true" + for field_name in block.fields: + if field_name.startswith("upstream") and block.fields[field_name].is_set_on(block): + olx_node.attrib[field_name] = str(getattr(block, field_name)) # Escape any CDATA special chars escaped_block_data = block.data.replace("]]>", "]]>") From a214cc4fe658fe382308179ef9ac42ecbd530e80 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 23:01:30 -0700 Subject: [PATCH 44/46] test: continue building out integration tests --- .../tests/test_downstream_sync_integration.py | 87 ++++++++++++++++++- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index b6601e90307c..56394794da54 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -6,7 +6,9 @@ from xml.etree import ElementTree import ddt +from opaque_keys.edx.keys import UsageKey +from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory from openedx.core.djangoapps.content_libraries.tests.base import ContentLibrariesRestApiTest @@ -68,8 +70,10 @@ def _get_course_block_olx(self, usage_key: str): # def _get_course_block_fields(self, usage_key: str): # return self._api('get', f'/xblock/{usage_key}', {}, expect_response=200) - # def _get_course_block_children(self, usage_key: str): - # return self._api('get', f'/xblock/container/{usage_key}', {}, expect_response=200) + def _get_course_block_children(self, usage_key: str) -> list[str]: + """ Get the IDs of the child XBlocks of the given XBlock """ + # TODO: is there really no REST API to get the children of an XBlock in Studio? + return [str(k) for k in modulestore().get_item(UsageKey.from_string(usage_key), depth=0).children] def _create_block_from_upstream( self, @@ -218,6 +222,9 @@ def test_unit_sync(self): """ # 1️⃣ Create a "vertical" block in the course based on a "unit" container: downstream_unit = self._create_block_from_upstream( + # The API consumer needs to specify "vertical" here, even though upstream is "unit". + # In the future we could create a nicer REST API endpoint for this that's not part of + # the messy '/xblock/' API and which auto-detects the types based on the upstream_key. block_category="vertical", parent_usage_key=str(self.course_subsection.usage_key), upstream_key=self.upstream_unit["id"], @@ -271,3 +278,79 @@ def test_unit_sync(self): >multi select... """) + + # 2️⃣ Now, lets modify the upstream problem 1: + + self._set_library_block_olx( + self.upstream_problem1["id"], + 'multiple choice v2...' + ) + self._publish_container(self.upstream_unit["id"]) + + status = self._get_sync_status(downstream_unit["locator"]) + self.assertDictContainsEntries(status, { + 'upstream_ref': self.upstream_unit["id"], # e.g. 'lct:CL-TEST:testlib:unit:u1' + 'version_available': 2, # <--- not updated since we didn't directly modify the unit + 'version_synced': 2, + 'version_declined': None, + # FIXME: ready_to_sync should be true, since a child block needs syncing. + # This may need to be fixed post-Teak, as syncing the children directly is still possible. + 'ready_to_sync': False, + 'error_message': None, + }) + + # Check the upstream/downstream status of [one of] the children + + downstream_problem1 = self._get_course_block_children(downstream_unit["locator"])[1] + assert "type@problem" in downstream_problem1 + self.assertDictContainsEntries(self._get_sync_status(downstream_problem1), { + 'upstream_ref': self.upstream_problem1["id"], + 'version_available': 3, # <--- updated since we modified the problem + 'version_synced': 2, + 'version_declined': None, + 'ready_to_sync': True, # <--- updated + 'error_message': None, + }) + + # 3️⃣ Now, sync and check the resulting OLX of the downstream + + self._sync_downstream(downstream_unit["locator"]) + + self.assertXmlEqual(self._get_course_block_olx(downstream_unit["locator"]), f""" + + This is the HTML. + + multiple choice v2... + multi select... + + """) + + # TODO: tests where + # (1) a unit is synced to a course + # (2) an upstream [or downstream] block is deleted or added, and changes are published. + # (3) the unit is synced From a20baec026c92666d3e79fd4d20839c4476848b4 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 23:02:35 -0700 Subject: [PATCH 45/46] chore: quality fix --- .../views/tests/test_downstream_sync_integration.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py index 56394794da54..1872a1b0e495 100644 --- a/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py +++ b/cms/djangoapps/contentstore/rest_api/v2/views/tests/test_downstream_sync_integration.py @@ -76,12 +76,12 @@ def _get_course_block_children(self, usage_key: str) -> list[str]: return [str(k) for k in modulestore().get_item(UsageKey.from_string(usage_key), depth=0).children] def _create_block_from_upstream( - self, - block_category: str, - parent_usage_key: str, - upstream_key: str, - expect_response: int = 200, - ): + self, + block_category: str, + parent_usage_key: str, + upstream_key: str, + expect_response: int = 200, + ): """ Call the CMS API for inserting an XBlock that's cloned from a library item. i.e. copy a *published* library block into a course, and create an From ecdeafd6a00c3e70d1f49b223c15b5882752fd19 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 23 Apr 2025 23:04:27 -0700 Subject: [PATCH 46/46] chore: updated query count in test, from serializer/dataclass change --- .../content_tagging/tests/test_objecttag_export_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openedx/core/djangoapps/content_tagging/tests/test_objecttag_export_helpers.py b/openedx/core/djangoapps/content_tagging/tests/test_objecttag_export_helpers.py index f196549dd1a9..eaeea3bda225 100644 --- a/openedx/core/djangoapps/content_tagging/tests/test_objecttag_export_helpers.py +++ b/openedx/core/djangoapps/content_tagging/tests/test_objecttag_export_helpers.py @@ -442,7 +442,7 @@ def test_build_library_object_tree(self) -> None: """ Test if we can export a library """ - with self.assertNumQueries(8): + with self.assertNumQueries(11): tagged_library = build_object_tree_with_objecttags(self.library.key, self.all_library_object_tags) assert tagged_library == self.expected_library_tagged_xblock