diff --git a/tests/common/db/packaging.py b/tests/common/db/packaging.py index 3b97b20cbd92..470f233a34c4 100644 --- a/tests/common/db/packaging.py +++ b/tests/common/db/packaging.py @@ -120,7 +120,8 @@ class Meta: lambda o: hashlib.blake2b(o.filename.encode("utf8"), digest_size=32).hexdigest() ) upload_time = factory.Faker( - "date_time_between_dates", datetime_start=datetime.datetime(2008, 1, 1) + "date_time_between_dates", + datetime_start=datetime.datetime(2008, 1, 1), ) path = factory.LazyAttribute( lambda o: "/".join( diff --git a/tests/functional/_fixtures/README.md b/tests/functional/_fixtures/README.md index f959f57bc7ee..5293d4b3a172 100644 --- a/tests/functional/_fixtures/README.md +++ b/tests/functional/_fixtures/README.md @@ -6,3 +6,11 @@ This is from https://pypi.org/project/sampleproject/3.0.0/#files, get it with: ``` $ wget https://files.pythonhosted.org/packages/67/2a/9f056e5fa36e43ef1037ff85581a2963cde420457de0ef29c779d41058ca/sampleproject-3.0.0.tar.gz ``` + +## `sampleproject-3.0.0-py3-none-any.whl` + +This is from https://pypi.org/project/sampleproject/3.0.0/#files, get it with: + +``` +$ wget https://files.pythonhosted.org/packages/ec/a8/5ec62d18adde798d33a170e7f72930357aa69a60839194c93eb0fb05e59c/sampleproject-3.0.0-py3-none-any.whl +``` diff --git a/tests/functional/_fixtures/sampleproject-3.0.0-py3-none-any.whl b/tests/functional/_fixtures/sampleproject-3.0.0-py3-none-any.whl new file mode 100644 index 000000000000..c36a12ca4c94 Binary files /dev/null and b/tests/functional/_fixtures/sampleproject-3.0.0-py3-none-any.whl differ diff --git a/tests/functional/forklift/test_legacy.py b/tests/functional/forklift/test_legacy.py index 3f0bc7dbd684..f10320ffb94b 100644 --- a/tests/functional/forklift/test_legacy.py +++ b/tests/functional/forklift/test_legacy.py @@ -75,7 +75,14 @@ def test_remove_doc_upload(webtest): ("/legacy/", {":action": "file_upload", "protocol_version": "1"}), ], ) -def test_file_upload(webtest, upload_url, additional_data): +@pytest.mark.parametrize( + "staged_release", + [ + True, + False, + ], +) +def test_file_upload(webtest, upload_url, additional_data, staged_release): user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password") # Construct the macaroon @@ -118,9 +125,15 @@ def test_file_upload(webtest, upload_url, additional_data): params.add("classifiers", "Programming Language :: Python :: 3.10") params.add("classifiers", "Programming Language :: Python :: 3.11") + headers = { + "Authorization": f"Basic {credentials}", + } + if staged_release: + headers["X-PyPI-Is-Staged"] = "1" + webtest.post( upload_url, - headers={"Authorization": f"Basic {credentials}"}, + headers=headers, params=params, upload_files=[("content", "sampleproject-3.0.0.tar.gz", content)], status=HTTPStatus.OK, @@ -134,6 +147,116 @@ def test_file_upload(webtest, upload_url, additional_data): assert len(project.releases) == 1 release = project.releases[0] assert release.version == "3.0.0" + assert release.published != staged_release + + +@pytest.mark.parametrize( + "stage_first_file", + [ + True, + False, + ], +) +@pytest.mark.parametrize( + "stage_second_file", + [ + True, + False, + ], +) +def test_stage_release(webtest, stage_first_file, stage_second_file): + user = UserFactory.create(with_verified_primary_email=True, clear_pwd="password") + + # Construct the macaroon + dm = MacaroonFactory.create( + user_id=user.id, + caveats=[caveats.RequestUser(user_id=str(user.id))], + ) + + m = pymacaroons.Macaroon( + location="localhost", + identifier=str(dm.id), + key=dm.key, + version=pymacaroons.MACAROON_V2, + ) + for caveat in dm.caveats: + m.add_first_party_caveat(caveats.serialize(caveat)) + serialized_macaroon = f"pypi-{m.serialize()}" + + credentials = base64.b64encode(f"__token__:{serialized_macaroon}".encode()).decode( + "utf-8" + ) + + with open("./tests/functional/_fixtures/sampleproject-3.0.0.tar.gz", "rb") as f: + first_file = f.read() + + with open( + "./tests/functional/_fixtures/sampleproject-3.0.0-py3-none-any.whl", "rb" + ) as f: + second_file = f.read() + + webtest.post( + "/legacy/?:action=file_upload", + headers={ + "Authorization": f"Basic {credentials}", + **({"X-PyPI-Is-Staged": "1"} if stage_first_file else {}), + }, + params=MultiDict( + { + "name": "sampleproject", + "sha256_digest": ( + "117ed88e5db073bb92969a7545745fd977ee85b7019706dd256a64058f70963d" + ), + "filetype": "sdist", + "metadata_version": "2.1", + "version": "3.0.0", + "classifiers": "Programming Language :: Python :: 3.11", + } + ), + upload_files=[("content", "sampleproject-3.0.0.tar.gz", first_file)], + status=HTTPStatus.OK, + ) + + assert user.projects + assert len(user.projects) == 1 + project = user.projects[0] + assert project.name == "sampleproject" + assert project.releases + assert len(project.releases) == 1 + release = project.releases[0] + + assert release.published != stage_first_file + + second_request_status = ( + HTTPStatus.BAD_REQUEST + if stage_second_file and not stage_first_file + else HTTPStatus.OK + ) + webtest.post( + "/legacy/?:action=file_upload", + headers={ + "Authorization": f"Basic {credentials}", + **({"X-PyPI-Is-Staged": "1"} if stage_second_file else {}), + }, + params=MultiDict( + { + "name": "sampleproject", + "sha256_digest": ( + "2e52702990c22cf1ce50206606b769fe0dbd5646a32873916144bd5aec5473b3" + ), + "filetype": "bdist_wheel", + "metadata_version": "2.1", + "version": "3.0.0", + "pyversion": "3.11", + "classifiers": "Programming Language :: Python :: 3.11", + } + ), + upload_files=[("content", "sampleproject-3.0.0-py3-none-any.whl", second_file)], + status=second_request_status, + ) + + if second_request_status == HTTPStatus.OK: + assert release.published != stage_second_file def test_duplicate_file_upload_error(webtest): diff --git a/tests/unit/forklift/test_legacy.py b/tests/unit/forklift/test_legacy.py index 3ba814a87bef..f941d9080b6a 100644 --- a/tests/unit/forklift/test_legacy.py +++ b/tests/unit/forklift/test_legacy.py @@ -39,8 +39,11 @@ from warehouse.admin.flags import AdminFlag, AdminFlagValue from warehouse.attestations.interfaces import IIntegrityService from warehouse.classifiers.models import Classifier +from warehouse.events.models import HasEvents +from warehouse.events.tags import EventTag from warehouse.forklift import legacy, metadata from warehouse.macaroons import IMacaroonService, caveats, security_policy +from warehouse.metrics import IMetricsService from warehouse.oidc.interfaces import SignedClaims from warehouse.oidc.utils import PublisherTokenContext from warehouse.packaging.interfaces import IFileStorage, IProjectService @@ -567,6 +570,44 @@ def test_is_duplicate_false(self, pyramid_config, db_request): assert legacy._is_duplicate_file(db_request.db, filename, wrong_hashes) is False +# Deduplication helpers +def fileadd_event(filename, identity, release, project): + """Constructs the FileAdd event mapping.""" + test_with_user = not hasattr(identity, "publisher") + return { + "filename": filename, + "submitted_by": ( + identity.username if test_with_user else "OpenID created token" + ), + "canonical_version": release.canonical_version, + "publisher_url": ( + f"{identity.publisher.publisher_url()}/commit/somesha" + if not test_with_user + else None + ), + "project_id": str(project.id), + "uploaded_via_trusted_publisher": not test_with_user, + } + + +def release_event(identity, release): + """Constructs the Release event mapping.""" + test_with_user = not hasattr(identity, "publisher") + return { + "submitted_by": ( + identity.username if test_with_user else "OpenID created token" + ), + "canonical_version": release.canonical_version, + "publisher_url": ( + f"{identity.publisher.publisher_url()}/commit/somesha" + if not test_with_user + else None + ), + "uploaded_via_trusted_publisher": not test_with_user, + "published": release.published, + } + + class TestFileUpload: def test_fails_disallow_new_upload(self, pyramid_config, pyramid_request): pyramid_request.flags = pretend.stub( @@ -3457,9 +3498,6 @@ def test_upload_succeeds_creates_release( expected_version, test_with_user, ): - from warehouse.events.models import HasEvents - from warehouse.events.tags import EventTag - project = ProjectFactory.create() if test_with_user: identity = UserFactory.create() @@ -3575,46 +3613,18 @@ def test_upload_succeeds_creates_release( ] # Ensure that all of our events have been created - release_event = { - "submitted_by": ( - identity.username if test_with_user else "OpenID created token" - ), - "canonical_version": release.canonical_version, - "publisher_url": ( - f"{identity.publisher.publisher_url()}/commit/somesha" - if not test_with_user - else None - ), - "uploaded_via_trusted_publisher": not test_with_user, - } - - fileadd_event = { - "filename": filename, - "submitted_by": ( - identity.username if test_with_user else "OpenID created token" - ), - "canonical_version": release.canonical_version, - "publisher_url": ( - f"{identity.publisher.publisher_url()}/commit/somesha" - if not test_with_user - else None - ), - "project_id": str(project.id), - "uploaded_via_trusted_publisher": not test_with_user, - } - assert record_event.calls == [ pretend.call( mock.ANY, tag=EventTag.Project.ReleaseAdd, request=db_request, - additional=release_event, + additional=release_event(identity, release), ), pretend.call( mock.ANY, tag=EventTag.File.FileAdd, request=db_request, - additional=fileadd_event, + additional=fileadd_event(filename, identity, release, project), ), ] @@ -3625,8 +3635,6 @@ def test_upload_succeeds_with_valid_attestation( db_request, integrity_service, ): - from warehouse.events.models import HasEvents - project = ProjectFactory.create() version = "1.0" publisher = GitHubPublisherFactory.create(projects=[project]) @@ -3730,8 +3738,6 @@ def test_upload_fails_attestation_error( db_request, invalid_attestations, ): - from warehouse.events.models import HasEvents - project = ProjectFactory.create() version = "1.0" publisher = GitHubPublisherFactory.create(projects=[project]) @@ -5282,6 +5288,351 @@ def test_upload_fails_when_license_and_license_expression_are_present( ) +class TestStagedRelease: + @staticmethod + def get_identity(test_with_user, project, db_request, pyramid_config): + if test_with_user: + identity = UserFactory.create() + EmailFactory.create(user=identity) + RoleFactory.create(user=identity, project=project) + db_request.user = identity + else: + publisher = GitHubPublisherFactory.create(projects=[project]) + claims = {"sha": "somesha"} + identity = PublisherTokenContext(publisher, SignedClaims(claims)) + db_request.oidc_publisher = identity.publisher + db_request.oidc_claims = identity.claims + + db_request.user_agent = "warehouse-tests/6.6.6" + + pyramid_config.testing_securitypolicy(identity=identity) + return identity + + @pytest.mark.parametrize( + "test_with_user", + [ + True, + False, + ], + ) + def test_upload_succeeds_with_stage_header( + self, test_with_user, monkeypatch, db_request, pyramid_config, metrics + ): + project = ProjectFactory.create() + identity = self.get_identity( + test_with_user, project, db_request, pyramid_config + ) + + filename = "{}-{}.tar.gz".format( + project.normalized_name.replace("-", "_"), "1.0" + ) + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": "1.0", + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + db_request.headers["X-PyPI-Is-Staged"] = "1" + + storage_service = pretend.stub(store=lambda path, filepath, meta: None) + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: storage_service, + IMetricsService: metrics, + }.get(svc) + + record_event = pretend.call_recorder( + lambda self, *, tag, request=None, additional: None + ) + monkeypatch.setattr(HasEvents, "record_event", record_event) + + resp = legacy.file_upload(db_request) + assert resp.status_code == 200 + + # Ensure that a Release object has been created. + release = ( + db_request.db.query(Release) + .filter((Release.project == project) & (Release.version == "1.0")) + .execution_options(include_staged=True) + .one() + ) + + assert not release.published + + # Ensure that a File object has been created. + db_request.db.query(File).filter( + (File.release == release) & (File.filename == filename) + ).one() + + # Ensure that all of our journal entries have been created + journals = ( + db_request.db.query(JournalEntry) + .options(joinedload(JournalEntry.submitted_by)) + .order_by("submitted_date", "id") + .all() + ) + assert [(j.name, j.version, j.action, j.submitted_by) for j in journals] == [ + ( + release.project.name, + release.version, + "new release", + identity if test_with_user else None, + ), + ( + release.project.name, + release.version, + f"add source file {filename}", + identity if test_with_user else None, + ), + ] + + # Ensure that all of our events have been created + assert record_event.calls == [ + pretend.call( + mock.ANY, + tag=EventTag.Project.ReleaseAdd, + request=db_request, + additional=release_event(identity, release), + ), + pretend.call( + mock.ANY, + tag=EventTag.File.FileAdd, + request=db_request, + additional=fileadd_event(filename, identity, release, project), + ), + ] + + @pytest.mark.parametrize( + "test_with_user", + [ + True, + False, + ], + ) + def test_upload_succeeds_on_staged_release( + self, test_with_user, monkeypatch, db_request, pyramid_config, metrics + ): + project = ProjectFactory.create() + identity = self.get_identity( + test_with_user, project, db_request, pyramid_config + ) + + # Create a release and add a file + release = ReleaseFactory.create(project=project, version="1.0", published=False) + FileFactory.create(release=release, packagetype="bdist_wheel") + + filename = "{}-{}.tar.gz".format( + project.normalized_name.replace("-", "_"), "1.0" + ) + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": "1.0", + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: pretend.stub(store=lambda path, filepath, meta: None), + IMetricsService: metrics, + }.get(svc) + + record_event = pretend.call_recorder( + lambda self, *, tag, request=None, additional: None + ) + monkeypatch.setattr(HasEvents, "record_event", record_event) + + assert not release.published + + resp = legacy.file_upload(db_request) + assert resp.status_code == 200 + + # Ensure that a Release object has been created. + release = ( + db_request.db.query(Release) + .filter((Release.project == project) & (Release.version == "1.0")) + .one() + ) + + assert release.published + + # Ensure that a File object has been created. + db_request.db.query(File).filter( + (File.release == release) & (File.filename == filename) + ).one() + + # Ensure that all of our journal entries have been created + journals = ( + db_request.db.query(JournalEntry) + .options(joinedload(JournalEntry.submitted_by)) + .order_by("submitted_date", "id") + .all() + ) + assert [(j.name, j.version, j.action, j.submitted_by) for j in journals] == [ + ( + release.project.name, + release.version, + f"add source file {filename}", + identity if test_with_user else None, + ), + ( + release.project.name, + release.version, + "publish release", + identity if test_with_user else None, + ), + ] + + # Ensure that all of our events have been created + assert record_event.calls == [ + pretend.call( + mock.ANY, + tag=EventTag.File.FileAdd, + request=db_request, + additional=fileadd_event(filename, identity, release, project), + ), + pretend.call( + mock.ANY, + tag=EventTag.Project.ReleasePublish, + request=db_request, + additional={ + "submitted_by": ( + identity.username if test_with_user else "OpenID created token" + ), + "uploaded_via_trusted_publisher": not test_with_user, + "canonical_version": release.canonical_version, + }, + ), + ] + + @pytest.mark.parametrize( + "test_with_user", + [ + True, + False, + ], + ) + def test_upload_succeeds_on_staged_release_with_stage_header( + self, test_with_user, monkeypatch, db_request, pyramid_config, metrics + ): + project = ProjectFactory.create() + identity = self.get_identity( + test_with_user, project, db_request, pyramid_config + ) + + release = ReleaseFactory.create(project=project, version="1.0", published=False) + FileFactory.create(release=release, packagetype="bdist_wheel") + + filename = "{}-{}.tar.gz".format( + project.normalized_name.replace("-", "_"), "1.0" + ) + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": "1.0", + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + db_request.headers["X-PyPI-Is-Staged"] = "1" + + db_request.find_service = lambda svc, name=None, context=None: { + IFileStorage: pretend.stub(store=lambda path, filepath, meta: None), + IMetricsService: metrics, + }.get(svc) + + record_event = pretend.call_recorder( + lambda self, *, tag, request=None, additional: None + ) + monkeypatch.setattr(HasEvents, "record_event", record_event) + + resp = legacy.file_upload(db_request) + assert resp.status_code == 200 + + # Ensure that the release is still not published + release = ( + db_request.db.query(Release) + .filter((Release.project == project) & (Release.version == "1.0")) + .execution_options(include_staged=True) + .one() + ) + assert not release.published + + # Ensure that a File object has been created. + db_request.db.query(File).filter( + (File.release == release) & (File.filename == filename) + ).one() + + assert record_event.calls == [ + pretend.call( + mock.ANY, + tag=EventTag.File.FileAdd, + request=db_request, + additional=fileadd_event(filename, identity, release, project), + ), + ] + + def test_upload_fails_with_staged_on_existing_release( + self, monkeypatch, pyramid_config, db_request + ): + project = ProjectFactory.create() + self.get_identity(True, project, db_request, pyramid_config) + ReleaseFactory.create(project=project, version="1.0", published=True) + filename = "{}-{}.tar.gz".format( + project.normalized_name.replace("-", "_"), "1.0" + ) + + db_request.POST = MultiDict( + { + "metadata_version": "1.2", + "name": project.name, + "version": "1.0", + "summary": "This is my summary!", + "filetype": "sdist", + "md5_digest": _TAR_GZ_PKG_MD5, + "content": pretend.stub( + filename=filename, + file=io.BytesIO(_TAR_GZ_PKG_TESTDATA), + type="application/tar", + ), + } + ) + db_request.headers["X-PyPI-Is-Staged"] = "1" + + with pytest.raises(HTTPBadRequest) as excinfo: + legacy.file_upload(db_request) + + resp = excinfo.value + assert resp.status_code == 400 + assert resp.status == "400 Release already published." + + def test_submit(pyramid_request): resp = legacy.submit(pyramid_request) diff --git a/tests/unit/legacy/api/test_json.py b/tests/unit/legacy/api/test_json.py index 3a7b029e8c7e..23973fa4bb73 100644 --- a/tests/unit/legacy/api/test_json.py +++ b/tests/unit/legacy/api/test_json.py @@ -118,6 +118,91 @@ def test_all_non_prereleases_yanked(self, monkeypatch, db_request): db_request.matchdict = {"name": project.normalized_name} assert json.latest_release_factory(db_request) == release + def test_with_staged(self, db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + ReleaseFactory.create(project=project, version="2.0", published=False) + db_request.matchdict = {"name": project.normalized_name} + assert json.latest_release_factory(db_request) == release + + def test_only_staged(self, db_request): + project = ProjectFactory.create() + ReleaseFactory.create(project=project, version="1.0", published=False) + db_request.matchdict = {"name": project.normalized_name} + resp = json.latest_release_factory(db_request) + + assert isinstance(resp, HTTPNotFound) + _assert_has_cors_headers(resp.headers) + + @pytest.mark.parametrize( + ("release0_state", "release1_state", "release2_state", "latest_release"), + [ + ("published", "published", "published", 2), + ("published", "published", "staged", 1), + ("published", "published", "yanked", 1), + ("published", "staged", "published", 2), + ("published", "staged", "staged", 0), + ("published", "staged", "yanked", 0), + ("published", "yanked", "published", 2), + ("published", "yanked", "staged", 0), + ("published", "yanked", "yanked", 0), + ("staged", "published", "published", 2), + ("staged", "published", "staged", 1), + ("staged", "published", "yanked", 1), + ("staged", "staged", "published", 2), + ("staged", "staged", "staged", -1), + ("staged", "staged", "yanked", 2), # Same endpoint as none yanked + ("staged", "yanked", "published", 2), + ("staged", "yanked", "staged", 1), + ("staged", "yanked", "yanked", 2), + ("yanked", "published", "published", 2), + ("yanked", "published", "staged", 1), + ("yanked", "published", "yanked", 1), + ("yanked", "staged", "published", 2), + ("yanked", "staged", "staged", 0), + ("yanked", "staged", "yanked", 2), + ("yanked", "yanked", "published", 2), + ("yanked", "yanked", "staged", 1), + ("yanked", "yanked", "yanked", 2), + ], + ) + def test_with_mixed_states( + self, db_request, release0_state, release1_state, release2_state, latest_release + ): + project = ProjectFactory.create() + + releases = [] + for version, state in [ + ("1.0", release0_state), + ("1.1", release1_state), + ("2.0", release2_state), + ]: + if state == "published": + releases.append( + ReleaseFactory.create( + project=project, version=version, published=True + ) + ) + elif state == "staged": + releases.append( + ReleaseFactory.create( + project=project, version=version, published=False + ) + ) + else: + releases.append( + ReleaseFactory.create(project=project, version=version, yanked=True) + ) + + db_request.matchdict = {"name": project.normalized_name} + + resp = json.latest_release_factory(db_request) + if latest_release >= 0: + assert resp == releases[latest_release] + else: + assert isinstance(resp, HTTPNotFound) + _assert_has_cors_headers(resp.headers) + def test_project_quarantined(self, monkeypatch, db_request): project = ProjectFactory.create( lifecycle_status=LifecycleStatus.QuarantineEnter @@ -191,6 +276,15 @@ def test_renders(self, pyramid_config, db_request, db_session): ) ] + ReleaseFactory.create( + project=project, + version="3.1", + description=DescriptionFactory.create( + content_type=description_content_type + ), + published=False, + ) + for urlspec in project_urls: label, _, purl = urlspec.partition(",") db_session.add( diff --git a/tests/unit/packaging/test_models.py b/tests/unit/packaging/test_models.py index 8a80d58129e2..46f5138b9a99 100644 --- a/tests/unit/packaging/test_models.py +++ b/tests/unit/packaging/test_models.py @@ -28,6 +28,7 @@ Project, ProjectFactory, ProjectMacaroonWarningAssociation, + Release, ReleaseURL, ) @@ -1216,6 +1217,25 @@ def test_description_relationship(self, db_request): assert description in db_request.db.deleted +@pytest.mark.parametrize( + "published", + [ + True, + False, + ], +) +def test_filter_staged_releases(db_request, published): + DBReleaseFactory.create(published=published) + assert db_request.db.query(Release).count() == (1 if published else 0) + + +def test_filter_staged_releases_with_staged(db_request): + DBReleaseFactory.create(published=False) + assert ( + db_request.db.query(Release).execution_options(include_staged=True).count() == 1 + ) + + class TestFile: def test_requires_python(self, db_session): """ diff --git a/tests/unit/packaging/test_views.py b/tests/unit/packaging/test_views.py index f914c3e97b1b..8eba115cf851 100644 --- a/tests/unit/packaging/test_views.py +++ b/tests/unit/packaging/test_views.py @@ -135,6 +135,19 @@ def test_only_yanked_release(self, monkeypatch, db_request): assert resp is response assert release_detail.calls == [pretend.call(release, db_request)] + def test_with_staged(self, monkeypatch, db_request): + project = ProjectFactory.create() + release = ReleaseFactory.create(project=project, version="1.0") + ReleaseFactory.create(project=project, version="1.1", published=False) + + response = pretend.stub() + release_detail = pretend.call_recorder(lambda ctx, request: response) + monkeypatch.setattr(views, "release_detail", release_detail) + + resp = views.project_detail(project, db_request) + assert resp is response + assert release_detail.calls == [pretend.call(release, db_request)] + class TestReleaseDetail: def test_normalizing_name_redirects(self, db_request): @@ -202,6 +215,19 @@ def test_detail_rendered(self, db_request): yanked_reason="plaintext yanked reason", ) ] + + # Add a staged version + staged_release = ReleaseFactory.create( + project=project, + version="5.1", + description=DescriptionFactory.create( + raw="unrendered description", + html="rendered description", + content_type="text/html", + ), + published=False, + ) + files = [ FileFactory.create( release=r, @@ -209,7 +235,7 @@ def test_detail_rendered(self, db_request): python_version="source", packagetype="sdist", ) - for r in releases + for r in releases + [staged_release] ] # Create a role for each user @@ -226,6 +252,7 @@ def test_detail_rendered(self, db_request): "bdists": [], "description": "rendered description", "latest_version": project.latest_version, + # Non published version are not listed here "all_versions": [ (r.version, r.created, r.is_prerelease, r.yanked, r.yanked_reason) for r in reversed(releases) @@ -324,6 +351,10 @@ def test_long_singleline_license(self, db_request): "characters, it's really so lo..." ) + def test_created_with_published(self, db_request): + release = ReleaseFactory.create() + assert release.published is True + class TestPEP740AttestationViewer: diff --git a/warehouse/events/tags.py b/warehouse/events/tags.py index 22eaaaec92f9..eb50d51491f8 100644 --- a/warehouse/events/tags.py +++ b/warehouse/events/tags.py @@ -133,6 +133,7 @@ class Project(EventTagEnum): ReleaseRemove = "project:release:remove" ReleaseUnyank = "project:release:unyank" ReleaseYank = "project:release:yank" + ReleasePublish = "project:release:publish" RoleAdd = "project:role:add" RoleChange = "project:role:change" RoleDeclineInvite = "project:role:decline_invite" diff --git a/warehouse/forklift/legacy.py b/warehouse/forklift/legacy.py index 1dbddefd09c5..0fca7d687f92 100644 --- a/warehouse/forklift/legacy.py +++ b/warehouse/forklift/legacy.py @@ -60,6 +60,7 @@ from warehouse.forklift import metadata from warehouse.forklift.forms import UploadForm, _filetype_extension_mapping from warehouse.macaroons.models import Macaroon +from warehouse.metrics import IMetricsService from warehouse.packaging.interfaces import IFileStorage, IProjectService from warehouse.packaging.metadata_verification import verify_email, verify_url from warehouse.packaging.models import ( @@ -426,6 +427,40 @@ def _sort_releases(request: Request, project: Project): r._pypi_ordering = i +def publish_staged_release(request, project, release): + """ + Publish a staged release. + + This method assumes all preconditions are met and only modifies the release + state to 'published', creating the appropriate events. + """ + metrics = request.find_service(IMetricsService, context=None) + + request.db.add( + JournalEntry( + name=project.name, + action="publish release", + version=release.version, + submitted_by=request.user if request.user else None, + ) + ) + + project.record_event( + tag=EventTag.Project.ReleasePublish, + request=request, + additional={ + "submitted_by": ( + request.user.username if request.user else "OpenID created token" + ), + "uploaded_via_trusted_publisher": bool(request.oidc_publisher), + "canonical_version": release.canonical_version, + }, + ) + release.published = True + + metrics.increment("warehouse.publish.ok") + + @view_config( route_name="forklift.legacy.file_upload", uses_session=True, @@ -788,6 +823,9 @@ def file_upload(request): ), ) from None + # Is the current release a staged release + staged_release = bool(request.headers.get("X-PyPI-Is-Staged", False)) + # Verify any verifiable URLs project_urls = ( {} @@ -851,6 +889,7 @@ def file_upload(request): (Release.project == project) & (Release.canonical_version == canonical_version) ) + .execution_options(include_staged=True) .one() ) except MultipleResultsFound: @@ -940,6 +979,7 @@ def file_upload(request): }, uploader=request.user if request.user else None, uploaded_via=request.user_agent, + published=not staged_release, ) request.db.add(release) is_new_release = True @@ -970,6 +1010,7 @@ def file_upload(request): else None ), "uploaded_via_trusted_publisher": bool(request.oidc_publisher), + "published": not staged_release, }, ) @@ -978,6 +1019,9 @@ def file_upload(request): # at least this should be some sort of hook or trigger. _sort_releases(request, project) + if release.published is True and staged_release: + raise _exc_with_message(HTTPBadRequest, "Release already published.") + # Pull the filename out of our POST data. filename = request.POST["content"].filename @@ -1565,20 +1609,26 @@ def file_upload(request): # For existing releases, we check if any of the existing project URLs are unverified # and have been verified in the current upload. In that case, we mark them as # verified. - if not is_new_release and project_urls: - for name, release_url in release._project_urls.items(): - if ( - not release_url.verified - and name in project_urls - and project_urls[name]["url"] == release_url.url - and project_urls[name]["verified"] - ): - release_url.verified = True + if not is_new_release: + if project_urls: + for name, release_url in release._project_urls.items(): + if ( + not release_url.verified + and name in project_urls + and project_urls[name]["url"] == release_url.url + and project_urls[name]["verified"] + ): + release_url.verified = True + + if home_page_verified and not release.home_page_verified: + release.home_page_verified = True + if download_url_verified and not release.download_url_verified: + release.download_url_verified = True - if home_page_verified and not release.home_page_verified: - release.home_page_verified = True - if download_url_verified and not release.download_url_verified: - release.download_url_verified = True + # If we had a staged release and this request does not include the header, that + # means we are good to publish + if not staged_release and release.published is False: + publish_staged_release(request, project, release) request.db.flush() # flush db now so server default values are populated for celery diff --git a/warehouse/packaging/models.py b/warehouse/packaging/models.py index 8f6fa0d63bd3..0aea9d6241f2 100644 --- a/warehouse/packaging/models.py +++ b/warehouse/packaging/models.py @@ -52,10 +52,12 @@ from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import ( Mapped, + ORMExecuteState, attribute_keyed_dict, declared_attr, mapped_column, validates, + with_loader_criteria, ) from urllib3.exceptions import LocationParseError from urllib3.util import parse_url @@ -1058,6 +1060,21 @@ def ensure_monotonic_journals(config, session, flush_context, instances): return +@db.listens_for(db.Session, "do_orm_execute") +def filter_staged_release(_, state: ORMExecuteState): + if ( + state.is_select + and not state.is_column_load + and not state.is_relationship_load + and not state.statement.get_execution_options().get("include_staged", False) + ): + state.statement = state.statement.options( + with_loader_criteria( + Release, lambda cls: cls.published, include_aliases=True + ) + ) + + class ProhibitedProjectName(db.Model): __tablename__ = "prohibited_project_names" __table_args__ = ( diff --git a/warehouse/templates/manage/project/history.html b/warehouse/templates/manage/project/history.html index 6ca65478a265..fb7b01ff040d 100644 --- a/warehouse/templates/manage/project/history.html +++ b/warehouse/templates/manage/project/history.html @@ -53,6 +53,14 @@

{% trans %}Security history{% endtrans %}

{{ event.additional.publisher_url }} {% endif %} + + {% trans %}Published:{% endtrans %} + {% if event.additional.published is defined and event.additional.published is false %} + {% trans %}No{% endtrans %} + {% else %} + {% trans %}Yes{% endtrans %} + {% endif %} + {% elif event.tag == EventTag.Project.ReleaseRemove %} {# No link to removed release #} @@ -75,7 +83,18 @@

{% trans %}Security history{% endtrans %}

{% trans %}Yanked by:{% endtrans %} {{ event.additional.submitted_by }} - + {% elif event.tag == EventTag.Project.ReleasePublish %} + + {% trans href=request.route_path('manage.project.release', project_name=project.name, version=event.additional.canonical_version), version=event.additional.canonical_version %}Version {{ version }} published{% endtrans %} + + + {% trans %}Published by:{% endtrans %} + {% if event.additional.uploaded_via_trusted_publisher or event.additional.publisher_url %} + {{ event.additional.submitted_by }} + {% else %} + {{ event.additional.submitted_by }} + {% endif %} + {# Display file events #} {% elif event.tag == EventTag.File.FileAdd %}