Skip to content

Warn on upload about non-normalized wheel distribution name #17378

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Feb 28, 2025
88 changes: 88 additions & 0 deletions tests/unit/email/test_init.py
Original file line number Diff line number Diff line change
Expand Up @@ -6316,6 +6316,94 @@ def test_environment_ignored_in_trusted_publisher_emails(
),
]

def test_pep427_emails(
self,
pyramid_request,
pyramid_config,
monkeypatch,
):
stub_user = pretend.stub(
id="id",
username="username",
name="",
email="[email protected]",
primary_email=pretend.stub(email="[email protected]", verified=True),
)
subject_renderer = pyramid_config.testing_add_renderer(
"email/pep427-name-email/subject.txt"
)
subject_renderer.string_response = "Email Subject"
body_renderer = pyramid_config.testing_add_renderer(
"email/pep427-name-email/body.txt"
)
body_renderer.string_response = "Email Body"
html_renderer = pyramid_config.testing_add_renderer(
"email/pep427-name-email/body.html"
)
html_renderer.string_response = "Email HTML Body"

send_email = pretend.stub(
delay=pretend.call_recorder(lambda *args, **kwargs: None)
)
pyramid_request.task = pretend.call_recorder(lambda *args, **kwargs: send_email)
monkeypatch.setattr(email, "send_email", send_email)

pyramid_request.db = pretend.stub(
query=lambda a: pretend.stub(
filter=lambda *a: pretend.stub(
one=lambda: pretend.stub(user_id=stub_user.id)
)
),
)
pyramid_request.user = stub_user
pyramid_request.registry.settings = {"mail.sender": "[email protected]"}

project_name = "Test_Project"
filename = "Test_Project-1.0-py3-none-any.whl"

result = email.send_pep427_name_email(
pyramid_request,
stub_user,
project_name=project_name,
filename=filename,
normalized_name="test_project",
)

assert result == {
"project_name": project_name,
"normalized_name": "test_project",
"filename": filename,
}
subject_renderer.assert_(project_name=project_name)
body_renderer.assert_(project_name=project_name)
html_renderer.assert_(project_name=project_name)

assert pyramid_request.task.calls == [pretend.call(send_email)]
assert send_email.delay.calls == [
pretend.call(
f"{stub_user.username} <{stub_user.email}>",
{
"sender": None,
"subject": "Email Subject",
"body_text": "Email Body",
"body_html": (
"<html>\n<head></head>\n"
"<body><p>Email HTML Body</p></body>\n</html>\n"
),
},
{
"tag": "account:email:sent",
"user_id": stub_user.id,
"additional": {
"from_": "[email protected]",
"to": stub_user.email,
"subject": "Email Subject",
"redact_ip": False,
},
},
)
]


class TestUserTermsOfServiceUpdateEmail:
def test_user_terms_of_service_updated(
Expand Down
122 changes: 108 additions & 14 deletions tests/unit/forklift/test_legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@
from warehouse.classifiers.models import Classifier
from warehouse.forklift import legacy, metadata
from warehouse.macaroons import IMacaroonService, caveats, security_policy
from warehouse.metrics import IMetricsService
from warehouse.oidc.interfaces import SignedClaims
from warehouse.oidc.utils import PublisherTokenContext
from warehouse.packaging.interfaces import IFileStorage, IProjectService
Expand Down Expand Up @@ -2762,8 +2763,12 @@ def test_upload_succeeds_with_wheel(
release = ReleaseFactory.create(project=project, version="1.0")
RoleFactory.create(user=user, project=project)

filename = f"{project.name}-{release.version}-cp34-none-{plat}.whl"
filebody = _get_whl_testdata(name=project.name, version=release.version)
filename = "{}-{}-cp34-none-{}.whl".format(
project.normalized_name.replace("-", "_"), release.version, plat
)
filebody = _get_whl_testdata(
name=project.normalized_name.replace("-", "_"), version=release.version
)
filestoragehash = _storage_hash(filebody)

pyramid_config.testing_securitypolicy(identity=user)
Expand Down Expand Up @@ -3112,15 +3117,20 @@ def test_upload_succeeds_with_wheel_after_sdist(
EmailFactory.create(user=user)
project = ProjectFactory.create()
release = ReleaseFactory.create(project=project, version="1.0")
FileFactory.create(
release=release,
packagetype="sdist",
filename=f"{project.name}-{release.version}.tar.gz",
filename = "{}-{}.tar.gz".format(
project.normalized_name.replace("-", "_"),
release.version,
)
FileFactory.create(release=release, packagetype="sdist", filename=filename)
RoleFactory.create(user=user, project=project)

filename = f"{project.name}-{release.version}-cp34-none-any.whl"
filebody = _get_whl_testdata(name=project.name, version=release.version)
filename = "{}-{}-cp34-none-any.whl".format(
project.normalized_name.replace("-", "_"),
release.version,
)
filebody = _get_whl_testdata(
name=project.normalized_name.replace("-", "_"), version=release.version
)
filestoragehash = _storage_hash(filebody)

pyramid_config.testing_securitypolicy(identity=user)
Expand Down Expand Up @@ -3363,7 +3373,10 @@ def test_upload_fails_with_missing_metadata_wheel(
with zipfile.ZipFile(file=temp_f, mode="w") as zfp:
zfp.writestr("some_file", "some_data")

filename = f"{project.name}-{release.version}-cp34-none-any.whl"
filename = "{}-{}-cp34-none-any.whl".format(
project.normalized_name.replace("-", "_"),
release.version,
)
filebody = temp_f.getvalue()

db_request.POST = MultiDict(
Expand Down Expand Up @@ -4888,6 +4901,76 @@ def test_upload_with_token_api_warns_if_trusted_publisher_configured(
if not warning_already_sent:
assert not warning_exists

@pytest.mark.parametrize("project_name", ["Some_Thing", "some.thing"])
def test_upload_warns_pep427(
self,
monkeypatch,
pyramid_config,
db_request,
metrics,
project_service,
macaroon_service,
project_name,
):
project = ProjectFactory.create(name=project_name)
owner = UserFactory.create()
maintainer = UserFactory.create()
RoleFactory.create(user=owner, project=project, role_name="Owner")
RoleFactory.create(user=maintainer, project=project, role_name="Maintainer")

pyramid_config.testing_securitypolicy(identity=owner)

filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
data = _get_whl_testdata(name=project.name, version="1.0")
digest = hashlib.md5(data).hexdigest()

db_request.POST = MultiDict(
{
"metadata_version": "1.2",
"name": project.name,
"version": "1.0",
"filetype": "bdist_wheel",
"pyversion": "py3",
"md5_digest": digest,
"content": pretend.stub(
filename=filename,
file=io.BytesIO(data),
type="application/zip",
),
}
)

storage_service = pretend.stub(store=lambda path, filepath, meta: None)

db_request.find_service = lambda svc, name=None, context=None: {
IFileStorage: storage_service,
IMacaroonService: macaroon_service,
IMetricsService: metrics,
IProjectService: project_service,
}.get(svc)
db_request.user_agent = "warehouse-tests/6.6.6"
db_request.help_url = pretend.call_recorder(lambda **kw: "/the/help/url/")

send_email = pretend.call_recorder(lambda *a, **kw: None)
monkeypatch.setattr(legacy, "send_pep427_name_email", send_email)
monkeypatch.setattr(
legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
)

resp = legacy.file_upload(db_request)

assert resp.status_code == 200

assert send_email.calls == [
pretend.call(
db_request,
{owner, maintainer},
project_name=project.name,
filename=filename,
normalized_name="some_thing",
),
]

@pytest.mark.parametrize(
("filename", "function_name", "extra_kwargs"),
[
Expand Down Expand Up @@ -5069,8 +5152,12 @@ def test_upload_succeeds_creates_release_metadata_2_4(
digest = _ZIP_PKG_MD5
data = _ZIP_PKG_TESTDATA
elif filetype == "bdist_wheel":
filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
data = _get_whl_testdata(name=project.name, version="1.0")
filename = "{}-{}-py3-none-any.whl".format(
project.normalized_name.replace("-", "_"), "1.0"
)
data = _get_whl_testdata(
name=project.normalized_name.replace("-", "_"), version="1.0"
)
digest = hashlib.md5(data).hexdigest()
monkeypatch.setattr(
legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
Expand Down Expand Up @@ -5184,13 +5271,20 @@ def test_upload_fails_missing_license_file_metadata_2_4(
data = _ZIP_PKG_TESTDATA
license_filename = "fake_package-1.0/LICENSE"
elif filetype == "bdist_wheel":
filename = "{}-{}-py3-none-any.whl".format(project.name, "1.0")
data = _get_whl_testdata(name=project.name, version="1.0")
filename = "{}-{}-py3-none-any.whl".format(
project.normalized_name.replace("-", "_"),
"1.0",
)
data = _get_whl_testdata(
name=project.normalized_name.replace("-", "_"), version="1.0"
)
digest = hashlib.md5(data).hexdigest()
monkeypatch.setattr(
legacy, "_is_valid_dist_file", lambda *a, **kw: (True, None)
)
license_filename = f"{project.name}-1.0.dist-info/licenses/LICENSE"
license_filename = "{}-1.0.dist-info/licenses/LICENSE".format(
project.normalized_name.replace("-", "_")
)

pyramid_config.testing_securitypolicy(identity=user)
db_request.user = user
Expand Down
9 changes: 9 additions & 0 deletions warehouse/email/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -1111,6 +1111,15 @@ def send_user_terms_of_service_updated(request, user):
}


@_email("pep427-name-email")
def send_pep427_name_email(request, users, project_name, filename, normalized_name):
return {
"project_name": project_name,
"filename": filename,
"normalized_name": normalized_name,
}


def includeme(config):
email_sending_class = config.maybe_dotted(config.registry.settings["mail.backend"])
config.register_service_factory(email_sending_class.create_service, IEmailSender)
Expand Down
20 changes: 20 additions & 0 deletions warehouse/forklift/legacy.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
from warehouse.constants import MAX_FILESIZE, MAX_PROJECT_SIZE, ONE_GIB, ONE_MIB
from warehouse.email import (
send_api_token_used_in_trusted_publisher_project_email,
send_pep427_name_email,
send_pep625_extension_email,
send_pep625_name_email,
send_pep625_version_email,
Expand Down Expand Up @@ -1372,6 +1373,25 @@ def file_upload(request):
f"{canonical_name.replace('-', '_')!r}.",
)

# The parse_wheel_filename function does not enforce lowercasing,
# and also returns a normalized name, so we must get the original
# distribution name from the filename manually
name_from_filename, _ = filename.split("-", 1)

# PEP 427 / PEP 503: Enforcement of project name normalization.
# Filenames that do not start with the fully normalized project name
# will not be permitted.
# https://packaging.python.org/en/latest/specifications/binary-distribution-format/#escaping-and-unicode
normalized_name = project.normalized_name.replace("-", "_")
if name_from_filename != normalized_name:
send_pep427_name_email(
request,
set(project.users),
project_name=project.name,
filename=filename,
normalized_name=normalized_name,
)

if meta.version != version:
request.metrics.increment(
"warehouse.upload.failed",
Expand Down
45 changes: 45 additions & 0 deletions warehouse/templates/email/pep427-name-email/body.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
{#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#}
{% extends "email/_base/body.html" %}

{% set domain = request.registry.settings.get('warehouse.domain') %}

{% block content %}
<p>
This email is notifying you of an upcoming deprecation that we have
determined may affect you as a result of your recent upload to
'{{ project_name }}'.
</p>
<p>
In the future, PyPI will require all newly uploaded binary distribution
filenames to comply with the <a href="https://packaging.python.org/en/latest/specifications/binary-distribution-format/">binary distribution format</a>.
Any binary distributions already uploaded will remain in place
as-is and do not need to be updated.
</p>
<p>
Specifically, your recent upload of '{{ filename }}' is incompatible with
the distribution format specification because the filename does not contain
the normalized project name '{{ normalized_name }}'.
</p>
<p>
In most cases, this can be resolved by upgrading the version of your build
tooling to a later version that fully supports the specification and
produces compliant filenames. You do not need to remove the file.
</p>
<p>
If you have questions, you can email
<a href="mailto:[email protected]">[email protected]</a> to communicate with the PyPI
[email protected] to communicate with the PyPI administrators.
</p>
{% endblock %}
28 changes: 28 additions & 0 deletions warehouse/templates/email/pep427-name-email/body.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
{#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-#}
{% extends "email/_base/body.txt" %}

{% block content %}

This email is notifying you of an upcoming deprecation that we have determined may affect you as a result of your recent upload to '{{ project_name }}'.

In the future, PyPI will require all newly uploaded binary distribution filenames to comply with the <a href="https://packaging.python.org/en/latest/specifications/binary-distribution-format/">binary distribution format</a>. Any binary distributions already uploaded will remain in place as-is and do not need to be updated.

Specifically, your recent upload of '{{ filename }}' is incompatible with the distribution format specification because the filename does not contain the normalized project name '{{ normalized_name }}'.

In most cases, this can be resolved by upgrading the version of your build tooling to a later version that fully supports the specification and produces compliant filenames. You do not need to remove the file.

If you have questions, you can email [email protected] to communicate with the PyPI administrators.

{% endblock %}
Loading