diff --git a/pyproject.toml b/pyproject.toml index ef3fcae..65736e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dev = [ "upstream-protobuf", "buf-bin==1.71.0", + "googleapis-googleapis-bufbuild-py==0.1.1.1.20260414192239+c17df5b2beca", "poethepoet==0.46.0", "protoc-runner==35.1", "pyright==1.1.410", @@ -441,6 +442,11 @@ exclude = [ reinstall-package = ["protobuf-py-ext"] no-build-isolation-package = ["protobuf-py-ext"] +[[tool.uv.index]] +name = "buf" +url = "https://buf.build/gen/python" +explicit = true + [tool.uv.build-backend] module-name = ["protobuf"] @@ -454,6 +460,8 @@ license-header = { workspace = true } protobuf-py-ext = { workspace = true } protobuf-py-bench = { workspace = true } +googleapis-googleapis-bufbuild-py = { index = "buf" } + [tool.uv.workspace] members = [ "packages/protobuf-py-ext", diff --git a/src/protobuf/plugin/_file.py b/src/protobuf/plugin/_file.py index f957404..620f085 100644 --- a/src/protobuf/plugin/_file.py +++ b/src/protobuf/plugin/_file.py @@ -334,12 +334,15 @@ def _to_el(self, v: object) -> str | Ident: return ident def _relativize(self, ident: Ident) -> Ident: + type_only = ident.type_only or self._type_checking + if ( + ident._desc is not None + and _desc_file(ident._desc).name not in self._file_to_generate + ): + return _absolute_ident(ident, type_only=type_only) + if not _is_relative(ident.module): - return Ident( - ident.name, - ident.module, - type_only=ident.type_only or self._type_checking, - ) + return Ident(ident.name, ident.module, type_only=type_only) self_segments = _module_segments(self.module) import_segments = _module_segments(ident.module) @@ -352,9 +355,7 @@ def _relativize(self, ident: Ident) -> Ident: shared = _shared_prefix_len(package_segments, import_segments) leading_dots = len(package_segments) - shared + 1 path = "." * leading_dots + ".".join(import_segments[shared:]) - return Module(path).ident( - ident.name, type_only=ident.type_only or self._type_checking - ) + return Module(path).ident(ident.name, type_only=type_only) def write(file: _File, path: str, *, no_fmt_off: bool = False) -> str: @@ -465,6 +466,19 @@ def _desc_ident( return ident +def _desc_file(desc: DescEnum | DescMessage | DescExtension | DescFile) -> DescFile: + return desc if isinstance(desc, DescFile) else desc.file + + +def _absolute_ident(ident: Ident, *, type_only: bool) -> Ident: + module_path = ident.module.path.removeprefix(".") + if module_path == "" and isinstance(ident._desc, DescFile): + module_path = ident.name + return Ident( + ident.name, Module(module_path), type_only=type_only, _desc=ident._desc + ) + + def _use_wkt_module(desc: DescFile, file_to_generate: frozenset[str]) -> bool: """Return True if the descriptor should be imported from protobuf.wkt.""" # Well-known types are imported from protobuf.wkt unless the @@ -483,8 +497,24 @@ def _write_imports( # Ruff splits the imports into groups of std, global, and relative. We also do the same: for group in _group_and_sort_imports(imports): for module, idents in group.items(): - deduped = sorted({aliases.resolve_import(ident) for ident in idents}) - lines.append(f"{indent}from {module.path} import {', '.join(deduped)}") + module_imports = [ + ident for ident in idents if _is_module_import(module, ident) + ] + lines.extend( + f"{indent}import {import_}" + for import_ in sorted( + {aliases.resolve_import(ident) for ident in module_imports} + ) + ) + + from_imports = [ + ident for ident in idents if not _is_module_import(module, ident) + ] + if from_imports: + deduped = sorted( + {aliases.resolve_import(ident) for ident in from_imports} + ) + lines.append(f"{indent}from {module.path} import {', '.join(deduped)}") lines.append("") @@ -597,6 +627,11 @@ def _is_relative(module: Module) -> bool: return module.path.startswith(".") +def _is_module_import(module: Module, ident: Ident) -> bool: + """Return True if the identifier imports a top-level module itself.""" + return isinstance(ident._desc, DescFile) and module.path == ident.name + + def _module_segments(module: Module) -> list[str]: path = module.path.removeprefix(".") return path.split(".") if path else [] diff --git a/tests/buf.lock b/tests/buf.lock new file mode 100644 index 0000000..8447589 --- /dev/null +++ b/tests/buf.lock @@ -0,0 +1,6 @@ +# Generated by buf. DO NOT EDIT. +version: v2 +deps: + - name: buf.build/googleapis/googleapis + commit: c17df5b2beca46928cc87d5656bd5343 + digest: b5:648a01e0170d4512dea7d564016165decd1ed6e34bef79fe54753e51ad7e27545709ad9157d7551270147d551155c595a2fb0bf5bb33b1c83040ddbce915c604 diff --git a/tests/buf.yaml b/tests/buf.yaml index 9acfb79..4354aa8 100644 --- a/tests/buf.yaml +++ b/tests/buf.yaml @@ -1,3 +1,5 @@ version: v2 modules: - path: proto_buf +deps: + - buf.build/googleapis/googleapis diff --git a/tests/gen_buf/__init__.py b/tests/gen_buf/__init__.py new file mode 100644 index 0000000..e7c353f --- /dev/null +++ b/tests/gen_buf/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations diff --git a/tests/gen_buf/local_dep/__init__.py b/tests/gen_buf/local_dep/__init__.py new file mode 100644 index 0000000..e7c353f --- /dev/null +++ b/tests/gen_buf/local_dep/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations diff --git a/tests/gen_buf/local_dep/dep_pb.py b/tests/gen_buf/local_dep/dep_pb.py new file mode 100644 index 0000000..f5c2401 --- /dev/null +++ b/tests/gen_buf/local_dep/dep_pb.py @@ -0,0 +1,82 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated from local_dep/dep.proto. DO NOT EDIT. +# Generated by protoc-gen-py v0.1.1 with parameter "". +# ruff: noqa: PGH004 +# ruff: noqa +# fmt: off + +from __future__ import annotations + +from typing import Literal, TYPE_CHECKING, TypeAlias + +from google.rpc import status_pb +from protobuf import Message +from protobuf._codegen import file_desc + +if TYPE_CHECKING: + from google.rpc.status_pb import Status + from protobuf import DescFile + + +_DepFields: TypeAlias = Literal["value", "status"] + +class Dep(Message[_DepFields]): + """ + ```proto + message local_dep.Dep + ``` + + Attributes: + value: + ```proto + string value = 1; + ``` + status: + ```proto + optional google.rpc.Status status = 2; + ``` + """ + + __slots__ = ("value", "status") + + if TYPE_CHECKING: + + def __init__( + self, + *, + value: str = "", + status: Status | None = None, + ) -> None: + pass + + value: str + status: Status | None + + +_DESC = file_desc( + b'\n\x13local_dep/dep.proto\x12\tlocal_dep\x1a\x17google/rpc/status.proto"G\n\x03Dep\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\x12*\n\x06status\x18\x02 \x01(\x0b2\x12.google.rpc.StatusR\x06statusb\x06proto3', + [ + status_pb.desc(), + ], + { + "Dep": Dep, + }, +) + + +def desc() -> DescFile: + """Returns the descriptor for the file `local_dep/dep.proto`.""" + return _DESC diff --git a/tests/gen_buf/local_import/__init__.py b/tests/gen_buf/local_import/__init__.py new file mode 100644 index 0000000..e7c353f --- /dev/null +++ b/tests/gen_buf/local_import/__init__.py @@ -0,0 +1,14 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations diff --git a/tests/gen_buf/local_import/importer_pb.py b/tests/gen_buf/local_import/importer_pb.py new file mode 100644 index 0000000..332cd2e --- /dev/null +++ b/tests/gen_buf/local_import/importer_pb.py @@ -0,0 +1,76 @@ +# Copyright (c) 2025-2026 Buf Technologies, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated from local_import/importer.proto. DO NOT EDIT. +# Generated by protoc-gen-py v0.1.1 with parameter "". +# ruff: noqa: PGH004 +# ruff: noqa +# fmt: off + +from __future__ import annotations + +from typing import Literal, TYPE_CHECKING, TypeAlias + +from local_dep import dep_pb +from protobuf import Message +from protobuf._codegen import file_desc + +if TYPE_CHECKING: + from local_dep.dep_pb import Dep + from protobuf import DescFile + + +_ImporterFields: TypeAlias = Literal["dep"] + +class Importer(Message[_ImporterFields]): + """ + ```proto + message local_import.Importer + ``` + + Attributes: + dep: + ```proto + optional local_dep.Dep dep = 1; + ``` + """ + + __slots__ = ("dep",) + + if TYPE_CHECKING: + + def __init__( + self, + *, + dep: Dep | None = None, + ) -> None: + pass + + dep: Dep | None + + +_DESC = file_desc( + b'\n\x1blocal_import/importer.proto\x12\x0clocal_import\x1a\x13local_dep/dep.proto",\n\x08Importer\x12 \n\x03dep\x18\x01 \x01(\x0b2\x0e.local_dep.DepR\x03depb\x06proto3', + [ + dep_pb.desc(), + ], + { + "Importer": Importer, + }, +) + + +def desc() -> DescFile: + """Returns the descriptor for the file `local_import/importer.proto`.""" + return _DESC diff --git a/tests/plugin/test_file.py b/tests/plugin/test_file.py index 04d9b79..015fd6a 100644 --- a/tests/plugin/test_file.py +++ b/tests/plugin/test_file.py @@ -688,7 +688,107 @@ def desc(): file=_File( path="test_pb.py", module=Module.for_desc(desc, "_pb"), - file_to_generate=frozenset(), + file_to_generate=frozenset(["input.proto", "b.proto", "pkg/b.proto"]), + plugin_name="test", + plugin_version="0.0.0", + parameter="", + ), + ) + + def test_dependency_not_generated(self, protoc: Protoc) -> None: + files = protoc.compile( + { + "app/main.proto": """ + syntax = "proto3"; + package app; + import "buf/validate/validate.proto"; + message Main { + buf.validate.Rule rule = 1; + } + """, + "buf/validate/validate.proto": """ + syntax = "proto3"; + package buf.validate; + message Rule {} + """, + }, + "include_imports", + "retain_options", + "experimental_editions", + "include_source_info", + ) + desc = files["app/main.proto"] + dep = files["buf/validate/validate.proto"] + + def write(f: File) -> None: + f.print(dep, ".desc()") + f.print("rule: ", dep.messages[0]) + + _test_generated_file( + write, + """\ + from __future__ import annotations + + from buf.validate import validate_pb + from buf.validate.validate_pb import Rule + + + validate_pb.desc() + rule: Rule + """, + file=_File( + path="app/main_pb.py", + module=Module.for_desc(desc, "_pb"), + file_to_generate=frozenset([desc.name]), + plugin_name="test", + plugin_version="0.0.0", + parameter="", + ), + ) + + def test_root_dependency_not_generated(self, protoc: Protoc) -> None: + files = protoc.compile( + { + "main.proto": """ + syntax = "proto3"; + import "dep.proto"; + message Main { + Dep dep = 1; + } + """, + "dep.proto": """ + syntax = "proto3"; + message Dep {} + """, + }, + "include_imports", + "retain_options", + "experimental_editions", + "include_source_info", + ) + desc = files["main.proto"] + dep = files["dep.proto"] + + def write(f: File) -> None: + f.print(dep, ".desc()") + f.print("dep: ", dep.messages[0]) + + _test_generated_file( + write, + """\ + from __future__ import annotations + + import dep_pb + from dep_pb import Dep + + + dep_pb.desc() + dep: Dep + """, + file=_File( + path="main_pb.py", + module=Module.for_desc(desc, "_pb"), + file_to_generate=frozenset([desc.name]), plugin_name="test", plugin_version="0.0.0", parameter="", @@ -724,7 +824,7 @@ class Bar(Message): file=_File( path="test_pb.py", module=Module.for_desc(desc, "_pb"), - file_to_generate=frozenset(), + file_to_generate=frozenset([desc.name]), plugin_name="test", plugin_version="0.0.0", parameter="", @@ -758,7 +858,9 @@ class Bar(Message): file=_File( path="test_pb.py", module=Module.for_desc(desc, "_pb"), - file_to_generate=frozenset(["google/protobuf/timestamp.proto"]), + file_to_generate=frozenset( + [desc.name, "google/protobuf/timestamp.proto"] + ), plugin_name="test", plugin_version="0.0.0", parameter="", @@ -798,7 +900,7 @@ class Bar(Message): file=_File( path="test_pb.py", module=Module.for_desc(desc, "_pb"), - file_to_generate=frozenset(), + file_to_generate=frozenset([desc.name]), plugin_name="test", plugin_version="0.0.0", parameter="", diff --git a/tests/plugin/test_protoc_gen_py.py b/tests/plugin/test_protoc_gen_py.py index 18a8f8e..10d9e1b 100644 --- a/tests/plugin/test_protoc_gen_py.py +++ b/tests/plugin/test_protoc_gen_py.py @@ -128,3 +128,32 @@ def desc() -> DescFile: ''' ).lstrip() ) + + def test_external_dependency(self, protoc: Protoc) -> None: + resp = protoc.run_plugin( + Plugin(_generate, options=_Options), + { + "app/main.proto": """ + syntax = "proto3"; + package app; + import "buf/validate/validate.proto"; + message Main { + buf.validate.Rule rule = 1; + } + """, + "buf/validate/validate.proto": """ + syntax = "proto3"; + package buf.validate; + message Rule {} + """, + }, + files_to_generate=["app/main.proto"], + ) + + assert resp.error == "" + files = {f.name: f.content for f in resp.file} + assert set(files) == {"__init__.py", "app/__init__.py", "app/main_pb.py"} + main = files["app/main_pb.py"] + assert "from buf.validate import validate_pb" in main + assert "from buf.validate.validate_pb import Rule" in main + assert "from ..buf.validate" not in main diff --git a/tests/plugin/test_schema.py b/tests/plugin/test_schema.py index d427664..ed68ca3 100644 --- a/tests/plugin/test_schema.py +++ b/tests/plugin/test_schema.py @@ -91,7 +91,9 @@ def generate(schema: Schema[None]) -> None: f.print(dep_file.messages[0]) resp = protoc.run_plugin( - Plugin(generate), _PROTO3_FILES, files_to_generate=["input.proto"] + Plugin(generate), + _PROTO3_FILES, + files_to_generate=["input.proto", "dep.proto"], ) assert resp.error == "" out_file = next( @@ -147,11 +149,16 @@ def generate(schema: Schema[None]) -> None: def test_not_py(self, protoc: Protoc) -> None: def generate(schema: Schema[None]) -> None: dep_file = next(f for f in schema.all_files if f.name == "dep.proto") - f = schema.generate_file(schema.files_to_generate[0], "_pb.pyi") + input_file = next( + f for f in schema.files_to_generate if f.name == "input.proto" + ) + f = schema.generate_file(input_file, "_pb.pyi") f.print(dep_file.messages[0]) resp = protoc.run_plugin( - Plugin(generate), _PROTO3_FILES, files_to_generate=["input.proto"] + Plugin(generate), + _PROTO3_FILES, + files_to_generate=["input.proto", "dep.proto"], ) assert resp.error == "" out_file = next((f for f in resp.file if f.name == "input_pb.pyi"), None) diff --git a/tests/proto_buf/local_dep/dep.proto b/tests/proto_buf/local_dep/dep.proto new file mode 100644 index 0000000..f001aac --- /dev/null +++ b/tests/proto_buf/local_dep/dep.proto @@ -0,0 +1,24 @@ +// Copyright (c) 2025-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto3"; + +package local_dep; + +import "google/rpc/status.proto"; + +message Dep { + string value = 1; + + google.rpc.Status status = 2; +} diff --git a/tests/proto_buf/local_import/importer.proto b/tests/proto_buf/local_import/importer.proto new file mode 100644 index 0000000..ede2225 --- /dev/null +++ b/tests/proto_buf/local_import/importer.proto @@ -0,0 +1,22 @@ +// Copyright (c) 2025-2026 Buf Technologies, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +syntax = "proto3"; + +package local_import; + +import "local_dep/dep.proto"; + +message Importer { + local_dep.Dep dep = 1; +} diff --git a/uv.lock b/uv.lock index 2b7bb7f..0f511c6 100644 --- a/uv.lock +++ b/uv.lock @@ -228,7 +228,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -259,6 +259,17 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/ec/67fbef5d497f86283db54c22eec6f6140243aae73265799baaaa19cd17fb/ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619", size = 11034, upload-time = "2022-05-02T15:47:14.552Z" }, ] +[[package]] +name = "googleapis-googleapis-bufbuild-py" +version = "0.1.1.1.20260414192239+c17df5b2beca" +source = { registry = "https://buf.build/gen/python" } +dependencies = [ + { name = "protobuf-py" }, +] +wheels = [ + { url = "https://buf.build/gen/python/googleapis-googleapis-bufbuild-py/googleapis_googleapis_bufbuild_py-0.1.1.1.20260414192239+c17df5b2beca-py3-none-any.whl", upload-time = "2026-06-24T23:41:41Z" }, +] + [[package]] name = "griffelib" version = "2.1.0" @@ -605,6 +616,7 @@ dev = [ { name = "example" }, { name = "example-plugin" }, { name = "fix-protobuf-imports" }, + { name = "googleapis-googleapis-bufbuild-py" }, { name = "license-header" }, { name = "maturin" }, { name = "poethepoet" }, @@ -636,6 +648,7 @@ dev = [ { name = "example", editable = "examples/protobuf" }, { name = "example-plugin", editable = "examples/plugin" }, { name = "fix-protobuf-imports", specifier = "==0.1.7" }, + { name = "googleapis-googleapis-bufbuild-py", specifier = "==0.1.1.1.20260414192239+c17df5b2beca", index = "https://buf.build/gen/python" }, { name = "license-header", editable = "packages/license-header" }, { name = "maturin", specifier = "==1.14.1" }, { name = "poethepoet", specifier = "==0.46.0" },