Skip to content

Commit e9fa82a

Browse files
EricRahmEricRahm
committed
Add C++ golden file test suite
This change introduces a robust, granular testing infrastructure for the C++ backend using golden files. This test suite serves as a critical safety net to enable safe and efficient refactoring of the code generator in the future. By comparing generated code against a set of "golden" (known-good) header files, we can quickly detect any unintended changes or regressions. Key Features: * **Granular Tests:** Each golden file comparison is a standalone `py_test` target, allowing for parallel execution and precise failure identification. * **`cpp_golden_test` Macro:** A new Starlark macro, `cpp_golden_test`, has been added to `compiler/back_end/cpp/build_defs.bzl`. This simplifies adding new golden tests by abstracting away the underlying `py_test` implementation details. * **Debugging:** To simplify the diagnosis of regressions, test failures produce a `diff` of the generated file against its golden counterpart. * **Full Coverage:** The suite includes tests for all `.emb` files in the `testdata` directory, including those with complex dependencies and imports. Usage: **To run all C++ golden tests:** ``` bazel test //compiler/back_end/cpp/... --test_tag_filters=golden ``` **To run a single golden test:** ``` bazel test //compiler/back_end/cpp:bits_golden_test ``` **To add a new golden test:** 1. Add your `.emb` file to `testdata/`. 2. Generate the corresponding golden `.emb.h` file (e.g., by temporarily modifying and running the `generate_golden_files.sh` script via a `sh_binary`). 3. Copy the new golden file into `testdata/golden_cpp/`. 4. Add a `cpp_golden_test()` rule to `compiler/back_end/cpp/BUILD`.
1 parent e301b74 commit e9fa82a

40 files changed

Lines changed: 128279 additions & 2 deletions

‎compiler/back_end/cpp/BUILD‎

Lines changed: 211 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@
1616

1717
load("@rules_python//python:py_binary.bzl", "py_binary")
1818
load("@rules_python//python:py_library.bzl", "py_library")
19-
load("@rules_python//python:py_test.bzl", "py_test")
20-
load(":build_defs.bzl", "emboss_cc_test")
19+
load(":build_defs.bzl", "cpp_golden_test", "emboss_cc_test")
2120

2221
package(
2322
default_visibility = [
@@ -388,3 +387,213 @@ emboss_cc_test(
388387
"@com_google_googletest//:gtest_main",
389388
],
390389
)
390+
391+
# New golden test infrastructure
392+
py_library(
393+
name = "one_golden_test_lib",
394+
srcs = ["one_golden_test.py"],
395+
)
396+
397+
py_binary(
398+
name = "run_one_golden_test",
399+
srcs = ["run_one_golden_test.py"],
400+
deps = [":one_golden_test_lib"],
401+
)
402+
403+
cpp_golden_test(
404+
name = "anonymous_bits_golden_test",
405+
emb_file = "//testdata:anonymous_bits.emb",
406+
golden_file = "//testdata/golden_cpp:anonymous_bits.emb.h",
407+
)
408+
409+
cpp_golden_test(
410+
name = "bits_golden_test",
411+
emb_file = "//testdata:bits.emb",
412+
golden_file = "//testdata/golden_cpp:bits.emb.h",
413+
)
414+
415+
cpp_golden_test(
416+
name = "absolute_cpp_namespace_golden_test",
417+
emb_file = "//testdata:absolute_cpp_namespace.emb",
418+
golden_file = "//testdata/golden_cpp:absolute_cpp_namespace.emb.h",
419+
)
420+
421+
cpp_golden_test(
422+
name = "alignments_golden_test",
423+
emb_file = "//testdata:alignments.emb",
424+
golden_file = "//testdata/golden_cpp:alignments.emb.h",
425+
)
426+
427+
cpp_golden_test(
428+
name = "auto_array_size_golden_test",
429+
emb_file = "//testdata:auto_array_size.emb",
430+
golden_file = "//testdata/golden_cpp:auto_array_size.emb.h",
431+
)
432+
433+
cpp_golden_test(
434+
name = "bcd_golden_test",
435+
emb_file = "//testdata:bcd.emb",
436+
golden_file = "//testdata/golden_cpp:bcd.emb.h",
437+
)
438+
439+
cpp_golden_test(
440+
name = "complex_offset_golden_test",
441+
emb_file = "//testdata:complex_offset.emb",
442+
golden_file = "//testdata/golden_cpp:complex_offset.emb.h",
443+
)
444+
445+
cpp_golden_test(
446+
name = "complex_structure_golden_test",
447+
emb_file = "//testdata:complex_structure.emb",
448+
golden_file = "//testdata/golden_cpp:complex_structure.emb.h",
449+
)
450+
451+
cpp_golden_test(
452+
name = "condition_golden_test",
453+
emb_file = "//testdata:condition.emb",
454+
golden_file = "//testdata/golden_cpp:condition.emb.h",
455+
)
456+
457+
cpp_golden_test(
458+
name = "cpp_namespace_golden_test",
459+
emb_file = "//testdata:cpp_namespace.emb",
460+
golden_file = "//testdata/golden_cpp:cpp_namespace.emb.h",
461+
)
462+
463+
cpp_golden_test(
464+
name = "dynamic_size_golden_test",
465+
emb_file = "//testdata:dynamic_size.emb",
466+
golden_file = "//testdata/golden_cpp:dynamic_size.emb.h",
467+
)
468+
469+
cpp_golden_test(
470+
name = "enum_case_golden_test",
471+
emb_file = "//testdata:enum_case.emb",
472+
golden_file = "//testdata/golden_cpp:enum_case.emb.h",
473+
)
474+
475+
cpp_golden_test(
476+
name = "enum_golden_test",
477+
emb_file = "//testdata:enum.emb",
478+
golden_file = "//testdata/golden_cpp:enum.emb.h",
479+
)
480+
481+
cpp_golden_test(
482+
name = "explicit_sizes_golden_test",
483+
emb_file = "//testdata:explicit_sizes.emb",
484+
golden_file = "//testdata/golden_cpp:explicit_sizes.emb.h",
485+
)
486+
487+
cpp_golden_test(
488+
name = "float_golden_test",
489+
emb_file = "//testdata:float.emb",
490+
golden_file = "//testdata/golden_cpp:float.emb.h",
491+
)
492+
493+
cpp_golden_test(
494+
name = "imported_golden_test",
495+
emb_file = "//testdata:imported.emb",
496+
golden_file = "//testdata/golden_cpp:imported.emb.h",
497+
)
498+
499+
cpp_golden_test(
500+
name = "inline_type_golden_test",
501+
emb_file = "//testdata:inline_type.emb",
502+
golden_file = "//testdata/golden_cpp:inline_type.emb.h",
503+
)
504+
505+
cpp_golden_test(
506+
name = "int_sizes_golden_test",
507+
emb_file = "//testdata:int_sizes.emb",
508+
golden_file = "//testdata/golden_cpp:int_sizes.emb.h",
509+
)
510+
511+
cpp_golden_test(
512+
name = "large_array_golden_test",
513+
emb_file = "//testdata:large_array.emb",
514+
golden_file = "//testdata/golden_cpp:large_array.emb.h",
515+
)
516+
517+
cpp_golden_test(
518+
name = "nested_structure_golden_test",
519+
emb_file = "//testdata:nested_structure.emb",
520+
golden_file = "//testdata/golden_cpp:nested_structure.emb.h",
521+
)
522+
523+
cpp_golden_test(
524+
name = "next_keyword_golden_test",
525+
emb_file = "//testdata:next_keyword.emb",
526+
golden_file = "//testdata/golden_cpp:next_keyword.emb.h",
527+
)
528+
529+
cpp_golden_test(
530+
name = "no_cpp_namespace_golden_test",
531+
emb_file = "//testdata:no_cpp_namespace.emb",
532+
golden_file = "//testdata/golden_cpp:no_cpp_namespace.emb.h",
533+
)
534+
535+
cpp_golden_test(
536+
name = "no_enum_traits_golden_test",
537+
emb_file = "//testdata:no_enum_traits.emb",
538+
golden_file = "//testdata/golden_cpp:no_enum_traits.emb.h",
539+
)
540+
541+
cpp_golden_test(
542+
name = "parameters_golden_test",
543+
emb_file = "//testdata:parameters.emb",
544+
golden_file = "//testdata/golden_cpp:parameters.emb.h",
545+
)
546+
547+
cpp_golden_test(
548+
name = "requires_golden_test",
549+
emb_file = "//testdata:requires.emb",
550+
golden_file = "//testdata/golden_cpp:requires.emb.h",
551+
)
552+
553+
cpp_golden_test(
554+
name = "start_size_range_golden_test",
555+
emb_file = "//testdata:start_size_range.emb",
556+
golden_file = "//testdata/golden_cpp:start_size_range.emb.h",
557+
)
558+
559+
cpp_golden_test(
560+
name = "subtypes_golden_test",
561+
emb_file = "//testdata:subtypes.emb",
562+
golden_file = "//testdata/golden_cpp:subtypes.emb.h",
563+
)
564+
565+
cpp_golden_test(
566+
name = "text_format_golden_test",
567+
emb_file = "//testdata:text_format.emb",
568+
golden_file = "//testdata/golden_cpp:text_format.emb.h",
569+
)
570+
571+
cpp_golden_test(
572+
name = "uint_sizes_golden_test",
573+
emb_file = "//testdata:uint_sizes.emb",
574+
golden_file = "//testdata/golden_cpp:uint_sizes.emb.h",
575+
)
576+
577+
cpp_golden_test(
578+
name = "virtual_field_golden_test",
579+
emb_file = "//testdata:virtual_field.emb",
580+
golden_file = "//testdata/golden_cpp:virtual_field.emb.h",
581+
)
582+
583+
cpp_golden_test(
584+
name = "importer_golden_test",
585+
emb_file = "//testdata:importer.emb",
586+
golden_file = "//testdata/golden_cpp:importer.emb.h",
587+
)
588+
589+
cpp_golden_test(
590+
name = "importer2_golden_test",
591+
emb_file = "//testdata:importer2.emb",
592+
golden_file = "//testdata/golden_cpp:importer2.emb.h",
593+
)
594+
595+
cpp_golden_test(
596+
name = "imported_genfiles_golden_test",
597+
emb_file = "//testdata:imported_genfiles.emb",
598+
golden_file = "//testdata/golden_cpp:imported_genfiles.emb.h",
599+
)

‎compiler/back_end/cpp/build_defs.bzl‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
# vim:set ft=blazebuild:
1717
"""Rule to generate cc_tests with and without system-specific optimizations."""
1818

19+
load("@rules_python//python:py_test.bzl", "py_test")
20+
1921
def emboss_cc_test(name, copts = None, no_w_sign_compare = False, **kwargs):
2022
"""Generates cc_test rules with and without -DEMBOSS_NO_OPTIMIZATIONS."""
2123
native.cc_test(
@@ -43,3 +45,33 @@ def emboss_cc_test(name, copts = None, no_w_sign_compare = False, **kwargs):
4345
] + ([] if no_w_sign_compare else ["-Wsign-compare"]) + (copts or []),
4446
**kwargs
4547
)
48+
49+
def cpp_golden_test(name, emb_file, golden_file, import_dirs = []):
50+
"""Defines a C++ golden file test.
51+
52+
Args:
53+
name: The name of the test.
54+
emb_file: The .emb file to test.
55+
golden_file: The golden .h file.
56+
import_dirs: A list of import directories.
57+
"""
58+
py_test(
59+
name = name,
60+
main = ":run_one_golden_test.py",
61+
srcs = [":run_one_golden_test.py", ":one_golden_test.py"],
62+
tags = ["golden"],
63+
args = [
64+
"$(location //compiler/front_end:emboss_front_end)",
65+
"$(location :emboss_codegen_cpp)",
66+
"$(location %s)" % emb_file,
67+
"$(location %s)" % golden_file,
68+
] + ["--import-dir=" + d for d in import_dirs],
69+
data = [
70+
"//compiler/front_end:emboss_front_end",
71+
":emboss_codegen_cpp",
72+
emb_file,
73+
golden_file,
74+
"//testdata:test_embs",
75+
] + import_dirs,
76+
deps = [":one_golden_test_lib"],
77+
)

‎compiler/back_end/cpp/one_golden_test.py‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# Copyright 2020 The Emboss Authors
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# https://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
import difflib
16+
import os
17+
import subprocess
18+
import sys
19+
import unittest
20+
21+
22+
class OneGoldenTest(unittest.TestCase):
23+
def __init__(
24+
self,
25+
emboss_front_end,
26+
emboss_compiler,
27+
emb_file,
28+
golden_file,
29+
include_dirs=None,
30+
):
31+
super(OneGoldenTest, self).__init__("test_golden_file")
32+
self.emboss_front_end = emboss_front_end
33+
self.emboss_compiler = emboss_compiler
34+
self.emb_file = emb_file
35+
self.golden_file = golden_file
36+
self.include_dirs = include_dirs if include_dirs is not None else []
37+
38+
def test_golden_file(self):
39+
temp_dir = os.environ.get("TEST_TMPDIR", "")
40+
ir_path = os.path.join(temp_dir, "ir.json")
41+
output_path = os.path.join(temp_dir, os.path.basename(self.golden_file))
42+
43+
front_end_args = [
44+
self.emboss_front_end,
45+
self.emb_file,
46+
"--output-file",
47+
ir_path,
48+
]
49+
for include_dir in self.include_dirs:
50+
front_end_args.extend(["--import-dir", include_dir])
51+
52+
process = subprocess.run(front_end_args, capture_output=True, text=True)
53+
self.assertEqual(
54+
process.returncode, 0, f"Front end failed with error:\n{process.stderr}"
55+
)
56+
57+
compiler_args = [
58+
self.emboss_compiler,
59+
"--input-file",
60+
ir_path,
61+
"--output-file",
62+
output_path,
63+
]
64+
65+
process = subprocess.run(compiler_args, capture_output=True, text=True)
66+
67+
self.assertEqual(
68+
process.returncode, 0, f"Compiler failed with error:\n{process.stderr}"
69+
)
70+
71+
with open(output_path, "r") as f:
72+
generated_contents = f.read()
73+
74+
with open(self.golden_file, "r") as f:
75+
golden_contents = f.read()
76+
77+
self.assertMultiLineEqual(
78+
golden_contents,
79+
generated_contents,
80+
msg="Generated file does not match golden file. Diff:\n"
81+
+ "".join(
82+
difflib.unified_diff(
83+
golden_contents.splitlines(keepends=True),
84+
generated_contents.splitlines(keepends=True),
85+
fromfile=self.golden_file,
86+
tofile="Generated C++ Header",
87+
)
88+
),
89+
)
90+
91+
92+
if __name__ == "__main__":
93+
# This script is not intended to be run directly.
94+
# It should be invoked by a test runner.
95+
pass

0 commit comments

Comments
 (0)