Insert checks for enum discriminants when debug assertions are enabled

[1c3t3a]

Similar to the existing null-pointer and alignment checks, this checks for valid enum discriminants on creation of enums through unsafe transmutes. Essentially this sanitizes patterns like the following:

let val: MyEnum = unsafe { std::mem::transmute<u32, MyEnum>(42) };

An extension of this check will be done in a follow-up that explicitly sanitizes for extern enum values that come into Rust from e.g. C/C++.

This check is similar to Miri's capabilities of checking for valid construction of enum values.

This PR is inspired by saethlin@'s PR
#104862. Thank you so much for keeping this code up and the detailed comments!

I also pair-programmed large parts of this together with vabr-g@.

r? @saethlin

[rustbot]

Some changes occurred to MIR optimizations

cc @rust-lang/wg-mir-opt

This PR changes MIR

cc @oli-obk, @RalfJung, @JakobDegen, @davidtwco, @vakaras

Some changes occurred in compiler/rustc_codegen_ssa

cc @WaffleLapkin

Some changes occurred in compiler/rustc_codegen_cranelift

cc @bjorn3

Some changes occurred to the CTFE machinery

cc @RalfJung, @oli-obk, @lcnr

rust-analyzer is developed in its own repository. If possible, consider making this change to rust-lang/rust-analyzer instead.

cc @rust-lang/rust-analyzer

[rustbot]

The Miri subtree was changed

cc @rust-lang/miri

[rust-log-analyzer]

The job x86_64-gnu-llvm-19 failed! Check out the build log: (web) (plain)

Click to see the possible cause of the failure (guessed by this bot)

#18 exporting to docker image format
#18 sending tarball 27.3s done
#18 DONE 35.1s
##[endgroup]
Setting extra environment values for docker:  --env ENABLE_GCC_CODEGEN=1 --env GCC_EXEC_PREFIX=/usr/lib/gcc/
[CI_JOB_NAME=x86_64-gnu-llvm-19]
[CI_JOB_NAME=x86_64-gnu-llvm-19]
debug: `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` configured.
---
sccache: Listening on address 127.0.0.1:4226
##[group]Configure the build
configure: processing command line
configure: 
configure: build.configure-args := ['--build=x86_64-unknown-linux-gnu', '--llvm-root=/usr/lib/llvm-19', '--enable-llvm-link-shared', '--set', 'rust.randomize-layout=true', '--set', 'rust.thin-lto-import-instr-limit=10', '--set', 'build.print-step-timings', '--enable-verbose-tests', '--set', 'build.metrics', '--enable-verbose-configure', '--enable-sccache', '--disable-manage-submodules', '--enable-locked-deps', '--enable-cargo-native-static', '--set', 'rust.codegen-units-std=1', '--set', 'dist.compression-profile=balanced', '--dist-compression-formats=xz', '--set', 'rust.lld=false', '--disable-dist-src', '--release-channel=nightly', '--enable-debug-assertions', '--enable-overflow-checks', '--enable-llvm-assertions', '--set', 'rust.verify-llvm-ir', '--set', 'rust.codegen-backends=llvm,cranelift,gcc', '--set', 'llvm.static-libstdcpp', '--set', 'gcc.download-ci-gcc=true', '--enable-new-symbol-mangling']
configure: build.build          := x86_64-unknown-linux-gnu
configure: target.x86_64-unknown-linux-gnu.llvm-config := /usr/lib/llvm-19/bin/llvm-config
configure: llvm.link-shared     := True
configure: rust.randomize-layout := True
configure: rust.thin-lto-import-instr-limit := 10
---
test [ui] tests/ui/mir/enable_passes_validation.rs#all_known ... ok
test [ui] tests/ui/mir/enable_passes_validation.rs#unprefixed ... ok
test [ui] tests/ui/mir/enable_passes_validation.rs#all_unknown ... ok
test [ui] tests/ui/mir/enable_passes_validation.rs#mixed ... ok
test [ui] tests/ui/mir/enum/convert_non_enum_niche_ok.rs ... ok
test [ui] tests/ui/mir/enum/convert_non_enum_ok.rs ... ok
test [ui] tests/ui/mir/enum/niche_option_tuple_break.rs ... ok
test [ui] tests/ui/mir/enum/niche_option_tuple_ok.rs ... ok
test [ui] tests/ui/mir/enum/numbered_variants_ok.rs ... ok
test [ui] tests/ui/mir/enum/convert_non_enum_niche_break.rs ... ok
test [ui] tests/ui/mir/enum/convert_non_enum_break.rs ... ok
test [ui] tests/ui/mir/enum/option_with_bigger_niche_break.rs ... ok
test [ui] tests/ui/mir/enum/option_with_bigger_niche_ok.rs ... ok
test [ui] tests/ui/mir/enum/single_ok.rs ... ok
test [ui] tests/ui/mir/enum/plain_no_data_ok.rs ... ok
test [ui] tests/ui/mir/enum/numbered_variants_break.rs ... ok
test [ui] tests/ui/mir/enum/single_with_repr_ok.rs ... ok
test [ui] tests/ui/mir/enum/with_niche_int_ok.rs ... ok
test [ui] tests/ui/mir/enum/plain_no_data_break.rs ... ok
test [ui] tests/ui/mir/enum/with_niche_ptr_ok.rs ... FAILED
test [ui] tests/ui/mir/enum/wrap_ok.rs ... ok
test [ui] tests/ui/mir/enum/with_niche_int_break.rs ... ok
test [ui] tests/ui/mir/enum/single_with_repr_break.rs ... ok
test [ui] tests/ui/mir/field-projection-invariant.rs ... ok
test [ui] tests/ui/mir/enum/wrap_break.rs ... ok
---
test [ui] tests/ui/zero-sized/zero-sized-tuple-struct.rs ... ok

failures:

---- [ui] tests/ui/mir/enum/with_niche_ptr_ok.rs stdout ----

error: test compilation failed although it shouldn't!
status: exit status: 1
command: env -u RUSTC_LOG_COLOR RUSTC_ICE="0" RUST_BACKTRACE="short" "/checkout/obj/build/x86_64-unknown-linux-gnu/stage2/bin/rustc" "/checkout/tests/ui/mir/enum/with_niche_ptr_ok.rs" "-Zthreads=1" "-Zsimulate-remapped-rust-src-base=/rustc/FAKE_PREFIX" "-Ztranslate-remapped-path-to-local-path=no" "-Z" "ignore-directory-in-diagnostics-source-blocks=/cargo" "-Z" "ignore-directory-in-diagnostics-source-blocks=/checkout/vendor" "--sysroot" "/checkout/obj/build/x86_64-unknown-linux-gnu/stage2" "--target=i686-unknown-linux-gnu" "--check-cfg" "cfg(test,FALSE)" "--error-format" "json" "--json" "future-incompat" "-Ccodegen-units=1" "-Zui-testing" "-Zdeduplicate-diagnostics=no" "-Zwrite-long-types-to-disk=no" "-Cstrip=debuginfo" "--emit" "metadata" "-C" "prefer-dynamic" "--out-dir" "/checkout/obj/build/x86_64-unknown-linux-gnu/test/ui/mir/enum/with_niche_ptr_ok" "-A" "internal_features" "-Crpath" "-Cdebuginfo=0" "-Lnative=/checkout/obj/build/i686-unknown-linux-gnu/native/rust-test-helpers" "-Clinker=x86_64-linux-gnu-gcc" "-C" "debug-assertions"
stdout: none
--- stderr -------------------------------
error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
##[error]  --> /checkout/tests/ui/mir/enum/with_niche_ptr_ok.rs:21:25
   |
LL |     let _val = unsafe { std::mem::transmute::<u64, Option<unsafe extern "C" fn()>>(0) };
   |                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
   = note: source type: `u64` (64 bits)
   = note: target type: `Option<unsafe extern "C" fn()>` (32 bits)

error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
##[error]  --> /checkout/tests/ui/mir/enum/with_niche_ptr_ok.rs:22:25
   |
LL |     let _val = unsafe { std::mem::transmute::<u64, Option<unsafe extern "C" fn()>>(1) };
   |                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
   = note: source type: `u64` (64 bits)
   = note: target type: `Option<unsafe extern "C" fn()>` (32 bits)

error[E0512]: cannot transmute between types of different sizes, or dependently-sized types
##[error]  --> /checkout/tests/ui/mir/enum/with_niche_ptr_ok.rs:23:25
   |
LL |     let _val = unsafe { std::mem::transmute::<u64, Option<unsafe extern "C" fn()>>(u64::MAX) };
   |                         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
   |
   = note: source type: `u64` (64 bits)
   = note: target type: `Option<unsafe extern "C" fn()>` (32 bits)

error: aborting due to 3 previous errors

For more information about this error, try `rustc --explain E0512`.
------------------------------------------



failures:
    [ui] tests/ui/mir/enum/with_niche_ptr_ok.rs

test result: FAILED. 18878 passed; 1 failed; 366 ignored; 0 measured; 0 filtered out; finished in 538.02s

Some tests failed in compiletest suite=ui pass_mode=check mode=ui host=x86_64-unknown-linux-gnu target=i686-unknown-linux-gnu
Build completed unsuccessfully in 0:10:37