Fix #8598: don't emit *.with.overflow intrinsics for DXIL

InstCombine folds the "widened multiply/add compared against a max value"
overflow-check idioms into llvm.umul.with.overflow / llvm.sadd.with.overflow.
These intrinsics (and the extractvalue of their result struct) are not legal
in DXIL, so shaders using them failed validation once optimizations were on.

Guard ProcessUMulZExtIdiom and ProcessUGT_ADDCST_ADD to bail out for the DXIL
target, mirroring the existing isDXIL() guard on bswap matching. The plain
multiply/add and compare are kept, which is what DXIL wants anyway.

Adds .ll regression tests for both idioms plus an end-to-end HLSL test built
from the issue repro.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: damyanp

lib/Transforms/InstCombine/InstCombineCompares.cpp

@@ -14,6 +14,7 @@
 #include "InstCombineInternal.h"
 #include "llvm/ADT/APSInt.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/Triple.h" // HLSL Change
 #include "llvm/Analysis/ConstantFolding.h"
 #include "llvm/Analysis/InstructionSimplify.h"
 #include "llvm/Analysis/MemoryBuiltins.h"
@@ -2053,6 +2054,10 @@ Instruction *InstCombiner::visitICmpInstWithCastAndCast(ICmpInst &ICI) {
 static Instruction *ProcessUGT_ADDCST_ADD(ICmpInst &I, Value *A, Value *B,
                                           ConstantInt *CI2, ConstantInt *CI1,
                                           InstCombiner &IC) {
+  // HLSL Change begin - DXIL has no sadd.with.overflow intrinsic.
+  if (Triple(I.getModule()->getTargetTriple()).isDXIL())
+    return nullptr;
+  // HLSL Change end
   // The transformation we're trying to do here is to transform this into an
   // llvm.sadd.with.overflow.  To do this, we have to replace the original add
   // with a narrower add, and discard the add-with-constant that is part of the
@@ -2245,6 +2250,10 @@ bool InstCombiner::OptimizeOverflowCheck(OverflowCheckFlavor OCF, Value *LHS,
 ///          replacement required.
 static Instruction *ProcessUMulZExtIdiom(ICmpInst &I, Value *MulVal,
                                          Value *OtherVal, InstCombiner &IC) {
+  // HLSL Change begin - DXIL has no umul.with.overflow intrinsic.
+  if (Triple(I.getModule()->getTargetTriple()).isDXIL())
+    return nullptr;
+  // HLSL Change end
   // Don't bother doing this transformation for pointers, don't do it for
   // vectors.
   if (!isa<IntegerType>(MulVal->getType()))

tools/clang/test/DXC/Passes/InstructionCombining/instcombine-no-sadd-overflow-dxil.ll

@@ -0,0 +1,24 @@
+; RUN: %dxopt %s -instcombine -S | FileCheck %s
+
+; Regression test for https://github.com/microsoft/DirectXShaderCompiler/issues/8598
+;
+; InstCombine used to fold this signed add overflow-check idiom into
+; llvm.sadd.with.overflow, which is not a legal DXIL intrinsic. For DXIL the
+; add/compare must be left alone.
+
+target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64"
+target triple = "dxil-ms-dx"
+
+; CHECK-LABEL: define i1 @test_sadd
+; CHECK-NOT: sadd.with.overflow
+; CHECK-NOT: extractvalue
+; CHECK: add {{(nsw )?}}i32
+; CHECK: icmp ugt i32
+define i1 @test_sadd(i8 %a8, i8 %b8) {
+  %a = sext i8 %a8 to i32
+  %b = sext i8 %b8 to i32
+  %add = add i32 %a, %b
+  %addcst = add i32 %add, 128
+  %cmp = icmp ugt i32 %addcst, 255
+  ret i1 %cmp
+}

tools/clang/test/DXC/Passes/InstructionCombining/instcombine-no-umul-overflow-dxil.ll

@@ -0,0 +1,23 @@
+; RUN: %dxopt %s -instcombine -S | FileCheck %s
+
+; Regression test for https://github.com/microsoft/DirectXShaderCompiler/issues/8598
+;
+; InstCombine used to fold the "widened multiply compared against UINT_MAX"
+; idiom into llvm.umul.with.overflow, which is not a legal DXIL intrinsic and
+; failed validation. For DXIL the multiply/compare must be left alone.
+
+target datalayout = "e-m:e-p:32:32-i1:32-i8:32-i16:32-i32:32-i64:64-f16:32-f32:32-f64:64-n8:16:32:64"
+target triple = "dxil-ms-dx"
+
+; CHECK-LABEL: define i1 @test
+; CHECK-NOT: umul.with.overflow
+; CHECK-NOT: extractvalue
+; CHECK: mul {{(nuw )?}}i64 %xw, %yw
+; CHECK: icmp ugt i64
+define i1 @test(i32 %x, i32 %y) {
+  %xw = zext i32 %x to i64
+  %yw = zext i32 %y to i64
+  %mul = mul i64 %xw, %yw
+  %cmp = icmp ugt i64 %mul, 4294967295
+  ret i1 %cmp
+}

tools/clang/test/HLSLFileCheck/hlsl/intrinsics/mul/mul_uint64_overflow_check.hlsl

@@ -0,0 +1,23 @@
+// RUN: %dxc -E main -T cs_6_0 %s | FileCheck %s
+
+// Regression test for https://github.com/microsoft/DirectXShaderCompiler/issues/8598
+//
+// The 32x32->64 bit multiply overflow check below used to be folded by
+// InstCombine into llvm.umul.with.overflow.i32, an intrinsic that is not legal
+// in DXIL and failed validation with optimizations enabled. Make sure it
+// compiles and validates, keeping a plain 64-bit multiply.
+
+// CHECK-NOT: umul.with.overflow
+// CHECK: mul nuw i64
+
+RWStructuredBuffer<uint> buf : register(u0);
+
+[numthreads(1, 1, 1)]
+void main()
+{
+    uint x = buf[0];
+    uint y = buf[1];
+    if (((uint64_t)x) * ((uint64_t)y) > 0xffffffffull) {
+        buf[2] = 1;
+    }
+}