Skip to content

Commit a700005

Browse files
a74nhkunalspathak
a74nh
and
kunalspathak
authored
JIT ARM64-SVE: Add AddAcross (#101674)
* JIT ARM64-SVE: Add AddAcross * Remove enum changes * Fix SVE tests max vector size to 512bit * fix zip test cases --------- Co-authored-by: Kunal Pathak <[email protected]>
1 parent b0c5a3d commit a700005

File tree

11 files changed

+767
-147
lines changed

11 files changed

+767
-147
lines changed

‎src/coreclr/jit/codegenarm64test.cpp

+8-8
Original file line numberDiff line numberDiff line change
@@ -5314,11 +5314,11 @@ void CodeGen::genArm64EmitterUnitTestsSve()
53145314
#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED
53155315

53165316
// IF_SVE_AI_3A
5317-
theEmitter->emitIns_R_R_R(INS_sve_saddv, EA_1BYTE, REG_V1, REG_P4, REG_V2,
5317+
theEmitter->emitIns_R_R_R(INS_sve_saddv, EA_SCALABLE, REG_V1, REG_P4, REG_V2,
53185318
INS_OPTS_SCALABLE_B); // SADDV <Dd>, <Pg>, <Zn>.<T>
5319-
theEmitter->emitIns_R_R_R(INS_sve_saddv, EA_2BYTE, REG_V2, REG_P5, REG_V3,
5319+
theEmitter->emitIns_R_R_R(INS_sve_saddv, EA_SCALABLE, REG_V2, REG_P5, REG_V3,
53205320
INS_OPTS_SCALABLE_H); // SADDV <Dd>, <Pg>, <Zn>.<T>
5321-
theEmitter->emitIns_R_R_R(INS_sve_uaddv, EA_4BYTE, REG_V3, REG_P6, REG_V4,
5321+
theEmitter->emitIns_R_R_R(INS_sve_uaddv, EA_SCALABLE, REG_V3, REG_P6, REG_V4,
53225322
INS_OPTS_SCALABLE_S); // UADDV <Dd>, <Pg>, <Zn>.<T>
53235323

53245324
// IF_SVE_AJ_3A
@@ -6768,15 +6768,15 @@ void CodeGen::genArm64EmitterUnitTestsSve()
67686768
#endif // ALL_ARM64_EMITTER_UNIT_TESTS_SVE_UNSUPPORTED
67696769

67706770
// IF_SVE_HE_3A
6771-
theEmitter->emitIns_R_R_R(INS_sve_faddv, EA_2BYTE, REG_V21, REG_P7, REG_V7,
6771+
theEmitter->emitIns_R_R_R(INS_sve_faddv, EA_SCALABLE, REG_V21, REG_P7, REG_V7,
67726772
INS_OPTS_SCALABLE_H); // FADDV <V><d>, <Pg>, <Zn>.<T>
6773-
theEmitter->emitIns_R_R_R(INS_sve_fmaxnmv, EA_2BYTE, REG_V22, REG_P6, REG_V6,
6773+
theEmitter->emitIns_R_R_R(INS_sve_fmaxnmv, EA_SCALABLE, REG_V22, REG_P6, REG_V6,
67746774
INS_OPTS_SCALABLE_H); // FMAXNMV <V><d>, <Pg>, <Zn>.<T>
6775-
theEmitter->emitIns_R_R_R(INS_sve_fmaxv, EA_4BYTE, REG_V23, REG_P5, REG_V5,
6775+
theEmitter->emitIns_R_R_R(INS_sve_fmaxv, EA_SCALABLE, REG_V23, REG_P5, REG_V5,
67766776
INS_OPTS_SCALABLE_S); // FMAXV <V><d>, <Pg>, <Zn>.<T>
6777-
theEmitter->emitIns_R_R_R(INS_sve_fminnmv, EA_8BYTE, REG_V24, REG_P4, REG_V4,
6777+
theEmitter->emitIns_R_R_R(INS_sve_fminnmv, EA_SCALABLE, REG_V24, REG_P4, REG_V4,
67786778
INS_OPTS_SCALABLE_D); // FMINNMV <V><d>, <Pg>, <Zn>.<T>
6779-
theEmitter->emitIns_R_R_R(INS_sve_fminv, EA_4BYTE, REG_V25, REG_P3, REG_V3,
6779+
theEmitter->emitIns_R_R_R(INS_sve_fminv, EA_SCALABLE, REG_V25, REG_P3, REG_V3,
67806780
INS_OPTS_SCALABLE_S); // FMINV <V><d>, <Pg>, <Zn>.<T>
67816781

67826782
// IF_SVE_HQ_3A

‎src/coreclr/jit/emitarm64sve.cpp

+23-6
Original file line numberDiff line numberDiff line change
@@ -3060,7 +3060,6 @@ void emitter::emitInsSve_R_R_R(instruction ins,
30603060
break;
30613061

30623062
case INS_sve_saddv:
3063-
case INS_sve_uaddv:
30643063
assert(isFloatReg(reg1));
30653064
assert(isLowPredicateRegister(reg2));
30663065
assert(isVectorRegister(reg3));
@@ -3069,6 +3068,15 @@ void emitter::emitInsSve_R_R_R(instruction ins,
30693068
fmt = IF_SVE_AI_3A;
30703069
break;
30713070

3071+
case INS_sve_uaddv:
3072+
assert(isFloatReg(reg1));
3073+
assert(isLowPredicateRegister(reg2));
3074+
assert(isVectorRegister(reg3));
3075+
assert(insOptsScalableStandard(opt));
3076+
assert(insScalableOptsNone(sopt));
3077+
fmt = IF_SVE_AI_3A;
3078+
break;
3079+
30723080
case INS_sve_addqv:
30733081
unreached(); // TODO-SVE: Not yet supported.
30743082
assert(isVectorRegister(reg1));
@@ -4059,7 +4067,7 @@ void emitter::emitInsSve_R_R_R(instruction ins,
40594067
assert(isLowPredicateRegister(reg2));
40604068
assert(isVectorRegister(reg3));
40614069
assert(insOptsScalableFloat(opt));
4062-
assert(isValidVectorElemsizeSveFloat(size));
4070+
assert(isScalableVectorSize(size));
40634071
assert(insScalableOptsNone(sopt));
40644072
fmt = IF_SVE_HE_3A;
40654073
break;
@@ -4069,7 +4077,7 @@ void emitter::emitInsSve_R_R_R(instruction ins,
40694077
assert(isLowPredicateRegister(reg2));
40704078
assert(isVectorRegister(reg3));
40714079
assert(insOptsScalableFloat(opt));
4072-
assert(isValidVectorElemsizeSveFloat(size));
4080+
assert(isScalableVectorSize(size));
40734081
assert(insScalableOptsNone(sopt));
40744082
fmt = IF_SVE_HJ_3A;
40754083
break;
@@ -12618,7 +12626,7 @@ void emitter::emitInsSveSanityCheck(instrDesc* id)
1261812626
assert(isVectorRegister(id->idReg1())); // ddddd
1261912627
assert(isLowPredicateRegister(id->idReg2())); // ggg
1262012628
assert(isVectorRegister(id->idReg3())); // mmmmm
12621-
assert(isValidVectorElemsizeSveFloat(id->idOpSize()));
12629+
assert(isScalableVectorSize(id->idOpSize()));
1262212630
break;
1262312631

1262412632
// Scalable to general register.
@@ -13211,11 +13219,20 @@ void emitter::emitInsSveSanityCheck(instrDesc* id)
1321113219

1321213220
// Scalable, widening to scalar SIMD.
1321313221
case IF_SVE_AI_3A: // ........xx...... ...gggnnnnnddddd -- SVE integer add reduction (predicated)
13214-
assert(insOptsScalableWide(id->idInsOpt())); // xx
13222+
switch (id->idIns())
13223+
{
13224+
case INS_sve_saddv:
13225+
assert(insOptsScalableWide(id->idInsOpt())); // xx
13226+
break;
13227+
13228+
default:
13229+
assert(insOptsScalableStandard(id->idInsOpt())); // xx
13230+
break;
13231+
}
1321513232
assert(isVectorRegister(id->idReg1())); // ddddd
1321613233
assert(isLowPredicateRegister(id->idReg2())); // ggg
1321713234
assert(isVectorRegister(id->idReg3())); // mmmmm
13218-
assert(isValidVectorElemsizeWidening(id->idOpSize()));
13235+
assert(isScalableVectorSize(id->idOpSize()));
1321913236
break;
1322013237

1322113238
// Scalable, possibly FP.

‎src/coreclr/jit/hwintrinsiclistarm64sve.h

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
// Sve
2020
HARDWARE_INTRINSIC(Sve, Abs, -1, -1, false, {INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_abs, INS_invalid, INS_sve_fabs, INS_sve_fabs}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation)
2121
HARDWARE_INTRINSIC(Sve, Add, -1, -1, false, {INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_add, INS_sve_fadd, INS_sve_fadd}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
22+
HARDWARE_INTRINSIC(Sve, AddAcross, -1, 1, true, {INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_saddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_uaddv, INS_sve_faddv, INS_sve_faddv}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation)
2223
HARDWARE_INTRINSIC(Sve, ConditionalSelect, -1, 3, true, {INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel, INS_sve_sel}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_SupportsContainment)
2324
HARDWARE_INTRINSIC(Sve, Count16BitElements, 0, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cnth, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed)
2425
HARDWARE_INTRINSIC(Sve, Count32BitElements, 0, 1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_cntw, INS_invalid, INS_invalid, INS_invalid}, HW_Category_Scalar, HW_Flag_Scalable|HW_Flag_HasEnumOperand|HW_Flag_SpecialCodeGen|HW_Flag_NoFloatingPointUsed)

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs

+62
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,68 @@ internal Arm64() { }
148148
/// </summary>
149149
public static unsafe Vector<double> Add(Vector<double> left, Vector<double> right) { throw new PlatformNotSupportedException(); }
150150

151+
/// AddAcross : Add reduction
152+
153+
/// <summary>
154+
/// float64_t svaddv[_f64](svbool_t pg, svfloat64_t op)
155+
/// FADDV Dresult, Pg, Zop.D
156+
/// </summary>
157+
public static unsafe Vector<double> AddAcross(Vector<double> value) { throw new PlatformNotSupportedException(); }
158+
159+
/// <summary>
160+
/// int64_t svaddv[_s16](svbool_t pg, svint16_t op)
161+
/// SADDV Dresult, Pg, Zop.H
162+
/// </summary>
163+
public static unsafe Vector<long> AddAcross(Vector<short> value) { throw new PlatformNotSupportedException(); }
164+
165+
/// <summary>
166+
/// int64_t svaddv[_s32](svbool_t pg, svint32_t op)
167+
/// SADDV Dresult, Pg, Zop.S
168+
/// </summary>
169+
public static unsafe Vector<long> AddAcross(Vector<int> value) { throw new PlatformNotSupportedException(); }
170+
171+
/// <summary>
172+
/// int64_t svaddv[_s8](svbool_t pg, svint8_t op)
173+
/// SADDV Dresult, Pg, Zop.B
174+
/// </summary>
175+
public static unsafe Vector<long> AddAcross(Vector<sbyte> value) { throw new PlatformNotSupportedException(); }
176+
177+
/// <summary>
178+
/// int64_t svaddv[_s64](svbool_t pg, svint64_t op)
179+
/// UADDV Dresult, Pg, Zop.D
180+
/// </summary>
181+
public static unsafe Vector<long> AddAcross(Vector<long> value) { throw new PlatformNotSupportedException(); }
182+
183+
/// <summary>
184+
/// float32_t svaddv[_f32](svbool_t pg, svfloat32_t op)
185+
/// FADDV Sresult, Pg, Zop.S
186+
/// </summary>
187+
public static unsafe Vector<float> AddAcross(Vector<float> value) { throw new PlatformNotSupportedException(); }
188+
189+
/// <summary>
190+
/// uint64_t svaddv[_u8](svbool_t pg, svuint8_t op)
191+
/// UADDV Dresult, Pg, Zop.B
192+
/// </summary>
193+
public static unsafe Vector<ulong> AddAcross(Vector<byte> value) { throw new PlatformNotSupportedException(); }
194+
195+
/// <summary>
196+
/// uint64_t svaddv[_u16](svbool_t pg, svuint16_t op)
197+
/// UADDV Dresult, Pg, Zop.H
198+
/// </summary>
199+
public static unsafe Vector<ulong> AddAcross(Vector<ushort> value) { throw new PlatformNotSupportedException(); }
200+
201+
/// <summary>
202+
/// uint64_t svaddv[_u32](svbool_t pg, svuint32_t op)
203+
/// UADDV Dresult, Pg, Zop.S
204+
/// </summary>
205+
public static unsafe Vector<ulong> AddAcross(Vector<uint> value) { throw new PlatformNotSupportedException(); }
206+
207+
/// <summary>
208+
/// uint64_t svaddv[_u64](svbool_t pg, svuint64_t op)
209+
/// UADDV Dresult, Pg, Zop.D
210+
/// </summary>
211+
public static unsafe Vector<ulong> AddAcross(Vector<ulong> value) { throw new PlatformNotSupportedException(); }
212+
151213

152214
/// ConditionalSelect : Conditionally select elements
153215

‎src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs

+63
Original file line numberDiff line numberDiff line change
@@ -177,6 +177,69 @@ internal Arm64() { }
177177
public static unsafe Vector<double> Add(Vector<double> left, Vector<double> right) => Add(left, right);
178178

179179

180+
/// AddAcross : Add reduction
181+
182+
/// <summary>
183+
/// float64_t svaddv[_f64](svbool_t pg, svfloat64_t op)
184+
/// FADDV Dresult, Pg, Zop.D
185+
/// </summary>
186+
public static unsafe Vector<double> AddAcross(Vector<double> value) => AddAcross(value);
187+
188+
/// <summary>
189+
/// int64_t svaddv[_s16](svbool_t pg, svint16_t op)
190+
/// SADDV Dresult, Pg, Zop.H
191+
/// </summary>
192+
public static unsafe Vector<long> AddAcross(Vector<short> value) => AddAcross(value);
193+
194+
/// <summary>
195+
/// int64_t svaddv[_s32](svbool_t pg, svint32_t op)
196+
/// SADDV Dresult, Pg, Zop.S
197+
/// </summary>
198+
public static unsafe Vector<long> AddAcross(Vector<int> value) => AddAcross(value);
199+
200+
/// <summary>
201+
/// int64_t svaddv[_s8](svbool_t pg, svint8_t op)
202+
/// SADDV Dresult, Pg, Zop.B
203+
/// </summary>
204+
public static unsafe Vector<long> AddAcross(Vector<sbyte> value) => AddAcross(value);
205+
206+
/// <summary>
207+
/// int64_t svaddv[_s64](svbool_t pg, svint64_t op)
208+
/// UADDV Dresult, Pg, Zop.D
209+
/// </summary>
210+
public static unsafe Vector<long> AddAcross(Vector<long> value) => AddAcross(value);
211+
212+
/// <summary>
213+
/// float32_t svaddv[_f32](svbool_t pg, svfloat32_t op)
214+
/// FADDV Sresult, Pg, Zop.S
215+
/// </summary>
216+
public static unsafe Vector<float> AddAcross(Vector<float> value) => AddAcross(value);
217+
218+
/// <summary>
219+
/// uint64_t svaddv[_u8](svbool_t pg, svuint8_t op)
220+
/// UADDV Dresult, Pg, Zop.B
221+
/// </summary>
222+
public static unsafe Vector<ulong> AddAcross(Vector<byte> value) => AddAcross(value);
223+
224+
/// <summary>
225+
/// uint64_t svaddv[_u16](svbool_t pg, svuint16_t op)
226+
/// UADDV Dresult, Pg, Zop.H
227+
/// </summary>
228+
public static unsafe Vector<ulong> AddAcross(Vector<ushort> value) => AddAcross(value);
229+
230+
/// <summary>
231+
/// uint64_t svaddv[_u32](svbool_t pg, svuint32_t op)
232+
/// UADDV Dresult, Pg, Zop.S
233+
/// </summary>
234+
public static unsafe Vector<ulong> AddAcross(Vector<uint> value) => AddAcross(value);
235+
236+
/// <summary>
237+
/// uint64_t svaddv[_u64](svbool_t pg, svuint64_t op)
238+
/// UADDV Dresult, Pg, Zop.D
239+
/// </summary>
240+
public static unsafe Vector<ulong> AddAcross(Vector<ulong> value) => AddAcross(value);
241+
242+
180243
/// ConditionalSelect : Conditionally select elements
181244

182245
/// <summary>

‎src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs

+10
Original file line numberDiff line numberDiff line change
@@ -4187,6 +4187,16 @@ internal Arm64() { }
41874187
public static System.Numerics.Vector<ulong> Add(System.Numerics.Vector<ulong> left, System.Numerics.Vector<ulong> right) { throw null; }
41884188
public static System.Numerics.Vector<float> Add(System.Numerics.Vector<float> left, System.Numerics.Vector<float> right) { throw null; }
41894189
public static System.Numerics.Vector<double> Add(System.Numerics.Vector<double> left, System.Numerics.Vector<double> right) { throw null; }
4190+
public static System.Numerics.Vector<double> AddAcross(System.Numerics.Vector<double> value) { throw null; }
4191+
public static System.Numerics.Vector<long> AddAcross(System.Numerics.Vector<short> value) { throw null; }
4192+
public static System.Numerics.Vector<long> AddAcross(System.Numerics.Vector<int> value) { throw null; }
4193+
public static System.Numerics.Vector<long> AddAcross(System.Numerics.Vector<sbyte> value) { throw null; }
4194+
public static System.Numerics.Vector<long> AddAcross(System.Numerics.Vector<long> value) { throw null; }
4195+
public static System.Numerics.Vector<float> AddAcross(System.Numerics.Vector<float> value) { throw null; }
4196+
public static System.Numerics.Vector<ulong> AddAcross(System.Numerics.Vector<byte> value) { throw null; }
4197+
public static System.Numerics.Vector<ulong> AddAcross(System.Numerics.Vector<ushort> value) { throw null; }
4198+
public static System.Numerics.Vector<ulong> AddAcross(System.Numerics.Vector<uint> value) { throw null; }
4199+
public static System.Numerics.Vector<ulong> AddAcross(System.Numerics.Vector<ulong> value) { throw null; }
41904200
public static ulong Count16BitElements([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; }
41914201
public static ulong Count32BitElements([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; }
41924202
public static ulong Count64BitElements([ConstantExpected] SveMaskPattern pattern = SveMaskPattern.All) { throw null; }

0 commit comments

Comments
 (0)