Remove the extra-inhabitant value witness functions.

This is essentially a long-belated follow-up to Arnold's swiftlang#12606.
The key observation here is that the enum-tag-single-payload witnesses
are strictly more powerful than the XI witnesses: you can simulate
the XI witnesses by using an extra case count that's <= the XI count.
Of course the result is less efficient than the XI witnesses, but
that's less important than overall code size, and we can work on
fast-paths for that.

The extra inhabitant count is stored in a 32-bit field (always present)
following the ValueWitnessFlags, which now occupy a fixed 32 bits.
This inflates non-XI VWTs on 32-bit targets by a word, but the net effect
on XI VWTs is to shrink them by two words, which is likely to be the
more important change.  Also, being able to access the XI count directly
should be a nice win.

Author: rjmccall

docs/Runtime.md

@@ -260,9 +260,7 @@ optimization.
 0000000000023a40 T _swift_assignExistentialWithCopy
 000000000001dbf0 T _swift_copyPOD
 000000000001c560 T _swift_getEnumCaseMultiPayload
-000000000001be60 T _swift_getEnumCaseSinglePayload
 000000000001c400 T _swift_storeEnumTagMultiPayload
-000000000001bf90 T _swift_storeEnumTagSinglePayload
 ```
 
 ## Type metadata lookup

include/swift/ABI/Metadata.h

@@ -295,10 +295,9 @@ class TargetValueWitnessTypes {
   // Handle the data witnesses explicitly so we can use more specific
   // types for the flags enums.
   typedef size_t size;
-  typedef ValueWitnessFlags flags;
   typedef size_t stride;
-  typedef ExtraInhabitantFlags extraInhabitantFlags;
-
+  typedef ValueWitnessFlags flags;
+  typedef uint32_t extraInhabitantCount;
 };
 
 struct TypeLayout;
@@ -376,16 +375,9 @@ template <typename Runtime> struct TargetValueWitnessTable {
 
   /// The number of extra inhabitants, that is, bit patterns that do not form
   /// valid values of the type, in this type's binary representation.
-  unsigned getNumExtraInhabitants() const;
-
-  /// Assert that this value witness table is an extra-inhabitants
-  /// value witness table and return it as such.
-  ///
-  /// This has an awful name because it's supposed to be internal to
-  /// this file.  Code outside this file should use LLVM's cast/dyn_cast.
-  /// We don't want to use those here because we need to avoid accidentally
-  /// introducing ABI dependencies on LLVM structures.
-  const struct ExtraInhabitantsValueWitnessTable *_asXIVWT() const;
+  unsigned getNumExtraInhabitants() const {
+    return extraInhabitantCount;
+  }
 
   /// Assert that this value witness table is an enum value witness table
   /// and return it as such.
@@ -608,13 +600,6 @@ struct TargetMetadata {
   #define DATA_VALUE_WITNESS(LOWER, UPPER, TYPE)
   #include "swift/ABI/ValueWitness.def"
 
-  int vw_getExtraInhabitantIndex(const OpaqueValue *value) const  {
-    return getValueWitnesses()->_asXIVWT()->getExtraInhabitantIndex(value, this);
-  }
-  void vw_storeExtraInhabitant(OpaqueValue *value, int index) const {
-    getValueWitnesses()->_asXIVWT()->storeExtraInhabitant(value, index, this);
-  }
-
   unsigned vw_getEnumTag(const OpaqueValue *value) const {
     return getValueWitnesses()->_asEVWT()->getEnumTag(const_cast<OpaqueValue*>(value), this);
   }
@@ -637,6 +622,10 @@ struct TargetMetadata {
     return getValueWitnesses()->getStride();
   }
 
+  unsigned vw_getNumExtraInhabitants() const {
+    return getValueWitnesses()->getNumExtraInhabitants();
+  }
+
   /// Allocate an out-of-line buffer if values of this type don't fit in the
   /// ValueBuffer.
   /// NOTE: This is not a box for copy-on-write existentials.

include/swift/ABI/MetadataValues.h

@@ -121,24 +121,27 @@ class TargetValueWitnessFlags {
   // flags of the field types can be mostly bitwise-or'ed together to derive the
   // flags for the struct. (The "non-inline" and "has-extra-inhabitants" bits
   // still require additional fixup.)
-  enum : int_type {
+  enum : uint32_t {
     AlignmentMask =       0x000000FF,
+    // unused             0x0000FF00,
     IsNonPOD =            0x00010000,
     IsNonInline =         0x00020000,
-    HasExtraInhabitants = 0x00040000,
+    // unused             0x00040000,
     HasSpareBits =        0x00080000,
     IsNonBitwiseTakable = 0x00100000,
     HasEnumWitnesses =    0x00200000,
     Incomplete =          0x00400000,
-
-    // Everything else is reserved.
+    // unused             0xFF800000,
   };
 
+  static constexpr const uint32_t MaxNumExtraInhabitants = 0x7FFFFFFF;
+
 private:
-  int_type Data;
+  uint32_t Data;
+
+  explicit constexpr TargetValueWitnessFlags(uint32_t data) : Data(data) {}
 
 public:
-  explicit constexpr TargetValueWitnessFlags(int_type data) : Data(data) {}
   constexpr TargetValueWitnessFlags() : Data(0) {}
 
   /// The required alignment of the first byte of an object of this
@@ -192,22 +195,12 @@ class TargetValueWitnessFlags {
     return TargetValueWitnessFlags((Data & ~IsNonBitwiseTakable) |
                                    (isBT ? 0 : IsNonBitwiseTakable));
   }
-  /// True if this type's binary representation has extra inhabitants, that is,
-  /// bit patterns that do not form valid values of the type.
-  ///
-  /// If true, then the extra inhabitant value witness table entries are
-  /// available in this type's value witness table.
-  bool hasExtraInhabitants() const { return Data & HasExtraInhabitants; }
+
   /// True if this type's binary representation is that of an enum, and the
   /// enum value witness table entries are available in this type's value
   /// witness table.
   bool hasEnumWitnesses() const { return Data & HasEnumWitnesses; }
   constexpr TargetValueWitnessFlags
-  withExtraInhabitants(bool hasExtraInhabitants) const {
-    return TargetValueWitnessFlags((Data & ~HasExtraInhabitants) |
-                               (hasExtraInhabitants ? HasExtraInhabitants : 0));
-  }
-  constexpr TargetValueWitnessFlags
   withEnumWitnesses(bool hasEnumWitnesses) const {
     return TargetValueWitnessFlags((Data & ~HasEnumWitnesses) |
                                    (hasEnumWitnesses ? HasEnumWitnesses : 0));
@@ -223,44 +216,12 @@ class TargetValueWitnessFlags {
                                    (isIncomplete ? Incomplete : 0));
   }
 
-  constexpr int_type getOpaqueValue() const { return Data; }
-  static constexpr TargetValueWitnessFlags getFromOpaqueValue(int_type data) {
-    return TargetValueWitnessFlags(data);
+  constexpr uint32_t getOpaqueValue() const {
+    return Data;
   }
 };
 using ValueWitnessFlags = TargetValueWitnessFlags<size_t>;
 
-/// Flags stored in a value-witness table with extra inhabitants.
-template <typename int_type>
-class TargetExtraInhabitantFlags {
-public:
-  enum : int_type {
-    NumExtraInhabitantsMask = 0x7FFFFFFFU,
-    ExtraInhabitantFlags
-  };
-  int_type Data;
-
-  constexpr TargetExtraInhabitantFlags(int_type data) : Data(data) {}
-
-public:
-  constexpr TargetExtraInhabitantFlags() : Data(0) {}
-  /// The number of extra inhabitants in the type's representation.
-  int getNumExtraInhabitants() const { return Data & NumExtraInhabitantsMask; }
-
-  constexpr TargetExtraInhabitantFlags
-  withNumExtraInhabitants(unsigned numExtraInhabitants) const {
-    return TargetExtraInhabitantFlags((Data & ~NumExtraInhabitantsMask) |
-                                      numExtraInhabitants);
-  }
-
-  constexpr int_type getOpaqueValue() const { return Data; }
-  static constexpr TargetExtraInhabitantFlags getFromOpaqueValue(int_type data){
-    return TargetExtraInhabitantFlags(data);
-  }
-};
-using ExtraInhabitantFlags =
-  TargetExtraInhabitantFlags<size_t>;
-
 /// Flags for dynamic-cast operations.
 enum class DynamicCastFlags : size_t {
   /// All flags clear.

include/swift/ABI/ValueWitness.def

@@ -23,39 +23,27 @@
 #if defined(WANT_ALL_VALUE_WITNESSES)
 #undef WANT_ALL_VALUE_WITNESSES
 #define WANT_REQUIRED_VALUE_WITNESSES         1
-#define WANT_EXTRA_INHABITANT_VALUE_WITNESSES 1
 #define WANT_ENUM_VALUE_WITNESSES             1
 
 /// WANT_ONLY_REQUIRED_VALUE_WITNESSES
 ///   Define this to expand only the required value witnesses.
 #elif defined(WANT_ONLY_REQUIRED_VALUE_WITNESSES)
 #undef WANT_ONLY_REQUIRED_VALUE_WITNESSES
 #define WANT_REQUIRED_VALUE_WITNESSES         1
-#define WANT_EXTRA_INHABITANT_VALUE_WITNESSES 0
-#define WANT_ENUM_VALUE_WITNESSES             0
-
-/// WANT_ONLY_EXTRA_INHABITANT_VALUE_WITNESSES
-///   Define this to expand only the extra-inhabitant value witnesses.
-#elif defined(WANT_ONLY_EXTRA_INHABITANT_VALUE_WITNESSES)
-#undef WANT_ONLY_EXTRA_INHABITANT_VALUE_WITNESSES
-#define WANT_REQUIRED_VALUE_WITNESSES         0
-#define WANT_EXTRA_INHABITANT_VALUE_WITNESSES 1
 #define WANT_ENUM_VALUE_WITNESSES             0
 
 /// WANT_ONLY_ENUM_VALUE_WITNESSES
 ///   Define this to expand only the enum value witnesses.
 #elif  defined(WANT_ONLY_ENUM_VALUE_WITNESSES)
 #undef  WANT_ONLY_ENUM_VALUE_WITNESSES
 #define WANT_REQUIRED_VALUE_WITNESSES         0
-#define WANT_EXTRA_INHABITANT_VALUE_WITNESSES 0
 #define WANT_ENUM_VALUE_WITNESSES             1
 
 /// WANT_REQUIRED_VALUE_WITNESSES
-/// WANT_EXTRA_INHABITANT_VALUE_WITNESSES
 /// WANT_ENUM_VALUE_WITNESSES
 ///   Define all of these to control exactly what to expand.
 #else
-#if !defined(WANT_REQUIRED_VALUE_WITNESSES) ||  !defined(WANT_EXTRA_INHABITANT_VALUE_WITNESSES) || !defined(WANT_ENUM_VALUE_WITNESSES)
+#if !defined(WANT_REQUIRED_VALUE_WITNESSES) || !defined(WANT_ENUM_VALUE_WITNESSES)
 #error failed to define a WANT macro; possible typo?
 #endif
 #endif
@@ -206,7 +194,16 @@ BEGIN_VALUE_WITNESS_RANGE(TypeLayoutWitness,
 BEGIN_VALUE_WITNESS_RANGE(RequiredTypeLayoutWitness,
                           Size)
 
-///   SIZE_TYPE flags;
+///   SIZE_TYPE stride;
+///
+/// The required size per element of an array of this type. It is at least
+/// one, even for zero-sized types, like the empty tuple.
+DATA_VALUE_WITNESS(stride,
+                   Stride,
+                   SIZE_TYPE)
+
+
+///   UINT_TYPE flags;
 ///
 /// The ValueWitnessAlignmentMask bits represent the required
 /// alignment of the first byte of an object of this type, expressed
@@ -222,97 +219,35 @@ BEGIN_VALUE_WITNESS_RANGE(RequiredTypeLayoutWitness,
 /// The ValueWitnessIsNonInline bit is set if the type cannot be
 /// represented in a fixed-size buffer or if it is not bitwise takable.
 ///
-/// The Enum_HasExtraInhabitants bit is set if the type's binary
-/// representation has "extra inhabitants" that do not form valid values of
-/// the type, and the value witness table contains the ExtraInhabitantWitness
-/// entries.
+/// The ExtraInhabitantsMask bits represent the number of "extra inhabitants"
+/// of the bit representation of the value that do not form valid values of
+/// the type.
 ///
 /// The Enum_HasSpareBits bit is set if the type's binary representation
 /// has unused bits.
 ///
 /// The HasEnumWitnesses bit is set if the type is an enum type.
 DATA_VALUE_WITNESS(flags,
                    Flags,
-                   SIZE_TYPE)
+                   UINT_TYPE)
 
-///   SIZE_TYPE stride;
+///   UINT_TYPE extraInhabitantCount;
 ///
-/// The required size per element of an array of this type. It is at least
-/// one, even for zero-sized types, like the empty tuple.
-DATA_VALUE_WITNESS(stride,
-                   Stride,
-                   SIZE_TYPE)
+/// The number of extra inhabitants in the type.
+DATA_VALUE_WITNESS(extraInhabitantCount,
+                   ExtraInhabitantCount,
+                   UINT_TYPE)
 
 END_VALUE_WITNESS_RANGE(RequiredTypeLayoutWitness,
-                        Stride)
+                        ExtraInhabitantCount)
 
 END_VALUE_WITNESS_RANGE(RequiredValueWitness,
-                        Stride)
-
-#endif /* WANT_REQUIRED_VALUE_WITNESSES */
-
-#if WANT_EXTRA_INHABITANT_VALUE_WITNESSES
-
-// The following value witnesses are conditionally present based on
-// the Enum_HasExtraInhabitants bit of the flags.
-
-///   SIZE_TYPE extraInhabitantFlags;
-///
-/// These bits are always present if the extra inhabitants witnesses are:
-///
-/// - The NumExtraInhabitantsMask bits contain the number of extra
-///   inhabitants of the type representation.
-///
-/// If the Enum_HasSpareBits flag is set in the value witness flags, these
-/// additional flags are available:
-///
-/// - The NumSpareBitsMask bits contain the number of (host-endian) contiguous
-///   spare bits in the type representation.
-/// - The SpareBitsShiftMask bits contain the (host-endian) bit offset of the
-///   lowest spare bit.
-DATA_VALUE_WITNESS(extraInhabitantFlags,
-                   ExtraInhabitantFlags,
-                   SIZE_TYPE)
-
-BEGIN_VALUE_WITNESS_RANGE(ExtraInhabitantValueWitness,
-                          ExtraInhabitantFlags)
+                        ExtraInhabitantCount)
 
 END_VALUE_WITNESS_RANGE(TypeLayoutWitness,
-                        ExtraInhabitantFlags)
-
-///   void (*storeExtraInhabitant)(T *obj, int index, M *self);
-///
-/// Given an invalid object of this type, store the representation of an
-/// extra inhabitant of the type. The object will remain invalid, because
-/// an extra inhabitant is by definition an invalid representation of the
-/// type. index must be less than numExtraInhabitants.
-FUNCTION_VALUE_WITNESS(storeExtraInhabitant,
-                       StoreExtraInhabitant,
-                       VOID_TYPE,
-                       (MUTABLE_VALUE_TYPE, INT_TYPE, TYPE_TYPE))
-
-BEGIN_VALUE_WITNESS_RANGE(ExtraInhabitantValueWitnessFunction,
-                          StoreExtraInhabitant)
-
-///   int (*getExtraInhabitantIndex)(T *obj, M *self);
-///
-/// Given an invalid object of this type with an extra inhabitant
-/// representation, returns the index of the extra inhabitant representation.
-/// Returns -1 if the object is a valid value of the type. If non-negative,
-/// the return value is the same index that can be passed to
-/// storeExtraInhabitant to reproduce the representation.
-FUNCTION_VALUE_WITNESS(getExtraInhabitantIndex,
-                       GetExtraInhabitantIndex,
-                       INT_TYPE,
-                       (IMMUTABLE_VALUE_TYPE, TYPE_TYPE))
-
-END_VALUE_WITNESS_RANGE(ExtraInhabitantValueWitnessFunction,
-                        GetExtraInhabitantIndex)
+                        ExtraInhabitantCount)
 
-END_VALUE_WITNESS_RANGE(ExtraInhabitantValueWitness,
-                        GetExtraInhabitantIndex)
-
-#endif /* WANT_EXTRA_INHABITANT_VALUE_WITNESSES */
+#endif /* WANT_REQUIRED_VALUE_WITNESSES */
 
 #if WANT_ENUM_VALUE_WITNESSES
 
@@ -372,9 +307,7 @@ END_VALUE_WITNESS_RANGE(ValueWitness,
 #undef FUNCTION_VALUE_WITNESS
 #undef VALUE_WITNESS
 #undef ENUM_VALUE_WITNESS
-#undef EXTRA_INHABITANT_VALUE_WITNESS
 #undef NON_REQUIRED_VALUE_WITNESS
 #undef REQUIRED_VALUE_WITNESS
 #undef WANT_ENUM_VALUE_WITNESSES
-#undef WANT_EXTRA_INHABITANT_VALUE_WITNESSES
 #undef WANT_REQUIRED_VALUE_WITNESSES