[HipDNN] Layernorm bwd frontend and CPU reference

projects/hipdnn/backend/include/HipdnnBackendAttributeName.h

@@ -57,6 +57,8 @@
  * - 3200-3299: Reduction operation attributes
  * - 3300-3399: Resample forward operation attributes
  * - 3400-3499: Shared resample descriptor attributes
+ * - 3500-3599: RMSNorm backward operation attributes
+ * - 3600-3699: Layernorm backward operation attributes
  * - 60000-60099: Knob info serialized value extension attributes
  * - 60100-60199: Knob choice serialized value extension attributes
  * - 60200-60299: Operation type extension attributes
@@ -1257,6 +1259,47 @@ typedef enum
 
     /** @} */
 
+    /**
+     * @name Layernorm Backward Operation Attributes (3600-3699)
+     * Attributes for HIPDNN_BACKEND_OPERATION_LAYERNORM_BACKWARD_DESCRIPTOR_EXT
+     * @{
+     */
+
+    /** @brief Output gradient tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DY_EXT = 3600,
+
+    /** @brief Input tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_X_EXT = 3601,
+
+    /** @brief Scale tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_SCALE_EXT = 3602,
+
+    /** @brief Mean tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_MEAN_EXT = 3603,
+
+    /** @brief Inverse variance tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_INV_VARIANCE_EXT = 3604,
+
+    /** @brief Epsilon tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_EPSILON_EXT = 3605,
+
+    /** @brief Input gradient tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DX_EXT = 3606,
+
+    /** @brief Scale gradient tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DSCALE_EXT = 3607,
+
+    /** @brief Bias gradient tensor for backward layernorm */
+    HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DBIAS_EXT = 3608,
+
+    /** @brief Number of normalized dimensions for backward layernorm */
+    HIPDNN_ATTR_LAYERNORM_BACKWARD_NORMALIZED_DIM_COUNT_EXT = 3609,
+
+    /** @brief Compute type for backward layernorm */
+    HIPDNN_ATTR_LAYERNORM_BACKWARD_COMP_TYPE_EXT = 3610,
+
+    /** @} */
+
     /**
      * @name Extension Attributes (60000+)
      * hipDNN-specific extension attributes

projects/hipdnn/backend/include/HipdnnBackendDescriptorType.h

@@ -313,4 +313,14 @@ typedef enum
      */
     HIPDNN_BACKEND_PROFILING_CONTROL_EXT = 34,
 
+    /**
+     * @brief Layernorm backward operation descriptor
+     *
+     * Represents a backward layer normalization operation with output
+     * gradient (DY), input (X), scale, mean, inverse variance, input
+     * gradient (DX), scale gradient and mean gradient tensors, a backward
+     * layernorm operator, and a compute data type
+     */
+    HIPDNN_BACKEND_OPERATION_LAYERNORM_BACKWARD_DESCRIPTOR_EXT = 35,
+
 } hipdnnBackendDescriptorType_t;

projects/hipdnn/backend/include/HipdnnOperationType.h

@@ -42,4 +42,5 @@ typedef enum
     HIPDNN_OPERATION_TYPE_REDUCTION_EXT = 17, ///< Reduction operation
     HIPDNN_OPERATION_TYPE_RESAMPLE_FWD = 18, ///< Resample forward operation
     HIPDNN_OPERATION_TYPE_RMSNORM_BACKWARD_EXT = 19, ///< RMS normalization backward
+    HIPDNN_OPERATION_TYPE_LAYERNORM_BACKWARD_EXT = 20, ///< Layer normalization backward
 } hipdnnOperationType_ext_t;

projects/hipdnn/backend/src/BackendEnumStringUtils.hpp

@@ -255,6 +255,8 @@ inline const char* hipdnnGetBackendDescriptorTypeName(hipdnnBackendDescriptorTyp
         return "HIPDNN_BACKEND_OPERATION_SDPA_FWD_DESCRIPTOR";
     case HIPDNN_BACKEND_OPERATION_LAYERNORM_DESCRIPTOR_EXT:
         return "HIPDNN_BACKEND_OPERATION_LAYERNORM_DESCRIPTOR_EXT";
+    case HIPDNN_BACKEND_OPERATION_LAYERNORM_BACKWARD_DESCRIPTOR_EXT:
+        return "HIPDNN_BACKEND_OPERATION_LAYERNORM_BACKWARD_DESCRIPTOR_EXT";
     case HIPDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR:
         return "HIPDNN_BACKEND_OPERATION_BLOCK_SCALE_QUANTIZE_DESCRIPTOR";
     case HIPDNN_BACKEND_OPERATION_BATCHNORM_DESCRIPTOR_EXT:
@@ -898,6 +900,30 @@ inline const char* hipdnnGetAttributeNameString(hipdnnBackendAttributeName_t att
     case HIPDNN_ATTR_RMSNORM_BACKWARD_COMP_TYPE_EXT:
         return "HIPDNN_ATTR_RMSNORM_BACKWARD_COMP_TYPE_EXT";
 
+    // Layernorm backward operation attributes
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DY_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DY_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_X_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_X_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_SCALE_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_SCALE_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_MEAN_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_MEAN_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_INV_VARIANCE_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_INV_VARIANCE_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_EPSILON_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_EPSILON_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DX_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DX_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DSCALE_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DSCALE_EXT";
+    case HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DBIAS_EXT:
+        return "HIPDNN_ATTR_OPERATION_LAYERNORM_BACKWARD_DBIAS_EXT";
+    case HIPDNN_ATTR_LAYERNORM_BACKWARD_NORMALIZED_DIM_COUNT_EXT:
+        return "HIPDNN_ATTR_LAYERNORM_BACKWARD_NORMALIZED_DIM_COUNT_EXT";
+    case HIPDNN_ATTR_LAYERNORM_BACKWARD_COMP_TYPE_EXT:
+        return "HIPDNN_ATTR_LAYERNORM_BACKWARD_COMP_TYPE_EXT";
+
     // Operation extension attributes
     case HIPDNN_ATTR_OPERATION_NAME_EXT:
         return "HIPDNN_ATTR_OPERATION_NAME_EXT";
@@ -953,6 +979,8 @@ inline const char* hipdnnGetOperationTypeString(hipdnnOperationType_ext_t type)
         return "HIPDNN_OPERATION_TYPE_CUSTOM_OP_EXT";
     case HIPDNN_OPERATION_TYPE_LAYERNORM_EXT:
         return "HIPDNN_OPERATION_TYPE_LAYERNORM_EXT";
+    case HIPDNN_OPERATION_TYPE_LAYERNORM_BACKWARD_EXT:
+        return "HIPDNN_OPERATION_TYPE_LAYERNORM_BACKWARD_EXT";
     case HIPDNN_OPERATION_TYPE_MATMUL_EXT:
         return "HIPDNN_OPERATION_TYPE_MATMUL_EXT";
     case HIPDNN_OPERATION_TYPE_NOT_SET_EXT:

projects/hipdnn/backend/src/CMakeLists.txt

@@ -37,6 +37,7 @@ add_library(
     descriptors/KnobDescriptor.cpp
     descriptors/KnobSettingDescriptor.cpp
     descriptors/LayernormOperationDescriptor.cpp
+    descriptors/LayernormBackwardOperationDescriptor.cpp
     descriptors/MatmulOperationDescriptor.cpp
     descriptors/PointwiseOperationDescriptor.cpp
     descriptors/ProfilingControlDescriptor.cpp

projects/hipdnn/backend/src/descriptors/DescriptorFactory.cpp

@@ -21,6 +21,7 @@
 #include "HipdnnException.hpp"
 #include "KnobDescriptor.hpp"
 #include "KnobSettingDescriptor.hpp"
+#include "LayernormBackwardOperationDescriptor.hpp"
 #include "LayernormOperationDescriptor.hpp"
 #include "MatmulOperationDescriptor.hpp"
 #include "PointwiseOperationDescriptor.hpp"
@@ -117,6 +118,9 @@ void DescriptorFactory::create(hipdnnBackendDescriptorType_t descriptorType,
     case HIPDNN_BACKEND_OPERATION_LAYERNORM_DESCRIPTOR_EXT:
         privateDesc = std::make_shared<LayernormOperationDescriptor>();
         break;
+    case HIPDNN_BACKEND_OPERATION_LAYERNORM_BACKWARD_DESCRIPTOR_EXT:
+        privateDesc = std::make_shared<LayernormBackwardOperationDescriptor>();
+        break;
     case HIPDNN_BACKEND_OPERATION_BATCHNORM_DESCRIPTOR_EXT:
         privateDesc = std::make_shared<BatchnormOperationDescriptor>();
         break;