Enhancements to YBD payload after assessment refactoring (#2238)

* Bumped up the payload version to 1.2
* Updated the AssessmentIssue payload for YBD to be in sync with standard struct format
* Adding MigrationComplexityExplanation to ybd assessment payload

Author: sanyamsinghal

yb-voyager/cmd/assessMigrationCommand.go

@@ -166,7 +166,7 @@ func packAndSendAssessMigrationPayload(status string, errMsg string) {
 	// whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format.
 	explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "")
 	if err != nil {
-		log.Errorf("failed to build migration complexity explanation: %v", err)
+		log.Errorf("failed to build migration complexity explanation for callhome assessment payload: %v", err)
 	}
 
 	var obfuscatedIssues []callhome.AssessmentIssueCallhome
@@ -462,15 +462,22 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
 		utils.PrintAndLog("failed to calculate the total sharded table size from tableIndexStats: %v", err)
 	}
 
-	assessmentIssues := flattenAssessmentReportToAssessmentIssues(assessmentReport)
+	assessmentIssues := convertAssessmentIssueToYugabyteDAssessmentIssue(assessmentReport)
+	// we will build this twice for json and html reports both, at the time of report generation.
+	// whatever happens later will be stored in the struct's field. so to be on safer side, we will build it again here as per required format.
+	explanation, err := buildMigrationComplexityExplanation(source.DBType, assessmentReport, "")
+	if err != nil {
+		log.Errorf("failed to build migration complexity explanation for yugabyted assessment payload: %v", err)
+	}
 
 	payload := AssessMigrationPayload{
-		PayloadVersion:      ASSESS_MIGRATION_YBD_PAYLOAD_VERSION,
-		VoyagerVersion:      assessmentReport.VoyagerVersion,
-		TargetDBVersion:     assessmentReport.TargetDBVersion,
-		MigrationComplexity: assessmentReport.MigrationComplexity,
-		SchemaSummary:       assessmentReport.SchemaSummary,
-		AssessmentIssues:    assessmentIssues,
+		PayloadVersion:                 ASSESS_MIGRATION_YBD_PAYLOAD_VERSION,
+		VoyagerVersion:                 assessmentReport.VoyagerVersion,
+		TargetDBVersion:                assessmentReport.TargetDBVersion,
+		MigrationComplexity:            assessmentReport.MigrationComplexity,
+		MigrationComplexityExplanation: explanation,
+		SchemaSummary:                  assessmentReport.SchemaSummary,
+		AssessmentIssues:               assessmentIssues,
 		SourceSizeDetails: SourceDBSizeDetails{
 			TotalIndexSize:     assessmentReport.GetTotalIndexSize(),
 			TotalTableSize:     assessmentReport.GetTotalTableSize(),
@@ -510,84 +517,25 @@ func createMigrationAssessmentCompletedEvent() *cp.MigrationAssessmentCompletedE
 	return ev
 }
 
-// flatten UnsupportedDataTypes, UnsupportedFeatures, MigrationCaveats
-func flattenAssessmentReportToAssessmentIssues(ar AssessmentReport) []AssessmentIssueYugabyteD {
-	var issues []AssessmentIssueYugabyteD
-
-	var dataTypesDocsLink string
-	switch source.DBType {
-	case POSTGRESQL:
-		dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK
-	case ORACLE:
-		dataTypesDocsLink = UNSUPPORTED_DATATYPES_DOC_LINK_ORACLE
-	}
-	for _, unsupportedDataType := range ar.UnsupportedDataTypes {
-		issues = append(issues, AssessmentIssueYugabyteD{
-			Type:            constants.DATATYPE,
-			TypeDescription: GetCategoryDescription(constants.DATATYPE),
-			Subtype:         unsupportedDataType.DataType,
-			ObjectName:      fmt.Sprintf("%s.%s.%s", unsupportedDataType.SchemaName, unsupportedDataType.TableName, unsupportedDataType.ColumnName),
-			SqlStatement:    "",
-			DocsLink:        dataTypesDocsLink,
-		})
-	}
-
-	for _, unsupportedFeature := range ar.UnsupportedFeatures {
-		for _, object := range unsupportedFeature.Objects {
-			issues = append(issues, AssessmentIssueYugabyteD{
-				Type:                   constants.FEATURE,
-				TypeDescription:        GetCategoryDescription(constants.FEATURE),
-				Subtype:                unsupportedFeature.FeatureName,
-				SubtypeDescription:     unsupportedFeature.FeatureDescription, // TODO: test payload once we add desc for unsupported features
-				ObjectName:             object.ObjectName,
-				SqlStatement:           object.SqlStatement,
-				DocsLink:               unsupportedFeature.DocsLink,
-				MinimumVersionsFixedIn: unsupportedFeature.MinimumVersionsFixedIn,
-			})
-		}
-	}
-
-	for _, migrationCaveat := range ar.MigrationCaveats {
-		for _, object := range migrationCaveat.Objects {
-			issues = append(issues, AssessmentIssueYugabyteD{
-				Type:                   constants.MIGRATION_CAVEATS,
-				TypeDescription:        GetCategoryDescription(constants.MIGRATION_CAVEATS),
-				Subtype:                migrationCaveat.FeatureName,
-				SubtypeDescription:     migrationCaveat.FeatureDescription,
-				ObjectName:             object.ObjectName,
-				SqlStatement:           object.SqlStatement,
-				DocsLink:               migrationCaveat.DocsLink,
-				MinimumVersionsFixedIn: migrationCaveat.MinimumVersionsFixedIn,
-			})
-		}
-	}
-
-	for _, uqc := range ar.UnsupportedQueryConstructs {
-		issues = append(issues, AssessmentIssueYugabyteD{
-			Type:                   constants.QUERY_CONSTRUCT,
-			TypeDescription:        GetCategoryDescription(constants.QUERY_CONSTRUCT),
-			Subtype:                uqc.ConstructTypeName,
-			SqlStatement:           uqc.Query,
-			DocsLink:               uqc.DocsLink,
-			MinimumVersionsFixedIn: uqc.MinimumVersionsFixedIn,
-		})
-	}
-
-	for _, plpgsqlObjects := range ar.UnsupportedPlPgSqlObjects {
-		for _, object := range plpgsqlObjects.Objects {
-			issues = append(issues, AssessmentIssueYugabyteD{
-				Type:                   constants.PLPGSQL_OBJECT,
-				TypeDescription:        GetCategoryDescription(constants.PLPGSQL_OBJECT),
-				Subtype:                plpgsqlObjects.FeatureName,
-				SubtypeDescription:     plpgsqlObjects.FeatureDescription,
-				ObjectName:             object.ObjectName,
-				SqlStatement:           object.SqlStatement,
-				DocsLink:               plpgsqlObjects.DocsLink,
-				MinimumVersionsFixedIn: plpgsqlObjects.MinimumVersionsFixedIn,
-			})
+func convertAssessmentIssueToYugabyteDAssessmentIssue(ar AssessmentReport) []AssessmentIssueYugabyteD {
+	var result []AssessmentIssueYugabyteD
+	for _, issue := range ar.Issues {
+		ybdIssue := AssessmentIssueYugabyteD{
+			Category:               issue.Category,
+			CategoryDescription:    issue.CategoryDescription,
+			Type:                   issue.Type, // Ques: should we be just sending Name in AssessmentIssueYugabyteD payload
+			Name:                   issue.Name,
+			Description:            issue.Description,
+			Impact:                 issue.Impact,
+			ObjectType:             issue.ObjectType,
+			ObjectName:             issue.ObjectName,
+			SqlStatement:           issue.SqlStatement,
+			DocsLink:               issue.DocsLink,
+			MinimumVersionsFixedIn: issue.MinimumVersionsFixedIn,
 		}
+		result = append(result, ybdIssue)
 	}
-	return issues
+	return result
 }
 
 func runAssessment() error {
@@ -1229,10 +1177,10 @@ func fetchUnsupportedPlPgSQLObjects(schemaAnalysisReport utils.SchemaReport) []U
 			})
 		}
 		feature := UnsupportedFeature{
-			FeatureName: issueName,
-			DisplayDDL:  true,
-			DocsLink:    docsLink,
-			Objects:     objects,
+			FeatureName:            issueName,
+			DisplayDDL:             true,
+			DocsLink:               docsLink,
+			Objects:                objects,
 			MinimumVersionsFixedIn: minVersionsFixedIn,
 		}
 		unsupportedPlpgSqlObjects = append(unsupportedPlpgSqlObjects, feature)

yb-voyager/cmd/common.go

@@ -1129,27 +1129,37 @@ type AssessMigrationDBConfig struct {
 // =============== for yugabyted controlplane ==============//
 // TODO: see if this can be accommodated in controlplane pkg, facing pkg cyclic dependency issue
 
-var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.1" // version(s) till now: 1.0, 1.1
+/*
+Version History
+1.0: Introduced AssessmentIssue field for storing assessment issues in flattened format
+1.1: Added TargetDBVersion and AssessmentIssueYugabyteD.MinimumVersionFixedIn
+1.2: Syncing it with original AssessmentIssue(adding fields Category, CategoryDescription, Type, Name, Description, Impact, ObjectType) and MigrationComplexityExplanation;
+*/
+var ASSESS_MIGRATION_YBD_PAYLOAD_VERSION = "1.2"
 
 type AssessMigrationPayload struct {
-	PayloadVersion        string
-	VoyagerVersion        string
-	TargetDBVersion       *ybversion.YBVersion
-	MigrationComplexity   string
-	SchemaSummary         utils.SchemaSummary
-	AssessmentIssues      []AssessmentIssueYugabyteD
-	SourceSizeDetails     SourceDBSizeDetails
-	TargetRecommendations TargetSizingRecommendations
-	ConversionIssues      []utils.AnalyzeSchemaIssue
+	PayloadVersion                 string
+	VoyagerVersion                 string
+	TargetDBVersion                *ybversion.YBVersion
+	MigrationComplexity            string
+	MigrationComplexityExplanation string
+	SchemaSummary                  utils.SchemaSummary
+	AssessmentIssues               []AssessmentIssueYugabyteD
+	SourceSizeDetails              SourceDBSizeDetails
+	TargetRecommendations          TargetSizingRecommendations
+	ConversionIssues               []utils.AnalyzeSchemaIssue
 	// Depreacted: AssessmentJsonReport is depricated; use the fields directly inside struct
 	AssessmentJsonReport AssessmentReportYugabyteD
 }
 
 type AssessmentIssueYugabyteD struct {
-	Type                   string                          `json:"Type"`                   // Feature, DataType, MigrationCaveat, UQC
-	TypeDescription        string                          `json:"TypeDescription"`        // Based on AssessmentIssue type
-	Subtype                string                          `json:"Subtype"`                // GIN Indexes, Advisory Locks etc
-	SubtypeDescription     string                          `json:"SubtypeDescription"`     // description based on subtype
+	Category               string                          `json:"Category"` // expected values: unsupported_features, unsupported_query_constructs, migration_caveats, unsupported_plpgsql_objects, unsupported_datatype
+	CategoryDescription    string                          `json:"CategoryDescription"`
+	Type                   string                          `json:"Type"`                   // Ex: GIN_INDEXES, SECURITY_INVOKER_VIEWS, STORED_GENERATED_COLUMNS
+	Name                   string                          `json:"Name"`                   // Ex: GIN Indexes, Security Invoker Views, Stored Generated Columns
+	Description            string                          `json:"Description"`            // description based on type/name
+	Impact                 string                          `json:"Impact"`                 // // Level-1, Level-2, Level-3 (no default: need to be assigned for each issue)
+	ObjectType             string                          `json:"ObjectType"`             // For datatype category, ObjectType will be datatype (for eg "geometry")
 	ObjectName             string                          `json:"ObjectName"`             // Fully qualified object name(empty if NA, eg UQC)
 	SqlStatement           string                          `json:"SqlStatement"`           // DDL or DML(UQC)
 	DocsLink               string                          `json:"DocsLink"`               // docs link based on the subtype