Feat/support emr (#738)

* backup

* [feat]create emr

* test create emr over

* [feat] support emr

* update emr doc

* [fix] review update

* [fix] review update

* [feat] support emr update

Co-authored-by: mikatong <[email protected]>

Author: tongyiming

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/cynosdb v1.0.199
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dayu v1.0.199
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.199
+	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr v1.0.287
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.294
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es v1.0.199
 	github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/gaap v1.0.199

go.sum

@@ -481,6 +481,8 @@ github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dayu v1.0.199 h1:CzZEt6
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dayu v1.0.199/go.mod h1:pz4s3nOhoB9cY0+uWzifuwr7lfh/Gvi1rv0ADxpPzD4=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.199 h1:7ShREKvI8ik2YNtLF42JR9x2YEeZS/gZvhIRfpsI8T0=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc v1.0.199/go.mod h1:5WGSrlIZJOhwIqPjjafb6vzrPEZieSHPhPMjjGPXOSU=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr v1.0.287 h1:+9COBXAbQmL7aJ39Q/mF50Ykxq4m5kq/y6vbO+u3zgI=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr v1.0.287/go.mod h1:xRvd0xsyCoviCiMRfJMh5lODPnLx+bnTfENNx6GHzFA=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.286 h1:daMss6AZ7UrzvfzRFtBp+0YdhSlZljlzRYE3XshU+MI=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.286/go.mod h1:CuOaLxOQr477GhMWAQPYQFUJrsZbW+ZqkAgP2uHDZXg=
 github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod v1.0.294 h1:VKP8TRlMSDUIJ32GuFKr4a0Fv/oi4LksRVOfJoK9CVc=

tencentcloud/connectivity/client.go

@@ -32,6 +32,7 @@ import (
 	dayu "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dayu/v20180709"
 	dc "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dc/v20180410"
 	dnspod "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/dnspod/v20210323"
+	emr "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr/v20190103"
 	es "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/es/v20180416"
 	gaap "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/gaap/v20180529"
 	kms "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms/v20190118"
@@ -98,6 +99,7 @@ type TencentCloudClient struct {
 	kmsConn            *kms.Client
 	ssmConn            *ssm.Client
 	apiConn            *api.Client
+	emrConn            *emr.Client
 	clsConn            *cls.Client
 	dnsPodConn         *dnspod.Client
 }
@@ -626,6 +628,18 @@ func (me *TencentCloudClient) UseApiClient() *api.Client {
 	return me.apiConn
 }
 
+// UseEmrClient return EMR client for service
+func (me *TencentCloudClient) UseEmrClient() *emr.Client {
+	if me.emrConn != nil {
+		return me.emrConn
+	}
+	cpf := me.NewClientProfile(300)
+	me.emrConn, _ = emr.NewClient(me.Credential, me.Region, cpf)
+	me.emrConn.WithHttpTransport(&LogRoundTripper{})
+
+	return me.emrConn
+}
+
 // UseClsClient return CLS client for service
 func (me *TencentCloudClient) UseClsClient() *cls.Client {
 	if me.clsConn != nil {

tencentcloud/data_source_tc_emr.go

@@ -0,0 +1,192 @@
+/*
+Provides an available EMR for the user.
+
+The EMR data source fetch proper EMR from user's EMR pool.
+
+Example Usage
+
+```hcl
+data "tencentcloud_emr" "my_emr" {
+  filter {
+    name   = "address-status"
+    values = ["UNBIND"]
+  }
+}
+```
+*/
+package tencentcloud
+
+import (
+	"context"
+	"log"
+
+	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
+	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
+	emr "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr/v20190103"
+	"github.com/tencentcloudstack/terraform-provider-tencentcloud/tencentcloud/internal/helper"
+)
+
+func dataSourceTencentCloudEmr() *schema.Resource {
+	return &schema.Resource{
+		Read: dataSourceTencentCloudEmrRead,
+
+		Schema: map[string]*schema.Schema{
+			"display_strategy": {
+				Type:        schema.TypeString,
+				Required:    true,
+				Description: "Display strategy(e.g.:clusterList, monitorManage).",
+			},
+			"prefix_instance_ids": {
+				Type:        schema.TypeList,
+				Optional:    true,
+				Elem:        &schema.Schema{Type: schema.TypeString},
+				Description: "fetch all instances with same prefix(e.g.:emr-xxxxxx).",
+			},
+			"project_id": {
+				Type:        schema.TypeInt,
+				Optional:    true,
+				Description: "Fetch all instances which owner same project. Default 0 meaning use default project id.",
+			},
+			"result_output_file": {
+				Type:        schema.TypeString,
+				Optional:    true,
+				Description: "Used to save results.",
+			},
+			"clusters": {
+				Type:        schema.TypeList,
+				Computed:    true,
+				Description: "A list of clusters will be exported and its every element contains the following attributes:",
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"id": {
+							Type:        schema.TypeInt,
+							Computed:    true,
+							Description: "Id of instance.",
+						},
+						"cluster_id": {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Cluster id of instance.",
+						},
+						"ftitle": {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Title of instance.",
+						},
+						"cluster_name": {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Cluster name of instance.",
+						},
+						"region_id": {
+							Type:        schema.TypeInt,
+							Computed:    true,
+							Description: "Region id of instance.",
+						},
+						"zone_id": {
+							Type:        schema.TypeInt,
+							Computed:    true,
+							Description: "Zone id of instance.",
+						},
+						"zone": {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Zone of instance.",
+						},
+						"master_ip": {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Master ip of instance.",
+						},
+						"project_id": {
+							Type:        schema.TypeInt,
+							Computed:    true,
+							Description: "Project id of instance.",
+						},
+						"charge_type": {
+							Type:        schema.TypeInt,
+							Computed:    true,
+							Description: "Charge type of instance.",
+						},
+						"status": {
+							Type:        schema.TypeInt,
+							Computed:    true,
+							Description: "Status of instance.",
+						},
+						"add_time": {
+							Type:        schema.TypeString,
+							Computed:    true,
+							Description: "Add time of instance.",
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func dataSourceTencentCloudEmrRead(d *schema.ResourceData, meta interface{}) error {
+	defer logElapsed("data_source.tencentcloud_emr.read")()
+
+	logId := getLogId(contextNil)
+	ctx := context.WithValue(context.TODO(), logIdKey, logId)
+
+	emrServer := EMRService{
+		client: meta.(*TencentCloudClient).apiV3Conn,
+	}
+
+	filters := map[string]interface{}{}
+	if v, ok := d.GetOk("display_strategy"); ok {
+		filters["display_strategy"] = v.(string)
+	}
+	if v, ok := d.GetOk("prefix_instance_ids"); ok {
+		filters["prefix_instance_ids"] = v.(string)
+	}
+	if v, ok := d.GetOk("project_id"); ok {
+		filters["project_id"] = v.(int64)
+	}
+	var clusters []*emr.ClusterInstancesInfo
+	var errRet error
+	err := resource.Retry(readRetryTimeout, func() *resource.RetryError {
+		clusters, errRet = emrServer.DescribeInstances(ctx, filters)
+		if errRet != nil {
+			return retryError(errRet, InternalError)
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+
+	emr_instances := make([]map[string]interface{}, 0, len(clusters))
+	ids := make([]string, 0, len(clusters))
+	for _, cluster := range clusters {
+		mapping := map[string]interface{}{
+			"cluster_id":   cluster.ClusterId,
+			"cluster_name": cluster.ClusterName,
+			"ftitle":       cluster.Ftitle,
+			"status":       cluster.Status,
+			"region_id":    cluster.RegionId,
+			"zone_id":      cluster.ZoneId,
+			"zone":         cluster.Zone,
+			"charge_type":  cluster.ChargeType,
+			"master_ip":    cluster.MasterIp,
+			"add_time":     cluster.AddTime,
+		}
+		emr_instances = append(emr_instances, mapping)
+		ids = append(ids, (string)(*cluster.Id))
+	}
+	d.SetId(helper.DataResourceIdsHash(ids))
+	err = d.Set("clusters", emr_instances)
+	if err != nil {
+		log.Printf("[CRITAL]%s provider set cluster list fail, reason:%s\n ", logId, err.Error())
+		return err
+	}
+	output, ok := d.GetOk("result_output_file")
+	if ok && output.(string) != "" {
+		if err := writeToFile(output.(string), emr_instances); err != nil {
+			return err
+		}
+	}
+	return nil
+}

tencentcloud/extension_emr.go

@@ -0,0 +1,105 @@
+package tencentcloud
+
+import (
+	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
+	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
+	emr "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/emr/v20190103"
+)
+
+const (
+	EmrInternetStatusCreated int64 = 2
+	EmrInternetStatusDeleted int64 = 201
+)
+
+const (
+	DisplayStrategyIsclusterList = "clusterList"
+)
+
+func buildResourceSpecSchema() *schema.Schema {
+	return &schema.Schema{
+		Type:     schema.TypeList,
+		Optional: true,
+		ForceNew: true,
+		MaxItems: 1,
+		Elem: &schema.Resource{
+			Schema: map[string]*schema.Schema{
+				"spec":         {Type: schema.TypeString, Optional: true},
+				"storage_type": {Type: schema.TypeInt, Optional: true},
+				"disk_type":    {Type: schema.TypeString, Optional: true},
+				"mem_size":     {Type: schema.TypeInt, Optional: true},
+				"cpu":          {Type: schema.TypeInt, Optional: true},
+				"disk_size":    {Type: schema.TypeInt, Optional: true},
+			},
+		},
+	}
+}
+
+func ParseMultiDisks(_multiDisks []map[string]interface{}) []*emr.MultiDisk {
+	multiDisks := make([]*emr.MultiDisk, len(_multiDisks))
+	for _, item := range _multiDisks {
+		var diskType string
+		var volume int64
+		var count int64
+		for subK, subV := range item {
+			if subK == "disk_type" {
+				diskType = subV.(string)
+			} else if subK == "volume" {
+				volume = subV.(int64)
+			} else if subK == "count" {
+				count = subV.(int64)
+			}
+		}
+		multiDisks = append(multiDisks,
+			&emr.MultiDisk{
+				DiskType: common.StringPtr(diskType),
+				Volume:   common.Int64Ptr(volume),
+				Count:    common.Int64Ptr(count),
+			})
+	}
+
+	return multiDisks
+}
+
+func ParseTags(_tags []map[string]string) []*emr.Tag {
+	tags := make([]*emr.Tag, len(_tags))
+	for _, item := range _tags {
+		for k, v := range item {
+			tags = append(tags, &emr.Tag{TagKey: &k, TagValue: &v})
+		}
+	}
+	return tags
+}
+
+func ParseResource(_resource map[string]interface{}) *emr.Resource {
+	resultResource := &emr.Resource{}
+	for k, v := range _resource {
+		if k == "spec" {
+			resultResource.Spec = common.StringPtr(v.(string))
+		} else if k == "storage_type" {
+			resultResource.StorageType = common.Int64Ptr((int64)(v.(int)))
+		} else if k == "disk_type" {
+			resultResource.DiskType = common.StringPtr(v.(string))
+		} else if k == "mem_size" {
+			resultResource.MemSize = common.Int64Ptr((int64)(v.(int)))
+		} else if k == "cpu" {
+			resultResource.Cpu = common.Int64Ptr((int64)(v.(int)))
+		} else if k == "disk_size" {
+			resultResource.DiskSize = common.Int64Ptr((int64)(v.(int)))
+		} else if k == "root_size" {
+			resultResource.RootSize = common.Int64Ptr((int64)(v.(int)))
+		} else if k == "multi_disks" {
+			multiDisks := v.([]map[string]interface{})
+			resultResource.MultiDisks = ParseMultiDisks(multiDisks)
+		} else if k == "tags" {
+			tags := v.([]map[string]string)
+			resultResource.Tags = ParseTags(tags)
+		} else if k == "instance_type" {
+			resultResource.InstanceType = common.StringPtr(v.(string))
+		} else if k == "local_disk_num" {
+			resultResource.LocalDiskNum = common.Uint64Ptr(v.(uint64))
+		} else if k == "DiskNum" {
+			resultResource.DiskNum = common.Uint64Ptr(v.(uint64))
+		}
+	}
+	return resultResource
+}

tencentcloud/provider.go

@@ -550,6 +550,13 @@ VPN
     tencentcloud_vpn_gateway_route
     tencentcloud_vpn_connection
 
+EMR
+  Data Source
+    tencentcloud_emr
+
+  Resource
+    tencentcloud_emr_cluster
+
 DNSPOD
   Resource
     tencentcloud_dnspod_domain_instance
@@ -667,6 +674,7 @@ func Provider() terraform.ResourceProvider {
 
 		DataSourcesMap: map[string]*schema.Resource{
 			"tencentcloud_availability_regions":                     dataSourceTencentCloudAvailabilityRegions(),
+			"tencentcloud_emr":                                      dataSourceTencentCloudEmr(),
 			"tencentcloud_availability_zones":                       dataSourceTencentCloudAvailabilityZones(),
 			"tencentcloud_availability_zones_by_product":            dataSourceTencentCloudAvailabilityZonesByProduct(),
 			"tencentcloud_instances":                                dataSourceTencentCloudInstances(),
@@ -825,6 +833,7 @@ func Provider() terraform.ResourceProvider {
 		},
 
 		ResourcesMap: map[string]*schema.Resource{
+			"tencentcloud_emr_cluster":                             resourceTencentCloudEmrCluster(),
 			"tencentcloud_instance":                                resourceTencentCloudInstance(),
 			"tencentcloud_reserved_instance":                       resourceTencentCloudReservedInstance(),
 			"tencentcloud_key_pair":                                resourceTencentCloudKeyPair(),

tencentcloud/resource_tc_clb_instance_topic_test.go

@@ -43,9 +43,8 @@ func testAccCheckClbInstanceTopicExists(n string) resource.TestCheckFunc {
 		clsService := ClsService{
 			client: testAccProvider.Meta().(*TencentCloudClient).apiV3Conn,
 		}
-
-		instance, err := clsService.DescribeTopicsById(ctx, rs.Primary.ID)
-
+		topicName := rs.Primary.Attributes["topic_name"]
+		instance, err := clsService.DescribeTopicsById(ctx, topicName)
 		if err != nil {
 			return err
 		}