From 7440ec745fd4b6172541bce9fe93d28ba0fd91ac Mon Sep 17 00:00:00 2001 From: Anatoliy Puzyrevsky Date: Mon, 20 Oct 2025 16:42:57 +0200 Subject: [PATCH 1/2] implement crawlers --- codegen/recipes/glue.go | 13 +++++-------- docs/tables/aws_glue_crawlers.md | 6 ++++-- go.mod | 2 +- go.sum | 2 ++ resources/services/glue/crawlers.go | 17 +++++++++-------- resources/services/glue/crawlers_fetch.go | 20 ++++++++++++++++---- 6 files changed, 37 insertions(+), 23 deletions(-) diff --git a/codegen/recipes/glue.go b/codegen/recipes/glue.go index ba980112..9237f478 100644 --- a/codegen/recipes/glue.go +++ b/codegen/recipes/glue.go @@ -52,9 +52,11 @@ func GlueResources() []*Resource { }...), }, { - SubService: "crawlers", - Struct: &types.Crawler{}, - SkipFields: []string{}, + SubService: "crawlers", + Struct: &ohaws.GlueCrawler{}, + PreResourceResolver: "getGlueCrawler", + SkipFields: []string{}, + UnwrapEmbeddedStructs: true, ExtraColumns: append( defaultRegionalColumns, []codegen.ColumnDefinition{ @@ -64,11 +66,6 @@ func GlueResources() []*Resource { Resolver: `resolveGlueCrawlerArn`, Options: schema.ColumnCreationOptions{PrimaryKey: true}, }, - { - Name: "tags", - Type: schema.TypeJSON, - Resolver: `resolveGlueCrawlerTags`, - }, }...), }, { diff --git a/docs/tables/aws_glue_crawlers.md b/docs/tables/aws_glue_crawlers.md index 8e59dfcf..b538b8b3 100644 --- a/docs/tables/aws_glue_crawlers.md +++ b/docs/tables/aws_glue_crawlers.md @@ -1,6 +1,8 @@ # Table: aws_glue_crawlers +This table shows data for Glue Crawlers. +https://docs.aws.amazon.com/glue/latest/webapi/API_Crawler.html The primary key for this table is **arn**. @@ -16,7 +18,6 @@ The primary key for this table is **arn**. |account_id|String| |region|String| |arn (PK)|String| -|tags|JSON| |classifiers|StringArray| |configuration|String| |crawl_elapsed_time|Int| @@ -36,4 +37,5 @@ The primary key for this table is **arn**. |state|String| |table_prefix|String| |targets|JSON| -|version|Int| \ No newline at end of file +|version|Int| +|tags|JSON| \ No newline at end of file diff --git a/go.mod b/go.mod index 988e2383..dda834bd 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/OpsHelmInc/cloudquery go 1.24 require ( - github.com/OpsHelmInc/ohaws v0.11.1 + github.com/OpsHelmInc/ohaws v0.11.3-0.20251020143521-3a46f4c79769 github.com/aws/aws-sdk-go-v2 v1.37.2 github.com/aws/aws-sdk-go-v2/config v1.30.3 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.61 diff --git a/go.sum b/go.sum index fc1ca3fd..7d4bb0f2 100644 --- a/go.sum +++ b/go.sum @@ -35,6 +35,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/OpsHelmInc/ohaws v0.11.1 h1:r45US5566tRIKBHf5DlUHffeaEpiT3h+HnxYi1ETRrQ= github.com/OpsHelmInc/ohaws v0.11.1/go.mod h1:oBn8DAX9YQwbvVAGxthUb7idOJB9KZ+h2IOhABM+EaI= +github.com/OpsHelmInc/ohaws v0.11.3-0.20251020143521-3a46f4c79769 h1:ljq3JssPTovEjckAVwZtR13MzVsT89Kzgs4oNfTsRY4= +github.com/OpsHelmInc/ohaws v0.11.3-0.20251020143521-3a46f4c79769/go.mod h1:oBn8DAX9YQwbvVAGxthUb7idOJB9KZ+h2IOhABM+EaI= github.com/OpsHelmInc/pkg v0.0.0-20250806141928-951fad7f7312 h1:PF8jGY+FRmusqhdo+z8VEiiY9Sml2MxVT7WVWjQei74= github.com/OpsHelmInc/pkg v0.0.0-20250806141928-951fad7f7312/go.mod h1:CO1WtHvUbiIdXfssnArlXCodHEyyZrmUUJ29D+HZR8E= github.com/avast/retry-go/v4 v4.3.1 h1:Mtg11F9PdAIMkMiio2RKcYauoVHjl2aB3zQJJlzD4cE= diff --git a/resources/services/glue/crawlers.go b/resources/services/glue/crawlers.go index 0980f8eb..a07ac903 100644 --- a/resources/services/glue/crawlers.go +++ b/resources/services/glue/crawlers.go @@ -9,9 +9,10 @@ import ( func Crawlers() *schema.Table { return &schema.Table{ - Name: "aws_glue_crawlers", - Resolver: fetchGlueCrawlers, - Multiplex: client.ServiceAccountRegionMultiplexer("glue"), + Name: "aws_glue_crawlers", + Resolver: fetchGlueCrawlers, + PreResourceResolver: getGlueCrawler, + Multiplex: client.ServiceAccountRegionMultiplexer("glue"), Columns: []schema.Column{ { Name: "account_id", @@ -31,11 +32,6 @@ func Crawlers() *schema.Table { PrimaryKey: true, }, }, - { - Name: "tags", - Type: schema.TypeJSON, - Resolver: resolveGlueCrawlerTags, - }, { Name: "classifiers", Type: schema.TypeStringArray, @@ -136,6 +132,11 @@ func Crawlers() *schema.Table { Type: schema.TypeInt, Resolver: schema.PathResolver("Version"), }, + { + Name: "tags", + Type: schema.TypeJSON, + Resolver: schema.PathResolver("Tags"), + }, }, } } diff --git a/resources/services/glue/crawlers_fetch.go b/resources/services/glue/crawlers_fetch.go index c0e0995c..1bd3f91b 100644 --- a/resources/services/glue/crawlers_fetch.go +++ b/resources/services/glue/crawlers_fetch.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/OpsHelmInc/cloudquery/client" + "github.com/OpsHelmInc/ohaws" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/aws/arn" "github.com/aws/aws-sdk-go-v2/service/glue" @@ -32,20 +33,31 @@ func fetchGlueCrawlers(ctx context.Context, meta schema.ClientMeta, parent *sche } func resolveGlueCrawlerArn(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { cl := meta.(*client.Client) - return resource.Set(c.Name, crawlerARN(cl, aws.ToString(resource.Item.(types.Crawler).Name))) + return resource.Set(c.Name, crawlerARN(cl, aws.ToString(resource.Item.(*ohaws.GlueCrawler).Name))) } -func resolveGlueCrawlerTags(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource, c schema.Column) error { + +func getGlueCrawler(ctx context.Context, meta schema.ClientMeta, resource *schema.Resource) error { cl := meta.(*client.Client) svc := cl.Services().Glue + c := resource.Item.(types.Crawler) input := glue.GetTagsInput{ - ResourceArn: aws.String(crawlerARN(cl, aws.ToString(resource.Item.(types.Crawler).Name))), + ResourceArn: aws.String(crawlerARN(cl, aws.ToString(c.Name))), } response, err := svc.GetTags(ctx, &input) if err != nil { + if cl.IsNotFoundError(err) { + return nil + } return err } - return resource.Set(c.Name, response.Tags) + + resource.Item = &ohaws.GlueCrawler{ + Crawler: c, + Tags: response.Tags, + } + + return nil } // ==================================================================================================================== From e403b17c31e9517216b536833bbefc1c01fd8c34 Mon Sep 17 00:00:00 2001 From: Anatoliy Puzyrevsky Date: Mon, 20 Oct 2025 16:53:38 +0200 Subject: [PATCH 2/2] fix doc --- docs/tables/aws_glue_crawlers.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/tables/aws_glue_crawlers.md b/docs/tables/aws_glue_crawlers.md index b538b8b3..ff244a0a 100644 --- a/docs/tables/aws_glue_crawlers.md +++ b/docs/tables/aws_glue_crawlers.md @@ -1,8 +1,6 @@ # Table: aws_glue_crawlers -This table shows data for Glue Crawlers. -https://docs.aws.amazon.com/glue/latest/webapi/API_Crawler.html The primary key for this table is **arn**.