Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor asset inventory #2879

Merged
merged 30 commits into from
Jan 16, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
20a6c86
Adopt ECS Fields
romulets Jan 2, 2025
1f8f4b8
Refactor asset inventory aws fetchers
romulets Jan 2, 2025
a5157d0
Add event.kind asset
romulets Jan 2, 2025
7adafdf
re-classify Azure resources
kubasobon Jan 2, 2025
3796e2b
re-do Azure fetchers
kubasobon Jan 2, 2025
3b74e43
add license headers
kubasobon Jan 2, 2025
cbaa2d2
fix Azure test cases after migration
kubasobon Jan 2, 2025
e5bd3ed
re-classify GCP resources
kubasobon Jan 2, 2025
00250e0
redo GCP fetchers
kubasobon Jan 2, 2025
113deed
fix GCP test cases
kubasobon Jan 2, 2025
40bed30
update integration test functions
kubasobon Jan 3, 2025
136c41c
update integration test cases
kubasobon Jan 3, 2025
dd9585f
fix integration test cases and functions
kubasobon Jan 3, 2025
ef9311f
Clean up Asset categorization
romulets Jan 3, 2025
d0239c6
update GCP definitions once again
kubasobon Jan 3, 2025
db22fad
Refactor aws integration tests
romulets Jan 3, 2025
0305896
Map ecs fields manually
romulets Jan 3, 2025
e0e2c2c
Fix tests
romulets Jan 6, 2025
eca77ed
Fix trailing whitespaces
romulets Jan 6, 2025
b785901
Update aws host mapping
romulets Jan 8, 2025
8207ddf
Update aws iam role mapping
romulets Jan 8, 2025
c58a61e
Update aws iam user mapping
romulets Jan 8, 2025
2315634
update ASSETS.md
kubasobon Jan 8, 2025
9348e98
update ASSETS.md
kubasobon Jan 8, 2025
7c0fdb0
remove unused func
kubasobon Jan 9, 2025
ae5f84f
Remove repetitive fields initialization
romulets Jan 9, 2025
b54676f
Add removed ids to related item ids
romulets Jan 9, 2025
b347fcc
Merge branch 'main' into refactor-asset-inventory
romulets Jan 9, 2025
f951b16
Merge branch 'main' into refactor-asset-inventory
kubasobon Jan 14, 2025
d2cd321
Merge branch 'main' into refactor-asset-inventory
kubasobon Jan 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
386 changes: 211 additions & 175 deletions internal/inventory/ASSETS.md

Large diffs are not rendered by default.

438 changes: 148 additions & 290 deletions internal/inventory/asset.go

Large diffs are not rendered by default.

78 changes: 29 additions & 49 deletions internal/inventory/awsfetcher/fetcher_ec2_instance.go
Original file line number Diff line number Diff line change
Expand Up @@ -58,66 +58,46 @@ func (e *ec2InstanceFetcher) Fetch(ctx context.Context, assetChannel chan<- inve
return
}

for _, instance := range instances {
if instance == nil {
for _, i := range instances {
if i == nil {
continue
}

iamFetcher := inventory.EmptyEnricher()
if instance.IamInstanceProfile != nil {
iamFetcher = inventory.WithIAM(inventory.AssetIAM{
Id: instance.IamInstanceProfile.Id,
Arn: instance.IamInstanceProfile.Arn,
if i.IamInstanceProfile != nil {
iamFetcher = inventory.WithUser(inventory.User{
ID: pointers.Deref(i.IamInstanceProfile.Arn),
})
}

subnetIds := []string{}
if id := pointers.Deref(instance.SubnetId); id != "" {
subnetIds = append(subnetIds, id)
}
assetChannel <- inventory.NewAssetEvent(
inventory.AssetClassificationAwsEc2Instance,
[]string{instance.GetResourceArn(), pointers.Deref(instance.InstanceId)},
instance.GetResourceName(),
i.GetResourceArn(),
pointers.Deref(i.PrivateDnsName),

inventory.WithRawAsset(instance),
inventory.WithTags(e.getTags(instance)),
inventory.WithCloud(inventory.AssetCloud{
inventory.WithRelatedAssetIds([]string{pointers.Deref(i.InstanceId)}),
inventory.WithRawAsset(i),
inventory.WithLabels(e.getTags(i)),
inventory.WithCloud(inventory.Cloud{
Provider: inventory.AwsCloudProvider,
Region: instance.Region,
AvailabilityZone: e.getAvailabilityZone(instance),
Account: inventory.AssetCloudAccount{
Id: e.AccountId,
Name: e.AccountName,
},
Instance: &inventory.AssetCloudInstance{
Id: pointers.Deref(instance.InstanceId),
Name: instance.GetResourceName(),
},
Machine: &inventory.AssetCloudMachine{
MachineType: string(instance.InstanceType),
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
Region: i.Region,
AvailabilityZone: e.getAvailabilityZone(i),
AccountID: e.AccountId,
AccountName: e.AccountName,
InstanceID: pointers.Deref(i.InstanceId),
InstanceName: i.GetResourceName(),
MachineType: string(i.InstanceType),
ServiceName: "AWS EC2",
}),
inventory.WithHost(inventory.AssetHost{
Architecture: string(instance.Architecture),
ImageId: instance.ImageId,
InstanceType: string(instance.InstanceType),
Platform: string(instance.Platform),
PlatformDetails: instance.PlatformDetails,
inventory.WithHost(inventory.Host{
ID: pointers.Deref(i.InstanceId),
Name: pointers.Deref(i.PrivateDnsName),
Architecture: string(i.Architecture),
Type: string(i.InstanceType),
IP: pointers.Deref(i.PublicIpAddress),
MacAddress: i.GetResourceMacAddresses(),
}),
iamFetcher,
inventory.WithNetwork(inventory.AssetNetwork{
NetworkId: instance.VpcId,
SubnetIds: subnetIds,
Ipv6Address: instance.Ipv6Address,
PublicIpAddress: instance.PublicIpAddress,
PrivateIpAddress: instance.PrivateIpAddress,
PublicDnsName: instance.PublicDnsName,
PrivateDnsName: instance.PrivateDnsName,
}),
)
}
}
Expand All @@ -134,10 +114,10 @@ func (e *ec2InstanceFetcher) getTags(instance *ec2.Ec2Instance) map[string]strin
return tags
}

func (e *ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) *string {
func (e *ec2InstanceFetcher) getAvailabilityZone(instance *ec2.Ec2Instance) string {
if instance.Placement == nil {
return nil
return ""
}

return instance.Placement.AvailabilityZone
return pointers.Deref(instance.Placement.AvailabilityZone)
}
102 changes: 44 additions & 58 deletions internal/inventory/awsfetcher/fetcher_ec2_instance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
Placement: &types.Placement{
AvailabilityZone: pointers.Ref("1a"),
},
NetworkInterfaces: []types.InstanceNetworkInterface{
{
MacAddress: pointers.Ref("mac1"),
},
{
MacAddress: pointers.Ref("mac2"),
},
},
},
Region: "us-east",
}
Expand All @@ -78,77 +86,55 @@ func TestEC2InstanceFetcher_Fetch(t *testing.T) {
expected := []inventory.AssetEvent{
inventory.NewAssetEvent(
inventory.AssetClassificationAwsEc2Instance,
[]string{"arn:aws:ec2:us-east::ec2/234567890", "234567890"},
"test-server",
"arn:aws:ec2:us-east::ec2/234567890",
"private-dns",
inventory.WithRelatedAssetIds([]string{"234567890"}),
inventory.WithRawAsset(instance1),
inventory.WithTags(map[string]string{"Name": "test-server", "key": "value"}),
inventory.WithCloud(inventory.AssetCloud{
inventory.WithLabels(map[string]string{"Name": "test-server", "key": "value"}),
inventory.WithCloud(inventory.Cloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
AvailabilityZone: pointers.Ref("1a"),
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Instance: &inventory.AssetCloudInstance{
Id: "234567890",
Name: "test-server",
},
Machine: &inventory.AssetCloudMachine{
MachineType: "instance-type",
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
AvailabilityZone: "1a",
AccountID: "123",
AccountName: "alias",
InstanceID: "234567890",
InstanceName: "test-server",
MachineType: "instance-type",
ServiceName: "AWS EC2",
}),
inventory.WithHost(inventory.AssetHost{
Architecture: string(types.ArchitectureValuesX8664),
ImageId: pointers.Ref("image-id"),
InstanceType: "instance-type",
Platform: "linux",
PlatformDetails: pointers.Ref("ubuntu"),
inventory.WithHost(inventory.Host{
ID: "234567890",
Name: "private-dns",
Architecture: string(types.ArchitectureValuesX8664),
Type: "instance-type",
IP: "public-ip-addr",
MacAddress: []string{"mac1", "mac2"},
}),
inventory.WithIAM(inventory.AssetIAM{
Id: pointers.Ref("a123123"),
Arn: pointers.Ref("123123:123123:123123"),
}),
inventory.WithNetwork(inventory.AssetNetwork{
NetworkId: pointers.Ref("vpc-id"),
SubnetIds: []string{"subnetId"},
Ipv6Address: pointers.Ref("ipv6"),
PublicIpAddress: pointers.Ref("public-ip-addr"),
PrivateIpAddress: pointers.Ref("private-ip-addre"),
PublicDnsName: pointers.Ref("public-dns"),
PrivateDnsName: pointers.Ref("private-dns"),
inventory.WithUser(inventory.User{
ID: "123123:123123:123123",
}),
),

inventory.NewAssetEvent(
inventory.AssetClassificationAwsEc2Instance,
[]string{},
"",
"",
inventory.WithRawAsset(instance2),
inventory.WithTags(map[string]string{}),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Instance: &inventory.AssetCloudInstance{
Id: "",
Name: "",
},
Machine: &inventory.AssetCloudMachine{
MachineType: "",
},
Service: &inventory.AssetCloudService{
Name: "AWS EC2",
},
inventory.WithLabels(map[string]string{}),
inventory.WithCloud(inventory.Cloud{
Provider: inventory.AwsCloudProvider,
Region: "us-east",
AvailabilityZone: "",
AccountID: "123",
AccountName: "alias",
InstanceID: "",
InstanceName: "",
MachineType: "",
ServiceName: "AWS EC2",
}),
inventory.WithHost(inventory.Host{
MacAddress: []string{},
}),
inventory.WithHost(inventory.AssetHost{}),
inventory.WithNetwork(inventory.AssetNetwork{SubnetIds: []string{}}),
),
}

Expand Down
18 changes: 7 additions & 11 deletions internal/inventory/awsfetcher/fetcher_elb.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,19 +81,15 @@ func (f *elbFetcher) fetch(ctx context.Context, resourceName string, function el
for _, item := range awsResources {
assetChannel <- inventory.NewAssetEvent(
classification,
[]string{item.GetResourceArn()},
item.GetResourceArn(),
item.GetResourceName(),
inventory.WithRawAsset(item),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Region: item.GetRegion(),
Account: inventory.AssetCloudAccount{
Id: f.AccountId,
Name: f.AccountName,
},
Service: &inventory.AssetCloudService{
Name: "AWS Networking",
},
inventory.WithCloud(inventory.Cloud{
Provider: inventory.AwsCloudProvider,
Region: item.GetRegion(),
AccountID: f.AccountId,
AccountName: f.AccountName,
ServiceName: "AWS Networking",
}),
)
}
Expand Down
32 changes: 12 additions & 20 deletions internal/inventory/awsfetcher/fetcher_elb_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,18 +69,14 @@ func TestELBv1Fetcher_Fetch(t *testing.T) {
expected := []inventory.AssetEvent{
inventory.NewAssetEvent(
inventory.AssetClassificationAwsElbV1,
[]string{"arn:aws:elasticloadbalancing:::loadbalancer/my-elb-v1"},
"arn:aws:elasticloadbalancing:::loadbalancer/my-elb-v1",
"my-elb-v1",
inventory.WithRawAsset(asset),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Service: &inventory.AssetCloudService{
Name: "AWS Networking",
},
inventory.WithCloud(inventory.Cloud{
Provider: inventory.AwsCloudProvider,
AccountID: "123",
AccountName: "alias",
ServiceName: "AWS Networking",
}),
),
}
Expand Down Expand Up @@ -119,18 +115,14 @@ func TestELBv2Fetcher_Fetch(t *testing.T) {
expected := []inventory.AssetEvent{
inventory.NewAssetEvent(
inventory.AssetClassificationAwsElbV2,
[]string{"arn:aws:elasticloadbalancing:::loadbalancer/my-elb-v2"},
"arn:aws:elasticloadbalancing:::loadbalancer/my-elb-v2",
"my-elb-v2",
inventory.WithRawAsset(asset),
inventory.WithCloud(inventory.AssetCloud{
Provider: inventory.AwsCloudProvider,
Account: inventory.AssetCloudAccount{
Id: "123",
Name: "alias",
},
Service: &inventory.AssetCloudService{
Name: "AWS Networking",
},
inventory.WithCloud(inventory.Cloud{
Provider: inventory.AwsCloudProvider,
AccountID: "123",
AccountName: "alias",
ServiceName: "AWS Networking",
}),
),
}
Expand Down
Loading
Loading