Garbage collection (#201)

arianvp · web-flow · commit 6d824d1bdd92 · 2024-12-24T20:57:20.000+01:00
* bump flake.lock * Rename delete_images to delete_images_by_name * add garbage collection * use 24.11 because awscli2 build failure NixOS/nixpkgs#367876 * Fix * wihtespace * Update docs and actively deprecate * Disable test as we're not using this image
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -21,4 +21,4 @@ jobs:
       - uses: DeterminateSystems/nix-installer-action@7993355175c2765e5733dae74f3e0786fe0e5c4f # v12
       - uses: DeterminateSystems/magic-nix-cache-action@87b14cf437d03d37989d87f0fa5ce4f5dc1a330b # v8
       - run: nix build .#amazonImage -L --system ${{ matrix.runs-on.system }}
-      - run: nix flake check -L --system ${{ matrix.runs-on.system }}
+      # - run: nix flake check -L --system ${{ matrix.runs-on.system }}
diff --git a/.github/workflows/upload-legacy-ami.yml b/.github/workflows/upload-legacy-ami.yml
@@ -86,6 +86,11 @@ jobs:
             --copy-to-regions \
             --public
 
+      - name: Delete deprecated AMIs
+        if: github.ref == 'refs/heads/main'
+        run: |
+          nix run .#delete-deprecated-images 
+
   deploy-pages:
     name: Deploy images page
     if: github.ref == 'refs/heads/main'
diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -1,19 +1,19 @@
 {
   description = "A very basic flake";
 
-  inputs = {
-    nixpkgs.url = "github:NixOS/nixpkgs";
-  };
+  inputs = { nixpkgs.url = "github:NixOS/nixpkgs?ref=nixos-24.11"; };
 
   outputs = { self, nixpkgs, ... }:
-    let inherit (nixpkgs) lib; in
+    let inherit (nixpkgs) lib;
 
-    {
+    in {
       nixosModules = {
         ec2-instance-connect = ./modules/ec2-instance-connect.nix;
 
-        legacyAmazonProfile = nixpkgs + "nixos/modules/virtualisation/amazon-image.nix";
-        legacyAmazonImage = nixpkgs + "/nixos/maintainers/scripts/ec2/amazon-image.nix";
+        legacyAmazonProfile = nixpkgs
+          + "nixos/modules/virtualisation/amazon-image.nix";
+        legacyAmazonImage = nixpkgs
+          + "/nixos/maintainers/scripts/ec2/amazon-image.nix";
 
         amazonProfile = ./modules/amazon-profile.nix;
         amazonImage = ./modules/amazon-image.nix;
@@ -27,11 +27,14 @@
         };
       };
 
-      lib.supportedSystems = [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" ];
+      lib.supportedSystems =
+        [ "aarch64-linux" "x86_64-linux" "aarch64-darwin" ];
 
       packages = lib.genAttrs self.lib.supportedSystems (system:
-        let pkgs = nixpkgs.legacyPackages.${system}; in {
-          ec2-instance-connect = pkgs.callPackage ./packages/ec2-instance-connect.nix { };
+        let pkgs = nixpkgs.legacyPackages.${system};
+        in {
+          ec2-instance-connect =
+            pkgs.callPackage ./packages/ec2-instance-connect.nix { };
           amazon-ec2-metadata-mock = pkgs.buildGoModule rec {
             pname = "amazon-ec2-metadata-mock";
             version = "1.11.2";
@@ -64,7 +67,10 @@
                 boot.loader.grub.enable = false;
                 boot.loader.systemd-boot.enable = true;
               }
-              { ec2.efi = true; amazonImage.sizeMB = "auto"; }
+              {
+                ec2.efi = true;
+                amazonImage.sizeMB = "auto";
+              }
               self.nixosModules.version
             ];
           }).config.system.build.amazonImage;
@@ -74,11 +80,12 @@
       apps = lib.genAttrs self.lib.supportedSystems (system:
         let
           upload-ami = self.packages.${system}.upload-ami;
-          mkApp = name: _: { type = "app"; program = "${upload-ami}/bin/${name}"; };
-        in
-          lib.mapAttrs mkApp self.packages.${system}.upload-ami.passthru.pyproject.project.scripts
-        );
-
+          mkApp = name: _: {
+            type = "app";
+            program = "${upload-ami}/bin/${name}";
+          };
+        in lib.mapAttrs mkApp
+        self.packages.${system}.upload-ami.passthru.pyproject.project.scripts);
 
       # TODO: unfortunately I don't have access to a aarch64-linux hardware with virtualisation support
       checks = lib.genAttrs [ "x86_64-linux" ] (system:
@@ -98,8 +105,7 @@
 
             };
           };
-        in
-        {
+        in {
           resize-partition = lib.nixos.runTest {
             hostPkgs = pkgs;
             imports = [ config ./tests/resize-partition.nix ];
@@ -110,13 +116,7 @@
           };
         });
 
-      devShells = lib.genAttrs [ "x86_64-linux" "aarch64-darwin" ] (system: {
-        default = let pkgs = nixpkgs.legacyPackages.${system}; in pkgs.mkShell {
-          nativeBuildInputs = [
-            pkgs.awscli2
-            pkgs.opentofu
-          ];
-        };
-      });
+      devShells = lib.genAttrs [ "x86_64-linux" "aarch64-darwin" ]
+        (system: { default = self.packages.${system}.upload-ami; });
     };
 }
diff --git a/site/index.html b/site/index.html
@@ -1,12 +1,11 @@
 <!DOCTYPE html>
 <html lang="en">
-
-<head>
+  <head>
     <meta charset="UTF-8">
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>NixOS Amazon Images / AMIs</title>
     <style>
-        table {
+      table {
             border-collapse: collapse;
         }
 
@@ -38,7 +37,7 @@
         }
     </style>
     <script type="module">
-        (async function () {
+      (async function () {
             try {
                 /**
                  * @typedef {Object} AWSImages
@@ -204,21 +203,18 @@
             }
         })()
     </script>
-</head>
-
-<body>
+  </head>
+  <body>
     <h1>Amazon Images / AMIs</h1>
     <p>
-        NixOS can be deployed to Amazon EC2 using our official AMI. We publish
+      NixOS can be deployed to Amazon EC2 using our official AMI. We publish
         AMIs to all AWS regions for both `x86_64` and `arm64` on a weekly basis.
     </p>
-    <p>We will start deprecating and garbage collecting images older than 90 days
-        in the future.
+    <p>We deprecate and garbage collecting images older than 90 days.
         This is why we suggest using a terraform data source or the AWS API to query
-        for the latest AMI.
-    </p>
-    <p>NixOS images are published under AWS Account ID <span id="owner-id"></span></p>
-
+        for the latest AMI.</p>
+    <p>NixOS images are published under AWS Account ID
+      <span id="owner-id"></span></p>
     <h2>Terraform / OpenTofu</h2>
     <p>You can use terraform to query for the latest image</p>
     <pre id="terraform">
@@ -249,52 +245,49 @@ <h2>AWS CLI</h2>
     <pre id="awscli2">
 aws ec2 describe-images --owners _OWNER_ID_  --filter 'Name=name,Values=nixos/24.11*' 'Name=architecture,Values=arm64' --query 'sort_by(Images, &CreationDate)'
 </pre>
-
     <h2>AMI table</h2>
     <p>Here are the latest NixOS images available in the Amazon cloud.</p>
     <table id="images-table">
-        <thead>
-            <tr>
-                <th class="region">
-                    <label for="search-regions">Region</label>
-                    <input type="search" class="region" list="regions-datalist" placeholder="Region">
-                    <datalist class="region" id="regions-datalist">
-                    </datalist>
-                </th>
-                <th class="architecture">
-                    <label for="search-architectures">Architecture</label>
-                    <input type="search" class="architecture" list="architectures-datalist" placeholder="Architecture">
-                    <datalist class="architecture" id="architectures-datalist">
-                    </datalist>
-                </th>
-                <th class="name">
-                    <label for="search-name">Name</label>
-                    <input type="search" class="name" placeholder="Name">
-                    <datalist class="name" id="names-datalist">
-                    </datalist>
-                </th>
-                <th class="creation-date" aria-sort="descending">
-                    Creation date
-                    <button class="sort">
-                        <span>▼</span>
-                    </button>
-                </th>
-                <th>Image ID</th>
-
-            </tr>
-        </thead>
-        <tbody>
-            <template id="row-template">
-                <tr>
-                    <td class="region"></td>
-                    <td class="architecture"></td>
-                    <td class="name"></td>
-                    <td class="creation-date"></td>
-                    <td class="image-id"></td>
-                </tr>
-            </template>
-        </tbody>
+      <thead>
+        <tr>
+          <th class="region">
+            <label for="search-regions">Region</label>
+            <input type="search" class="region" list="regions-datalist" placeholder="Region">
+            <datalist class="region" id="regions-datalist">
+            </datalist>
+          </th>
+          <th class="architecture">
+            <label for="search-architectures">Architecture</label>
+            <input type="search" class="architecture" list="architectures-datalist" placeholder="Architecture">
+            <datalist class="architecture" id="architectures-datalist">
+            </datalist>
+          </th>
+          <th class="name">
+            <label for="search-name">Name</label>
+            <input type="search" class="name" placeholder="Name">
+            <datalist class="name" id="names-datalist">
+            </datalist>
+          </th>
+          <th class="creation-date" aria-sort="descending">
+            Creation date
+            <button class="sort">
+              <span>▼</span>
+            </button>
+          </th>
+          <th>Image ID</th>
+        </tr>
+      </thead>
+      <tbody>
+        <template id="row-template">
+          <tr>
+            <td class="region"></td>
+            <td class="architecture"></td>
+            <td class="name"></td>
+            <td class="creation-date"></td>
+            <td class="image-id"></td>
+          </tr>
+        </template>
+      </tbody>
     </table>
-</body>
-
-</html>
+  </body>
+</html>
diff --git a/upload-ami/default.nix b/upload-ami/default.nix
@@ -1,5 +1,7 @@
 { buildPythonApplication
 , python3Packages
+, awscli2
+, opentofu
 , lib
 }:
 
@@ -33,6 +35,8 @@ buildPythonApplication {
   pyproject = true;
   nativeBuildInputs =
     map (name: python3Packages.${name}) pyproject.build-system.requires ++ [
+      opentofu
+      awscli2 
       python3Packages.mypy
       python3Packages.black
     ];
diff --git a/upload-ami/pyproject.toml b/upload-ami/pyproject.toml
@@ -17,6 +17,8 @@ disable-image-block-public-access = "upload_ami.disable_image_block_public_acces
 enable-regions = "upload_ami.enable_regions:main"
 request-public-ami-quota-increase  = "upload_ami.request_public_ami_quota_increase:main"
 describe-images = "upload_ami.describe_images:main"
-delete-images = "upload_ami.delete_images:main"
+delete-images-by-name = "upload_ami.delete_images_by_name:main"
+delete-deprecated-images = "upload_ami.delete_deprecated_images:main"
+delete-orphaned-snapshots = "upload_ami.delete_orphaned_snapshots:main"
 [tool.mypy]
 strict=true
diff --git a/upload-ami/src/upload_ami/delete_deprecated_images.py b/upload-ami/src/upload_ami/delete_deprecated_images.py
@@ -0,0 +1,78 @@
+import logging
+import boto3
+from mypy_boto3_ec2 import EC2Client
+import argparse
+import botocore.exceptions
+import datetime
+
+logger = logging.getLogger(__name__)
+
+
+def delete_deprecated_images(ec2: EC2Client, dry_run: bool) -> None:
+    """
+    Delete an image by its name.
+
+    Name can be a filter
+
+    Idempotent, unlike nuke
+    """
+
+    images_paginator = ec2.get_paginator("describe_images")
+    images_iterator = images_paginator.paginate(Owners=["self"])
+    for pages in images_iterator:
+        for image in pages["Images"]:
+            if "DeprecationTime" in image:
+                # HACK: As python can not parse ISO8601 strings with
+                # milliseconds, but it **can** produce them, instead of parsing
+                # the datetime from the API, we format the current time as an
+                # ISO8601 string and compare the strings. This works because
+                # ISO8601 strings are lexicographically comparable.
+                current_time = datetime.datetime.isoformat(
+                    datetime.datetime.now(), timespec="milliseconds"
+                )
+                if current_time >= image["DeprecationTime"]:
+                    assert "ImageId" in image
+                    assert "Name" in image
+                    logger.info(f"Deleting image {image['Name']} : {image['ImageId']}. DeprecationTime: {image['DeprecationTime']}")
+                    try:
+                        ec2.deregister_image(ImageId=image["ImageId"], DryRun=dry_run)
+                    except botocore.exceptions.ClientError as e:
+                        if "DryRunOperation" in str(e):
+                            logger.info(f"Would have deleted image {image['ImageId']}")
+                        else:
+                            raise
+                    assert "BlockDeviceMappings" in image
+                    assert "Ebs" in image["BlockDeviceMappings"][0]
+                    assert "SnapshotId" in image["BlockDeviceMappings"][0]["Ebs"]
+                    snapshot_id = image["BlockDeviceMappings"][0]["Ebs"]["SnapshotId"]
+                    logger.info(f"Deleting snapshot {snapshot_id}")
+                    try:
+                        ec2.delete_snapshot(SnapshotId=snapshot_id, DryRun=dry_run)
+                    except botocore.exceptions.ClientError as e:
+                        if "DryRunOperation" in str(e):
+                            logger.info(f"Would have deleted snapshot {snapshot_id}")
+                        else:
+                            raise
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="Do not actually delete anything, just log what would be deleted",
+    )
+    logging.basicConfig(level=logging.INFO)
+    ec2: EC2Client = boto3.client("ec2")
+
+    args = parser.parse_args()
+    regions = ec2.describe_regions()["Regions"]
+    for region in regions:
+        assert "RegionName" in region
+        ec2r = boto3.client("ec2", region_name=region["RegionName"])
+        logging.info(f"Checking region {region['RegionName']}")
+        delete_deprecated_images(ec2r, args.dry_run)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/upload-ami/src/upload_ami/delete_images_by_name.py b/upload-ami/src/upload_ami/delete_images_by_name.py
diff --git a/upload-ami/src/upload_ami/delete_orphaned_snapshots.py b/upload-ami/src/upload_ami/delete_orphaned_snapshots.py
