fixups hydra staging

non-critical-infra/hosts/staging-hydra/default.nix

@@ -50,4 +50,9 @@
     "[email protected] AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEPPocCK4JCbFWshVHMgICOm61LC6V2JAXThzKjXv7TSAAAABHNzaDo="
     "[email protected] AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIEWWZ8LjNo41679gFI4Iv4YtjFxwhSbMZVsvvYYaTXdxAAAABHNzaDo= picnoir@framework"
   ];
+
+  zramSwap = {
+    enable = true;
+    memoryPercent = 150;
+  };
 }

non-critical-infra/hosts/staging-hydra/hydra.nix

@@ -17,6 +17,16 @@ in
     9199 # hydra-notify metrics
   ];
 
+  services.postgresql.settings = {
+    log_min_duration_statement = 5000;
+    log_duration = "off";
+    log_statement = "none";
+
+    max_connections = 500;
+    work_mem = "20MB";
+    maintenance_work_mem = "2GB";
+  };
+
   # garbage collection
   nix.gc = {
     automatic = true;
@@ -27,7 +37,10 @@ in
   nix.settings = {
     # gc outputs as well, since they are served from the cache
     gc-keep-outputs = lib.mkForce false;
-    allowed-users = [ "hydra-www" ];
+    allowed-users = [
+      "hydra"
+      "hydra-www"
+    ];
   };
 
   # Don't rate-limit the journal.
@@ -80,22 +93,19 @@ in
 
         log_prefix = https://cache.nixos.org/
 
-        evaluator_workers = 1
+        evaluator_workers = 4
         evaluator_max_memory_size = 4096
 
         queue_runner_endpoint = http://localhost:8080
 
         max_concurrent_evals = 1
 
-        # increase the number of active compress slots (CPU is 48*2 on mimas)
-        max_local_worker_threads = 144
-
         max_unsupported_time = 86400
 
         allow_import_from_derivation = false
 
         max_output_size = 3821225472 # 3 << 30 + 600000000 = 3 GiB + 0.6 GB
-        max_db_connections = 350
+        max_db_connections = 50
 
         queue_runner_metrics_address = [::]:9198
 
@@ -110,19 +120,12 @@ in
 
     hydra-queue-runner-v2 = {
       enable = true;
-      settings.remoteStoreAddr = [
-        "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
-      ];
-    };
-
-    hydra-queue-builder-v2 = {
-      enable = true;
-      queueRunnerAddr = "https://queue-runner.staging-hydra.nixos.org";
-      mtls = {
-        serverRootCaCertPath = "${./ca.crt}";
-        clientCertPath = "${./client.crt}";
-        clientKeyPath = config.sops.secrets."queue-runner-client.key".path;
-        domainName = "queue-runner.staging-hydra.nixos.org";
+      settings = {
+        queueTriggerTimerInS = 300;
+        concurrentUploadLimit = 2;
+        remoteStoreAddr = [
+          "s3://nix-cache-staging?secret-key=${config.sops.secrets.signing-key.path}&ls-compression=br&log-compression=br"
+        ];
       };
     };
 
@@ -143,6 +146,7 @@ in
           # This is necessary so that grpc connections do not get closed early
           # see https://stackoverflow.com/a/67805465
           client_body_timeout 31536000s;
+          client_max_body_size 0;
 
           grpc_pass grpc://[::1]:50051;
 
@@ -168,11 +172,6 @@ in
       format = "binary";
       owner = config.systemd.services.nginx.serviceConfig.User;
     };
-    "queue-runner-client.key" = {
-      sopsFile = ../../secrets/queue-runner-client.key.staging-hydra;
-      format = "binary";
-      owner = config.systemd.services.hydra-queue-builder-v2.serviceConfig.User;
-    };
     hydra-users = {
       sopsFile = ../../secrets/hydra-users.staging-hydra;
       format = "binary";

non-critical-infra/modules/hydra-queue-builder-v2.nix

@@ -2,10 +2,12 @@
   config,
   pkgs,
   lib,
+  inputs,
   ...
 }:
 let
   cfg = config.services.hydra-queue-builder-v2;
+  unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
 in
 {
   options = {
@@ -98,6 +100,12 @@ in
         default = true;
       };
 
+      authorizationFile = lib.mkOption {
+        description = "Path to token authorization file if token auth should be used.";
+        type = lib.types.nullOr lib.types.path;
+        default = null;
+      };
+
       mtls = lib.mkOption {
         description = "mtls options";
         default = null;
@@ -127,7 +135,12 @@ in
 
       package = lib.mkOption {
         type = lib.types.package;
-        default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).builder;
+        default =
+          (pkgs.recurseIntoAttrs (
+            pkgs.callPackage ../packages/hydra-queue-runner {
+              inherit (unstable) nixVersions openssl;
+            }
+          )).builder;
       };
     };
   };
@@ -197,6 +210,10 @@ in
           ++ lib.optionals (cfg.useSubstitutes != null) [
             "--use-substitutes"
           ]
+          ++ lib.optionals (cfg.authorizationFile != null) [
+            "--authorization-file"
+            cfg.authorizationFile
+          ]
           ++ lib.optionals (cfg.mtls != null) [
             "--server-root-ca-cert-path"
             cfg.mtls.serverRootCaCertPath

non-critical-infra/modules/hydra-queue-runner-v2.nix

@@ -2,12 +2,14 @@
   config,
   pkgs,
   lib,
+  inputs,
   ...
 }:
 let
   cfg = config.services.hydra-queue-runner-v2;
 
   format = pkgs.formats.toml { };
+  unstable = import inputs.nixpkgs-unstable { inherit (pkgs) system; };
 in
 {
   options = {
@@ -101,6 +103,21 @@ in
               type = lib.types.int;
               default = 60;
             };
+            maxConcurrentDownloads = lib.mkOption {
+              description = "Max count of concurrent downloads per build. Increasing this will increase memory usage of the queue runner.";
+              type = lib.types.ints.positive;
+              default = 5;
+            };
+            concurrentUploadLimit = lib.mkOption {
+              description = "Concurrent limit for uploading to s3.";
+              type = lib.types.ints.positive;
+              default = 5;
+            };
+            tokenListPath = lib.mkOption {
+              description = "Path to a list of allowed authentication tokens.";
+              type = lib.types.nullOr lib.types.path;
+              default = null;
+            };
           };
         };
         default = { };
@@ -170,7 +187,12 @@ in
       };
       package = lib.mkOption {
         type = lib.types.package;
-        default = (pkgs.recurseIntoAttrs (pkgs.callPackage ../packages/hydra-queue-runner { })).runner;
+        default =
+          (pkgs.recurseIntoAttrs (
+            pkgs.callPackage ../packages/hydra-queue-runner {
+              inherit (unstable) nixVersions openssl;
+            }
+          )).runner;
       };
     };
   };
@@ -198,6 +220,7 @@ in
         HOME = "/var/lib/hydra/queue-runner";
       };
 
+      restartIfChanged = false;
       serviceConfig = {
         Type = "notify";
         Restart = "always";
@@ -244,6 +267,8 @@ in
           "~@privileged"
           "~@resources"
         ];
+        ManagedOOMPreference = "avoid";
+        LimitNOFILE = 65536;
 
         ProtectSystem = "strict";
         ProtectHome = true;

non-critical-infra/packages/hydra-queue-runner/default.nix

@@ -1,5 +1,5 @@
 {
-  rustPackages,
+  rustPackages_1_88,
   fetchFromGitHub,
   pkg-config,
   openssl,
@@ -13,15 +13,15 @@
   boost,
 }:
 let
-  version = "unstable-2025-08-07";
+  version = "unstable-2025-10-09";
   src = fetchFromGitHub {
     owner = "helsinki-systems";
     repo = "hydra-queue-runner";
-    rev = "54b3c9351d2ae10be5c4d1b97cc0f86300cd70ca";
-    hash = "sha256-gR2DzWkTykM9GdW3Nf/V8eRv68fl3aO+NW0zNPFSRT0=";
+    rev = "ad26924556d7ff6b4945a8f15c04271e8210e703";
+    hash = "sha256-hKZ//s4N+GPiKqRI8AzMO32HZOcJWYu6n5+uwhjx6vA=";
   };
   useFetchCargoVendor = true;
-  cargoHash = "sha256-oNUMmFfts4rjBX0k5mzsxpYA2JqgsRu1nMRFf/2rZa8=";
+  cargoHash = "sha256-eDkU+K2XvFU9OPhlPPcjqssabWwU8EqWzj7da0MuMAI=";
   nativeBuildInputs = [
     pkg-config
     protobuf
@@ -32,7 +32,7 @@ let
     zlib
     protobuf
 
-    nixVersions.nix_2_29
+    nixVersions.nix_2_31
     nlohmann_json
     libsodium
     boost
@@ -46,7 +46,7 @@ let
   };
 in
 {
-  runner = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
+  runner = rustPackages_1_88.rustPlatform.buildRustPackage (finalAttrs: {
     pname = "hydra-queue-runner";
     inherit version src;
     __structuredAttrs = true;
@@ -66,15 +66,17 @@ in
     cargoTestFlags = finalAttrs.cargoBuildFlags;
 
     postInstall = ''
-      wrapProgram $out/bin/queue-runner --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
+      wrapProgram $out/bin/queue-runner \
+        --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
+        --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
     '';
 
     meta = meta // {
       mainProgram = "queue-runner";
     };
   });
 
-  builder = rustPackages.rustPlatform.buildRustPackage (finalAttrs: {
+  builder = rustPackages_1_88.rustPlatform.buildRustPackage (finalAttrs: {
     pname = "hydra-queue-builder";
     inherit src version;
     __structuredAttrs = true;
@@ -94,7 +96,9 @@ in
     cargoTestFlags = finalAttrs.cargoBuildFlags;
 
     postInstall = ''
-      wrapProgram $out/bin/builder --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_29 ]}
+      wrapProgram $out/bin/builder \
+        --prefix PATH : ${lib.makeBinPath [ nixVersions.nix_2_31 ]} \
+        --set-default JEMALLOC_SYS_WITH_MALLOC_CONF "background_thread:true,narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0,abort_conf:true"
     '';
 
     meta = meta // {