Skip to content

Commit 0e0e27a

Browse files
defistardefistar
authored
Merge branch 'main' into feature/nil-bridge-unit-tests
2 parents 7eaa200 + 6bfba4b commit 0e0e27a

File tree

98 files changed

+1969
-886
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

98 files changed

+1969
-886
lines changed

‎.github/actions/setup-nix-macos/action.yml‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
name: "Setup Nix (macOS)"
2+
description: "Install Nix, hook S3 cache; macOS only"
3+
4+
inputs:
5+
aws_region:
6+
description: "AWS region"
7+
required: true
8+
default: eu-west-2
9+
aws_role_to_assume:
10+
description: AWS role to assume
11+
required: true
12+
default: "arn:aws:iam::070427263827:role/github-actions/gha_nix_cache"
13+
s3_location_nix_cache:
14+
description: S3 Nix cache location
15+
required: true
16+
default: "s3://nil-githhub-actions-nix-cache-qrba32i47dik503juihjai4x"
17+
nix_cache_key_pub:
18+
description: Nix cache public key
19+
required: true
20+
default: "nil-nix-cache:LX95txIkFncQOsRIXc3KjQkdjikbxDlSFISV/s9+aps="
21+
nix_cache_key_pub_ssm_parameter:
22+
description: Path to nix_cache_key_pub SSM parameter
23+
required: true
24+
default: "arn:aws:ssm:eu-west-2:070427263827:parameter/github-action-runners/nil-githhub-actions/runners/config/nix_cache_key_pub"
25+
nix_cache_key_sec_ssm_parameter:
26+
description: Path to nix_cache_key_sec SSM parameter
27+
required: true
28+
default: "arn:aws:ssm:eu-west-2:070427263827:parameter/github-action-runners/nil-githhub-actions/runners/config/nix_cache_key_sec"
29+
github_access_token:
30+
description: Configure nix to pull from github using the given github token.
31+
32+
runs:
33+
using: "composite"
34+
steps:
35+
- name: Try configure AWS credentials via OIDC
36+
continue-on-error: true
37+
uses: aws-actions/configure-aws-credentials@v4
38+
with:
39+
role-to-assume: ${{ inputs.aws_role_to_assume }}
40+
aws-region: ${{ inputs.aws_region }}
41+
role-duration-seconds: 7200 # 2h
42+
retry-max-attempts: 3
43+
44+
- name: Decide write flag
45+
id: calculate_write_flag
46+
run: if [[ -n "$AWS_ACCESS_KEY_ID" ]]; then echo "has_write=1" >> "$GITHUB_OUTPUT"; fi
47+
shell: bash
48+
49+
- name: Show AWS identity
50+
if: steps.calculate_write_flag.outputs.has_write
51+
run: aws sts get-caller-identity
52+
shell: bash
53+
54+
# The source of truth for nix_cache_key_pub is SSM. But we cannot read values from it anonymously.
55+
# Since this is a public key, it is acceptable to hardcode it directly in the workflow. Additionally, we can use
56+
# OIDC authorized runs to verify that our hardcoded value is not outdated.
57+
- name: Compare nix_cache_key_pub input with value saved in SSM
58+
if: steps.calculate_write_flag.outputs.has_write
59+
run: |
60+
param=$(aws ssm get-parameter --name ${{ inputs.nix_cache_key_pub_ssm_parameter }} --with-decryption | jq -r '.Parameter.Value')
61+
test "${{ inputs.nix_cache_key_pub }}" == "$param" || { \
62+
echo "ERROR: nix_cache_key_pub input value does not match" \
63+
"the reference stored in SSM ('$param'). The SSM value should be considered" \
64+
"authoritative. Please update the input value in the workflow file.";
65+
exit 1; \
66+
}
67+
shell: bash
68+
69+
- name: Get nix_cache_key_sec from SSM
70+
if: steps.calculate_write_flag.outputs.has_write
71+
id: get_nix_cache_key_sec
72+
run: |
73+
param=$(aws ssm get-parameter --name ${{ inputs.nix_cache_key_sec_ssm_parameter }} --with-decryption | jq -r '.Parameter.Value')
74+
echo "$param" | sudo cp /dev/stdin /private/nix-signing-key
75+
shell: bash
76+
77+
- name: Create /etc/nix/upload-to-cache.sh
78+
if: steps.calculate_write_flag.outputs.has_write
79+
run: |
80+
sudo mkdir -p /etc/nix
81+
sudo tee /etc/nix/upload-to-cache.sh <<EOL
82+
#!/bin/bash
83+
84+
set -f # disable globbing
85+
export IFS=' '
86+
echo "Signing and uploading paths" \$OUT_PATHS
87+
88+
exec /nix/var/nix/profiles/default/bin/nix copy --to '${{ inputs.s3_location_nix_cache }}?region=${{ inputs.aws_region }}&secret-key=/private/nix-signing-key' \$OUT_PATHS
89+
EOL
90+
sudo chmod a+x /etc/nix/upload-to-cache.sh
91+
shell: bash
92+
93+
- name: Expose AWS credentials to the nix-daemon
94+
if: steps.calculate_write_flag.outputs.has_write
95+
run: |
96+
sudo launchctl setenv AWS_ACCESS_KEY_ID "$AWS_ACCESS_KEY_ID"
97+
sudo launchctl setenv AWS_SECRET_ACCESS_KEY "$AWS_SECRET_ACCESS_KEY"
98+
sudo launchctl setenv AWS_SESSION_TOKEN "$AWS_SESSION_TOKEN"
99+
shell: bash
100+
101+
# https://github.com/NixOS/nix/issues/2242#issuecomment-2336841344
102+
- name: macOS 15 eDSRecordAlreadyExists workaround
103+
run: echo "NIX_FIRST_BUILD_UID=30001" >> "$GITHUB_ENV"
104+
shell: bash
105+
106+
- name: Install Nix
107+
uses: cachix/install-nix-action@v27
108+
with:
109+
github_access_token: ${{ inputs.github_access_token }}
110+
extra_nix_config: |
111+
max-jobs = 1
112+
extra-substituters = ${{ inputs.s3_location_nix_cache }}?region=${{ inputs.aws_region }}
113+
extra-trusted-public-keys = ${{ inputs.nix_cache_key_pub }}
114+
${{ steps.calculate_write_flag.outputs.has_write && 'post-build-hook = /etc/nix/upload-to-cache.sh' }}
115+
116+
- name: Show /etc/nix/nix.conf
117+
run: cat /etc/nix/nix.conf
118+
shell: bash

‎.github/workflows/build_artifacts.yml‎

Lines changed: 18 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,64 +1,31 @@
11
name: Build artifacts
22

33
on:
4-
pull_request:
54
workflow_dispatch:
6-
merge_group:
75
push:
86
branches:
97
- main
108

9+
permissions:
10+
id-token: write
11+
contents: read
12+
1113
concurrency:
1214
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
1315
cancel-in-progress: true
1416

1517
jobs:
16-
build_artifacts:
17-
name: Build artifacts
18-
runs-on: ["self-hosted", "aws_autoscaling"]
19-
environment: prod
20-
steps:
21-
- name: checkout local actions
22-
uses: actions/checkout@v3
23-
with:
24-
fetch-depth: 0
25-
26-
- name: Run formatters
27-
run: |
28-
./scripts/sh_format_all.sh -n -c
29-
./scripts/nix_format_all.sh -n -c
30-
31-
- name: Generate required go files
32-
run: nix develop -c make generated
33-
34-
- name: golangci-lint
35-
run: nix develop -c make golangci-lint
36-
37-
- name: Run checklocks
38-
run: nix develop -c make checklocks
39-
40-
- name: build
41-
run: nix build -L
42-
43-
- name: Upload nil binary as artifact
44-
if: github.event_name == 'workflow_dispatch'
45-
uses: actions/upload-artifact@v4
46-
with:
47-
name: nil-linux-x64
48-
path: |
49-
result/bin/nil
50-
51-
- name: upload packages to s3
52-
if: (github.event_name == 'push' && github.ref == 'refs/heads/main') || github.event_name == 'workflow_dispatch'
53-
env:
54-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
55-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
56-
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
57-
run: |
58-
export PATH=/home/ec2-user/.local/bin:${PATH}
59-
nix bundle --bundler . .#nil -L
60-
sudo yum update -y
61-
sudo yum install -y awscli python3-pip
62-
pip3 install -U mkrepo
63-
aws s3 cp "deb-package-nil/`ls deb-package-nil`" s3://nil-deb-repo/ubuntu/pool/main/d/nil/
64-
mkrepo s3://nil-deb-repo/ubuntu
18+
nix_check_macos:
19+
name: macOS
20+
uses: ./.github/workflows/nix_check_macos.yml
21+
if: github.event_name == 'workflow_dispatch'
22+
with:
23+
upload_artifacts: true
24+
25+
lint_and_build:
26+
name: Linux
27+
uses: ./.github/workflows/lint_and_build.yml
28+
secrets: inherit
29+
with:
30+
upload_artifacts: ${{ github.event_name == 'workflow_dispatch' }}
31+
upload_packages_to_s3: true

‎.github/workflows/cluster_ci.yml‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
name: Cluster CI
2+
3+
on:
4+
pull_request:
5+
workflow_dispatch:
6+
merge_group:
7+
8+
permissions:
9+
id-token: write
10+
contents: read
11+
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
14+
cancel-in-progress: true
15+
16+
jobs:
17+
nix_check_linux:
18+
name: Linux
19+
uses: ./.github/workflows/nix_check_linux.yml
20+
21+
nix_check_macos:
22+
name: macOS
23+
uses: ./.github/workflows/nix_check_macos.yml
24+
with:
25+
upload_artifacts: ${{ github.event_name == 'workflow_dispatch' }}
26+
27+
lint_and_build:
28+
name: Lint and build
29+
uses: ./.github/workflows/lint_and_build.yml
30+
secrets: inherit
31+
with:
32+
upload_artifacts: ${{ github.event_name == 'workflow_dispatch' }}
33+
upload_packages_to_s3: false

‎.github/workflows/lint_and_build.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
on:
2+
workflow_call:
3+
inputs:
4+
upload_artifacts:
5+
description: "Upload binaries as artifacts"
6+
required: false
7+
default: false
8+
type: boolean
9+
upload_packages_to_s3:
10+
description: "Upload packages to s3"
11+
required: false
12+
default: false
13+
type: boolean
14+
15+
jobs:
16+
lint_and_build:
17+
name: Lint and build
18+
runs-on: ["self-hosted", "aws_autoscaling"]
19+
environment: prod
20+
steps:
21+
- name: checkout local actions
22+
uses: actions/checkout@v4
23+
with:
24+
fetch-depth: 0
25+
26+
- name: Run formatters
27+
run: |
28+
./scripts/sh_format_all.sh -n -c
29+
./scripts/nix_format_all.sh -n -c
30+
31+
- name: Generate required go files
32+
run: nix develop -c make generated
33+
34+
- name: golangci-lint
35+
run: nix develop -c make golangci-lint
36+
37+
- name: Run checklocks
38+
run: nix develop -c make checklocks
39+
40+
- name: build
41+
run: nix build -L
42+
43+
- name: Upload nil binary as artifact
44+
if: inputs.upload_artifacts
45+
uses: actions/upload-artifact@v4
46+
with:
47+
name: nil-linux-x64
48+
path: |
49+
result/bin/nil
50+
51+
- name: upload packages to s3
52+
if: inputs.upload_packages_to_s3
53+
env:
54+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
55+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
56+
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
57+
run: |
58+
export PATH=/home/ec2-user/.local/bin:${PATH}
59+
nix bundle --bundler . .#nil -L
60+
sudo yum update -y
61+
sudo yum install -y awscli python3-pip
62+
pip3 install -U mkrepo
63+
aws s3 cp "deb-package-nil/`ls deb-package-nil`" s3://nil-deb-repo/ubuntu/pool/main/d/nil/
64+
mkrepo s3://nil-deb-repo/ubuntu

0 commit comments

Comments
 (0)