diff --git a/.github/workflows/containers-build-push.yml b/.github/workflows/containers-build-push.yml new file mode 100644 index 00000000..a8934d1a --- /dev/null +++ b/.github/workflows/containers-build-push.yml @@ -0,0 +1,118 @@ +name: Build and push containers + +on: + push: + # For push to main branch (PR merges) + branches: + - main + - master + paths: + - "elkserver/docker/**" + - ".github/workflows/containers-build-push.yml" + # For releases + tags: + - "*" + # For pull requests + pull_request: + branches: + - "*" + paths: + - "elkserver/docker/**" + - ".github/workflows/containers-build-push.yml" + +permissions: + contents: read + packages: write + +jobs: + docker: + runs-on: ubuntu-latest + strategy: + matrix: + container: + - base + - elasticsearch + - jupyter + - kibana + - logstash + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Prepare + id: prepare + run: | + if [[ "x$DOCKER_REPO" == "x" ]]; then + DOCKER_REPO=outflanknl + fi + DOCKER_IMAGE="${DOCKER_REPO}/redelk-${{ matrix.container }}" + echo "name=docker_image::${DOCKER_IMAGE}" >> $GITHUB_OUTPUT + echo "name=docker_repo::${DOCKER_REPO}" >> $GITHUB_OUTPUT + echo "name=docker_file::elkserver/docker/redelk-${{ matrix.container }}/Dockerfile" >> $GITHUB_OUTPUT + env: + DOCKER_REPO: ${{ secrets.DOCKER_REPO }} + + - name: Docker meta + id: docker_meta + uses: docker/metadata-action@v4 + with: + images: | + ${{ steps.prepare.outputs.docker_image }} + ghcr.io/${{ github.repository_owner }}/redelk-${{ matrix.container }} + labels: | + org.opencontainers.image.source=${{ github.event.repository.html_url }} + org.opencontainers.image.revision=${{ github.sha }} + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + # if: ${{ github.event_name != 'pull_request' }} + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Cache Docker layers + uses: actions/cache@v3 + id: cache + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ matrix.container }}-${{ hashFiles(steps.prepare.outputs.docker_file) }} + restore-keys: | + ${{ runner.os }}-buildx-${{ matrix.container }}-${{ hashFiles(steps.prepare.outputs.docker_file) }} + ${{ runner.os }}-buildx-${{ matrix.container }}- + ${{ runner.os }}-buildx- + + - name: Build and push + uses: docker/build-push-action@v4 + with: + context: elkserver/docker/redelk-${{ matrix.container }} + file: ./elkserver/docker/redelk-${{ matrix.container }}/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} + build-args: | + DOCKER_METADATA_OUTPUT_JSON + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + outputs: type=image,name=target,annotation-index.org.opencontainers.image.description=${{ matrix.container }} container for RedELK + + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/workflows/docker-build-dev-base.yml b/.github/workflows/docker-build-dev-base.yml deleted file mode 100644 index df65e455..00000000 --- a/.github/workflows/docker-build-dev-base.yml +++ /dev/null @@ -1,68 +0,0 @@ -name: Build docker base image (dev) - -on: - push: - branches: - - "*" - pull_request_target: - branches: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-base" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-base-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-base- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-base - file: ./elkserver/docker/redelk-base/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache - # tags: | - # ${{ steps.semver_parser.outputs.major }} - # ${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }} - # ${{ steps.semver_parser.outputs.major }}.${{ steps.semver_parser.outputs.minor }}.${{ steps.semver_parser.outputs.patch }} diff --git a/.github/workflows/docker-build-dev-elasticsearch.yml b/.github/workflows/docker-build-dev-elasticsearch.yml deleted file mode 100644 index f9fbefc1..00000000 --- a/.github/workflows/docker-build-dev-elasticsearch.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build docker elasticsearch image (dev) - -on: - push: - branches: - - "*" - pull_request_target: - branches: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-elasticsearch" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-elasticsearch-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-elasticsearch- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-elasticsearch - file: ./elkserver/docker/redelk-elasticsearch/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-dev-jupyter.yml b/.github/workflows/docker-build-dev-jupyter.yml deleted file mode 100644 index e7231bb3..00000000 --- a/.github/workflows/docker-build-dev-jupyter.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build docker jupyter image (dev) - -on: - push: - branches: - - "*" - pull_request_target: - branches: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-jupyter" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-jupyter-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-jupyter- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-jupyter - file: ./elkserver/docker/redelk-jupyter/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-dev-kibana.yml b/.github/workflows/docker-build-dev-kibana.yml deleted file mode 100644 index d77a03c1..00000000 --- a/.github/workflows/docker-build-dev-kibana.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build docker kibana image (dev) - -on: - push: - branches: - - "*" - pull_request_target: - branches: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-kibana" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-kibana-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-kibana- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-kibana - file: ./elkserver/docker/redelk-kibana/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-dev-logstash.yml b/.github/workflows/docker-build-dev-logstash.yml deleted file mode 100644 index c07f9ad2..00000000 --- a/.github/workflows/docker-build-dev-logstash.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build docker logstash image (dev) - -on: - push: - branches: - - "*" - pull_request_target: - branches: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-logstash" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-logstash-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-logstash- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-logstash - file: ./elkserver/docker/redelk-logstash/Dockerfile - platforms: linux/amd64,linux/arm64 - push: true - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-prd-base.yml b/.github/workflows/docker-build-prd-base.yml deleted file mode 100644 index 160830dc..00000000 --- a/.github/workflows/docker-build-prd-base.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build docker base image (prd) - -on: - push: - tags: - - "*" - paths: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-base" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-base-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-base- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-base - file: ./elkserver/docker/redelk-base/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-prd-elasticsearch.yml b/.github/workflows/docker-build-prd-elasticsearch.yml deleted file mode 100644 index 34427507..00000000 --- a/.github/workflows/docker-build-prd-elasticsearch.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build docker elasticsearch image (prd) - -on: - push: - tags: - - "*" - paths: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-elasticsearch" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-elasticsearch-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-elasticsearch- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-elasticsearch - file: ./elkserver/docker/redelk-elasticsearch/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-prd-jupyter.yml b/.github/workflows/docker-build-prd-jupyter.yml deleted file mode 100644 index 737bddda..00000000 --- a/.github/workflows/docker-build-prd-jupyter.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build docker jupyter image (prd) - -on: - push: - tags: - - "*" - paths: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-jupyter" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-jupyter-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-jupyter- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-jupyter - file: ./elkserver/docker/redelk-jupyter/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-prd-kibana.yml b/.github/workflows/docker-build-prd-kibana.yml deleted file mode 100644 index ce448ba3..00000000 --- a/.github/workflows/docker-build-prd-kibana.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build docker kibana image (prd) - -on: - push: - tags: - - "*" - paths: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-kibana" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-kibana-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-kibana- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-kibana - file: ./elkserver/docker/redelk-kibana/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/docker-build-prd-logstash.yml b/.github/workflows/docker-build-prd-logstash.yml deleted file mode 100644 index a9fdf134..00000000 --- a/.github/workflows/docker-build-prd-logstash.yml +++ /dev/null @@ -1,63 +0,0 @@ -name: Build docker logstash image (prd) - -on: - push: - tags: - - "*" - paths: - - "*" - -jobs: - docker: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Prepare - id: prepare - run: | - if [[ "x$DOCKER_REPO" == "x" ]]; then - DOCKER_REPO=outflanknl - fi - DOCKER_IMAGE="${DOCKER_REPO}/redelk-logstash" - echo ::set-output name=docker_image::${DOCKER_IMAGE} - echo ::set-output name=docker_repo::${DOCKER_REPO} - echo ::set-output name=build_args::BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ') \ - VCS_REF=${GITHUB_SHA::8} - env: - DOCKER_REPO: ${{ secrets.DOCKER_REPO }} - - name: Docker meta - id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 - with: - images: ${{ steps.prepare.outputs.docker_image }} - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - if: ${{ github.event_name != 'pull_request' }} - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - name: Cache Docker layers - uses: actions/cache@v2 - id: cache - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-logstash-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx-logstash- - - name: Build and push - uses: docker/build-push-action@v2 - with: - context: elkserver/docker/redelk-logstash - file: ./elkserver/docker/redelk-logstash/Dockerfile - platforms: linux/amd64,linux/arm64 - push: ${{ github.event_name != 'pull_request' }} - tags: ${{ steps.docker_meta.outputs.tags }} - labels: ${{ steps.docker_meta.outputs.labels }} - build-args: ${{ steps.prepare.outputs.build_args }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache diff --git a/.github/workflows/label.yml b/.github/workflows/label.yml index d8711056..1882e4f6 100644 --- a/.github/workflows/label.yml +++ b/.github/workflows/label.yml @@ -6,13 +6,14 @@ # https://github.com/actions/labeler name: Label PR -on: [pull_request_target] +on: + - pull_request_target jobs: label: + permissions: + contents: read + pull-requests: write runs-on: ubuntu-latest - steps: - - uses: actions/labeler@v2 - with: - repo-token: "${{ secrets.GITHUB_TOKEN }}" + - uses: actions/labeler@v4 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint-python.yml similarity index 62% rename from .github/workflows/lint.yml rename to .github/workflows/lint-python.yml index b8d4a88e..0e3e08e5 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint-python.yml @@ -4,11 +4,11 @@ on: push: paths: - "elkserver/docker/redelk-base/redelkinstalldata/scripts/**" - - .github/workflows/lint.yml + - .github/workflows/lint-python.yml pull_request: paths: - "elkserver/docker/redelk-base/redelkinstalldata/scripts/**" - - .github/workflows/lint.yml + - .github/workflows/lint-python.yml jobs: lint: @@ -17,16 +17,23 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v3 + - name: Set up Python 3.6 uses: actions/setup-python@v4 with: python-version: "3.6.9" cache: pip + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install black + pip install black pylint Pillow pip install -r elkserver/docker/redelk-base/redelkinstalldata/scripts/requirements.txt + - name: Analysing the code with black run: | black --check --diff --color elkserver/docker/redelk-base/redelkinstalldata/scripts + + - name: Analysing the code with pylint + run: | + find elkserver/docker/redelk-base/redelkinstalldata/scripts -not -path '*/Chameleon/*' -name '*.py' -print0 | xargs -0 -i sh -c 'echo pylint {}; PYLINTRC=$(pwd)/elkserver/docker/redelk-base/redelkinstalldata/scripts/.pylintrc pylint {}' diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 1802090e..c005fe63 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -10,6 +10,12 @@ on: jobs: update_release_draft: + permissions: + # write permission is required to create a github release + contents: write + # write permission is required for autolabeler + # otherwise, read permission is required at least + pull-requests: write runs-on: ubuntu-latest steps: - uses: release-drafter/release-drafter@v5 diff --git a/README.md b/README.md index ae0b88b5..f8bb4a2d 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,14 @@ -![Build docker base image (dev)](https://github.com/outflanknl/RedELK/workflows/Build%20docker%20base%20image%20(dev)/badge.svg?branch=maindev) -![Build docker elasticsearch image (dev)](https://github.com/outflanknl/RedELK/workflows/Build%20docker%20elasticsearch%20image%20(dev)/badge.svg?branch=maindev) -![Build docker jupyter image (dev)](https://github.com/outflanknl/RedELK/workflows/Build%20docker%20jupyter%20image%20(dev)/badge.svg?branch=maindev) -![Build docker kibana image (dev)](https://github.com/outflanknl/RedELK/workflows/Build%20docker%20kibana%20image%20(dev)/badge.svg?branch=maindev) -![Build docker logstash image (dev)](https://github.com/outflanknl/RedELK/workflows/Build%20docker%20logstash%20image%20(dev)/badge.svg?branch=maindev) +[![Build and push containers](https://github.com/fastlorenzo/RedELK/actions/workflows/containers-build-push.yml/badge.svg)](https://github.com/fastlorenzo/RedELK/actions/workflows/containers-build-push.yml) Red Team's SIEM - tool for Red Teams for tracking and alarming about Blue Team activities as well as enhanced usability in long term operations. 1. **Enhanced usability and overview** for the red team operators by creating a central location where all relevant _operational_ logs from multiple teamservers are collected and enriched. This is great for historic searching within the operation as well as giving a read-only view on the operation (e.g. for the White Team). Especially useful for multi-scenario, multi-teamserver, multi-member and multi-month operations. Also, super easy ways for viewing all screenshots, IOCs, keystrokes output, etc. \o/ 2. **Spot the Blue Team** by having a central location where all _traffic_ logs from redirectors are collected and enriched. Using specific queries its now possible to detect that the Blue Team is investigating your infrastructure. -# Background info # +# Background info + Check the [wiki](https://github.com/outflanknl/RedELK/wiki) for info on usage or one the blog posts or presentations listed below: + - Blog part 1: [Why we need RedELK](https://outflank.nl/blog/2019/02/14/introducing-redelk-part-1-why-we-need-it/) - Blog part 2: [Getting you up and running](https://outflank.nl/blog/2020/02/28/redelk-part-2-getting-you-up-and-running/) - Blog part 3: [Achieving operational oversight](https://outflank.nl/blog/2020/04/07/redelk-part-3-achieving-operational-oversight/) @@ -19,21 +17,24 @@ Check the [wiki](https://github.com/outflanknl/RedELK/wiki) for info on usage or - x33fcon 2019 Catching Blue Team OPSEC failures [video](https://www.youtube.com/watch?v=-CNMgh0yJag) and [slides](https://github.com/outflanknl/Presentations/blob/master/x33fcon2019_OutOfTheBlue-CatchingBlueTeamOPSECFailures_publicversion.pdf) - BruCon 2018: Using Blue Team techniques in Red Team ops [video](https://www.youtube.com/watch?v=OjtftdPts4g) and [slides](https://github.com/outflanknl/Presentations/blob/master/MirrorOnTheWall_BruCon2018_UsingBlueTeamTechniquesinRedTeamOps_Bergman-Smeets_FINAL.pdf) -# Installation # +# Installation + Check the [wiki](https://github.com/outflanknl/RedELK/wiki) for manual installation manual. There are also Ansible playbooks maintained by others: + - [RedELK Server playbook](https://github.com/fastlorenzo/redelk-server) - maintained by one of RedELK's developers - [RedELK Client playbook](https://github.com/fastlorenzo/redelk-client) - maintained by one of RedELK's developers - [ansible-redelk](https://github.com/curi0usJack/ansible-redelk) - maintained by curi0usJack/TrustedSec -# Conceptual overview # +# Conceptual overview Here's a conceptual overview of how RedELK works. ![](./images/redelk_overview.jpg) +# Authors and contribution -# Authors and contribution # This project is developed and maintained by: + - Marc Smeets (@MarcOverIP on [Github](https://github.com/MarcOverIP) and [Twitter](https://twitter.com/MarcOverIP)) - Mark Bergman (@xychix on [Github](https://github.com/xychix) and [Twitter](https://twitter.com/xychix)) - Lorenzo Bernardi (@fastlorenzo on [Github](https://github.com/fastlorenzo) and [Twitter](https://twitter.com/fastlorenzo)) diff --git a/elkserver/docker/redelk-kibana/Dockerfile b/elkserver/docker/redelk-kibana/Dockerfile index e69db533..97a18101 100755 --- a/elkserver/docker/redelk-kibana/Dockerfile +++ b/elkserver/docker/redelk-kibana/Dockerfile @@ -8,8 +8,8 @@ # FROM docker.elastic.co/kibana/kibana:7.16.3 -LABEL maintainer="Outflank B.V. / Marc Smeets" -LABEL description="RedELK Kibana" +LABEL maintainer "Outflank B.V. / Marc Smeets" +LABEL org.opencontainers.image.description "RedELK Kibana" #COPY redelk-7.10.0.zip /tmp/redelk-7.10.0.zip #RUN /usr/share/kibana/bin/kibana-plugin install file:/tmp/redelk-7.10.0.zip diff --git a/elkserver/docker/redelk-logstash/Dockerfile b/elkserver/docker/redelk-logstash/Dockerfile index 43dac1a0..ad8ab94b 100644 --- a/elkserver/docker/redelk-logstash/Dockerfile +++ b/elkserver/docker/redelk-logstash/Dockerfile @@ -8,8 +8,8 @@ # FROM docker.elastic.co/logstash/logstash:7.16.3 -LABEL maintainer="Outflank B.V. / Marc Smeets" -LABEL description="RedELK Logstash" +LABEL maintainer "Outflank B.V. / Marc Smeets" +LABEL org.opencontainers.image.description "RedELK Logstash" # Create relevant directories #RUN mkdir -p /usr/share/logstash/redelk-main/conf.d