diff --git a/.github/actions/phoenix-build/action.yml b/.github/actions/phoenix-build/action.yml index 2d70202e4..b110f8d7b 100644 --- a/.github/actions/phoenix-build/action.yml +++ b/.github/actions/phoenix-build/action.yml @@ -25,7 +25,7 @@ inputs: # action runner runs: using: 'docker' - image: 'phoenixrtos/build' + image: 'potrzebne/build' env: CONSOLE: 'serial' TARGET: ${{ inputs.target }} diff --git a/.github/actions/phoenix-clang-tidy/action.yml b/.github/actions/phoenix-clang-tidy/action.yml new file mode 100644 index 000000000..31b8c0c8b --- /dev/null +++ b/.github/actions/phoenix-clang-tidy/action.yml @@ -0,0 +1,25 @@ +# vim:sw=2:ts=2 +# action name +name: 'phoenix-clang-tidy' + +# action description +description: 'Runs clang-tidy' + +inputs: + files: + description: 'Specifies files to run clang-tidy' + required: false + + +# action clang-tidy +runs: + using: 'docker' + image: 'potrzebne/devel' + entrypoint: ./scripts/ci/phoenix_clang_tidy.py + args: + - ${{ inputs.files }} + +# branding +branding: + icon: terminal + color: green diff --git a/.github/workflows/ci-submodule.yml b/.github/workflows/ci-submodule.yml index 213fa9cbc..8000f27bd 100644 --- a/.github/workflows/ci-submodule.yml +++ b/.github/workflows/ci-submodule.yml @@ -13,7 +13,7 @@ on: build_params: type: string description: "parameters to build.sh script" - default: 'core fs test project image' # by default don't build ports + default: 'bear core fs test project image' # by default don't build ports required: false @@ -29,14 +29,15 @@ jobs: include: - target: 'ia32-generic-qemu' syspage: 'psh pc-ata uart16550' - - target: 'armv7a9-zynq7000-qemu' - additional_params: 'ports' + #- target: 'armv7a9-zynq7000-qemu' + # additional_params: 'ports' steps: # step 1: checkout phoenix-rtos-project repository code inside the workspace directory of the runner - name: Checkout phoenix-rtos-project - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: - repository: phoenix-rtos/phoenix-rtos-project + repository: jsarzy/phoenix-rtos-project + ref: clang submodules: recursive # step 2: update the submodule (recurse-sumbodules=no is needed for phoenix-rtos-lwip checkout not to fail, but it fails to update the submodule (TODO) @@ -73,23 +74,29 @@ jobs: _boot/${{ matrix.target }} rootfs-${{ matrix.target }}.tar - test-emu: + - name: Upload clang-tidy artifacts + uses: actions/upload-artifact@v2 + with: + name: clang-tidy-artifacts + path: | + _build/*/include + _build/*/compile_commands.json + + clang-tidy: + if: github.event_name == 'pull_request' needs: build - name: run tests on emulators + name: run clang-tidy runs-on: ubuntu-latest - outputs: - runner_result: ${{ steps.runner.outcome }} - strategy: - matrix: - target: ['armv7a9-zynq7000-qemu', 'host-generic-pc', 'ia32-generic-qemu'] steps: - name: Checkout phoenix-rtos-project - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: - repository: phoenix-rtos/phoenix-rtos-project + repository: jsarzy/phoenix-rtos-project + ref: clang submodules: recursive + # step 2: update the submodule (recurse-sumbodules=no is needed for phoenix-rtos-lwip checkout not to fail, but it fails to update the submodule (TODO) - name: Update submodule ${{ github.event.repository.name }} working-directory: ${{ github.event.repository.name }} run: | @@ -100,34 +107,69 @@ jobs: - name: Download build artifacts uses: actions/download-artifact@v2 with: - name: phoenix-rtos-${{ matrix.target }} + name: clang-tidy-artifacts + path: _build - - name: Untar rootfs - working-directory: _fs - run: tar -xvf ../rootfs-${{ matrix.target }}.tar + - name: Filter compile_commands.json + run: | + for file in _build/*/compile_commands.json; do + sed -i "s/-msingle-pic-base//g" $file + sed -i "s/-mno-pic-data-is-text-relative//g" $file + sed -i "s/-fstack-usage//g" $file + done - - name: Test runner - id: runner - uses: ./.github/actions/phoenix-runner + - name: Get changed and added files + id: changed_files + run: scripts/ci/changed_files.sh ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} ${{ github.event.repository.name }} + + - name: Echo added files + run: | + echo ${{ steps.changed_files.outputs.files }} + pwd + + + - name: Run phoenix_clang_tidy + # if: changed files are not empty + uses: ./.github/actions/phoenix-clang-tidy with: - target: ${{ matrix.target }} + files: ${{ steps.changed_files.outputs.files }} - # runs on self-hosted runners with HW targets attached - test-hw: + - name: Upload clang-tidy results + uses: actions/upload-artifact@v2 + with: + name: clang-tidy-result + path: result.json + + - name: Install reviewdog + run: | + wget http://misc.nalajcie.org/reviewdog -O /tmp/reviewdog 2>/dev/null + chmod +x /tmp/reviewdog + + - name: Reviewdog + env: + REVIEWDOG_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + cd ${{ github.event.repository.name }} + git diff -U0 HEAD^ + cat ../result.json | /tmp/reviewdog -f=rdjson -diff="git diff -U0 HEAD^" -name=clang-tidy -reporter=github-pr-review -fail-on-error + + + test-emu: needs: build - name: run tests on hardware - runs-on: ${{ matrix.target }} + name: run tests on emulators + runs-on: ubuntu-latest outputs: runner_result: ${{ steps.runner.outcome }} strategy: matrix: - target: ['armv7m7-imxrt106x-evk', 'armv7m7-imxrt117x-evk'] + target: ['armv7a9-zynq7000-qemu', 'host-generic-pc', 'ia32-generic-qemu'] steps: - name: Checkout phoenix-rtos-project - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: - repository: phoenix-rtos/phoenix-rtos-project + repository: jsarzy/phoenix-rtos-project + ref: clang submodules: recursive - name: Update submodule ${{ github.event.repository.name }} @@ -148,32 +190,72 @@ jobs: - name: Test runner id: runner - run: | - python3 ./phoenix-rtos-tests/runner.py -T${{ matrix.target }} - - mail-notification: - if: ${{ failure() }} - needs: ['build', 'test-emu', 'test-hw'] - name: notify ci team using e-mail - runs-on: ubuntu-latest - steps: - - name: Send mail - # If there is some failure, not caused by build step or test runner step send mail - if: >- - ${{ needs.build.outputs.build_result != 'failure' - && needs.test-emu.outputs.runner_result != 'failure' - && needs.test-hw.outputs.runner_result != 'failure' }} - uses: dawidd6/action-send-mail@v3 + uses: ./.github/actions/phoenix-runner with: - server_address: smtp.gmail.com - server_port: 465 - username: ${{ secrets.CI_BOT_EMAIL_USERNAME }} - password: ${{ secrets.CI_BOT_EMAIL_PASSWORD }} - subject: Github Actions Warning - to: ci@phoenix-rtos.com - # Required sender full name (address can be skipped): - from: Continuous Integration Bot - # Optional whether this connection use TLS (default is true if server_port is 465) - secure: true - body: There is some problem with GH Actions. https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + target: ${{ matrix.target }} + # runs on self-hosted runners with HW targets attached + # test-hw: + # needs: build + # name: run tests on hardware + # runs-on: ${{ matrix.target }} + # outputs: + # runner_result: ${{ steps.runner.outcome }} + # strategy: + # matrix: + # target: ['armv7m7-imxrt106x-evk', 'armv7m7-imxrt117x-evk'] + # + # steps: + # - name: Checkout phoenix-rtos-project + # uses: actions/checkout@v2 + # with: + # repository: phoenix-rtos/phoenix-rtos-project + # submodules: recursive + # + # - name: Update submodule ${{ github.event.repository.name }} + # working-directory: ${{ github.event.repository.name }} + # run: | + # git fetch --recurse-submodules=no --force ${{ github.event.repository.clone_url }} "+refs/heads/*:refs/remotes/origin/*" + # git fetch --recurse-submodules=no --force ${{ github.event.repository.clone_url }} "+refs/pull/*/head:refs/remotes/origin/pr/*" + # git checkout ${{ github.sha }} || git checkout ${{ github.event.pull_request.head.sha }} + # + # - name: Download build artifacts + # uses: actions/download-artifact@v2 + # with: + # name: phoenix-rtos-${{ matrix.target }} + # + # - name: Untar rootfs + # working-directory: _fs + # run: tar -xvf ../rootfs-${{ matrix.target }}.tar + # + # - name: Test runner + # id: runner + # run: | + # python3 ./phoenix-rtos-tests/runner.py -T${{ matrix.target }} + # + # mail-notification: + # if: ${{ failure() }} + # needs: ['build', 'test-emu', 'test-hw'] + # name: notify ci team using e-mail + # runs-on: ubuntu-latest + # steps: + # - name: Send mail + # # If there is some failure, not caused by build step or test runner step send mail + # if: >- + # ${{ needs.build.outputs.build_result != 'failure' + # && needs.test-emu.outputs.runner_result != 'failure' + # && needs.test-hw.outputs.runner_result != 'failure' }} + # uses: dawidd6/action-send-mail@v3 + # with: + # server_address: smtp.gmail.com + # server_port: 465 + # username: ${{ secrets.CI_BOT_EMAIL_USERNAME }} + # password: ${{ secrets.CI_BOT_EMAIL_PASSWORD }} + # subject: Github Actions Warning + # to: ci@phoenix-rtos.com + # # Required sender full name (address can be skipped): + # from: Continuous Integration Bot + # # Optional whether this connection use TLS (default is true if server_port is 465) + # secure: true + # body: There is some problem with GH Actions. https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + # diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59e207764..1013d0384 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,7 @@ on: branches: - master - 'feature/*' + # - clang pull_request: branches: - master @@ -22,11 +23,10 @@ jobs: strategy: matrix: target: ['armv7a7-imx6ull-evk', 'armv7a9-zynq7000-qemu', 'armv7a9-zynq7000-zedboard', 'armv7a9-zynq7000-zturn', 'armv7m4-stm32l4x6-nucleo', 'armv7m7-imxrt106x-evk', 'armv7m7-imxrt117x-evk', 'host-generic-pc', 'ia32-generic-pc', 'ia32-generic-qemu', 'riscv64-generic-qemu', 'riscv64-generic-spike'] + #target: ['ia32-generic-qemu'] include: - target: 'ia32-generic-qemu' syspage: 'psh pc-ata uart16550' - - target: 'armv7a9-zynq7000-qemu' - additional_params: 'ports' steps: # step 1: checkout repository code inside the workspace directory of the runner - name: Checkout the repository @@ -44,7 +44,7 @@ jobs: with: target: ${{ matrix.target }} syspage: ${{ matrix.syspage }} - params: core fs test project image ${{ matrix.additional_params }} + params: core fs test project image bear ${{ matrix.additional_params }} # step 3: tar rootfs - name: Tar rootfs @@ -60,46 +60,73 @@ jobs: _boot/${{ matrix.target }} rootfs-${{ matrix.target }}.tar - test-emu: + - name: Upload clang-tidy artifacts + uses: actions/upload-artifact@v2 + with: + name: clang-tidy-artifacts + path: | + _build/*/include + _build/*/compile_commands.json + + clang-tidy: needs: build - name: run tests on emulators + name: run clang-tidy runs-on: ubuntu-latest - outputs: - runner_result: ${{ steps.runner.outcome }} - strategy: - matrix: - target: ['armv7a9-zynq7000-qemu', 'host-generic-pc', 'ia32-generic-qemu'] steps: - name: Checkout the repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: - submodules: true + submodules: recursive - name: Download build artifacts uses: actions/download-artifact@v2 with: - name: phoenix-rtos-${{ matrix.target }} + name: clang-tidy-artifacts + path: _build - - name: Untar rootfs - working-directory: _fs - run: tar -xvf ../rootfs-${{ matrix.target }}.tar + - name: Filter compile_commands.json + run: | + for file in _build/*/compile_commands.json; do + sed -i "s/-msingle-pic-base//g" $file + sed -i "s/-mno-pic-data-is-text-relative//g" $file + sed -i "s/-fstack-usage//g" $file + done + + - name: Get changed and added files + id: files + uses: tj-actions/changed-files@v25 + with: + files: | + *.c + *.C + *.h + *.H - - name: Test runner - id: runner - uses: ./.github/actions/phoenix-runner + - name: Echo added files + run: | + echo ${{ steps.files.outputs.added_files }} + echo ${{ steps.files.outputs.modified_files }} + + - name: Run phoenix_clang_tidy + uses: ./.github/actions/phoenix-clang-tidy + + - name: Upload clang-tidy results + uses: actions/upload-artifact@v2 with: - target: ${{ matrix.target }} + name: clang-tidy-result + path: result.json - test-hw: + + test-emu: needs: build - name: run tests on hardware - runs-on: ${{ matrix.target }} + name: run tests on emulators + runs-on: ubuntu-latest outputs: runner_result: ${{ steps.runner.outcome }} strategy: matrix: - target: ['armv7m7-imxrt106x-evk', 'armv7m7-imxrt117x-evk'] + target: ['ia32-generic-qemu'] steps: - name: Checkout the repository @@ -118,31 +145,62 @@ jobs: - name: Test runner id: runner - run: | - python3 ./phoenix-rtos-tests/runner.py -T${{ matrix.target }} - - mail-notification: - if: ${{ failure() }} - needs: ['build', 'test-emu', 'test-hw'] - name: notify ci team using e-mail - runs-on: ubuntu-latest - steps: - - name: Send mail - # If there is some failure, not caused by build step or test runner step send mail - if: >- - ${{ needs.build.outputs.build_result != 'failure' - && needs.test-emu.outputs.runner_result != 'failure' - && needs.test-hw.outputs.runner_result != 'failure' }} - uses: dawidd6/action-send-mail@v3 + uses: ./.github/actions/phoenix-runner with: - server_address: smtp.gmail.com - server_port: 465 - username: ${{ secrets.CI_BOT_EMAIL_USERNAME }} - password: ${{ secrets.CI_BOT_EMAIL_PASSWORD }} - subject: Github Actions Warning - to: ci@phoenix-rtos.com - # Required sender full name (address can be skipped): - from: Continuous Integration Bot - # Optional whether this connection use TLS (default is true if server_port is 465) - secure: true - body: There is some problem with GH Actions. https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + target: ${{ matrix.target }} + + # test-hw: + # needs: build + # name: run tests on hardware + # runs-on: ${{ matrix.target }} + # outputs: + # runner_result: ${{ steps.runner.outcome }} + # strategy: + # matrix: + # target: ['armv7m7-imxrt106x-evk', 'armv7m7-imxrt117x-evk'] + # + # steps: + # - name: Checkout the repository + # uses: actions/checkout@v2 + # with: + # submodules: true + # + # - name: Download build artifacts + # uses: actions/download-artifact@v2 + # with: + # name: phoenix-rtos-${{ matrix.target }} + # + # - name: Untar rootfs + # working-directory: _fs + # run: tar -xvf ../rootfs-${{ matrix.target }}.tar + # + # - name: Test runner + # id: runner + # run: | + # python3 ./phoenix-rtos-tests/runner.py -T${{ matrix.target }} + # + # mail-notification: + # if: ${{ failure() }} + # needs: ['build', 'test-emu', 'test-hw'] + # name: notify ci team using e-mail + # runs-on: ubuntu-latest + # steps: + # - name: Send mail + # # If there is some failure, not caused by build step or test runner step send mail + # if: >- + # ${{ needs.build.outputs.build_result != 'failure' + # && needs.test-emu.outputs.runner_result != 'failure' + # && needs.test-hw.outputs.runner_result != 'failure' }} + # uses: dawidd6/action-send-mail@v3 + # with: + # server_address: smtp.gmail.com + # server_port: 465 + # username: ${{ secrets.CI_BOT_EMAIL_USERNAME }} + # password: ${{ secrets.CI_BOT_EMAIL_PASSWORD }} + # subject: Github Actions Warning + # to: ci@phoenix-rtos.com + # # Required sender full name (address can be skipped): + # from: Continuous Integration Bot + # # Optional whether this connection use TLS (default is true if server_port is 465) + # secure: true + # body: There is some problem with GH Actions. https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} diff --git a/.gitmodules b/.gitmodules index fb2e534dc..ab6963653 100644 --- a/.gitmodules +++ b/.gitmodules @@ -27,7 +27,7 @@ url = https://github.com/phoenix-rtos/phoenix-rtos-posixsrv [submodule "phoenix-rtos-tests"] path = phoenix-rtos-tests - url = https://github.com/phoenix-rtos/phoenix-rtos-tests + url = https://github.com/jsarzy/phoenix-rtos-tests [submodule "phoenix-rtos-corelibs"] path = phoenix-rtos-corelibs url = https://github.com/phoenix-rtos/phoenix-rtos-corelibs/ diff --git a/scripts/ci/changed_files.sh b/scripts/ci/changed_files.sh new file mode 100755 index 000000000..a0b1ff6ca --- /dev/null +++ b/scripts/ci/changed_files.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +set -o pipefail + +submodule='' + +if [ "$#" -ge 2 ]; then + base="$1" + head="$2" + if [ "$#" -eq 3 ]; then + submodule="$3" + fi +else + echo "need base sha and head sha" + exit 1 +fi + + +if [ "${submodule}" ]; then + cd "$submodule" || (echo "submodule ${submodule} doesn't exists" && exit 1) +fi + +echo "--------------------------" +echo "BASE: $base" +echo "HEAD: $head" +[ "${submodule}" ] && echo "SUBMODULE: $submodule" +echo "--------------------------" + +files=$(git diff "$base" "$head" --name-only --diff-filter=MA | egrep '\.c$|\.h$' | sed 's/^/'"$submodule"'\//' | tr '\n' ' ' ) + +# shellcheck disable=SC2181 +if [ "$?" -ne 0 ]; then + echo "diff command failed" + exit 1 +fi + +# Set github-actions variable +echo "::set-output name=files::$files" + +if [ "${files}" ]; then + echo "MODIFIED OR ADDED FILES: $files" +else + echo "MODIFIED OR ADDED FILES: there is no files with .c or .h suffix in diff" +fi + +[ "${submodule}" ] && cd .. diff --git a/scripts/ci/phoenix_clang_tidy.py b/scripts/ci/phoenix_clang_tidy.py new file mode 100755 index 000000000..7bc13872b --- /dev/null +++ b/scripts/ci/phoenix_clang_tidy.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 + +import argparse +import bisect +import collections +import dataclasses +import itertools +import json +import os +import re +import subprocess +import sys +import tempfile +import multiprocessing + +from typing import Any, Dict, List, Optional, Sized, Tuple +from pathlib import Path + +import yaml + +CLANG_CHECKS = [ + "-clang-analyzer-security.insecureAPI.DeprecatedOrUnsafeBufferHandling", + "-clang-analyzer-security.insecureAPI.strcpy", +] + +@dataclasses.dataclass(eq=True, frozen=True, repr=True) +class TidyPosition: + line: int + column: int + + def rdf(self) -> Dict[str, Any]: + return { + "line": self.line, + "column": self.column, + } + + +@dataclasses.dataclass(eq=True, frozen=True, repr=True) +class TidyReplacement: + text: str + start: TidyPosition + end: TidyPosition + + def rdf(self) -> Dict[str, Any]: + return { + "text": self.text, + "start": self.start.rdf(), + "end": self.end.rdf(), + } + + +@dataclasses.dataclass(eq=True, frozen=True, repr=True) +class TidyDiagnostic: + file_path: str + location: TidyPosition + diag_name: str + message: str + replacements: Tuple[TidyReplacement] + + # TODO remove this targets variable + def rdf(self, targets: Optional[List[str]] = None) -> Dict[str, Any]: + if targets and len(targets) <= 3: + msg = ','.join(targets) + '\n' + self.message + else: + msg = self.message + + return { + "message": msg, + "location": { + "path": self.file_path, + "range": { + "start": self.location.rdf(), + }, + }, + "code": { + "value": self.diag_name, + }, + "suggestions": [rep.rdf() for rep in self.replacements], + } + + +def diagnostics_to_rdfjson(diagnostics: Dict[TidyDiagnostic, List[str]]) -> str: + rdf = { + "source": { + "name": "clang-tidy", + }, + "severity": "WARNING", + "diagnostics": [diag.rdf(targets) for diag, targets in diagnostics.items()], + } + + return json.dumps(rdf, indent=4, sort_keys=True) + + +class OffsetFinder: + # TODO(kuba) make docs string from it + # The same idea as in tricium clang-tidy scriptself. + # Instead of keeping entires files in memory, we only save the end-of-line + # positions + + def __init__(self, stream: str): + # build eof positions table + self.eofs = [m.start() for m in re.finditer("\n", stream)] + + def find_position(self, offset: int) -> Tuple[int, int]: + line = bisect.bisect_left(self.eofs, offset) + prev_line = 0 if line == 0 else self.eofs[line - 1] + return line + 1, offset - prev_line # +1 ? + + +class FileOffsetCache: + def __init__(self): + self.finder: Dict[str, OffsetFinder] = {} + + def __contains__(self, item: Any) -> bool: + return item in self.finder + + def add(self, fpath: str): + with open(fpath) as f: + self.finder[fpath] = OffsetFinder(f.read()) + + def find(self, fpath: str, offset: int) -> Tuple[int, int]: + return self.finder[fpath].find_position(offset) + + +class PathResolver: + def __init__(self, compilation_db: Dict, files: Optional[List[str]] = None): + self.rel2abs: Dict[str, str] = {} + + for entry in compilation_db: + abs_path = entry["directory"] + "/" + entry["file"] + # TODO(kuba) add here cwd + self.add(entry["file"], abs_path) + + + def add(self, relpath: str, abspath: str): + if relpath in self.rel2abs and self.rel2abs[relpath] != abspath: + pass + # print('key', relpath, 'want', abspath, 'got', self.rel2abs[relpath]) + # TODO(Kuba) for now we do notraise error here, but maybe log some info and ignore it + + self.rel2abs[relpath] = abspath + + def find(self, fpath: str) -> str: + return self.rel2abs[fpath] + + +def parse_clang_tidy_yaml( + data: Dict, + offset_cache: FileOffsetCache, + path_resolver: PathResolver +) -> List[TidyDiagnostic]: + if data is None: + return [] + + diagnostics = [] + + for diag in data["Diagnostics"]: + msg = diag["DiagnosticMessage"] + file_path = msg["FilePath"] + if not file_path: + # TODO(kuba) + # if there are too many errors clang-tidy break fruther checks + continue + + try: + abs_file_path = path_resolver.find(file_path) + except KeyError: + # File is not in the compilation db, it may happen in few cases: + # 1. It's *.c source file that is built for another target + # 2. It's *.h header file + # We can't distinguish if the header file is used by our target, so + # include it anyway + if not file_path.endswith(".h"): + print(f"{file_path} missing in compilation database, omitting") + continue + + if os.path.isabs(file_path): + abs_file_path = file_path + path_resolver.add(file_path, abs_file_path) + else: + continue + + if abs_file_path not in offset_cache: + offset_cache.add(abs_file_path) + + line, column = offset_cache.find(abs_file_path, msg["FileOffset"]) + + replacements = [] + for rep in msg.get("Replacements", ()): + try: + rep_file_path = path_resolver.find(rep["FilePath"]) + except KeyError: + rep_file_path = rep["FilePath"] + + + if rep_file_path != abs_file_path: + print( + f"Replacement file path {rep_file_path} doesn't match \ + diagnosed file: {abs_file_path}" + ) + continue + + start_line, start_column = offset_cache.find(rep_file_path, rep["Offset"]) + end_line, end_column = offset_cache.find( + rep_file_path, rep["Offset"] + rep["Length"] + ) + + replacements.append( + TidyReplacement( + text=rep["ReplacementText"], + start=TidyPosition(line=start_line, column=start_column), + end=TidyPosition(line=end_line, column=end_column), + ) + ) + + if 'Notes' in diagnostics: + # Do not parse notes, just give a user information that they are available + # in the CI logs/output + msg["Message"] += "\nSee 'notes' in the clang-tidy output to trace the origin or warning/error" + + diagnostics.append( + TidyDiagnostic( + file_path=file_path, + location=TidyPosition(line=line, column=column), + diag_name=diag["DiagnosticName"], + message=msg["Message"], + replacements=tuple(replacements), + ) + ) + + return diagnostics + + +class ClangTidyError(Exception): + pass + + +def run_clang_tidy( + files: Optional[List[str]], + cdb_dir: str, + clang_checks: Optional[List[str]], + toolchain_headers: Optional[str] = None, +): + with tempfile.NamedTemporaryFile() as tidy_yml: + cmd = ["run-clang-tidy"] + + if clang_checks is not None: + cmd.append(f"-checks={','.join(clang_checks)}") + + if toolchain_headers is not None: + cmd.append(f"-extra-arg=-I{toolchain_headers}") + + cmd.append("-quiet") + cmd.extend(["-p", cdb_dir]) + cmd.extend(["-export-fixes", tidy_yml.name]) + + if files is not None: + cmd.extend(files) + + print(cdb_dir, cmd) + + try: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + ) + except FileNotFoundError as e: + raise ClangTidyError("Failed to run run-clang-tidy - make sure it's installed") from e + + cdb = cdb_dir + "/" + "compile_commands.json" + with open(cdb) as f: + try: + db = json.load(f) + except json.decoder.JSONDecodeError as e: + raise ClangTidyError(f"Error in json produced by bear {cdb}") from e + + try: + data = yaml.safe_load(tidy_yml.read()) + except yaml.YAMLError as e: + raise ClangTidyError("Error in yaml produced by clang-tidy") from e + + diagnostics = parse_clang_tidy_yaml(data, FileOffsetCache(), PathResolver(db, files)) + + return proc.stdout, diagnostics + + +class ToolchainError(Exception): + pass + + +def toolchain_headers_path(target: str) -> str: + if target.startswith("host"): + cc = "gcc" + elif target.startswith("arm"): + cc = "arm-phoenix-gcc" + elif target.startswith("ia32"): + cc = "i386-pc-phoenix-gcc" + elif target.startswith("riscv64"): + cc = "riscv64-phoenix-gcc" + else: + raise ToolchainError(f"Failed to find toolchain for given target {target}") + + try: + proc = subprocess.run( + [cc, "-print-sysroot"], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + check=True + ) + except FileNotFoundError as e: + raise ToolchainError(f"Failed to run {cc} - make sure it's installed") from e + except PermissionError as e: + raise ToolchainError(f"Failed to run {cc} - permission denied") from e + except subprocess.CalledProcessError as e: + raise ToolchainError(f"{cc} finished with an error") from e + + path = proc.stdout.rstrip() + path = path + "/usr/include/" + return os.path.abspath(path) + + +def run_clang_tidy_process(args: Tuple[str, List[str]]): + target, files = args + try: + toolchain_headers = toolchain_headers_path(target) + + #print(f"Running clang tidy for {target} files: {'all' if not files else ', '.join(files)}") + stdout, diagnostics = run_clang_tidy( + files=files, + cdb_dir=os.path.dirname(cdb_path(target)), + clang_checks=CLANG_CHECKS, + toolchain_headers=toolchain_headers, + ) + + return target, stdout, diagnostics + except Exception as e: + # TODO(Kuba) write nice comment + # As we running in multithread is better to catch any exception here + # and just log it to the user + import traceback + print(traceback.format_exc()) + + print_clang_tidy_output(target, f"run-clang-tidy process failed: {e}") + return None + + +def print_clang_tidy_output(target: str, output: str, diagnostics: Optional[Sized] = None): + is_github_actions = lambda: os.getenv("GITHUB_ACTIONS", False) + + if is_github_actions(): + print(f"::group::", end='') + + msg = f"Run clang-tidy on {target}" + msg += f" results: {len(diagnostics)}" if diagnostics is not None else "" + msg += "\n" + output + print(msg) + + if is_github_actions(): + print("::endgroup::") + + +def cdb_path(target: str) -> str: + return f"_build/{target}/compile_commands.json" + + +def absent_cdb(targets: List[str]) -> List[str]: + absent = [] + + for target in targets: + cdb = Path(cdb_path(target)) + if not cdb.exists(): + absent.append(str(cdb)) + + return absent + + +def parse_opts() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Blablabla for now") + + TARGETS = "armv7a7-imx6ull-evk,armv7a9-zynq7000-qemu,armv7a9-zynq7000-zedboard,armv7a9-zynq7000-zturn,armv7m4-stm32l4x6-nucleo,armv7m7-imxrt106x-evk,armv7m7-imxrt117x-evk,host-generic-pc,ia32-generic-pc,ia32-generic-qemu,riscv64-generic-qemu,riscv64-generic-spike" + #TARGETS = "ia32-generic-qemu,armv7m7-imxrt106x-evk" + + parser.add_argument("--targets", type=str, default=TARGETS) + parser.add_argument("--quiet", action="store_true", default=False) + parser.add_argument("-j", type=int, default=2) + parser.add_argument("files", nargs="*", default=None) + + opts = parser.parse_args() + opts.targets = opts.targets.split(",") + opts.j = max(opts.j, len(opts.targets)) + + # TODO + opts.files = [file for s in opts.files for file in s.split()] + print(opts.files) + #opts.files = opts.files.split() + + return opts + + +def main() -> None: + opts = parse_opts() + results = collections.defaultdict(list) + + print(f"Starting N={opts.j} multiprocesses for targets {opts.targets}") + + absent = absent_cdb(opts.targets) + if len(absent) != 0: + print("missing files:", " ".join(absent)) + sys.exit(1) + + with multiprocessing.Pool(opts.j) as pool: + clang_tidy_results = pool.imap_unordered( + run_clang_tidy_process, ((target, opts.files) for target in opts.targets) + ) + + failed = False + for result in clang_tidy_results: + if result is None: + # TODO(kuba) + # What to do here? Do we want post anyway other results? + failed = True + continue + + target, output, diagnostics = result + if not opts.quiet: + print_clang_tidy_output(target, output, diagnostics) + + for diag in diagnostics: + results[diag].append(target) + + print(f"target: {target} results: {len(results)}") + + if failed: + # We will not post results if job has failed + sys.exit(1) + + with open("result.json", "w+") as f: + f.write(diagnostics_to_rdfjson(results)) + + +main()