-
Notifications
You must be signed in to change notification settings - Fork 59
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #325 from Privado-Inc/dev
[Optimize] Parallel processing for comparison report.
- Loading branch information
Showing
1 changed file
with
98 additions
and
30 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,15 +1,40 @@ | ||
| name: Monitoring Stability and Comparing Results for privado | ||
| name: Monitoring Stability and Comparing Results for privado | ||
|
|
||
| # Triggers when a pull_request or a push action is configured on master branch | ||
| # Triggers when a pull_request is created | ||
| on: | ||
| pull_request_target: | ||
| branches: | ||
| - "**" | ||
|
|
||
| jobs: | ||
| start_workflow: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Send message to slack | ||
| id: initial-message | ||
| uses: archive/github-actions-slack@master | ||
| with: | ||
| slack-optional-parse: full | ||
| slack-bot-user-oauth-access-token: ${{ secrets.SLACK_TOKEN }} | ||
| slack-channel: ${{ secrets.SLACK_CHANNEL_ID }} | ||
| slack-text: "Comparison workflow started for ${{github.event.pull_request.html_url}}" | ||
|
|
||
| - name: Save output to env | ||
| id: save-output | ||
| run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT | ||
| outputs: | ||
| init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}} | ||
|
|
||
| setup_and_scan: | ||
| needs: start_workflow | ||
| strategy: | ||
| matrix: | ||
| language: ['java', 'python', 'js', 'ruby-1', 'ruby-2'] | ||
| continue-on-error: true | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
|
|
||
| - name: Install JDK-18 | ||
| uses: actions/setup-java@v3 | ||
| with: | ||
|
|
@@ -31,48 +56,91 @@ jobs: | |
| with: | ||
| repository: Privado-Inc/standalone-monitoring-stability | ||
| path: ./temp/standalone-monitoring-stability | ||
| ref: main | ||
| ref: main | ||
|
|
||
| - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}} | ||
| run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc | ||
| run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc | ||
|
|
||
| - name: Run aws-export | ||
| run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}} | ||
| run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}} | ||
|
|
||
| - name: Move results to a folder | ||
| run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt | ||
| run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt | ||
|
|
||
| - name: Zip the results | ||
| run: cd /home/runner/work/privado/privado/temp/standalone-monitoring-stability && zip result-${{matrix.language}}-${{github.event.number}}.zip -r ./results | ||
|
|
||
| - name: Set summary variable | ||
| run: | | ||
| echo "MESSAGE<<EOF" >> $GITHUB_ENV | ||
| echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV | ||
| echo "EOF" >> $GITHUB_ENV | ||
| - name: Post results to slack | ||
| uses: MeilCli/slack-upload-file@v3 | ||
| with: | ||
| thread_ts: ${{needs.start_workflow.outputs.init_message_ts}} | ||
| slack_token: ${{ secrets.SLACK_TOKEN }} | ||
| channel_id: ${{ secrets.SLACK_CHANNEL_ID }} | ||
| file_path: "/home/runner/work/privado/privado/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip" | ||
| initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}" | ||
| file_type: "zip" | ||
|
|
||
| - name: Export workflow output | ||
| run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt | ||
|
|
||
| - name: Upload output and result for next job | ||
| - name: Set summary variable | ||
| run: | | ||
| echo "MESSAGE<<EOF" >> $GITHUB_ENV | ||
| echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV | ||
| echo "EOF" >> $GITHUB_ENV | ||
| - name: Upload summary file | ||
| uses: actions/upload-artifact@master | ||
| with: | ||
| name: results | ||
| path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results | ||
| name: ${{matrix.language}} | ||
| path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt | ||
|
|
||
| send-result: | ||
| needs: setup_and_scan | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Workflow report analysis | ||
| if: ${{ env.MESSAGE != 'true' }} | ||
| run: exit 1 | ||
|
|
||
| - name: Download result folder | ||
| collate_summary: | ||
| needs: [start_workflow, setup_and_scan] | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Download summary file | ||
| uses: actions/download-artifact@master | ||
| with: | ||
| name: results | ||
| path: ./results | ||
|
|
||
| - name: Zip the results | ||
| run: zip result-${{github.event.number}}.zip -r ./results | ||
| path: ./language_summary | ||
|
|
||
| - name: Install Python 3.10 | ||
| uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: '3.10' | ||
|
|
||
| - name: Clone standalone-monitoring-stability/flow-test | ||
| uses: actions/checkout@v3 | ||
| with: | ||
| repository: Privado-Inc/standalone-monitoring-stability | ||
| path: ./temp/standalone-monitoring-stability | ||
| ref: main | ||
|
|
||
|
|
||
| - name: Collate summary | ||
| run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary | ||
|
|
||
| - name: Set summary variable | ||
| run: | | ||
| echo "MESSAGE<<EOF" >> $GITHUB_ENV | ||
| echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV | ||
| echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV | ||
| echo "EOF" >> $GITHUB_ENV | ||
| - name: Post results to slack | ||
| uses: MeilCli/slack-upload-file@v3 | ||
| - name: Send summary to slack | ||
| uses: slackapi/[email protected] | ||
| with: | ||
| slack_token: ${{ secrets.SLACK_TOKEN }} | ||
| channel_id: ${{ secrets.SLACK_CHANNEL_ID }} | ||
| file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip" | ||
| initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}" | ||
| file_type: "zip" | ||
| update-ts: ${{needs.start_workflow.outputs.init_message_ts}} | ||
| channel-id: ${{ secrets.SLACK_CHANNEL_ID }} | ||
| slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}" | ||
| env: | ||
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }} | ||