From a88a9ff3d9ca9883e4c07a277dba0fe85d37d8f7 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 15:08:48 +0800 Subject: [PATCH 01/12] update perf imports --- .github/workflows/performance.yml | 26 ++++++++++++++++--- noxfile.py | 42 +++++++++++++++---------------- 2 files changed, 42 insertions(+), 26 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 78b4064..17df8d3 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -1,9 +1,8 @@ name: performance on: -# push: -# branches: [ main ] -# paths: [ '**.py', 'src/**' ] + release: + types: [ published ] workflow_dispatch: concurrency: @@ -14,7 +13,9 @@ jobs: perf-import: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: + python: [ '3.8', '3.9', '3.10', '3.11', '3.12' ] os: [ ubuntu-latest, macOS-latest, windows-latest ] steps: @@ -28,9 +29,26 @@ jobs: python -m pip install nox pyperf - name: Perf importing third-party packages run: | - sh scripts/perf_import.sh + nox -s test_import_third_party_perf-${{ matrix.python }} + - uses: actions/upload-artifact@v3 + with: + name: perf-import + path: "*.json" + + perf-import-result: + runs-on: ubuntu-latest + needs: [ perf-import ] + steps: + - uses: actions/setup-python@v4 + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: perf-import + path: ./ pyperformance: + if: false runs-on: ${{ matrix.os }} strategy: matrix: diff --git a/noxfile.py b/noxfile.py index 695a7fc..0d714b3 100644 --- a/noxfile.py +++ b/noxfile.py @@ -43,6 +43,16 @@ def _py_version(session: nox.Session): return out +def _perf_config() -> t.List[str]: + result = [] + _run = os.environ.get('RUN', '').lower() + if _run in ('fast', 'short'): + result.append('--fast') + elif _run in ('long', 'rigorous'): + result.append('--rigorous') + return result + + def _self_tests(session: nox.Session): session.install(".") session.install("pytest") @@ -180,17 +190,18 @@ def test_import_third_party_perf(session: nox.Session, package): raw_out = f'perf-import-{session.python}-raw' cds_out = f'perf-import-{session.python}-cds' - session.run('pyperf', 'command', '--fast', f'--append={raw_out}.json', f'--name={package.name}', + base_cmd = ['pyperf', 'command'] + _perf_config() + [f'--name={package.name}'] + + session.run(*base_cmd, f'--append={raw_out}.json', 'python', '-c', package.import_stmt) - session.run('pyperf', 'command', '--fast', f'--append={cds_out}.json', f'--name={package.name}', - '--inherit-environ=PYCDSMODE,PYCDSARCHIVE', + session.run(*base_cmd, f'--append={cds_out}.json', '--inherit-environ=PYCDSMODE,PYCDSARCHIVE', 'python', '-c', package.import_stmt, env={'PYCDSMODE': 'SHARE', 'PYCDSARCHIVE': img}) ci_session_cleanup() -def _pyperformance(session: nox.Session, pyperformance_args=None): +def _pyperformance(session: nox.Session, pyperformance_cmd=None): session.install(CDS_PYPERFORMANCE) configs = [ @@ -205,15 +216,17 @@ def _pyperformance(session: nox.Session, pyperformance_args=None): tmp = session.create_tmp() session.chdir(tmp) - if pyperformance_args is None: - pyperformance_args = ['pyperformance', 'run'] + if pyperformance_cmd is None: + pyperformance_cmd = ['pyperformance', 'run'] + + pyperformance_cmd += _perf_config() cmd_exc = None for out, args in configs: if os.path.exists(out): session.run('mv', out, out + '.old') try: - session.run(*(pyperformance_args + args), f'--out={out}') + session.run(*(pyperformance_cmd + args), f'--out={out}') except CommandFailed as e: if cmd_exc is None: cmd_exc = e @@ -222,21 +235,6 @@ def _pyperformance(session: nox.Session, pyperformance_args=None): raise cmd_exc -@nox.session(venv_backend='venv') -def pyperformance_current(session: nox.Session): - _pyperformance(session, ['pyperformance', 'run', '--fast']) - - -@nox.session(python=SUPPORTED_PYTHONS) -def pyperformance(session: nox.Session): - _pyperformance(session, ['pyperformance', 'run', '--fast']) - - -@nox.session(python=SUPPORTED_PYTHONS) -def pyperformance_looong(session: nox.Session): - _pyperformance(session, ['pyperformance', 'run', '--rigorous']) - - @nox.session(venv_backend='venv') def build_sdist(session: nox.Session): """ From 8264e46c5dd614253ae14418d1fb0c2d2f3c19a6 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 15:12:43 +0800 Subject: [PATCH 02/12] test on upstream --- .github/workflows/performance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 17df8d3..70bb7b8 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -3,6 +3,8 @@ name: performance on: release: types: [ published ] + pull_request: + branches: [ main ] workflow_dispatch: concurrency: From 1d2999f799dfde3efe22de4ef865b262ed228cf9 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 15:15:11 +0800 Subject: [PATCH 03/12] skip function test --- .github/workflows/check.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 0a6303c..45ba597 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -12,6 +12,7 @@ concurrency: jobs: test-cpython: + if: false runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -35,6 +36,7 @@ jobs: run: nox -s tests_venv_current test-conda: + if: false runs-on: ${{ matrix.os }} strategy: matrix: @@ -61,6 +63,7 @@ jobs: uses: mxschmitt/action-tmate@v3 test-twine: + if: false runs-on: ubuntu-latest steps: From d8cd6f171cd4571ea2949db0e6f7fd6fed93c9c3 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 15:23:39 +0800 Subject: [PATCH 04/12] fix --- noxfile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/noxfile.py b/noxfile.py index 0d714b3..1a09cf8 100644 --- a/noxfile.py +++ b/noxfile.py @@ -187,8 +187,8 @@ def test_import_third_party_perf(session: nox.Session, package): session.run('python', '-c', package.import_stmt, env={'PYCDSMODE': 'SHARE', 'PYCDSARCHIVE': img}, log=False) logger.info(f'finish generating CDS archive for {package.name}') - raw_out = f'perf-import-{session.python}-raw' - cds_out = f'perf-import-{session.python}-cds' + raw_out = f'perf-import-{session.python}-{OS.lower()}-raw' + cds_out = f'perf-import-{session.python}-{OS.lower()}-cds' base_cmd = ['pyperf', 'command'] + _perf_config() + [f'--name={package.name}'] From b8810b24d8d71c21cb35e1e7c9a47fe7bccf66f9 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 15:48:55 +0800 Subject: [PATCH 05/12] tune linux --- .github/workflows/performance.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 70bb7b8..1010043 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -29,6 +29,10 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install nox pyperf + - name: Tune if possible + if: runner.os == 'Linux' + run: | + sudo pyperf system tune - name: Perf importing third-party packages run: | nox -s test_import_third_party_perf-${{ matrix.python }} From 823575ea1945fd82a7009c37711543e346b6612e Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 16:51:49 +0800 Subject: [PATCH 06/12] update --- .github/workflows/performance.yml | 3 +++ noxfile.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 1010043..3bcfca9 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -34,6 +34,9 @@ jobs: run: | sudo pyperf system tune - name: Perf importing third-party packages + continue-on-error: true + env: + RUN: LONG run: | nox -s test_import_third_party_perf-${{ matrix.python }} - uses: actions/upload-artifact@v3 diff --git a/noxfile.py b/noxfile.py index 1a09cf8..08d2685 100644 --- a/noxfile.py +++ b/noxfile.py @@ -130,7 +130,7 @@ def tests_multiple_conda(session: nox.Session): # conda-provided tf might require pypy and this is not what we want, # and pypi only provides tf for CPython <= 3.11 Package('tensorflow', skip=lambda _py: _py in ('3.12',)), - Package('seaborn', conda=True), + Package('seaborn', conda=True, skip=lambda _py: OS == 'Windows'), Package('azureml-core', module='azureml.core'), Package('opencv-python', module='cv2') From a4d638e64424702832d02d60f61e989876a2974b Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 17:01:31 +0800 Subject: [PATCH 07/12] update --- .github/workflows/performance.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 3bcfca9..86232fd 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -29,10 +29,6 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install nox pyperf - - name: Tune if possible - if: runner.os == 'Linux' - run: | - sudo pyperf system tune - name: Perf importing third-party packages continue-on-error: true env: From f4c8d3d479a877af09e285a6c6f04cf3ebb0f9c4 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 19:18:06 +0800 Subject: [PATCH 08/12] upload result --- .github/workflows/performance.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 86232fd..f236cf8 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -45,12 +45,29 @@ jobs: needs: [ perf-import ] steps: - uses: actions/setup-python@v4 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pyperf - name: Download artifact uses: actions/download-artifact@v3 with: name: perf-import path: ./ + - name: Result + run: | + for VERSION in 3.8 3.9 3.10 3.11 3.12 + do + for OS in linux macos windows + do + pyperf compare_to perf-import-$VERSION-$OS-raw.json perf-import-$VERSION-$OS-cds.json | tee perf-import-$VERSION-$OS.table + done + done + - uses: actions/upload-artifact@v3 + with: + name: perf-import + path: "*.table" pyperformance: if: false From 749fc80dab83f60c70bc59c511b99cc6f1a44a31 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Mon, 23 Oct 2023 21:24:34 +0800 Subject: [PATCH 09/12] fix --- .github/workflows/performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index f236cf8..a0e6eac 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -59,9 +59,9 @@ jobs: run: | for VERSION in 3.8 3.9 3.10 3.11 3.12 do - for OS in linux macos windows + for OS in linux darwin windows do - pyperf compare_to perf-import-$VERSION-$OS-raw.json perf-import-$VERSION-$OS-cds.json | tee perf-import-$VERSION-$OS.table + pyperf compare_to perf-import-$VERSION-$OS-raw.json perf-import-$VERSION-$OS-cds.json --table | tee perf-import-$VERSION-$OS.table done done - uses: actions/upload-artifact@v3 From e2897d7a013a59de392b52922bf5e3f889f2c5a6 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 24 Oct 2023 10:03:48 +0800 Subject: [PATCH 10/12] test pyperformance --- .github/workflows/performance.yml | 22 ++++++++++++++++------ noxfile.py | 8 +++++--- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index a0e6eac..6afd685 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -13,6 +13,7 @@ concurrency: jobs: perf-import: + if: false runs-on: ${{ matrix.os }} strategy: fail-fast: false @@ -28,7 +29,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install nox pyperf + python -m pip install nox - name: Perf importing third-party packages continue-on-error: true env: @@ -70,21 +71,30 @@ jobs: path: "*.table" pyperformance: - if: false runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: + python: [ '3.8', '3.9', '3.10', '3.11', '3.12' ] os: [ ubuntu-latest, macOS-latest, windows-latest ] steps: - uses: actions/checkout@v4 - - uses: s-weigand/setup-conda@v1 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 with: - conda-channels: conda-forge + python-version: ${{ matrix.python }} - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install nox pyperf + python -m pip install nox - name: Pyperformance tests against cds + continue-on-error: true + env: + RUN: LONG run: | - sh scripts/pyperformance.sh + nox -s pyperformance + - uses: actions/upload-artifact@v3 + with: + name: pyperformance + path: "*.json" diff --git a/noxfile.py b/noxfile.py index 08d2685..90080af 100644 --- a/noxfile.py +++ b/noxfile.py @@ -3,6 +3,7 @@ """ import os import platform +import shutil import typing as t import nox @@ -201,7 +202,8 @@ def test_import_third_party_perf(session: nox.Session, package): ci_session_cleanup() -def _pyperformance(session: nox.Session, pyperformance_cmd=None): +@nox.session(venv_backend='venv') +def pyperformance(session: nox.Session, pyperformance_cmd=None): session.install(CDS_PYPERFORMANCE) configs = [ @@ -224,9 +226,9 @@ def _pyperformance(session: nox.Session, pyperformance_cmd=None): cmd_exc = None for out, args in configs: if os.path.exists(out): - session.run('mv', out, out + '.old') + shutil.move(out, out + '.old') try: - session.run(*(pyperformance_cmd + args), f'--out={out}') + session.run(*(pyperformance_cmd + args), f'--out={repr(out)}') except CommandFailed as e: if cmd_exc is None: cmd_exc = e From 56a3b53e07708e93d29709da18b4afc02bf72905 Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Tue, 24 Oct 2023 11:36:35 +0800 Subject: [PATCH 11/12] test new pyperformance --- noxfile.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/noxfile.py b/noxfile.py index 90080af..c04fb51 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,7 +20,7 @@ GA = os.environ.get('GITHUB_ACTIONS') == 'true' -CDS_PYPERFORMANCE = 'git+https://github.com/oraluben/pyperformance.git@cds' +CDS_PYPERFORMANCE = 'git+https://github.com/oraluben/pyperformance.git@cds-dev' def _clean_nox(): @@ -203,11 +203,11 @@ def test_import_third_party_perf(session: nox.Session, package): @nox.session(venv_backend='venv') -def pyperformance(session: nox.Session, pyperformance_cmd=None): +def pyperformance(session: nox.Session): session.install(CDS_PYPERFORMANCE) configs = [ - (os.path.realpath(f'pyperformance-{_py_version(session)}-{config_name}.json'), config_args) + (os.path.realpath(f'pyperformance-{_py_version(session)}-{OS.lower()}-{config_name}.json'), config_args) for (config_name, config_args) in [ ('raw', []), ('cds-site', ['--install-cds', PYCDS_ROOT]), @@ -218,17 +218,14 @@ def pyperformance(session: nox.Session, pyperformance_cmd=None): tmp = session.create_tmp() session.chdir(tmp) - if pyperformance_cmd is None: - pyperformance_cmd = ['pyperformance', 'run'] - - pyperformance_cmd += _perf_config() + pyperformance_cmd = ['pyperformance', 'run'] + _perf_config() cmd_exc = None for out, args in configs: if os.path.exists(out): shutil.move(out, out + '.old') try: - session.run(*(pyperformance_cmd + args), f'--out={repr(out)}') + session.run(*(pyperformance_cmd + args), f'--out={out}') except CommandFailed as e: if cmd_exc is None: cmd_exc = e From 2a6e255b1fe7ea04b95460a19a4d726fbf9fd15c Mon Sep 17 00:00:00 2001 From: Yichen Yan Date: Fri, 27 Oct 2023 15:17:50 +0800 Subject: [PATCH 12/12] use normal runs --- .github/workflows/performance.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 6afd685..82e573f 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -90,8 +90,6 @@ jobs: python -m pip install nox - name: Pyperformance tests against cds continue-on-error: true - env: - RUN: LONG run: | nox -s pyperformance - uses: actions/upload-artifact@v3