From 886b3a8e0dec52965b3714ce5e5392e0c5086500 Mon Sep 17 00:00:00 2001 From: Stefan Baumann Date: Fri, 23 Jul 2021 18:38:36 +0200 Subject: [PATCH 1/6] Implement current MIREX key scoring method and warn if using old one --- mir_eval/key.py | 38 ++++++++++++++++++++++++++++++-------- 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/mir_eval/key.py b/mir_eval/key.py index f72e8ad3..28021f7b 100644 --- a/mir_eval/key.py +++ b/mir_eval/key.py @@ -19,6 +19,7 @@ ''' import collections +import warnings from . import util KEY_TO_SEMITONE = {'c': 0, 'c#': 1, 'db': 1, 'd': 2, 'd#': 3, 'eb': 3, 'e': 4, @@ -96,7 +97,8 @@ def split_key_string(key): return KEY_TO_SEMITONE[key.lower()], mode -def weighted_score(reference_key, estimated_key): +def weighted_score(reference_key, estimated_key, + allow_descending_fifths=False): """Computes a heuristic score which is weighted according to the relationship of the reference and estimated key, as follows: @@ -114,11 +116,16 @@ def weighted_score(reference_key, estimated_key): | Other | 0.0 | +------------------------------------------------------+-------+ + When specifying allow_descending_fifths=True, the scoring changes so that + keys that are a perfect fifth above or below the reference key score 0.5 + points. This is consistent with the scoring used for MIREX since 2017. + Examples -------- >>> ref_key = mir_eval.io.load_key('ref.txt') >>> est_key = mir_eval.io.load_key('est.txt') - >>> score = mir_eval.key.weighted_score(ref_key, est_key) + >>> score = mir_eval.key.weighted_score(ref_key, est_key, + ... allow_descending_fifths=True) Parameters ---------- @@ -126,12 +133,21 @@ def weighted_score(reference_key, estimated_key): Reference key string. estimated_key : str Estimated key string. + allow_descending_fifths : bool + Specifies whether to score descending fifth errors or not. Returns ------- score : float Score representing how closely related the keys are. """ + # Notify users of difference between default behaviour in mir_eval and + # the scoring used by MIREX since 2017 + if not allow_descending_fifths: + warnings.warn('The selected key scoring method does not match that '\ + 'currently used by MIREX. To use the same method, specify '\ + 'allow_descending_fifths=True.') + validate(reference_key, estimated_key) reference_key, reference_mode = split_key_string(reference_key) estimated_key, estimated_mode = split_key_string(estimated_key) @@ -142,10 +158,14 @@ def weighted_score(reference_key, estimated_key): # then the result is 'Other'. if reference_key is None or estimated_key is None: return 0. - # If keys are the same mode and a perfect fifth (differ by 7 semitones) + # If keys are the same mode and a perfect fifth up (7 semitones) if (estimated_mode == reference_mode and (estimated_key - reference_key) % 12 == 7): return 0.5 + # If keys are the same mode and a perfect fifth down (7 semitones) + if (allow_descending_fifths and estimated_mode == reference_mode and + (reference_key - estimated_key) % 12 == 7): + return 0.5 # Estimated key is relative minor of reference key (9 semitones) if (estimated_mode != reference_mode == 'major' and (estimated_key - reference_key) % 12 == 9): @@ -161,23 +181,25 @@ def weighted_score(reference_key, estimated_key): return 0. -def evaluate(reference_key, estimated_key, **kwargs): +def evaluate(reference_key, estimated_key, allow_descending_fifths=False, + **kwargs): """Compute all metrics for the given reference and estimated annotations. Examples -------- >>> ref_key = mir_eval.io.load_key('reference.txt') >>> est_key = mir_eval.io.load_key('estimated.txt') - >>> scores = mir_eval.key.evaluate(ref_key, est_key) + >>> scores = mir_eval.key.evaluate(ref_key, est_key + ... allow_descending_fifths=True) Parameters ---------- ref_key : str Reference key string. - ref_key : str Estimated key string. - + allow_descending_fifths : bool + Specifies whether to score descending fifth errors or not. kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -192,6 +214,6 @@ def evaluate(reference_key, estimated_key, **kwargs): scores = collections.OrderedDict() scores['Weighted Score'] = util.filter_kwargs( - weighted_score, reference_key, estimated_key) + weighted_score, reference_key, estimated_key, allow_descending_fifths) return scores From 31729d8154cfc98ef5b0687e130780c2a1f9c1f9 Mon Sep 17 00:00:00 2001 From: Stefan Baumann Date: Fri, 23 Jul 2021 19:41:59 +0200 Subject: [PATCH 2/6] Update key scoring warning to specify future default behaviour change --- mir_eval/key.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mir_eval/key.py b/mir_eval/key.py index 28021f7b..57b89089 100644 --- a/mir_eval/key.py +++ b/mir_eval/key.py @@ -119,6 +119,8 @@ def weighted_score(reference_key, estimated_key, When specifying allow_descending_fifths=True, the scoring changes so that keys that are a perfect fifth above or below the reference key score 0.5 points. This is consistent with the scoring used for MIREX since 2017. + In the future, the default behaviour will change to use the new method by + default. Examples -------- @@ -146,7 +148,8 @@ def weighted_score(reference_key, estimated_key, if not allow_descending_fifths: warnings.warn('The selected key scoring method does not match that '\ 'currently used by MIREX. To use the same method, specify '\ - 'allow_descending_fifths=True.') + 'allow_descending_fifths=True. The default behaviour will '\ + 'change to allow_descending_fifths=True in the future.') validate(reference_key, estimated_key) reference_key, reference_mode = split_key_string(reference_key) From 94d7676c6bba05823277643fb626419c4ce2c5e5 Mon Sep 17 00:00:00 2001 From: instr3 Date: Tue, 30 Sep 2025 23:13:43 +0800 Subject: [PATCH 3/6] Minor fix --- mir_eval/key.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mir_eval/key.py b/mir_eval/key.py index 57b89089..421481d7 100644 --- a/mir_eval/key.py +++ b/mir_eval/key.py @@ -197,9 +197,9 @@ def evaluate(reference_key, estimated_key, allow_descending_fifths=False, Parameters ---------- - ref_key : str + reference_key : str Reference key string. - ref_key : str + estimated_key : str Estimated key string. allow_descending_fifths : bool Specifies whether to score descending fifth errors or not. @@ -216,7 +216,7 @@ def evaluate(reference_key, estimated_key, allow_descending_fifths=False, # Compute all metrics scores = collections.OrderedDict() - scores['Weighted Score'] = util.filter_kwargs( + scores["Weighted Score"] = util.filter_kwargs( weighted_score, reference_key, estimated_key, allow_descending_fifths) return scores From 40bd8ec7ec1ba859d9f2981a4485b2efd0ddd50f Mon Sep 17 00:00:00 2001 From: instr3 Date: Tue, 30 Sep 2025 23:16:32 +0800 Subject: [PATCH 4/6] Merge remote-tracking branch 'remotes/origin2/main' into HEAD # Conflicts: # mir_eval/key.py --- .codespell_ignore_list | 5 + .coveragerc | 2 - .github/environment-docs.yml | 10 + .github/environment-lint.yml | 21 + .github/environment-minimal.yml | 12 + .github/environment.yml | 12 + .github/workflows/docs-ci.yml | 72 ++ .github/workflows/lint_python.yml | 71 ++ .github/workflows/release.yml | 43 + .github/workflows/test.yml | 91 ++ .gitignore | 6 +- .readthedocs.yaml | 25 + .travis.yml | 37 - README.rst | 26 +- docs/api/alignment.rst | 9 + docs/api/beat.rst | 9 + docs/api/chord.rst | 9 + docs/api/display.rst | 8 + docs/api/hierarchy.rst | 9 + docs/api/index.rst | 27 + docs/api/io.rst | 9 + docs/api/key.rst | 9 + docs/api/melody.rst | 9 + docs/api/multipitch.rst | 9 + docs/api/onset.rst | 9 + docs/api/pattern.rst | 9 + docs/api/segment.rst | 9 + docs/api/separation.rst | 9 + docs/api/sonify.rst | 9 + docs/api/tempo.rst | 9 + docs/api/transcription.rst | 9 + docs/api/transcription_velocity.rst | 9 + docs/api/util.rst | 9 + docs/changes.rst | 194 ++-- docs/conf.py | 178 ++-- docs/index.rst | 249 +---- mir_eval/__init__.py | 3 +- mir_eval/alignment.py | 357 +++++++ mir_eval/beat.py | 320 +++--- mir_eval/chord.py | 408 ++++---- mir_eval/display.py | 611 ++++++----- mir_eval/hierarchy.py | 275 +++-- mir_eval/io.py | 246 +++-- mir_eval/key.py | 82 +- mir_eval/melody.py | 317 +++--- mir_eval/multipitch.py | 164 +-- mir_eval/onset.py | 29 +- mir_eval/pattern.py | 254 +++-- mir_eval/segment.py | 474 +++++---- mir_eval/separation.py | 443 ++++---- mir_eval/sonify.py | 290 +++-- mir_eval/tempo.py | 60 +- mir_eval/transcription.py | 269 +++-- mir_eval/transcription_velocity.py | 191 +++- mir_eval/util.py | 262 ++--- setup.cfg | 72 ++ setup.py | 37 +- tests/baseline_images/test_display/events.png | Bin 9671 -> 0 bytes .../test_display/hierarchy_label.png | Bin 12968 -> 0 bytes .../test_display/hierarchy_nolabel.png | Bin 11832 -> 0 bytes .../test_display/labeled_events.png | Bin 9319 -> 0 bytes .../test_display/labeled_intervals.png | Bin 8411 -> 0 bytes .../labeled_intervals_compare.png | Bin 18373 -> 0 bytes .../labeled_intervals_compare_common.png | Bin 18447 -> 0 bytes .../labeled_intervals_compare_noextend.png | Bin 11731 -> 0 bytes .../labeled_intervals_noextend.png | Bin 8411 -> 0 bytes .../test_display/multipitch_hz_unvoiced.png | Bin 22829 -> 0 bytes .../test_display/multipitch_hz_voiced.png | Bin 22829 -> 0 bytes .../test_display/multipitch_midi.png | Bin 54981 -> 0 bytes .../test_display/piano_roll.png | Bin 12572 -> 0 bytes .../test_display/piano_roll_midi.png | Bin 12572 -> 0 bytes .../baseline_images/test_display/pitch_hz.png | Bin 29960 -> 0 bytes .../test_display/pitch_midi.png | Bin 25290 -> 0 bytes .../test_display/pitch_midi_hz.png | Bin 29977 -> 0 bytes .../baseline_images/test_display/segment.png | Bin 14440 -> 0 bytes .../test_display/segment_text.png | Bin 12498 -> 0 bytes .../test_display/separation.png | Bin 244783 -> 0 bytes .../test_display/separation_label.png | Bin 246824 -> 0 bytes .../test_display/test_display_events.png | Bin 0 -> 9219 bytes .../test_display_hierarchy_label.png | Bin 0 -> 12864 bytes .../test_display_hierarchy_nolabel.png | Bin 0 -> 11614 bytes .../test_display_labeled_events.png | Bin 0 -> 9268 bytes .../test_display_labeled_intervals.png | Bin 0 -> 7994 bytes ...test_display_labeled_intervals_compare.png | Bin 0 -> 17728 bytes ...splay_labeled_intervals_compare_common.png | Bin 0 -> 17801 bytes ...lay_labeled_intervals_compare_noextend.png | Bin 0 -> 11067 bytes ...est_display_labeled_intervals_noextend.png | Bin 0 -> 7994 bytes .../test_display_multipitch_hz_unvoiced.png | Bin 0 -> 22329 bytes .../test_display_multipitch_hz_voiced.png | Bin 0 -> 22329 bytes .../test_display_multipitch_midi.png | Bin 0 -> 55094 bytes .../test_display/test_display_piano_roll.png | Bin 0 -> 11991 bytes .../test_display_piano_roll_midi.png | Bin 0 -> 11991 bytes .../test_display/test_display_pitch_hz.png | Bin 0 -> 29670 bytes .../test_display/test_display_pitch_midi.png | Bin 0 -> 25212 bytes .../test_display_pitch_midi_hz.png | Bin 0 -> 29486 bytes .../test_display/test_display_segment.png | Bin 0 -> 14219 bytes .../test_display_segment_text.png | Bin 0 -> 11878 bytes .../test_display/test_display_separation.png | Bin 0 -> 245162 bytes .../test_display_separation_label.png | Bin 0 -> 247154 bytes .../test_display_ticker_midi_zoom.png | Bin 0 -> 22008 bytes .../test_display/ticker_midi_zoom.png | Bin 22852 -> 0 bytes tests/data/alignment/est00.txt | 10 + tests/data/alignment/est01.txt | 10 + tests/data/alignment/est02.txt | 11 + tests/data/alignment/est03_mirex.txt | 6 + tests/data/alignment/est04_mirex.txt | 3 + tests/data/alignment/output00.json | 1 + tests/data/alignment/output01.json | 1 + tests/data/alignment/output02.json | 1 + tests/data/alignment/output03_mirex.json | 1 + tests/data/alignment/output04_mirex.json | 1 + tests/data/alignment/ref00.txt | 10 + tests/data/alignment/ref01.txt | 10 + tests/data/alignment/ref02.txt | 11 + tests/data/alignment/ref03_mirex.txt | 6 + tests/data/alignment/ref04_mirex.txt | 3 + .../data/transcription_velocity/output2.json | 2 +- tests/generate_data.py | 99 +- tests/mpl_ic.py | 353 ------- tests/test_alignment.py | 84 ++ tests/test_beat.py | 200 ++-- tests/test_chord.py | 989 +++++++++--------- tests/test_display.py | 489 +++++---- tests/test_hierarchy.py | 302 +++--- tests/test_input_output.py | 194 ++-- tests/test_key.py | 110 +- tests/test_melody.py | 472 ++++----- tests/test_multipitch.py | 238 +++-- tests/test_onset.py | 132 +-- tests/test_pattern.py | 141 ++- tests/test_segment.py | 327 +++--- tests/test_separation.py | 550 ++++++---- tests/test_sonify.py | 200 +++- tests/test_tempo.py | 151 +-- tests/test_transcription.py | 333 +++--- tests/test_transcription_velocity.py | 156 +-- tests/test_util.py | 359 ++++--- 137 files changed, 7018 insertions(+), 5354 deletions(-) create mode 100644 .codespell_ignore_list delete mode 100644 .coveragerc create mode 100644 .github/environment-docs.yml create mode 100644 .github/environment-lint.yml create mode 100644 .github/environment-minimal.yml create mode 100644 .github/environment.yml create mode 100644 .github/workflows/docs-ci.yml create mode 100644 .github/workflows/lint_python.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml create mode 100644 .readthedocs.yaml delete mode 100644 .travis.yml create mode 100644 docs/api/alignment.rst create mode 100644 docs/api/beat.rst create mode 100644 docs/api/chord.rst create mode 100644 docs/api/display.rst create mode 100644 docs/api/hierarchy.rst create mode 100644 docs/api/index.rst create mode 100644 docs/api/io.rst create mode 100644 docs/api/key.rst create mode 100644 docs/api/melody.rst create mode 100644 docs/api/multipitch.rst create mode 100644 docs/api/onset.rst create mode 100644 docs/api/pattern.rst create mode 100644 docs/api/segment.rst create mode 100644 docs/api/separation.rst create mode 100644 docs/api/sonify.rst create mode 100644 docs/api/tempo.rst create mode 100644 docs/api/transcription.rst create mode 100644 docs/api/transcription_velocity.rst create mode 100644 docs/api/util.rst create mode 100644 mir_eval/alignment.py create mode 100644 setup.cfg delete mode 100644 tests/baseline_images/test_display/events.png delete mode 100644 tests/baseline_images/test_display/hierarchy_label.png delete mode 100644 tests/baseline_images/test_display/hierarchy_nolabel.png delete mode 100644 tests/baseline_images/test_display/labeled_events.png delete mode 100644 tests/baseline_images/test_display/labeled_intervals.png delete mode 100644 tests/baseline_images/test_display/labeled_intervals_compare.png delete mode 100644 tests/baseline_images/test_display/labeled_intervals_compare_common.png delete mode 100644 tests/baseline_images/test_display/labeled_intervals_compare_noextend.png delete mode 100644 tests/baseline_images/test_display/labeled_intervals_noextend.png delete mode 100644 tests/baseline_images/test_display/multipitch_hz_unvoiced.png delete mode 100644 tests/baseline_images/test_display/multipitch_hz_voiced.png delete mode 100644 tests/baseline_images/test_display/multipitch_midi.png delete mode 100644 tests/baseline_images/test_display/piano_roll.png delete mode 100644 tests/baseline_images/test_display/piano_roll_midi.png delete mode 100644 tests/baseline_images/test_display/pitch_hz.png delete mode 100644 tests/baseline_images/test_display/pitch_midi.png delete mode 100644 tests/baseline_images/test_display/pitch_midi_hz.png delete mode 100644 tests/baseline_images/test_display/segment.png delete mode 100644 tests/baseline_images/test_display/segment_text.png delete mode 100644 tests/baseline_images/test_display/separation.png delete mode 100644 tests/baseline_images/test_display/separation_label.png create mode 100644 tests/baseline_images/test_display/test_display_events.png create mode 100644 tests/baseline_images/test_display/test_display_hierarchy_label.png create mode 100644 tests/baseline_images/test_display/test_display_hierarchy_nolabel.png create mode 100644 tests/baseline_images/test_display/test_display_labeled_events.png create mode 100644 tests/baseline_images/test_display/test_display_labeled_intervals.png create mode 100644 tests/baseline_images/test_display/test_display_labeled_intervals_compare.png create mode 100644 tests/baseline_images/test_display/test_display_labeled_intervals_compare_common.png create mode 100644 tests/baseline_images/test_display/test_display_labeled_intervals_compare_noextend.png create mode 100644 tests/baseline_images/test_display/test_display_labeled_intervals_noextend.png create mode 100644 tests/baseline_images/test_display/test_display_multipitch_hz_unvoiced.png create mode 100644 tests/baseline_images/test_display/test_display_multipitch_hz_voiced.png create mode 100644 tests/baseline_images/test_display/test_display_multipitch_midi.png create mode 100644 tests/baseline_images/test_display/test_display_piano_roll.png create mode 100644 tests/baseline_images/test_display/test_display_piano_roll_midi.png create mode 100644 tests/baseline_images/test_display/test_display_pitch_hz.png create mode 100644 tests/baseline_images/test_display/test_display_pitch_midi.png create mode 100644 tests/baseline_images/test_display/test_display_pitch_midi_hz.png create mode 100644 tests/baseline_images/test_display/test_display_segment.png create mode 100644 tests/baseline_images/test_display/test_display_segment_text.png create mode 100644 tests/baseline_images/test_display/test_display_separation.png create mode 100644 tests/baseline_images/test_display/test_display_separation_label.png create mode 100644 tests/baseline_images/test_display/test_display_ticker_midi_zoom.png delete mode 100644 tests/baseline_images/test_display/ticker_midi_zoom.png create mode 100644 tests/data/alignment/est00.txt create mode 100644 tests/data/alignment/est01.txt create mode 100644 tests/data/alignment/est02.txt create mode 100644 tests/data/alignment/est03_mirex.txt create mode 100644 tests/data/alignment/est04_mirex.txt create mode 100644 tests/data/alignment/output00.json create mode 100644 tests/data/alignment/output01.json create mode 100644 tests/data/alignment/output02.json create mode 100644 tests/data/alignment/output03_mirex.json create mode 100644 tests/data/alignment/output04_mirex.json create mode 100755 tests/data/alignment/ref00.txt create mode 100755 tests/data/alignment/ref01.txt create mode 100755 tests/data/alignment/ref02.txt create mode 100755 tests/data/alignment/ref03_mirex.txt create mode 100755 tests/data/alignment/ref04_mirex.txt delete mode 100644 tests/mpl_ic.py create mode 100644 tests/test_alignment.py diff --git a/.codespell_ignore_list b/.codespell_ignore_list new file mode 100644 index 00000000..87defebc --- /dev/null +++ b/.codespell_ignore_list @@ -0,0 +1,5 @@ +nce +fpr +shepard +dum +theis diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 013dd202..00000000 --- a/.coveragerc +++ /dev/null @@ -1,2 +0,0 @@ -[report] -show_missing = True diff --git a/.github/environment-docs.yml b/.github/environment-docs.yml new file mode 100644 index 00000000..fe61a654 --- /dev/null +++ b/.github/environment-docs.yml @@ -0,0 +1,10 @@ +name: docs +channels: + - conda-forge + - defaults +dependencies: + - pip + - numpy >=1.15.4 + - scipy >=1.4.0 + - matplotlib-base>=3.3.0 + - numpydoc diff --git a/.github/environment-lint.yml b/.github/environment-lint.yml new file mode 100644 index 00000000..ba2e6838 --- /dev/null +++ b/.github/environment-lint.yml @@ -0,0 +1,21 @@ +name: lint +channels: + - conda-forge + - defaults +dependencies: + # required + - pip + - bandit + - codespell + - flake8 + - pytest + - pydocstyle + + # Dependencies for velin + - numpydoc>=1.1.0 + - sphinx>=5.1.0 + - pygments + - black + + - pip: + - velin diff --git a/.github/environment-minimal.yml b/.github/environment-minimal.yml new file mode 100644 index 00000000..6e5560da --- /dev/null +++ b/.github/environment-minimal.yml @@ -0,0 +1,12 @@ +name: test +channels: + - conda-forge + - defaults +dependencies: + - pip + - numpy ==1.15.4 + - scipy ==1.4.0 + - matplotlib-base==3.3.0 + - pytest + - pytest-cov + - pytest-mpl diff --git a/.github/environment.yml b/.github/environment.yml new file mode 100644 index 00000000..68641776 --- /dev/null +++ b/.github/environment.yml @@ -0,0 +1,12 @@ +name: test +channels: + - conda-forge + - defaults +dependencies: + - pip + - numpy >=1.15.4 + - scipy >=1.4.0 + - matplotlib-base>=3.3.0 + - pytest + - pytest-cov + - pytest-mpl diff --git a/.github/workflows/docs-ci.yml b/.github/workflows/docs-ci.yml new file mode 100644 index 00000000..92955a3c --- /dev/null +++ b/.github/workflows/docs-ci.yml @@ -0,0 +1,72 @@ +name: CI Docs + +on: + push: + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: True + +jobs: + setup: + name: "Doc environment setup" + runs-on: ${{ matrix.os }} + strategy: + fail-fast: true + matrix: + include: + - os: ubuntu-latest + python-version: "3.8" + channel-priority: "flexible" + envfile: ".github/environment-docs.yml" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache conda + uses: actions/cache@v4 + env: + # Increase this value to reset cache if etc/example-environment.yml has not changed + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles( matrix.envfile ) }} + + - name: Install Conda environment + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: false + python-version: ${{ matrix.python-version }} + add-pip-as-python-dependency: true + auto-activate-base: false + activate-environment: docs + channel-priority: ${{ matrix.channel-priority }} + environment-file: ${{ matrix.envfile }} + use-only-tar-bz2: false # IMPORTANT: This needs to be set for caching to work properly! + + - name: Conda info + shell: bash -l {0} + run: | + conda info -a + conda list + + - name: Install mir_eval + shell: bash -l {0} + run: python -m pip install --upgrade-strategy only-if-needed -e .[docs] + + - name: Build docs + shell: bash -l {0} + working-directory: docs + run: make html + + - name: Link checking + id: linkcheck + shell: bash -l {0} + working-directory: docs + run: make linkcheck diff --git a/.github/workflows/lint_python.yml b/.github/workflows/lint_python.yml new file mode 100644 index 00000000..48005966 --- /dev/null +++ b/.github/workflows/lint_python.yml @@ -0,0 +1,71 @@ +name: lint_python +on: [pull_request, push] +jobs: + lint_python: + name: "Lint and code analysis" + runs-on: ubuntu-latest + strategy: + fail-fast: true + matrix: + include: + - os: ubuntu-latest + python-version: "3.11" + channel-priority: "flexible" + envfile: ".github/environment-lint.yml" + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Cache conda + uses: actions/cache@v4 + env: + CACHE_NUMBER: 0 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-${{ matrix.python-version }}-conda-${{ env.CACHE_NUMBER }}-${{ hashFiles( matrix.envfile ) }} + - name: Install conda environmnent + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: false + python-version: ${{ matrix.python-version }} + add-pip-as-python-dependency: true + auto-activate-base: false + activate-environment: lint + # mamba-version: "*" + channel-priority: ${{ matrix.channel-priority }} + environment-file: ${{ matrix.envfile }} + use-only-tar-bz2: false + + - name: Conda info + shell: bash -l {0} + run: | + conda info -a + conda list + + - name: Spell check package + shell: bash -l {0} + run: codespell --ignore-words .codespell_ignore_list mir_eval + + - name: Security check + shell: bash -l {0} + run: bandit --recursive --skip B101,B110 . + + - name: Style check package + shell: bash -l {0} + run: python -m flake8 mir_eval + + - name: Format check package + shell: bash -l {0} + run: python -m black --check mir_eval + + - name: Format check tests + shell: bash -l {0} + run: python -m black --check tests + + - name: Docstring check + shell: bash -l {0} + run: python -m velin --check mir_eval + + - name: Docstring style check + shell: bash -l {0} + run: python -m pydocstyle mir_eval diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..eb2b6ade --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,43 @@ +name: Publish Python 🐍 distributions 📦 to PyPI + +on: + release: + types: [created] + + +jobs: + pypi-publish: + name: Upload release to PyPI + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/mir_eval + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + + - name: Install pypa/build + run: >- + python -m + pip install + build + --user + + - name: Build a binary wheel and a source tarball + run: >- + python -m + build + --sdist + --wheel + --outdir dist/ + . + + - name: Publish package distributions to PyPI + if: startsWith(github.ref, 'refs/tags') + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..28cdbd57 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,91 @@ +name: Test Python code + +on: + pull_request: + branches: + - main + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: True + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + python-version: ["3.8", "3.10", "3.11", "3.12"] + channel-priority: [strict] + envfile: [".github/environment.yml"] + include: + - python-version: "3.13" + os: macos-latest + - python-version: "3.13" + os: windows-latest + - python-version: "3.13" + os: ubuntu-latest + channel-priority: flexible + - os: ubuntu-latest + python-version: "3.7" + envfile: ".github/environment-minimal.yml" + channel-priority: "flexible" + name: "Minimal dependencies" + + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: Cache conda packages + uses: actions/cache@v4 + with: + path: ~/conda_pkgs_dir + key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles( matrix.envfile ) }} + + - name: Create conda environment + uses: conda-incubator/setup-miniconda@v3 + with: + python-version: ${{ matrix.python-version }} + auto-activate-base: false + channel-priority: ${{ matrix.channel-priority }} + environment-file: ${{ matrix.envfile }} + # Disabling bz2 to get more recent dependencies. + # NOTE: this breaks cache support, so CI will be slower. + use-only-tar-bz2: False # IMPORTANT: This needs to be set for caching to work properly! + + - name: Install package in development mode + shell: bash -l {0} + run: python -m pip install --upgrade-strategy=only-if-needed -e .[display,tests] + + - name: Log installed packages for debugging + shell: bash -l {0} + run: | + conda info -a + conda list + + - name: Show libraries in the system on which NumPy was built + shell: bash -l {0} + run: python -c "import numpy; numpy.show_config()" + + - name: Show libraries in the system on which SciPy was built + shell: bash -l {0} + run: python -c "import scipy; scipy.show_config()" + + - name: Run unit tests + shell: bash -l {0} + run: pytest + working-directory: tests + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: ./coverage.xml + flags: unittests + env_vars: OS,PYTHON + name: codecov-umbrella + fail_ci_if_error: true + verbose: true diff --git a/.gitignore b/.gitignore index 53e602e3..6eece87b 100644 --- a/.gitignore +++ b/.gitignore @@ -44,5 +44,9 @@ Thumbs.db # docs docs/_build/* -# matplotlib tsets +# matplotlib tests tests/result_images/* + +# coverage +coverage.xml + diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000..18d5429e --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,25 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the OS, Python version, and other tools you might need +build: + os: ubuntu-24.04 + tools: + python: "3.12" + +# Build documentation in the "docs/" directory with Sphinx +sphinx: + configuration: docs/conf.py + +# Optionally, but recommended, +# declare the Python requirements required to build your documentation +# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dd51460b..00000000 --- a/.travis.yml +++ /dev/null @@ -1,37 +0,0 @@ -language: python - -notifications: - email: false - -python: - - "3.5" - -before_install: - - if [[ "$TRAVIS_PYTHON_VERSION" == "2.7" ]]; then - wget http://repo.continuum.io/miniconda/Miniconda-3.8.3-Linux-x86_64.sh -O miniconda.sh; - else - wget http://repo.continuum.io/miniconda/Miniconda3-3.8.3-Linux-x86_64.sh -O miniconda.sh; - fi - - bash miniconda.sh -b -p $HOME/miniconda - - export PATH="$HOME/miniconda/bin:$PATH" - - hash -r - - conda config --set always_yes yes --set changeps1 no - - conda update -q conda - - conda info -a - - deps='pip atlas numpy scipy sphinx nose six future pep8 matplotlib>=2.1.0,<3 decorator' - - conda create -q -n test-environment "python=$TRAVIS_PYTHON_VERSION" $deps - - source activate test-environment - - pip install python-coveralls - - pip install numpydoc - -install: - - pip install -e .[display,testing] - -script: - - nosetests -v --with-coverage --cover-package=mir_eval -w tests - - pep8 mir_eval tests - - python setup.py build_sphinx - - python setup.py egg_info -b.dev sdist --formats gztar - -after_success: - - coveralls diff --git a/README.rst b/README.rst index a31a42ab..585c0187 100644 --- a/README.rst +++ b/README.rst @@ -1,22 +1,30 @@ -.. image:: https://travis-ci.org/craffel/mir_eval.svg?branch=master - :target: https://travis-ci.org/craffel/mir_eval -.. image:: https://coveralls.io/repos/craffel/mir_eval/badge.svg?branch=master&service=github - :target: https://coveralls.io/github/craffel/mir_eval?branch=master +.. image:: https://anaconda.org/conda-forge/mir_eval/badges/version.svg + :target: https://anaconda.org/conda-forge/mir_eval + +.. image:: https://img.shields.io/pypi/v/mir_eval.svg + :target: https://pypi.python.org/pypi/mir_eval + +.. image:: https://github.com/mir-evaluation/mir_eval/actions/workflows/test.yml/badge.svg + :target: https://github.com/mir-evaluation/mir_eval/actions/workflows/test.yml + +.. image:: https://codecov.io/gh/mir-evaluation/mir_eval/graph/badge.svg?token=OzRL3aW7TX + :target: https://codecov.io/gh/mir-evaluation/mir_eval + +.. image:: https://img.shields.io/pypi/l/mir_eval.svg + :target: https://github.com/mir-evaluation/mir_eval/blob/main/LICENSE.txt + mir_eval ======== Python library for computing common heuristic accuracy scores for various music/audio information retrieval/signal processing tasks. -Documentation, including installation and usage information: http://craffel.github.io/mir_eval/ - -If you're looking for the mir_eval web service, which you can use to run mir_eval without installing anything or writing any code, it can be found here: http://labrosa.ee.columbia.edu/mir_eval/ +Documentation, including installation and usage information: https://mir-evaluation.github.io/mir_eval/ Dependencies: * `Scipy/Numpy `_ -* future -* six +* `decorator `_ If you use mir_eval in a research project, please cite the following paper: diff --git a/docs/api/alignment.rst b/docs/api/alignment.rst new file mode 100644 index 00000000..9744b426 --- /dev/null +++ b/docs/api/alignment.rst @@ -0,0 +1,9 @@ +mir_eval.alignment +================== + +.. automodule:: mir_eval.alignment + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/beat.rst b/docs/api/beat.rst new file mode 100644 index 00000000..c79b793d --- /dev/null +++ b/docs/api/beat.rst @@ -0,0 +1,9 @@ +mir_eval.beat +============= + +.. automodule:: mir_eval.beat + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/chord.rst b/docs/api/chord.rst new file mode 100644 index 00000000..6e8ab10a --- /dev/null +++ b/docs/api/chord.rst @@ -0,0 +1,9 @@ +mir_eval.chord +============== + +.. automodule:: mir_eval.chord + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/display.rst b/docs/api/display.rst new file mode 100644 index 00000000..32bd3027 --- /dev/null +++ b/docs/api/display.rst @@ -0,0 +1,8 @@ +mir_eval.display +================ + +.. automodule:: mir_eval.display + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource diff --git a/docs/api/hierarchy.rst b/docs/api/hierarchy.rst new file mode 100644 index 00000000..2590ddc8 --- /dev/null +++ b/docs/api/hierarchy.rst @@ -0,0 +1,9 @@ +mir_eval.hierarchy +================== + +.. automodule:: mir_eval.hierarchy + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/index.rst b/docs/api/index.rst new file mode 100644 index 00000000..1e3e1448 --- /dev/null +++ b/docs/api/index.rst @@ -0,0 +1,27 @@ +API Reference +============= + +Below is the reference documentation for all **mir_eval** submodules: + +.. toctree:: + :maxdepth: 2 + :titlesonly: + + alignment + beat + chord + melody + multipitch + onset + pattern + segment + hierarchy + separation + tempo + transcription + transcription_velocity + key + util + io + sonify + display diff --git a/docs/api/io.rst b/docs/api/io.rst new file mode 100644 index 00000000..aa9d3364 --- /dev/null +++ b/docs/api/io.rst @@ -0,0 +1,9 @@ +mir_eval.io +=========== + +.. automodule:: mir_eval.io + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/key.rst b/docs/api/key.rst new file mode 100644 index 00000000..6be0024f --- /dev/null +++ b/docs/api/key.rst @@ -0,0 +1,9 @@ +mir_eval.key +============ + +.. automodule:: mir_eval.key + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/melody.rst b/docs/api/melody.rst new file mode 100644 index 00000000..444b346e --- /dev/null +++ b/docs/api/melody.rst @@ -0,0 +1,9 @@ +mir_eval.melody +=============== + +.. automodule:: mir_eval.melody + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/multipitch.rst b/docs/api/multipitch.rst new file mode 100644 index 00000000..32c11397 --- /dev/null +++ b/docs/api/multipitch.rst @@ -0,0 +1,9 @@ +mir_eval.multipitch +=================== + +.. automodule:: mir_eval.multipitch + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/onset.rst b/docs/api/onset.rst new file mode 100644 index 00000000..6e98b12c --- /dev/null +++ b/docs/api/onset.rst @@ -0,0 +1,9 @@ +mir_eval.onset +============== + +.. automodule:: mir_eval.onset + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/pattern.rst b/docs/api/pattern.rst new file mode 100644 index 00000000..36485cad --- /dev/null +++ b/docs/api/pattern.rst @@ -0,0 +1,9 @@ +mir_eval.pattern +================ + +.. automodule:: mir_eval.pattern + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/segment.rst b/docs/api/segment.rst new file mode 100644 index 00000000..8a4231a3 --- /dev/null +++ b/docs/api/segment.rst @@ -0,0 +1,9 @@ +mir_eval.segment +================ + +.. automodule:: mir_eval.segment + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/separation.rst b/docs/api/separation.rst new file mode 100644 index 00000000..4020bd1c --- /dev/null +++ b/docs/api/separation.rst @@ -0,0 +1,9 @@ +mir_eval.separation +=================== + +.. automodule:: mir_eval.separation + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/sonify.rst b/docs/api/sonify.rst new file mode 100644 index 00000000..ca101f04 --- /dev/null +++ b/docs/api/sonify.rst @@ -0,0 +1,9 @@ +mir_eval.sonify +=============== + +.. automodule:: mir_eval.sonify + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/tempo.rst b/docs/api/tempo.rst new file mode 100644 index 00000000..b6e164e6 --- /dev/null +++ b/docs/api/tempo.rst @@ -0,0 +1,9 @@ +mir_eval.tempo +============== + +.. automodule:: mir_eval.tempo + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/transcription.rst b/docs/api/transcription.rst new file mode 100644 index 00000000..9a7d5008 --- /dev/null +++ b/docs/api/transcription.rst @@ -0,0 +1,9 @@ +mir_eval.transcription +====================== + +.. automodule:: mir_eval.transcription + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/transcription_velocity.rst b/docs/api/transcription_velocity.rst new file mode 100644 index 00000000..c95b81cc --- /dev/null +++ b/docs/api/transcription_velocity.rst @@ -0,0 +1,9 @@ +mir_eval.transcription_velocity +=============================== + +.. automodule:: mir_eval.transcription_velocity + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/api/util.rst b/docs/api/util.rst new file mode 100644 index 00000000..2e9c01fb --- /dev/null +++ b/docs/api/util.rst @@ -0,0 +1,9 @@ +mir_eval.util +============= + +.. automodule:: mir_eval.util + :members: + :undoc-members: + :show-inheritance: + :member-order: bysource + diff --git a/docs/changes.rst b/docs/changes.rst index ca6c1a63..953ba439 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -1,6 +1,82 @@ Changes ======= +v0.8.2 +------ + +- `#418`_: Fixed a bug in time-frequency sonification with single-interval inputs. + +.. _#418: https://github.com/mir-evaluation/mir_eval/pull/418 + +v0.8.1 +------ + +- `#410`_: Fixed several errors with time-frequency sonification +- `#412`_: Removed unused import of deprecated scipy submodule +- `#416`_: `mir_eval.io` routines now support `pathlib.Path` (and generally `os.Pathlike`) inputs as well as strings or open file descriptors. +- `#416`_: `mir_eval.io.load_wav` is deprecated and will be removed in v0.9.' + +.. _#410: https://github.com/mir-evaluation/mir_eval/pull/410 +.. _#412: https://github.com/mir-evaluation/mir_eval/pull/412 +.. _#416: https://github.com/mir-evaluation/mir_eval/pull/416 + + +v0.8 +---- + +- `#397`_: Removed invalid chord types from `chord.QUALITIES` +- `#390`_: Migrate from personal to organization account +- `#388`_: Update to packaging workflow +- `#387`_: Various modernizations +- `#385`_: Fixed import error +- `#384`_: Rename `testing` to `tests` +- `#382`_: Deprecated the source separation module +- `#380`_: Matplotlib support modernization +- `#379`_: Support nans in pitch contour sonification +- `#378`_: Improved efficiency in chord validation +- `#374`_: Fixed length calculation error in `sonify.time_frequency` +- `#370`_: Test and CI modernization +- `#367`_: Added interpolation method to docs for `resample_multipitch` +- `#361`_: Added PyPI package release workflow +- `#360`_: Fixed broken link +- `#359`_: Remove support for python 2 +- `#356`_, `#357`_, `#358`_: Migrate testing to github actions +- `#355`_: Optimize and fix sonification +- `#353`_: Set numerical precision, fix support for modern numpy + +.. _#397: https://github.com/mir-evaluation/mir_eval/pull/397 +.. _#390: https://github.com/mir-evaluation/mir_eval/pull/390 +.. _#388: https://github.com/mir-evaluation/mir_eval/pull/388 +.. _#387: https://github.com/mir-evaluation/mir_eval/pull/387 +.. _#385: https://github.com/mir-evaluation/mir_eval/pull/385 +.. _#384: https://github.com/mir-evaluation/mir_eval/pull/384 +.. _#382: https://github.com/mir-evaluation/mir_eval/pull/382 +.. _#380: https://github.com/mir-evaluation/mir_eval/pull/380 +.. _#379: https://github.com/mir-evaluation/mir_eval/pull/379 +.. _#378: https://github.com/mir-evaluation/mir_eval/pull/378 +.. _#374: https://github.com/mir-evaluation/mir_eval/pull/374 +.. _#370: https://github.com/mir-evaluation/mir_eval/pull/370 +.. _#367: https://github.com/mir-evaluation/mir_eval/pull/367 +.. _#361: https://github.com/mir-evaluation/mir_eval/pull/361 +.. _#360: https://github.com/mir-evaluation/mir_eval/pull/360 +.. _#359: https://github.com/mir-evaluation/mir_eval/pull/359 +.. _#356: https://github.com/mir-evaluation/mir_eval/pull/356 +.. _#357: https://github.com/mir-evaluation/mir_eval/pull/357 +.. _#358: https://github.com/mir-evaluation/mir_eval/pull/358 +.. _#355: https://github.com/mir-evaluation/mir_eval/pull/355 +.. _#353: https://github.com/mir-evaluation/mir_eval/pull/353 + + + +v0.7 +---- + +- `#334`_: Support notation for unknown/ambiguous key or mode +- `#343`_: Add suite of alignment metrics + +.. _#334: https://github.com/mir-evaluation/mir_eval/pull/334 +.. _#343: https://github.com/mir-evaluation/mir_eval/pull/343 + v0.6 ---- @@ -18,19 +94,19 @@ v0.6 - `#327`_: Stop testing 2.7 - `#328`_: Cast n_voiced to int in display.multipitch -.. _#297: https://github.com/craffel/mir_eval/pull/297 -.. _#299: https://github.com/craffel/mir_eval/pull/299 -.. _#301: https://github.com/craffel/mir_eval/pull/301 -.. _#302: https://github.com/craffel/mir_eval/pull/302 -.. _#305: https://github.com/craffel/mir_eval/pull/305 -.. _#307: https://github.com/craffel/mir_eval/pull/307 -.. _#309: https://github.com/craffel/mir_eval/pull/309 -.. _#312: https://github.com/craffel/mir_eval/pull/312 -.. _#320: https://github.com/craffel/mir_eval/pull/320 -.. _#323: https://github.com/craffel/mir_eval/pull/323 -.. _#324: https://github.com/craffel/mir_eval/pull/324 -.. _#327: https://github.com/craffel/mir_eval/pull/327 -.. _#328: https://github.com/craffel/mir_eval/pull/328 +.. _#297: https://github.com/mir-evaluation/mir_eval/pull/297 +.. _#299: https://github.com/mir-evaluation/mir_eval/pull/299 +.. _#301: https://github.com/mir-evaluation/mir_eval/pull/301 +.. _#302: https://github.com/mir-evaluation/mir_eval/pull/302 +.. _#305: https://github.com/mir-evaluation/mir_eval/pull/305 +.. _#307: https://github.com/mir-evaluation/mir_eval/pull/307 +.. _#309: https://github.com/mir-evaluation/mir_eval/pull/309 +.. _#312: https://github.com/mir-evaluation/mir_eval/pull/312 +.. _#320: https://github.com/mir-evaluation/mir_eval/pull/320 +.. _#323: https://github.com/mir-evaluation/mir_eval/pull/323 +.. _#324: https://github.com/mir-evaluation/mir_eval/pull/324 +.. _#327: https://github.com/mir-evaluation/mir_eval/pull/327 +.. _#328: https://github.com/mir-evaluation/mir_eval/pull/328 v0.5 ---- @@ -58,28 +134,28 @@ v0.5 - `#282`_: remove evaluator scripts - `#283`_: add transcription eval with velocity -.. _#222: https://github.com/craffel/mir_eval/pull/222 -.. _#225: https://github.com/craffel/mir_eval/pull/225 -.. _#227: https://github.com/craffel/mir_eval/pull/227 -.. _#234: https://github.com/craffel/mir_eval/pull/234 -.. _#236: https://github.com/craffel/mir_eval/pull/236 -.. _#240: https://github.com/craffel/mir_eval/pull/240 -.. _#242: https://github.com/craffel/mir_eval/pull/242 -.. _#245: https://github.com/craffel/mir_eval/pull/245 -.. _#247: https://github.com/craffel/mir_eval/pull/247 -.. _#249: https://github.com/craffel/mir_eval/pull/249 -.. _#252: https://github.com/craffel/mir_eval/pull/252 -.. _#254: https://github.com/craffel/mir_eval/pull/254 -.. _#255: https://github.com/craffel/mir_eval/pull/255 -.. _#258: https://github.com/craffel/mir_eval/pull/258 -.. _#259: https://github.com/craffel/mir_eval/pull/259 -.. _#263: https://github.com/craffel/mir_eval/pull/263 -.. _#266: https://github.com/craffel/mir_eval/pull/266 -.. _#268: https://github.com/craffel/mir_eval/pull/268 -.. _#277: https://github.com/craffel/mir_eval/pull/277 -.. _#279: https://github.com/craffel/mir_eval/pull/279 -.. _#282: https://github.com/craffel/mir_eval/pull/282 -.. _#283: https://github.com/craffel/mir_eval/pull/283 +.. _#222: https://github.com/mir-evaluation/mir_eval/pull/222 +.. _#225: https://github.com/mir-evaluation/mir_eval/pull/225 +.. _#227: https://github.com/mir-evaluation/mir_eval/pull/227 +.. _#234: https://github.com/mir-evaluation/mir_eval/pull/234 +.. _#236: https://github.com/mir-evaluation/mir_eval/pull/236 +.. _#240: https://github.com/mir-evaluation/mir_eval/pull/240 +.. _#242: https://github.com/mir-evaluation/mir_eval/pull/242 +.. _#245: https://github.com/mir-evaluation/mir_eval/pull/245 +.. _#247: https://github.com/mir-evaluation/mir_eval/pull/247 +.. _#249: https://github.com/mir-evaluation/mir_eval/pull/249 +.. _#252: https://github.com/mir-evaluation/mir_eval/pull/252 +.. _#254: https://github.com/mir-evaluation/mir_eval/pull/254 +.. _#255: https://github.com/mir-evaluation/mir_eval/pull/255 +.. _#258: https://github.com/mir-evaluation/mir_eval/pull/258 +.. _#259: https://github.com/mir-evaluation/mir_eval/pull/259 +.. _#263: https://github.com/mir-evaluation/mir_eval/pull/263 +.. _#266: https://github.com/mir-evaluation/mir_eval/pull/266 +.. _#268: https://github.com/mir-evaluation/mir_eval/pull/268 +.. _#277: https://github.com/mir-evaluation/mir_eval/pull/277 +.. _#279: https://github.com/mir-evaluation/mir_eval/pull/279 +.. _#282: https://github.com/mir-evaluation/mir_eval/pull/282 +.. _#283: https://github.com/mir-evaluation/mir_eval/pull/283 v0.4 ---- @@ -94,15 +170,15 @@ v0.4 - `#212`_: added frame-wise blind-source separation evaluation - `#218`_: speed up `melody.resample_melody_series` when times are equivalent -.. _#189: https://github.com/craffel/mir_eval/issues/189 -.. _#195: https://github.com/craffel/mir_eval/issues/195 -.. _#196: https://github.com/craffel/mir_eval/issues/196 -.. _#203: https://github.com/craffel/mir_eval/issues/203 -.. _#205: https://github.com/craffel/mir_eval/issues/205 -.. _#208: https://github.com/craffel/mir_eval/issues/208 -.. _#210: https://github.com/craffel/mir_eval/issues/210 -.. _#212: https://github.com/craffel/mir_eval/issues/212 -.. _#218: https://github.com/craffel/mir_eval/pull/218 +.. _#189: https://github.com/mir-evaluation/mir_eval/issues/189 +.. _#195: https://github.com/mir-evaluation/mir_eval/issues/195 +.. _#196: https://github.com/mir-evaluation/mir_eval/issues/196 +.. _#203: https://github.com/mir-evaluation/mir_eval/issues/203 +.. _#205: https://github.com/mir-evaluation/mir_eval/issues/205 +.. _#208: https://github.com/mir-evaluation/mir_eval/issues/208 +.. _#210: https://github.com/mir-evaluation/mir_eval/issues/210 +.. _#212: https://github.com/mir-evaluation/mir_eval/issues/212 +.. _#218: https://github.com/mir-evaluation/mir_eval/pull/218 v0.3 ---- @@ -111,10 +187,10 @@ v0.3 - `#175`_: filter_kwargs passes through `**kwargs` - `#181`_: added key detection metrics -.. _#170: https://github.com/craffel/mir_eval/issues/170 -.. _#173: https://github.com/craffel/mir_eval/issues/173 -.. _#175: https://github.com/craffel/mir_eval/issues/175 -.. _#181: https://github.com/craffel/mir_eval/issues/181 +.. _#170: https://github.com/mir-evaluation/mir_eval/issues/170 +.. _#173: https://github.com/mir-evaluation/mir_eval/issues/173 +.. _#175: https://github.com/mir-evaluation/mir_eval/issues/175 +.. _#181: https://github.com/mir-evaluation/mir_eval/issues/181 v0.2 ---- @@ -131,17 +207,17 @@ v0.2 - `#159`_: fixed documentation error in `chord.tetrads` - `#160`_: fixed documentation error in `util.intervals_to_samples` -.. _#103: https://github.com/craffel/mir_eval/issues/103 -.. _#109: https://github.com/craffel/mir_eval/issues/109 -.. _#122: https://github.com/craffel/mir_eval/issues/122 -.. _#136: https://github.com/craffel/mir_eval/issues/136 -.. _#138: https://github.com/craffel/mir_eval/issues/138 -.. _#139: https://github.com/craffel/mir_eval/issues/139 -.. _#147: https://github.com/craffel/mir_eval/issues/147 -.. _#150: https://github.com/craffel/mir_eval/issues/150 -.. _#151: https://github.com/craffel/mir_eval/issues/151 -.. _#159: https://github.com/craffel/mir_eval/issues/159 -.. _#160: https://github.com/craffel/mir_eval/issues/160 +.. _#103: https://github.com/mir-evaluation/mir_eval/issues/103 +.. _#109: https://github.com/mir-evaluation/mir_eval/issues/109 +.. _#122: https://github.com/mir-evaluation/mir_eval/issues/122 +.. _#136: https://github.com/mir-evaluation/mir_eval/issues/136 +.. _#138: https://github.com/mir-evaluation/mir_eval/issues/138 +.. _#139: https://github.com/mir-evaluation/mir_eval/issues/139 +.. _#147: https://github.com/mir-evaluation/mir_eval/issues/147 +.. _#150: https://github.com/mir-evaluation/mir_eval/issues/150 +.. _#151: https://github.com/mir-evaluation/mir_eval/issues/151 +.. _#159: https://github.com/mir-evaluation/mir_eval/issues/159 +.. _#160: https://github.com/mir-evaluation/mir_eval/issues/160 v0.1 diff --git a/docs/conf.py b/docs/conf.py index a6e24e0f..72b7bdd6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # mir_eval documentation build configuration file, created by # sphinx-quickstart on Thu May 8 15:55:45 2014. @@ -14,217 +13,236 @@ import sys import os -sys.path.insert(0, os.path.abspath('..')) + +sys.path.insert(0, os.path.abspath("..")) +from mir_eval import __version__ as release +version = release # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ------------------------------------------------ # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx.ext.imgmath', - 'numpydoc', + "sphinx.ext.autodoc", + "sphinx.ext.imgmath", + "numpydoc", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = u'mir_eval' -copyright = u'2014, Colin Raffel et al.' +project = "mir_eval" +copyright = "2014, Colin Raffel et al." # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the # built documents. # # The short X.Y version. -version = '0.6' +#version = "0.7" # The full version, including alpha/beta/rc tags. -release = '0.6' +#release = "0.7" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all # documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # If true, keep warnings as "system message" paragraphs in the built documents. -#keep_warnings = False +# keep_warnings = False # -- Options for HTML output ---------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = 'default' +#html_theme = "default" +html_theme = "sphinx_rtd_theme" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +html_theme_options = { + "navigation_depth": 4, + "collapse_navigation": False, +} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ["_static"] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied # directly to the root of the documentation. -#html_extra_path = [] +# html_extra_path = [] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} +html_sidebars = { + '**': [ + 'globaltoc.html', # shows the full, global TOC from the master toctree + 'localtoc.html', # also shows the local toctree (optional) + 'relations.html', + 'sourcelink.html', + 'searchbox.html', + ] +} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'mir_evaldoc' +htmlhelp_basename = "mir_evaldoc" # -- Options for LaTeX output --------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', - -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', - -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - ('index', 'mir_eval.tex', u'mir\\_eval Documentation', - u'Colin Raffel et al.', 'manual'), + ( + "index", + "mir_eval.tex", + "mir\\_eval Documentation", + "Colin Raffel et al.", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output --------------------------------------- @@ -232,12 +250,11 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'mir_eval', u'mir_eval Documentation', - [u'Colin Raffel et al.'], 1) + ("index", "mir_eval", "mir_eval Documentation", ["Colin Raffel et al."], 1) ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------- @@ -246,21 +263,32 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'mir_eval', u'mir_eval Documentation', - u'Colin Raffel et al.', 'mir_eval', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "mir_eval", + "mir_eval Documentation", + "Colin Raffel et al.", + "mir_eval", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' # If true, do not generate a @detailmenu in the "Top" node's menu. -#texinfo_no_detailmenu = False +# texinfo_no_detailmenu = False -autodoc_member_order = 'bysource' +autodoc_member_order = "bysource" +linkcheck_allow_unauthorized = True +linkcheck_ignore = [ + # Sphinx has problems with the anchor on this one, but it does work + 'http://www.music-ir.org/mirex/wiki/2015:Multiple_Fundamental_Frequency_Estimation_%26_Tracking_Results_-_MIREX_Dataset#Task_2:Note_Tracking_.28NT.29' +] diff --git a/docs/index.rst b/docs/index.rst index 7a3b7a15..ea9492e3 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,231 +1,98 @@ -************************** -``mir_eval`` Documentation -************************** +.. mir_eval documentation master file -``mir_eval`` is a Python library which provides a transparent, standaridized, and straightforward way to evaluate Music Information Retrieval systems. +mir_eval Documentation +====================== -If you use ``mir_eval`` in a research project, please cite the following paper: +**mir_eval** is a Python library which provides a transparent, standardized, and straightforward way to evaluate Music Information Retrieval systems. -C. Raffel, B. McFee, E. J. Humphrey, J. Salamon, O. Nieto, D. Liang, and D. P. W. Ellis, `"mir_eval: A Transparent Implementation of Common MIR Metrics" `_, Proceedings of the 15th International Conference on Music Information Retrieval, 2014. +If you use **mir_eval** in a research project, please cite the following paper: -.. _installation: + C. Raffel, B. McFee, E. J. Humphrey, J. Salamon, O. Nieto, D. Liang, and D. P. W. Ellis, `"mir_eval: A Transparent Implementation of Common MIR Metrics" `_, Proceedings of the 15th International Conference on Music Information Retrieval, 2014. + +Installation +============ + +The simplest way to install **mir_eval** is via ``pip``: + +.. code-block:: console + + python -m pip install mir_eval + +If you use `conda` packages, **mir_eval** is available on conda-forge: + +.. code-block:: console -Installing ``mir_eval`` -======================= + conda install -c conda-forge mir_eval -The simplest way to install ``mir_eval`` is by using ``pip``, which will also install the required dependencies if needed. -To install ``mir_eval`` using ``pip``, simply run +Alternatively, you can install from source: -``pip install mir_eval`` +.. code-block:: console -Alternatively, you can install ``mir_eval`` from source by first installing the dependencies and then running + python setup.py install -``python setup.py install`` +If you don't use Python and want to get started as quickly as possible, you might consider using `Anaconda `_ which makes it easy to install a Python environment which can run **mir_eval**. -from the source directory. +Using mir_eval +============== -If you don't use Python and want to get started as quickly as possible, you might consider using `Anaconda `_ which makes it easy to install a Python environment which can run ``mir_eval``. +Once installed, you can import **mir_eval** in your code: -Using ``mir_eval`` -============================================= +.. code-block:: python -Once you've installed ``mir_eval`` (see :ref:`installation`), you can import it in your Python code as follows: + import mir_eval -``import mir_eval`` +For example, to evaluate beat tracking: -From here, you will typically either load in data and call the ``evaluate()`` function from the appropriate submodule like so:: +.. code-block:: python - reference_beats = mir_eval.io.load_events('reference_beats.txt') - estimated_beats = mir_eval.io.load_events('estimated_beats.txt') - # Scores will be a dict containing scores for all of the metrics - # implemented in mir_eval.beat. The keys are metric names - # and values are the scores achieved - scores = mir_eval.beat.evaluate(reference_beats, estimated_beats) + reference_beats = mir_eval.io.load_events('reference_beats.txt') + estimated_beats = mir_eval.io.load_events('estimated_beats.txt') + scores = mir_eval.beat.evaluate(reference_beats, estimated_beats) -or you'll load in the data, do some preprocessing, and call specific metric functions from the appropriate submodule like so:: +At the end of execution, ``scores`` will be a dict containing scores +for all of the metrics implemented in `mir_eval.beat`. +The keys are metric names and values are the scores achieved. - reference_beats = mir_eval.io.load_events('reference_beats.txt') - estimated_beats = mir_eval.io.load_events('estimated_beats.txt') - # Crop out beats before 5s, a common preprocessing step - reference_beats = mir_eval.beat.trim_beats(reference_beats) - estimated_beats = mir_eval.beat.trim_beats(estimated_beats) - # Compute the F-measure metric and store it in f_measure - f_measure = mir_eval.beat.f_measure(reference_beats, estimated_beats) +You can also load in the data, do some preprocessing, and call specific metric functions from the appropriate submodule like so: -The documentation for each metric function, found in the :ref:`mir_eval` section below, contains further usage information. +.. code-block:: python + + reference_beats = mir_eval.io.load_events('reference_beats.txt') + estimated_beats = mir_eval.io.load_events('estimated_beats.txt') + # Crop out beats before 5s, a common preprocessing step + reference_beats = mir_eval.beat.trim_beats(reference_beats) + estimated_beats = mir_eval.beat.trim_beats(estimated_beats) + # Compute the F-measure metric and store it in f_measure + f_measure = mir_eval.beat.f_measure(reference_beats, estimated_beats) Alternatively, you can use the evaluator scripts which allow you to run evaluation from the command line, without writing any code. These scripts are are available here: https://github.com/craffel/mir_evaluators -.. _mir_eval: -``mir_eval`` -============ +API Reference +============= -The structure of the ``mir_eval`` Python module is as follows: -Each MIR task for which evaluation metrics are included in ``mir_eval`` is given its own submodule, and each metric is defined as a separate function in each submodule. +The structure of the **mir_eval** Python module is as follows: +Each MIR task for which evaluation metrics are included in **mir_eval** is given its own submodule, and each metric is defined as a separate function in each submodule. Every metric function includes detailed documentation, example usage, input validation, and references to the original paper which defined the metric (see the subsections below). The task submodules also all contain a function ``evaluate()``, which takes as input reference and estimated annotations and returns a dictionary of scores for all of the metrics implemented (for casual users, this is the place to start). Finally, each task submodule also includes functions for common data pre-processing steps. -``mir_eval`` also includes the following additional submodules: +**mir_eval** also includes the following additional submodules: * :mod:`mir_eval.io` which contains convenience functions for loading in task-specific data from common file formats * :mod:`mir_eval.util` which includes miscellaneous functionality shared across the submodules * :mod:`mir_eval.sonify` which implements some simple methods for synthesizing annotations of various formats for "evaluation by ear". * :mod:`mir_eval.display` which provides functions for plotting annotations for various tasks. -The following subsections document each submodule. - -:mod:`mir_eval.beat` --------------------- -.. automodule:: mir_eval.beat - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.chord` ---------------------- -.. automodule:: mir_eval.chord - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.melody` ----------------------- -.. automodule:: mir_eval.melody - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.multipitch` --------------------------- -.. automodule:: mir_eval.multipitch - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.onset` ---------------------- -.. automodule:: mir_eval.onset - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.pattern` ------------------------ -.. automodule:: mir_eval.pattern - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.segment` ------------------------ -.. automodule:: mir_eval.segment - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.hierarchy` -------------------------- -.. automodule:: mir_eval.hierarchy - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.separation` --------------------------- -.. automodule:: mir_eval.separation - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.tempo` --------------------------- -.. automodule:: mir_eval.tempo - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.transcription` ------------------------------ -.. automodule:: mir_eval.transcription - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.transcription_velocity` --------------------------------------- -.. automodule:: mir_eval.transcription_velocity - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.key` ------------------------------ -.. automodule:: mir_eval.key - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.util` --------------------- -.. automodule:: mir_eval.util - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.io` ------------------- -.. automodule:: mir_eval.io - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.sonify` ----------------------- -.. automodule:: mir_eval.sonify - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -:mod:`mir_eval.display` ------------------------ -.. automodule:: mir_eval.display - :members: - :undoc-members: - :show-inheritance: - :member-order: bysource - -Changes -======= +Detailed API documentation for each submodule is available in the API Reference section. +See the :doc:`API Reference ` for full details. + .. toctree:: - :maxdepth: 1 + :caption: mir_eval + :maxdepth: 2 + api/index changes - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` - diff --git a/mir_eval/__init__.py b/mir_eval/__init__.py index 2b3a9eb2..fa5cdfae 100644 --- a/mir_eval/__init__.py +++ b/mir_eval/__init__.py @@ -2,6 +2,7 @@ """Top-level module for mir_eval""" # Import all submodules (for each task) +from . import alignment from . import beat from . import chord from . import io @@ -19,4 +20,4 @@ from . import transcription_velocity from . import key -__version__ = '0.6' +__version__ = "0.8.2" diff --git a/mir_eval/alignment.py b/mir_eval/alignment.py new file mode 100644 index 00000000..6c1b7687 --- /dev/null +++ b/mir_eval/alignment.py @@ -0,0 +1,357 @@ +""" +Alignment models are given a sequence of events along with a piece of audio, and then return a +sequence of timestamps, with one timestamp for each event, indicating the position of this event +in the audio. The events are listed in order of occurrence in the audio, so that output +timestamps have to be monotonically increasing. +Evaluation usually involves taking the series of predicted and ground truth timestamps and +comparing their distance, usually on a pair-wise basis, e.g. taking the median absolute error in +seconds. + +Conventions +----------- +Timestamps should be provided in the form of a 1-dimensional array of onset +times in seconds in increasing order. + +Metrics +------- +* :func:`mir_eval.alignment.absolute_error`: Median absolute error and average absolute error +* :func:`mir_eval.alignment.percentage_correct`: Percentage of correct timestamps, + where a timestamp is counted + as correct if it lies within a certain tolerance window around the ground truth timestamp +* :func:`mir_eval.alignment.pcs`: Percentage of correct segments: Percentage of overlap between + predicted segments and ground truth segments, where segments are defined by (start time, + end time) pairs +* :func:`mir_eval.alignment.perceptual_metric`: metric based on human synchronicity perception as + measured in the paper "User-centered evaluation of lyrics to audio alignment", + N. Lizé-Masclef, A. Vaglio, M. Moussallam, ISMIR 2021 + +References +---------- + .. [#lizemasclef2021] N. Lizé-Masclef, A. Vaglio, M. Moussallam. + "User-centered evaluation of lyrics to audio alignment", + International Society for Music Information Retrieval (ISMIR) conference, + 2021. + + .. [#mauch2010] M. Mauch, F: Hiromasa, M. Goto. + "Lyrics-to-audio alignment and phrase-level segmentation using + incomplete internet-style chord annotations", + Frontiers in Proceedings of the Sound Music Computing Conference (SMC), 2010. + + .. [#dzhambazov2017] G. Dzhambazov. + "Knowledge-Based Probabilistic Modeling For Tracking Lyrics In Music Audio Signals", + PhD Thesis, 2017. + + .. [#fujihara2011] H. Fujihara, M. Goto, J. Ogata, H. Okuno. + "LyricSynchronizer: Automatic synchronization system between musical audio signals and lyrics", + IEEE Journal of Selected Topics in Signal Processing, VOL. 5, NO. 6, 2011 + +""" + +import collections +from typing import Optional + +import numpy as np +from scipy.stats import skewnorm + +from mir_eval.util import filter_kwargs + + +def validate(reference_timestamps: np.ndarray, estimated_timestamps: np.ndarray): + """Check that the input annotations to a metric look like valid onset time + arrays, and throws helpful errors if not. + + Parameters + ---------- + reference_timestamps : np.ndarray + reference timestamp locations, in seconds + estimated_timestamps : np.ndarray + estimated timestamp locations, in seconds + """ + # We need to have 1D numpy arrays + if not isinstance(reference_timestamps, np.ndarray): + raise ValueError( + "Reference timestamps need to be a numpy array, but got" + f" {type(reference_timestamps)}" + ) + if not isinstance(estimated_timestamps, np.ndarray): + raise ValueError( + "Estimated timestamps need to be a numpy array, but got" + f" {type(estimated_timestamps)}" + ) + if reference_timestamps.ndim != 1: + raise ValueError( + "Reference timestamps need to be a one-dimensional vector, but got" + f" {reference_timestamps.ndim} dimensions" + ) + if estimated_timestamps.ndim != 1: + raise ValueError( + "Estimated timestamps need to be a one-dimensional vector, but got" + f" {estimated_timestamps.ndim} dimensions" + ) + + # If reference or estimated timestamps are empty, cannot compute metric + if reference_timestamps.size == 0: + raise ValueError("Reference timestamps are empty.") + if estimated_timestamps.size != reference_timestamps.size: + raise ValueError( + "Number of timestamps must be the same in prediction and ground" + f" truth, but found {estimated_timestamps.size} in prediction and" + f" {reference_timestamps.size} in ground truth" + ) + + # Check monotonicity + if not np.all(reference_timestamps[1:] - reference_timestamps[:-1] >= 0): + raise ValueError("Reference timestamps are not monotonically increasing!") + if not np.all(estimated_timestamps[1:] - estimated_timestamps[:-1] >= 0): + raise ValueError("Estimated timestamps are not monotonically increasing!") + + # Check positivity (need for correct PCS metric calculation) + if not np.all(reference_timestamps >= 0): + raise ValueError("Reference timestamps can not be below 0!") + if not np.all(estimated_timestamps >= 0): + raise ValueError("Estimated timestamps can not be below 0!") + + +def absolute_error(reference_timestamps, estimated_timestamps): + """Compute the absolute deviations between estimated and reference timestamps, + and then returns the median and average over all events + + Examples + -------- + >>> reference_timestamps = mir_eval.io.load_events('reference.txt') + >>> estimated_timestamps = mir_eval.io.load_events('estimated.txt') + >>> mae, aae = mir_eval.align.absolute_error(reference_onsets, estimated_timestamps) + + Parameters + ---------- + reference_timestamps : np.ndarray + reference timestamps, in seconds + estimated_timestamps : np.ndarray + estimated timestamps, in seconds + + Returns + ------- + mae : float + Median absolute error + aae: float + Average absolute error + """ + validate(reference_timestamps, estimated_timestamps) + deviations = np.abs(reference_timestamps - estimated_timestamps) + return np.median(deviations), np.mean(deviations) + + +def percentage_correct(reference_timestamps, estimated_timestamps, window=0.3): + """Compute the percentage of correctly predicted timestamps. A timestamp is predicted + correctly if its position doesn't deviate more than the window parameter from the ground + truth timestamp. + + Examples + -------- + >>> reference_timestamps = mir_eval.io.load_events('reference.txt') + >>> estimated_timestamps = mir_eval.io.load_events('estimated.txt') + >>> pc = mir_eval.align.percentage_correct(reference_onsets, estimated_timestamps, window=0.2) + + Parameters + ---------- + reference_timestamps : np.ndarray + reference timestamps, in seconds + estimated_timestamps : np.ndarray + estimated timestamps, in seconds + window : float + Window size, in seconds + (Default value = .3) + + Returns + ------- + pc : float + Percentage of correct timestamps + """ + validate(reference_timestamps, estimated_timestamps) + deviations = np.abs(reference_timestamps - estimated_timestamps) + return np.mean(deviations <= window) + + +def percentage_correct_segments( + reference_timestamps, estimated_timestamps, duration: Optional[float] = None +): + """Calculate the percentage of correct segments (PCS) metric. + + It constructs segments out of predicted and estimated timestamps separately + out of each given timestamp vector and calculates the percentage of overlap between correct + segments compared to the total duration. + + WARNING: This metrics behaves differently depending on whether "duration" is given! + + If duration is not given (default case), the computation follows the MIREX lyrics alignment + challenge 2020. For a timestamp vector with entries (t1,t2, ... tN), segments with + the following (start, end) boundaries are created: (t1, t2), ... (tN-1, tN). + After the segments are created, the overlap between the reference and estimated segments is + determined and divided by the total duration, which is the distance between the + first and last timestamp in the reference. + + If duration is given, the segment boundaries are instead (0, t1), (t1, t2), ... (tN, duration). + The overlap is computed in the same way, but then divided by the duration parameter given to + this function. + This method follows the original paper [#fujihara2011] more closely, where the metric was + proposed. + As a result, this variant of the metrics punishes cases where the first estimated timestamp + is too early or the last estimated timestamp is too late, whereas the MIREX variant does not. + On the other hand, the MIREX metric is invariant to how long the eventless beginning and end + parts of the audio are, which might be a desirable property. + + Examples + -------- + >>> reference_timestamps = mir_eval.io.load_events('reference.txt') + >>> estimated_timestamps = mir_eval.io.load_events('estimated.txt') + >>> pcs = mir_eval.align.percentage_correct_segments(reference_timestamps, estimated_timestamps) + + Parameters + ---------- + reference_timestamps : np.ndarray + reference timestamps, in seconds + estimated_timestamps : np.ndarray + estimated timestamps, in seconds + duration : float + Optional. Total duration of audio (seconds). WARNING: Metric is computed differently + depending on whether this is provided or not - see documentation above! + + Returns + ------- + pcs : float + Percentage of time where ground truth and predicted segments overlap + """ + validate(reference_timestamps, estimated_timestamps) + if duration is not None: + duration = float(duration) + if duration <= 0: + raise ValueError( + f"Positive duration needs to be provided, but got {duration}" + ) + if np.max(reference_timestamps) > duration: + raise ValueError( + "Expected largest reference timestamp" + f"{np.max(reference_timestamps)} to not be " + f"larger than duration {duration}" + ) + if np.max(estimated_timestamps) > duration: + raise ValueError( + "Expected largest estimated timestamp " + f"{np.max(estimated_timestamps)} to not be " + f"larger than duration {duration}" + ) + + ref_starts = np.concatenate([[0], reference_timestamps]) + ref_ends = np.concatenate([reference_timestamps, [duration]]) + est_starts = np.concatenate([[0], estimated_timestamps]) + est_ends = np.concatenate([estimated_timestamps, [duration]]) + else: + # MIREX lyrics alignment 2020 style: + # Ignore regions before start and after end reference timestamp + duration = reference_timestamps[-1] - reference_timestamps[0] + if duration <= 0: + raise ValueError( + f"Reference timestamps are all identical, can not compute PCS" + f" metric!" + ) + + ref_starts = reference_timestamps[:-1] + ref_ends = reference_timestamps[1:] + est_starts = estimated_timestamps[:-1] + est_ends = estimated_timestamps[1:] + + overlap_starts = np.maximum(ref_starts, est_starts) + overlap_ends = np.minimum(ref_ends, est_ends) + overlap_duration = np.sum(np.maximum(overlap_ends - overlap_starts, 0)) + return overlap_duration / duration + + +def karaoke_perceptual_metric(reference_timestamps, estimated_timestamps): + """Metric based on human synchronicity perception as measured in the paper + "User-centered evaluation of lyrics to audio alignment" [#lizemasclef2021] + + The parameters of this function were tuned on data collected through a user Karaoke-like + experiment + It reflects human judgment of how "synchronous" lyrics and audio stimuli are perceived + in that setup. + Beware that this metric is non-symmetrical and by construction it is also not equal to 1 at 0. + + Examples + -------- + >>> reference_timestamps = mir_eval.io.load_events('reference.txt') + >>> estimated_timestamps = mir_eval.io.load_events('estimated.txt') + >>> score = mir_eval.align.karaoke_perceptual_metric(reference_onsets, estimated_timestamps) + + Parameters + ---------- + reference_timestamps : np.ndarray + reference timestamps, in seconds + estimated_timestamps : np.ndarray + estimated timestamps, in seconds + + Returns + ------- + perceptual_score : float + Perceptual score, averaged over all timestamps + """ + validate(reference_timestamps, estimated_timestamps) + offsets = estimated_timestamps - reference_timestamps + + # Score offsets using a certain skewed normal distribution + skewness = 1.12244251 + localisation = -0.22270315 + scale = 0.29779424 + normalisation_factor = 1.6857 + perceptual_scores = (1.0 / normalisation_factor) * skewnorm.pdf( + offsets, skewness, loc=localisation, scale=scale + ) + + return np.mean(perceptual_scores) + + +def evaluate(reference_timestamps, estimated_timestamps, **kwargs): + """Compute all metrics for the given reference and estimated annotations. + + Examples + -------- + >>> reference_timestamps = mir_eval.io.load_events('reference.txt') + >>> estimated_timestamps = mir_eval.io.load_events('estimated.txt') + >>> duration = max(np.max(reference_timestamps), np.max(estimated_timestamps)) + 10 + >>> scores = mir_eval.align.evaluate(reference_onsets, estimated_timestamps, duration) + + Parameters + ---------- + reference_timestamps : np.ndarray + reference timestamp locations, in seconds + estimated_timestamps : np.ndarray + estimated timestamp locations, in seconds + **kwargs + Additional keyword arguments which will be passed to the + appropriate metric or preprocessing functions. + + Returns + ------- + scores : dict + Dictionary of scores, where the key is the metric name (str) and + the value is the (float) score achieved. + """ + # Compute all metrics + scores = collections.OrderedDict() + + scores["pc"] = filter_kwargs( + percentage_correct, reference_timestamps, estimated_timestamps, **kwargs + ) + scores["mae"], scores["aae"] = absolute_error( + reference_timestamps, estimated_timestamps + ) + scores["pcs"] = filter_kwargs( + percentage_correct_segments, + reference_timestamps, + estimated_timestamps, + **kwargs, + ) + scores["perceptual"] = karaoke_perceptual_metric( + reference_timestamps, estimated_timestamps + ) + + return scores diff --git a/mir_eval/beat.py b/mir_eval/beat.py index ec890f76..21211a07 100644 --- a/mir_eval/beat.py +++ b/mir_eval/beat.py @@ -1,4 +1,4 @@ -''' +r""" The aim of a beat detection algorithm is to report the times at which a typical human listener might tap their foot to a piece of music. As a result, most metrics for evaluating the performance of beat tracking systems involve @@ -42,7 +42,7 @@ * :func:`mir_eval.beat.information_gain`: The Information Gain of a normalized beat error histogram over a uniform distribution -''' +""" import numpy as np import collections @@ -51,11 +51,11 @@ # The maximum allowable beat time -MAX_TIME = 30000. +MAX_TIME = 30000.0 -def trim_beats(beats, min_beat_time=5.): - """Removes beats before min_beat_time. A common preprocessing step. +def trim_beats(beats, min_beat_time=5.0): + """Remove beats before min_beat_time. A common preprocessing step. Parameters ---------- @@ -75,7 +75,7 @@ def trim_beats(beats, min_beat_time=5.): def validate(reference_beats, estimated_beats): - """Checks that the input annotations to a metric look like valid beat time + """Check that the input annotations to a metric look like valid beat time arrays, and throws helpful errors if not. Parameters @@ -115,27 +115,25 @@ def _get_reference_beat_variations(reference_beats): Half tempo, odd beats half_even : np.ndarray Half tempo, even beats - """ - # Create annotations at twice the metric level - interpolated_indices = np.arange(0, reference_beats.shape[0]-.5, .5) + interpolated_indices = np.arange(0, reference_beats.shape[0] - 0.5, 0.5) original_indices = np.arange(0, reference_beats.shape[0]) - double_reference_beats = np.interp(interpolated_indices, - original_indices, - reference_beats) + double_reference_beats = np.interp( + interpolated_indices, original_indices, reference_beats + ) # Return metric variations: # True, off-beat, double tempo, half tempo odd, and half tempo even - return (reference_beats, - double_reference_beats[1::2], - double_reference_beats, - reference_beats[::2], - reference_beats[1::2]) + return ( + reference_beats, + double_reference_beats[1::2], + double_reference_beats, + reference_beats[::2], + reference_beats[1::2], + ) -def f_measure(reference_beats, - estimated_beats, - f_measure_threshold=0.07): +def f_measure(reference_beats, estimated_beats, f_measure_threshold=0.07): """Compute the F-measure of correct vs incorrectly predicted beats. "Correctness" is determined over a small window. @@ -167,20 +165,16 @@ def f_measure(reference_beats, validate(reference_beats, estimated_beats) # When estimated beats are empty, no beats are correct; metric is 0 if estimated_beats.size == 0 or reference_beats.size == 0: - return 0. + return 0.0 # Compute the best-case matching between reference and estimated locations - matching = util.match_events(reference_beats, - estimated_beats, - f_measure_threshold) + matching = util.match_events(reference_beats, estimated_beats, f_measure_threshold) - precision = float(len(matching))/len(estimated_beats) - recall = float(len(matching))/len(reference_beats) + precision = float(len(matching)) / len(estimated_beats) + recall = float(len(matching)) / len(reference_beats) return util.f_measure(precision, recall) -def cemgil(reference_beats, - estimated_beats, - cemgil_sigma=0.04): +def cemgil(reference_beats, estimated_beats, cemgil_sigma=0.04): """Cemgil's score, computes a gaussian error of each estimated beat. Compares against the original beat times and all metrical variations. @@ -213,7 +207,7 @@ def cemgil(reference_beats, validate(reference_beats, estimated_beats) # When estimated beats are empty, no beats are correct; metric is 0 if estimated_beats.size == 0 or reference_beats.size == 0: - return 0., 0. + return 0.0, 0.0 # We'll compute Cemgil's accuracy for each variation accuracies = [] for reference_beats in _get_reference_beat_variations(reference_beats): @@ -223,9 +217,9 @@ def cemgil(reference_beats, # Find the error for the closest beat to the reference beat beat_diff = np.min(np.abs(beat - estimated_beats)) # Add gaussian error into the accuracy - accuracy += np.exp(-(beat_diff**2)/(2.0*cemgil_sigma**2)) + accuracy += np.exp(-(beat_diff**2) / (2.0 * cemgil_sigma**2)) # Normalize the accuracy - accuracy /= .5*(estimated_beats.shape[0] + reference_beats.shape[0]) + accuracy /= 0.5 * (estimated_beats.shape[0] + reference_beats.shape[0]) # Add it to our list of accuracy scores accuracies.append(accuracy) # Return raw accuracy with non-varied annotations @@ -233,11 +227,9 @@ def cemgil(reference_beats, return accuracies[0], np.max(accuracies) -def goto(reference_beats, - estimated_beats, - goto_threshold=0.35, - goto_mu=0.2, - goto_sigma=0.2): +def goto( + reference_beats, estimated_beats, goto_threshold=0.35, goto_mu=0.2, goto_sigma=0.2 +): """Calculate Goto's score, a binary 1 or 0 depending on some specific heuristic criteria @@ -275,25 +267,26 @@ def goto(reference_beats, validate(reference_beats, estimated_beats) # When estimated beats are empty, no beats are correct; metric is 0 if estimated_beats.size == 0 or reference_beats.size == 0: - return 0. + return 0.0 # Error for each beat beat_error = np.ones(reference_beats.shape[0]) # Flag for whether the reference and estimated beats are paired paired = np.zeros(reference_beats.shape[0]) # Keep track of Goto's three criteria goto_criteria = 0 - for n in range(1, reference_beats.shape[0]-1): + for n in range(1, reference_beats.shape[0] - 1): # Get previous inner-reference-beat-interval - previous_interval = 0.5*(reference_beats[n] - reference_beats[n-1]) + previous_interval = 0.5 * (reference_beats[n] - reference_beats[n - 1]) # Window start - in the middle of the current beat and the previous window_min = reference_beats[n] - previous_interval # Next inter-reference-beat-interval - next_interval = 0.5*(reference_beats[n+1] - reference_beats[n]) + next_interval = 0.5 * (reference_beats[n + 1] - reference_beats[n]) # Window end - in the middle of the current beat and the next window_max = reference_beats[n] + next_interval # Get estimated beats in the window - beats_in_window = np.logical_and((estimated_beats >= window_min), - (estimated_beats < window_max)) + beats_in_window = np.logical_and( + (estimated_beats >= window_min), (estimated_beats < window_max) + ) # False negative/positive if beats_in_window.sum() == 0 or beats_in_window.sum() > 1: paired[n] = 0 @@ -305,39 +298,36 @@ def goto(reference_beats, offset = estimated_beats[beats_in_window] - reference_beats[n] # Scale by previous or next interval if offset < 0: - beat_error[n] = offset/previous_interval + beat_error[n] = offset[0] / previous_interval else: - beat_error[n] = offset/next_interval + beat_error[n] = offset[0] / next_interval # Get indices of incorrect beats incorrect_beats = np.flatnonzero(np.abs(beat_error) > goto_threshold) # All beats are correct (first and last will be 0 so always correct) if incorrect_beats.shape[0] < 3: # Get the track of correct beats - track = beat_error[incorrect_beats[0] + 1:incorrect_beats[-1] - 1] + track = beat_error[incorrect_beats[0] + 1 : incorrect_beats[-1] - 1] goto_criteria = 1 else: # Get the track of maximal length track_len = np.max(np.diff(incorrect_beats)) track_start = np.flatnonzero(np.diff(incorrect_beats) == track_len)[0] # Is the track length at least 25% of the song? - if track_len - 1 > .25*(reference_beats.shape[0] - 2): + if track_len - 1 > 0.25 * (reference_beats.shape[0] - 2): goto_criteria = 1 start_beat = incorrect_beats[track_start] end_beat = incorrect_beats[track_start + 1] - track = beat_error[start_beat:end_beat + 1] + track = beat_error[start_beat : end_beat + 1] # If we have a track if goto_criteria: # Are mean and std of the track less than the required thresholds? - if np.mean(np.abs(track)) < goto_mu \ - and np.std(track, ddof=1) < goto_sigma: + if np.mean(np.abs(track)) < goto_mu and np.std(track, ddof=1) < goto_sigma: goto_criteria = 3 # If all criteria are met, score is 100%! - return 1.0*(goto_criteria == 3) + return 1.0 * (goto_criteria == 3) -def p_score(reference_beats, - estimated_beats, - p_score_threshold=0.2): +def p_score(reference_beats, estimated_beats, p_score_threshold=0.2): """Get McKinney's P-score. Based on the autocorrelation of the reference and estimated beats @@ -370,52 +360,59 @@ def p_score(reference_beats, # Warn when only one beat is provided for either estimated or reference, # report a warning if reference_beats.size == 1: - warnings.warn("Only one reference beat was provided, so beat intervals" - " cannot be computed.") + warnings.warn( + "Only one reference beat was provided, so beat intervals" + " cannot be computed." + ) if estimated_beats.size == 1: - warnings.warn("Only one estimated beat was provided, so beat intervals" - " cannot be computed.") + warnings.warn( + "Only one estimated beat was provided, so beat intervals" + " cannot be computed." + ) # When estimated or reference beats have <= 1 beats, can't compute the # metric, so return 0 if estimated_beats.size <= 1 or reference_beats.size <= 1: - return 0. + return 0.0 # Quantize beats to 10ms - sampling_rate = int(1.0/0.010) + sampling_rate = int(1.0 / 0.010) # Shift beats so that the minimum in either sequence is zero offset = min(estimated_beats.min(), reference_beats.min()) estimated_beats = np.array(estimated_beats - offset) reference_beats = np.array(reference_beats - offset) # Get the largest time index - end_point = np.int(np.ceil(np.max([np.max(estimated_beats), - np.max(reference_beats)]))) + end_point = np.int64( + np.ceil(np.max([np.max(estimated_beats), np.max(reference_beats)])) + ) # Make impulse trains with impulses at beat locations - reference_train = np.zeros(end_point*sampling_rate + 1) - beat_indices = np.ceil(reference_beats*sampling_rate).astype(np.int) + reference_train = np.zeros(end_point * sampling_rate + 1) + beat_indices = np.ceil(reference_beats * sampling_rate).astype(np.int64) reference_train[beat_indices] = 1.0 - estimated_train = np.zeros(end_point*sampling_rate + 1) - beat_indices = np.ceil(estimated_beats*sampling_rate).astype(np.int) + estimated_train = np.zeros(end_point * sampling_rate + 1) + beat_indices = np.ceil(estimated_beats * sampling_rate).astype(np.int64) estimated_train[beat_indices] = 1.0 # Window size to take the correlation over # defined as .2*median(inter-annotation-intervals) annotation_intervals = np.diff(np.flatnonzero(reference_train)) - win_size = int(np.round(p_score_threshold*np.median(annotation_intervals))) + win_size = int(np.round(p_score_threshold * np.median(annotation_intervals))) # Get full correlation - train_correlation = np.correlate(reference_train, estimated_train, 'full') + train_correlation = np.correlate(reference_train, estimated_train, "full") # Get the middle element - note we are rounding down on purpose here - middle_lag = train_correlation.shape[0]//2 + middle_lag = train_correlation.shape[0] // 2 # Truncate to only valid lags (those corresponding to the window) start = middle_lag - win_size end = middle_lag + win_size + 1 train_correlation = train_correlation[start:end] # Compute and return the P-score n_beats = np.max([estimated_beats.shape[0], reference_beats.shape[0]]) - return np.sum(train_correlation)/n_beats + return np.sum(train_correlation) / n_beats -def continuity(reference_beats, - estimated_beats, - continuity_phase_threshold=0.175, - continuity_period_threshold=0.175): +def continuity( + reference_beats, + estimated_beats, + continuity_phase_threshold=0.175, + continuity_period_threshold=0.175, +): """Get metrics based on how much of the estimated beat sequence is continually correct. @@ -458,23 +455,26 @@ def continuity(reference_beats, # Warn when only one beat is provided for either estimated or reference, # report a warning if reference_beats.size == 1: - warnings.warn("Only one reference beat was provided, so beat intervals" - " cannot be computed.") + warnings.warn( + "Only one reference beat was provided, so beat intervals" + " cannot be computed." + ) if estimated_beats.size == 1: - warnings.warn("Only one estimated beat was provided, so beat intervals" - " cannot be computed.") + warnings.warn( + "Only one estimated beat was provided, so beat intervals" + " cannot be computed." + ) # When estimated or reference beats have <= 1 beats, can't compute the # metric, so return 0 if estimated_beats.size <= 1 or reference_beats.size <= 1: - return 0., 0., 0., 0. + return 0.0, 0.0, 0.0, 0.0 # Accuracies for each variation continuous_accuracies = [] total_accuracies = [] # Get accuracy for each variation for reference_beats in _get_reference_beat_variations(reference_beats): # Annotations that have been used - n_annotations = np.max([reference_beats.shape[0], - estimated_beats.shape[0]]) + n_annotations = np.max([reference_beats.shape[0], estimated_beats.shape[0]]) used_annotations = np.zeros(n_annotations) # Whether or not we are continuous at any given point beat_successes = np.zeros(n_annotations) @@ -494,13 +494,15 @@ def continuity(reference_beats, # How far is the estimated beat from the reference beat, # relative to the inter-annotation-interval? if nearest + 1 < reference_beats.shape[0]: - reference_interval = (reference_beats[nearest + 1] - - reference_beats[nearest]) + reference_interval = ( + reference_beats[nearest + 1] - reference_beats[nearest] + ) else: # Special case when nearest + 1 is too large - use the # previous interval instead - reference_interval = (reference_beats[nearest] - - reference_beats[nearest - 1]) + reference_interval = ( + reference_beats[nearest] - reference_beats[nearest - 1] + ) # Handle this special case when beats are not unique if reference_interval == 0: if min_difference == 0: @@ -508,17 +510,15 @@ def continuity(reference_beats, else: phase = np.inf else: - phase = np.abs(min_difference/reference_interval) + phase = np.abs(min_difference / reference_interval) # How close is the inter-beat-interval # to the inter-annotation-interval? if m + 1 < estimated_beats.shape[0]: - estimated_interval = (estimated_beats[m + 1] - - estimated_beats[m]) + estimated_interval = estimated_beats[m + 1] - estimated_beats[m] else: # Special case when m + 1 is too large - use the # previous interval - estimated_interval = (estimated_beats[m] - - estimated_beats[m - 1]) + estimated_interval = estimated_beats[m] - estimated_beats[m - 1] # Handle this special case when beats are not unique if reference_interval == 0: if estimated_interval == 0: @@ -526,10 +526,11 @@ def continuity(reference_beats, else: period = np.inf else: - period = \ - np.abs(1 - estimated_interval/reference_interval) - if phase < continuity_phase_threshold and \ - period < continuity_period_threshold: + period = np.abs(1 - estimated_interval / reference_interval) + if ( + phase < continuity_phase_threshold + and period < continuity_period_threshold + ): # Set this annotation as used used_annotations[nearest] = 1 # This beat is matched @@ -538,18 +539,21 @@ def continuity(reference_beats, else: # How far is the estimated beat from the reference beat, # relative to the inter-annotation-interval? - reference_interval = (reference_beats[nearest] - - reference_beats[nearest - 1]) - phase = np.abs(min_difference/reference_interval) + reference_interval = ( + reference_beats[nearest] - reference_beats[nearest - 1] + ) + phase = np.abs(min_difference / reference_interval) # How close is the inter-beat-interval # to the inter-annotation-interval? - estimated_interval = (estimated_beats[m] - - estimated_beats[m - 1]) - reference_interval = (reference_beats[nearest] - - reference_beats[nearest - 1]) - period = np.abs(1 - estimated_interval/reference_interval) - if phase < continuity_phase_threshold and \ - period < continuity_period_threshold: + estimated_interval = estimated_beats[m] - estimated_beats[m - 1] + reference_interval = ( + reference_beats[nearest] - reference_beats[nearest - 1] + ) + period = np.abs(1 - estimated_interval / reference_interval) + if ( + phase < continuity_phase_threshold + and period < continuity_period_threshold + ): # Set this annotation as used used_annotations[nearest] = 1 # This beat is matched @@ -565,21 +569,21 @@ def continuity(reference_beats, beat_successes = beat_successes[1:-1] # Get the continuous accuracy as the longest track of successful beats longest_track = np.max(np.diff(beat_failures)) - 1 - continuous_accuracy = longest_track/(1.0*beat_successes.shape[0]) + continuous_accuracy = longest_track / (1.0 * beat_successes.shape[0]) continuous_accuracies.append(continuous_accuracy) # Get the total accuracy - all sequences - total_accuracy = np.sum(beat_successes)/(1.0*beat_successes.shape[0]) + total_accuracy = np.sum(beat_successes) / (1.0 * beat_successes.shape[0]) total_accuracies.append(total_accuracy) # Grab accuracy scores - return (continuous_accuracies[0], - total_accuracies[0], - np.max(continuous_accuracies), - np.max(total_accuracies)) + return ( + continuous_accuracies[0], + total_accuracies[0], + np.max(continuous_accuracies), + np.max(total_accuracies), + ) -def information_gain(reference_beats, - estimated_beats, - bins=41): +def information_gain(reference_beats, estimated_beats, bins=41): """Get the information gain - K-L divergence of the beat error histogram to a uniform histogram @@ -611,20 +615,25 @@ def information_gain(reference_beats, # If an even number of bins is provided, # there will be no bin centered at zero, so warn the user. if not bins % 2: - warnings.warn("bins parameter is even, " - "so there will not be a bin centered at zero.") + warnings.warn( + "bins parameter is even, " "so there will not be a bin centered at zero." + ) # Warn when only one beat is provided for either estimated or reference, # report a warning if reference_beats.size == 1: - warnings.warn("Only one reference beat was provided, so beat intervals" - " cannot be computed.") + warnings.warn( + "Only one reference beat was provided, so beat intervals" + " cannot be computed." + ) if estimated_beats.size == 1: - warnings.warn("Only one estimated beat was provided, so beat intervals" - " cannot be computed.") + warnings.warn( + "Only one estimated beat was provided, so beat intervals" + " cannot be computed." + ) # When estimated or reference beats have <= 1 beats, can't compute the # metric, so return 0 if estimated_beats.size <= 1 or reference_beats.size <= 1: - return 0. + return 0.0 # Get entropy for reference beats->estimated beats # and estimated beats->reference beats forward_entropy = _get_entropy(reference_beats, estimated_beats, bins) @@ -633,15 +642,18 @@ def information_gain(reference_beats, norm = np.log2(bins) if forward_entropy > backward_entropy: # Note that the beat evaluation toolbox does not normalize - information_gain_score = (norm - forward_entropy)/norm + information_gain_score = (norm - forward_entropy) / norm else: - information_gain_score = (norm - backward_entropy)/norm + information_gain_score = (norm - backward_entropy) / norm return information_gain_score def _get_entropy(reference_beats, estimated_beats, bins): - """Helper function for information gain - (needs to be run twice - once backwards, once forwards) + """Compute the entropy of the beat error histogram. + + This is a helper function for the information gain + metric, and needs to be run twice: once backwards, once + forwards. Parameters ---------- @@ -656,7 +668,6 @@ def _get_entropy(reference_beats, estimated_beats, bins): ------- entropy : float Entropy of beat error histogram - """ beat_error = np.zeros(estimated_beats.shape[0]) for n in range(estimated_beats.shape[0]): @@ -667,34 +678,34 @@ def _get_entropy(reference_beats, estimated_beats, bins): # If the first annotation is closest... if closest_beat == 0: # Inter-annotation interval - space between first two beats - interval = .5*(reference_beats[1] - reference_beats[0]) + interval = 0.5 * (reference_beats[1] - reference_beats[0]) # If last annotation is closest... if closest_beat == (reference_beats.shape[0] - 1): - interval = .5*(reference_beats[-1] - reference_beats[-2]) + interval = 0.5 * (reference_beats[-1] - reference_beats[-2]) else: if absolute_error < 0: # Closest annotation is the one before the current beat # so look at previous inner-annotation-interval start = reference_beats[closest_beat] end = reference_beats[closest_beat - 1] - interval = .5*(start - end) + interval = 0.5 * (start - end) else: # Closest annotation is the one after the current beat # so look at next inner-annotation-interval start = reference_beats[closest_beat + 1] end = reference_beats[closest_beat] - interval = .5*(start - end) + interval = 0.5 * (start - end) # The actual error of this beat - beat_error[n] = .5*absolute_error/interval + beat_error[n] = 0.5 * absolute_error / interval # Put beat errors in range (-.5, .5) - beat_error = np.mod(beat_error + .5, -1) + .5 + beat_error = np.mod(beat_error + 0.5, -1) + 0.5 # Note these are slightly different the beat evaluation toolbox # (they are uniform) - histogram_bin_edges = np.linspace(-.5, .5, bins + 1) + histogram_bin_edges = np.linspace(-0.5, 0.5, bins + 1) # Get the histogram raw_bin_values = np.histogram(beat_error, histogram_bin_edges)[0] # Turn into a proper probability distribution - raw_bin_values = raw_bin_values/(1.0*np.sum(raw_bin_values)) + raw_bin_values = raw_bin_values / (1.0 * np.sum(raw_bin_values)) # Set zero-valued bins to 1 to make the entropy calculation well-behaved raw_bin_values[raw_bin_values == 0] = 1 # Calculate entropy @@ -716,7 +727,7 @@ def evaluate(reference_beats, estimated_beats, **kwargs): Reference beat times, in seconds estimated_beats : np.ndarray Query beat times, in seconds - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -727,7 +738,6 @@ def evaluate(reference_beats, estimated_beats, **kwargs): the value is the (float) score achieved. """ - # Trim beat times at the beginning of the annotations reference_beats = util.filter_kwargs(trim_beats, reference_beats, **kwargs) estimated_beats = util.filter_kwargs(trim_beats, estimated_beats, **kwargs) @@ -737,34 +747,36 @@ def evaluate(reference_beats, estimated_beats, **kwargs): scores = collections.OrderedDict() # F-Measure - scores['F-measure'] = util.filter_kwargs(f_measure, reference_beats, - estimated_beats, **kwargs) + scores["F-measure"] = util.filter_kwargs( + f_measure, reference_beats, estimated_beats, **kwargs + ) # Cemgil - scores['Cemgil'], scores['Cemgil Best Metric Level'] = \ - util.filter_kwargs(cemgil, reference_beats, estimated_beats, **kwargs) + scores["Cemgil"], scores["Cemgil Best Metric Level"] = util.filter_kwargs( + cemgil, reference_beats, estimated_beats, **kwargs + ) # Goto - scores['Goto'] = util.filter_kwargs(goto, reference_beats, - estimated_beats, **kwargs) + scores["Goto"] = util.filter_kwargs( + goto, reference_beats, estimated_beats, **kwargs + ) # P-Score - scores['P-score'] = util.filter_kwargs(p_score, reference_beats, - estimated_beats, **kwargs) + scores["P-score"] = util.filter_kwargs( + p_score, reference_beats, estimated_beats, **kwargs + ) # Continuity metrics - (scores['Correct Metric Level Continuous'], - scores['Correct Metric Level Total'], - scores['Any Metric Level Continuous'], - scores['Any Metric Level Total']) = util.filter_kwargs(continuity, - reference_beats, - estimated_beats, - **kwargs) + ( + scores["Correct Metric Level Continuous"], + scores["Correct Metric Level Total"], + scores["Any Metric Level Continuous"], + scores["Any Metric Level Total"], + ) = util.filter_kwargs(continuity, reference_beats, estimated_beats, **kwargs) # Information gain - scores['Information gain'] = util.filter_kwargs(information_gain, - reference_beats, - estimated_beats, - **kwargs) + scores["Information gain"] = util.filter_kwargs( + information_gain, reference_beats, estimated_beats, **kwargs + ) return scores diff --git a/mir_eval/chord.py b/mir_eval/chord.py index 0124d5b6..a5b2d64d 100644 --- a/mir_eval/chord.py +++ b/mir_eval/chord.py @@ -1,4 +1,4 @@ -r''' +r""" Chord estimation algorithms produce a list of intervals and labels which denote the chord being played over each timespan. They are evaluated by comparing the estimated chord labels to some reference, usually using a mapping to a chord @@ -67,7 +67,7 @@ entire quality in closed voicing, i.e. spanning only a single octave; extended chords (9's, 11's and 13's) are rolled into a single octave with any upper voices included as extensions. For example, ('A:7', 'A:9') are - equivlent but ('A:7', 'A:maj7') are not. + equivalent but ('A:7', 'A:maj7') are not. * :func:`mir_eval.chord.tetrads_inv`: Same as above, with inversions (bass relationships). @@ -93,7 +93,7 @@ .. [#harte2010towards] C. Harte. Towards Automatic Extraction of Harmony Information from Music Signals. PhD thesis, Queen Mary University of London, August 2010. -''' +""" import numpy as np import warnings @@ -105,35 +105,34 @@ BITMAP_LENGTH = 12 NO_CHORD = "N" -NO_CHORD_ENCODED = -1, np.array([0]*BITMAP_LENGTH), -1 +NO_CHORD_ENCODED = -1, np.array([0] * BITMAP_LENGTH), -1 X_CHORD = "X" -X_CHORD_ENCODED = -1, np.array([-1]*BITMAP_LENGTH), -1 +X_CHORD_ENCODED = -1, np.array([-1] * BITMAP_LENGTH), -1 class InvalidChordException(Exception): - r'''Exception class for suspect / invalid chord labels''' + r"""Exception class for suspect / invalid chord labels""" - def __init__(self, message='', chord_label=None): + def __init__(self, message="", chord_label=None): self.message = message self.chord_label = chord_label self.name = self.__class__.__name__ - super(InvalidChordException, self).__init__(message) + super().__init__(message) # --- Chord Primitives --- def _pitch_classes(): - r'''Map from pitch class (str) to semitone (int).''' - pitch_classes = ['C', 'D', 'E', 'F', 'G', 'A', 'B'] + r"""Map from pitch class (str) to semitone (int).""" + pitch_classes = ["C", "D", "E", "F", "G", "A", "B"] semitones = [0, 2, 4, 5, 7, 9, 11] - return dict([(c, s) for c, s in zip(pitch_classes, semitones)]) + return {c: s for c, s in zip(pitch_classes, semitones)} def _scale_degrees(): - r'''Mapping from scale degrees (str) to semitones (int).''' - degrees = ['1', '2', '3', '4', '5', '6', '7', - '8', '9', '10', '11', '12', '13'] + r"""Map scale degrees (str) to semitones (int).""" + degrees = ["1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13"] semitones = [0, 2, 4, 5, 7, 9, 11, 12, 14, 16, 17, 19, 21] - return dict([(d, s) for d, s in zip(degrees, semitones)]) + return {d: s for d, s in zip(degrees, semitones)} # Maps pitch classes (strings) to semitone indexes (ints). @@ -141,7 +140,7 @@ def _scale_degrees(): def pitch_class_to_semitone(pitch_class): - r'''Convert a pitch class to semitone. + r"""Convert a pitch class to semitone. Parameters ---------- @@ -153,18 +152,19 @@ def pitch_class_to_semitone(pitch_class): semitone : int Semitone value of the pitch class. - ''' + """ semitone = 0 for idx, char in enumerate(pitch_class): - if char == '#' and idx > 0: + if char == "#" and idx > 0: semitone += 1 - elif char == 'b' and idx > 0: + elif char == "b" and idx > 0: semitone -= 1 elif idx == 0: semitone = PITCH_CLASSES.get(char) else: raise InvalidChordException( - "Pitch class improperly formed: %s" % pitch_class) + "Pitch class improperly formed: %s" % pitch_class + ) return semitone % 12 @@ -177,7 +177,7 @@ def scale_degree_to_semitone(scale_degree): Parameters ---------- - scale degree : str + scale_degree : str Spelling of a relative scale degree, e.g. 'b3', '7', '#5' Returns @@ -194,15 +194,17 @@ def scale_degree_to_semitone(scale_degree): if scale_degree.startswith("#"): offset = scale_degree.count("#") scale_degree = scale_degree.strip("#") - elif scale_degree.startswith('b'): + elif scale_degree.startswith("b"): offset = -1 * scale_degree.count("b") scale_degree = scale_degree.strip("b") semitone = SCALE_DEGREES.get(scale_degree, None) if semitone is None: raise InvalidChordException( - "Scale degree improperly formed: {}, expected one of {}." - .format(scale_degree, list(SCALE_DEGREES.keys()))) + "Scale degree improperly formed: {}, expected one of {}.".format( + scale_degree, list(SCALE_DEGREES.keys()) + ) + ) return semitone + offset @@ -242,35 +244,32 @@ def scale_degree_to_bitmap(scale_degree, modulo=False, length=BITMAP_LENGTH): # semitones, i.e. vector[0] is the tonic. QUALITIES = { # 1 2 3 4 5 6 7 - 'maj': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0], - 'min': [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0], - 'aug': [1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0], - 'dim': [1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0], - 'sus4': [1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0], - 'sus2': [1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0], - '7': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - 'maj7': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1], - 'min7': [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], - 'minmaj7': [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], - 'maj6': [1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0], - 'min6': [1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0], - 'dim7': [1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0], - 'hdim7': [1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0], - 'maj9': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1], - 'min9': [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], - '9': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - 'b9': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - '#9': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - 'min11': [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], - '11': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - '#11': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - 'maj13': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1], - 'min13': [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], - '13': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - 'b13': [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], - '1': [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], - '5': [1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], - '': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]} + "maj": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0], + "min": [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0], + "aug": [1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0], + "dim": [1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0], + "sus4": [1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0], + "sus2": [1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0], + "7": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], + "maj7": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1], + "min7": [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], + "minmaj7": [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1], + "maj6": [1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0], + "min6": [1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0], + "dim7": [1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0], + "hdim7": [1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0], + "maj9": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1], + "min9": [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], + "9": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], + "min11": [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], + "11": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], + "maj13": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1], + "min13": [1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0], + "13": [1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0], + "1": [1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + "5": [1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0], + "": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], +} def quality_to_bitmap(quality): @@ -290,7 +289,8 @@ def quality_to_bitmap(quality): if quality not in QUALITIES: raise InvalidChordException( "Unsupported chord quality shorthand: '%s' " - "Did you mean to reduce extended chords?" % quality) + "Did you mean to reduce extended chords?" % quality + ) return np.array(QUALITIES[quality]) @@ -299,19 +299,20 @@ def quality_to_bitmap(quality): # TODO(ejhumphrey): Revisit how minmaj7's are mapped. This is how TMC did it, # but MMV handles it like a separate quality (rather than an add7). EXTENDED_QUALITY_REDUX = { - 'minmaj7': ('min', set(['7'])), - 'maj9': ('maj7', set(['9'])), - 'min9': ('min7', set(['9'])), - '9': ('7', set(['9'])), - 'b9': ('7', set(['b9'])), - '#9': ('7', set(['#9'])), - '11': ('7', set(['9', '11'])), - '#11': ('7', set(['9', '#11'])), - '13': ('7', set(['9', '11', '13'])), - 'b13': ('7', set(['9', '11', 'b13'])), - 'min11': ('min7', set(['9', '11'])), - 'maj13': ('maj7', set(['9', '11', '13'])), - 'min13': ('min7', set(['9', '11', '13']))} + "minmaj7": ("min", {"7"}), + "maj9": ("maj7", {"9"}), + "min9": ("min7", {"9"}), + "9": ("7", {"9"}), + "b9": ("7", {"b9"}), + "#9": ("7", {"#9"}), + "11": ("7", {"9", "11"}), + "#11": ("7", {"9", "#11"}), + "13": ("7", {"9", "11", "13"}), + "b13": ("7", {"9", "11", "b13"}), + "min11": ("min7", {"9", "11"}), + "maj13": ("maj7", {"9", "11", "13"}), + "min13": ("min7", {"9", "11", "13"}), +} def reduce_extended_quality(quality): @@ -335,25 +336,24 @@ def reduce_extended_quality(quality): # --- Chord Label Parsing --- +# This monster regexp is pulled from the JAMS chord namespace, +# which is in turn derived from the context-free grammar of +# Harte et al., 2005. +CHORD_RE = re.compile( + r"""^((N|X)|(([A-G](b*|#*))((:(maj|min|dim|aug|1|5|sus2|sus4|maj6|min6|7|maj7|min7|dim7|hdim7|minmaj7|aug7|9|maj9|min9|11|maj11|min11|13|maj13|min13)(\((\*?((b*|#*)([1-9]|1[0-3]?))(,\*?((b*|#*)([1-9]|1[0-3]?)))*)\))?)|(:\((\*?((b*|#*)([1-9]|1[0-3]?))(,\*?((b*|#*)([1-9]|1[0-3]?)))*)\)))?((/((b*|#*)([1-9]|1[0-3]?)))?)?))$""" +) # nopep8 + + def validate_chord_label(chord_label): """Test for well-formedness of a chord label. Parameters ---------- - chord : str + chord_label : str Chord label to validate. - """ - - # This monster regexp is pulled from the JAMS chord namespace, - # which is in turn derived from the context-free grammar of - # Harte et al., 2005. - - pattern = re.compile(r'''^((N|X)|(([A-G](b*|#*))((:(maj|min|dim|aug|1|5|sus2|sus4|maj6|min6|7|maj7|min7|dim7|hdim7|minmaj7|aug7|9|maj9|min9|11|maj11|min11|13|maj13|min13)(\((\*?((b*|#*)([1-9]|1[0-3]?))(,\*?((b*|#*)([1-9]|1[0-3]?)))*)\))?)|(:\((\*?((b*|#*)([1-9]|1[0-3]?))(,\*?((b*|#*)([1-9]|1[0-3]?)))*)\)))?((/((b*|#*)([1-9]|1[0-3]?)))?)?))$''') # nopep8 - - if not pattern.match(chord_label): - raise InvalidChordException('Invalid chord label: ' - '{}'.format(chord_label)) + if not CHORD_RE.match(chord_label): + raise InvalidChordException("Invalid chord label: " "{}".format(chord_label)) pass @@ -392,9 +392,9 @@ def split(chord_label, reduce_extended_chords=False): chord_label = str(chord_label) validate_chord_label(chord_label) if chord_label == NO_CHORD: - return [chord_label, '', set(), ''] + return [chord_label, "", set(), ""] - bass = '1' + bass = "1" if "/" in chord_label: chord_label, bass = chord_label.split("/") @@ -404,7 +404,7 @@ def split(chord_label, reduce_extended_chords=False): chord_label, scale_degrees = chord_label.split("(") omission = "*" in scale_degrees scale_degrees = scale_degrees.strip(")") - scale_degrees = set([i.strip() for i in scale_degrees.split(",")]) + scale_degrees = {i.strip() for i in scale_degrees.split(",")} # Note: Chords lacking quality AND added interval information are major. # If a quality shorthand is specified, it is returned. @@ -413,8 +413,9 @@ def split(chord_label, reduce_extended_chords=False): # Intervals specifying omissions MUST have a quality. if omission and ":" not in chord_label: raise InvalidChordException( - "Intervals specifying omissions MUST have a quality.") - quality = '' if scale_degrees else 'maj' + "Intervals specifying omissions MUST have a quality." + ) + quality = "" if scale_degrees else "maj" if ":" in chord_label: chord_root, quality_name = chord_label.split(":") # Extended chords (with ":"s) may not explicitly have Major qualities, @@ -431,7 +432,7 @@ def split(chord_label, reduce_extended_chords=False): return [chord_root, quality, scale_degrees, bass] -def join(chord_root, quality='', extensions=None, bass=''): +def join(chord_root, quality="", extensions=None, bass=""): r"""Join the parts of a chord into a complete chord label. Parameters @@ -459,15 +460,14 @@ def join(chord_root, quality='', extensions=None, bass=''): chord_label += ":%s" % quality if extensions: chord_label += "(%s)" % ",".join(extensions) - if bass and bass != '1': + if bass and bass != "1": chord_label += "/%s" % bass validate_chord_label(chord_label) return chord_label # --- Chords to Numerical Representations --- -def encode(chord_label, reduce_extended_chords=False, - strict_bass_intervals=False): +def encode(chord_label, reduce_extended_chords=False, strict_bass_intervals=False): """Translate a chord label to numerical representations for evaluation. Parameters @@ -490,15 +490,14 @@ def encode(chord_label, reduce_extended_chords=False, 12-dim vector of relative semitones in the chord spelling. bass_number : int Relative semitone of the chord's bass note, e.g. 0=root, 7=fifth, etc. - """ - if chord_label == NO_CHORD: return NO_CHORD_ENCODED if chord_label == X_CHORD: return X_CHORD_ENCODED chord_root, quality, scale_degrees, bass = split( - chord_label, reduce_extended_chords=reduce_extended_chords) + chord_label, reduce_extended_chords=reduce_extended_chords + ) root_number = pitch_class_to_semitone(chord_root) bass_number = scale_degree_to_semitone(bass) % 12 @@ -507,14 +506,14 @@ def encode(chord_label, reduce_extended_chords=False, semitone_bitmap[0] = 1 for scale_degree in scale_degrees: - semitone_bitmap += scale_degree_to_bitmap(scale_degree, - reduce_extended_chords) + semitone_bitmap += scale_degree_to_bitmap(scale_degree, reduce_extended_chords) - semitone_bitmap = (semitone_bitmap > 0).astype(np.int) + semitone_bitmap = (semitone_bitmap > 0).astype(np.int64) if not semitone_bitmap[bass_number] and strict_bass_intervals: raise InvalidChordException( - "Given bass scale degree is absent from this chord: " - "%s" % chord_label, chord_label) + "Given bass scale degree is absent from this chord: " "%s" % chord_label, + chord_label, + ) else: semitone_bitmap[bass_number] = 1 return root_number, semitone_bitmap, bass_number @@ -544,8 +543,8 @@ def encode_many(chord_labels, reduce_extended_chords=False): """ num_items = len(chord_labels) - roots, basses = np.zeros([2, num_items], dtype=np.int) - semitones = np.zeros([num_items, 12], dtype=np.int) + roots, basses = np.zeros([2, num_items], dtype=np.int64) + semitones = np.zeros([num_items, 12], dtype=np.int64) local_cache = dict() for i, label in enumerate(chord_labels): result = local_cache.get(label, None) @@ -557,7 +556,7 @@ def encode_many(chord_labels, reduce_extended_chords=False): def rotate_bitmap_to_root(bitmap, chord_root): - """Circularly shift a relative bitmap to its asbolute pitch classes. + """Circularly shift a relative bitmap to its absolute pitch classes. For clarity, the best explanation is an example. Given 'G:Maj', the root and quality map are as follows:: @@ -592,15 +591,15 @@ def rotate_bitmap_to_root(bitmap, chord_root): def rotate_bitmaps_to_roots(bitmaps, roots): - """Circularly shift a relative bitmaps to asbolute pitch classes. + """Circularly shift a relative bitmaps to absolute pitch classes. See :func:`rotate_bitmap_to_root` for more information. Parameters ---------- - bitmap : np.ndarray, shape=(N, 12) + bitmaps : np.ndarray, shape=(N, 12) Bitmap of active notes, relative to the given root. - root : np.ndarray, shape=(N,) + roots : np.ndarray, shape=(N,) Absolute pitch class number. Returns @@ -617,7 +616,7 @@ def rotate_bitmaps_to_roots(bitmaps, roots): # --- Comparison Routines --- def validate(reference_labels, estimated_labels): - """Checks that the input annotations to a comparison function look like + """Check that the input annotations to a comparison function look like valid chord labels. Parameters @@ -626,22 +625,22 @@ def validate(reference_labels, estimated_labels): Reference chord labels to score against. estimated_labels : list, len=n Estimated chord labels to score against. - """ N = len(reference_labels) M = len(estimated_labels) if N != M: raise ValueError( "Chord comparison received different length lists: " - "len(reference)=%d\tlen(estimates)=%d" % (N, M)) + "len(reference)=%d\tlen(estimates)=%d" % (N, M) + ) for labels in [reference_labels, estimated_labels]: for chord_label in labels: validate_chord_label(chord_label) # When either label list is empty, warn the user if len(reference_labels) == 0: - warnings.warn('Reference labels are empty') + warnings.warn("Reference labels are empty") if len(estimated_labels) == 0: - warnings.warn('Estimated labels are empty') + warnings.warn("Estimated labels are empty") def weighted_accuracy(comparisons, weights): @@ -684,29 +683,32 @@ def weighted_accuracy(comparisons, weights): N = len(comparisons) # There should be as many weights as comparisons if weights.shape[0] != N: - raise ValueError('weights and comparisons should be of the same' - ' length. len(weights) = {} but len(comparisons)' - ' = {}'.format(weights.shape[0], N)) + raise ValueError( + "weights and comparisons should be of the same" + " length. len(weights) = {} but len(comparisons)" + " = {}".format(weights.shape[0], N) + ) if (weights < 0).any(): - raise ValueError('Weights should all be positive.') + raise ValueError("Weights should all be positive.") if np.sum(weights) == 0: - warnings.warn('No nonzero weights, returning 0') + warnings.warn("No nonzero weights, returning 0") return 0 # Find all comparison scores which are valid - valid_idx = (comparisons >= 0) + valid_idx = comparisons >= 0 # If no comparable chords were provided, warn and return 0 if valid_idx.sum() == 0: - warnings.warn("No reference chords were comparable " - "to estimated chords, returning 0.") + warnings.warn( + "No reference chords were comparable " "to estimated chords, returning 0." + ) return 0 # Remove any uncomparable labels comparisons = comparisons[valid_idx] weights = weights[valid_idx] # Normalize the weights total_weight = float(np.sum(weights)) - normalized_weights = np.asarray(weights, dtype=float)/total_weight + normalized_weights = np.asarray(weights, dtype=float) / total_weight # Score is the sum of all weighted comparisons - return np.sum(comparisons*normalized_weights) + return np.sum(comparisons * normalized_weights) def thirds(reference_labels, estimated_labels): @@ -749,7 +751,7 @@ def thirds(reference_labels, estimated_labels): eq_roots = ref_roots == est_roots eq_thirds = ref_semitones[:, 3] == est_semitones[:, 3] - comparison_scores = (eq_roots * eq_thirds).astype(np.float) + comparison_scores = (eq_roots * eq_thirds).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -797,7 +799,7 @@ def thirds_inv(reference_labels, estimated_labels): eq_root = ref_roots == est_roots eq_bass = ref_bass == est_bass eq_third = ref_semitones[:, 3] == est_semitones[:, 3] - comparison_scores = (eq_root * eq_third * eq_bass).astype(np.float) + comparison_scores = (eq_root * eq_third * eq_bass).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -843,9 +845,8 @@ def triads(reference_labels, estimated_labels): est_roots, est_semitones = encode_many(estimated_labels, False)[:2] eq_roots = ref_roots == est_roots - eq_semitones = np.all( - np.equal(ref_semitones[:, :8], est_semitones[:, :8]), axis=1) - comparison_scores = (eq_roots * eq_semitones).astype(np.float) + eq_semitones = np.all(np.equal(ref_semitones[:, :8], est_semitones[:, :8]), axis=1) + comparison_scores = (eq_roots * eq_semitones).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -892,9 +893,8 @@ def triads_inv(reference_labels, estimated_labels): eq_roots = ref_roots == est_roots eq_basses = ref_bass == est_bass - eq_semitones = np.all( - np.equal(ref_semitones[:, :8], est_semitones[:, :8]), axis=1) - comparison_scores = (eq_roots * eq_semitones * eq_basses).astype(np.float) + eq_semitones = np.all(np.equal(ref_semitones[:, :8], est_semitones[:, :8]), axis=1) + comparison_scores = (eq_roots * eq_semitones * eq_basses).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -941,7 +941,7 @@ def tetrads(reference_labels, estimated_labels): eq_roots = ref_roots == est_roots eq_semitones = np.all(np.equal(ref_semitones, est_semitones), axis=1) - comparison_scores = (eq_roots * eq_semitones).astype(np.float) + comparison_scores = (eq_roots * eq_semitones).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -989,7 +989,7 @@ def tetrads_inv(reference_labels, estimated_labels): eq_roots = ref_roots == est_roots eq_basses = ref_bass == est_bass eq_semitones = np.all(np.equal(ref_semitones, est_semitones), axis=1) - comparison_scores = (eq_roots * eq_semitones * eq_basses).astype(np.float) + comparison_scores = (eq_roots * eq_semitones * eq_basses).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -1029,13 +1029,11 @@ def root(reference_labels, estimated_labels): comparison_scores : np.ndarray, shape=(n,), dtype=float Comparison scores, in [0.0, 1.0], or -1 if the comparison is out of gamut. - """ - validate(reference_labels, estimated_labels) ref_roots, ref_semitones = encode_many(reference_labels, False)[:2] est_roots = encode_many(estimated_labels, False)[0] - comparison_scores = (ref_roots == est_roots).astype(np.float) + comparison_scores = (ref_roots == est_roots).astype(np.float64) # Ignore 'X' chords comparison_scores[np.any(ref_semitones < 0, axis=1)] = -1.0 @@ -1087,7 +1085,7 @@ def mirex(reference_labels, estimated_labels): eq_chroma = (ref_chroma * est_chroma).sum(axis=-1) # Chroma matching for set bits - comparison_scores = (eq_chroma >= min_intersection).astype(np.float) + comparison_scores = (eq_chroma >= min_intersection).astype(np.float64) # No-chord matching; match -1 roots, SKIP_CHORDS dropped next no_root = np.logical_and(ref_data[0] == -1, est_data[0] == -1) @@ -1096,8 +1094,9 @@ def mirex(reference_labels, estimated_labels): # Skip chords where the number of active semitones `n` is # 0 < n < `min_intersection`. ref_semitone_count = (ref_data[1] > 0).sum(axis=1) - skip_idx = np.logical_and(ref_semitone_count > 0, - ref_semitone_count < min_intersection) + skip_idx = np.logical_and( + ref_semitone_count > 0, ref_semitone_count < min_intersection + ) # Also ignore 'X' chords. np.logical_or(skip_idx, np.any(ref_data[1] < 0, axis=1), skip_idx) comparison_scores[skip_idx] = -1.0 @@ -1141,16 +1140,15 @@ def majmin(reference_labels, estimated_labels): """ validate(reference_labels, estimated_labels) - maj_semitones = np.array(QUALITIES['maj'][:8]) - min_semitones = np.array(QUALITIES['min'][:8]) + maj_semitones = np.array(QUALITIES["maj"][:8]) + min_semitones = np.array(QUALITIES["min"][:8]) ref_roots, ref_semitones, _ = encode_many(reference_labels, False) est_roots, est_semitones, _ = encode_many(estimated_labels, False) eq_root = ref_roots == est_roots - eq_quality = np.all(np.equal(ref_semitones[:, :8], - est_semitones[:, :8]), axis=1) - comparison_scores = (eq_root * eq_quality).astype(np.float) + eq_quality = np.all(np.equal(ref_semitones[:, :8], est_semitones[:, :8]), axis=1) + comparison_scores = (eq_root * eq_quality).astype(np.float64) # Test for Major / Minor / No-chord is_maj = np.all(np.equal(ref_semitones[:, :8], maj_semitones), axis=1) @@ -1208,16 +1206,15 @@ def majmin_inv(reference_labels, estimated_labels): """ validate(reference_labels, estimated_labels) - maj_semitones = np.array(QUALITIES['maj'][:8]) - min_semitones = np.array(QUALITIES['min'][:8]) + maj_semitones = np.array(QUALITIES["maj"][:8]) + min_semitones = np.array(QUALITIES["min"][:8]) ref_roots, ref_semitones, ref_bass = encode_many(reference_labels, False) est_roots, est_semitones, est_bass = encode_many(estimated_labels, False) eq_root_bass = (ref_roots == est_roots) * (ref_bass == est_bass) - eq_semitones = np.all(np.equal(ref_semitones[:, :8], - est_semitones[:, :8]), axis=1) - comparison_scores = (eq_root_bass * eq_semitones).astype(np.float) + eq_semitones = np.all(np.equal(ref_semitones[:, :8], est_semitones[:, :8]), axis=1) + comparison_scores = (eq_root_bass * eq_semitones).astype(np.float64) # Test for Major / Minor / No-chord is_maj = np.all(np.equal(ref_semitones[:, :8], maj_semitones), axis=1) @@ -1272,7 +1269,7 @@ def sevenths(reference_labels, estimated_labels): """ validate(reference_labels, estimated_labels) - seventh_qualities = ['maj', 'min', 'maj7', '7', 'min7', ''] + seventh_qualities = ["maj", "min", "maj7", "7", "min7", ""] valid_semitones = np.array([QUALITIES[name] for name in seventh_qualities]) ref_roots, ref_semitones = encode_many(reference_labels, False)[:2] @@ -1280,11 +1277,15 @@ def sevenths(reference_labels, estimated_labels): eq_root = ref_roots == est_roots eq_semitones = np.all(np.equal(ref_semitones, est_semitones), axis=1) - comparison_scores = (eq_root * eq_semitones).astype(np.float) + comparison_scores = (eq_root * eq_semitones).astype(np.float64) # Test for reference chord inclusion - is_valid = np.array([np.all(np.equal(ref_semitones, semitones), axis=1) - for semitones in valid_semitones]) + is_valid = np.array( + [ + np.all(np.equal(ref_semitones, semitones), axis=1) + for semitones in valid_semitones + ] + ) # Drop if NOR comparison_scores[np.sum(is_valid, axis=0) == 0] = -1 return comparison_scores @@ -1327,7 +1328,7 @@ def sevenths_inv(reference_labels, estimated_labels): """ validate(reference_labels, estimated_labels) - seventh_qualities = ['maj', 'min', 'maj7', '7', 'min7', ''] + seventh_qualities = ["maj", "min", "maj7", "7", "min7", ""] valid_semitones = np.array([QUALITIES[name] for name in seventh_qualities]) ref_roots, ref_semitones, ref_basses = encode_many(reference_labels, False) @@ -1335,11 +1336,15 @@ def sevenths_inv(reference_labels, estimated_labels): eq_roots_basses = (ref_roots == est_roots) * (ref_basses == est_basses) eq_semitones = np.all(np.equal(ref_semitones, est_semitones), axis=1) - comparison_scores = (eq_roots_basses * eq_semitones).astype(np.float) + comparison_scores = (eq_roots_basses * eq_semitones).astype(np.float64) # Test for Major / Minor / No-chord - is_valid = np.array([np.all(np.equal(ref_semitones, semitones), axis=1) - for semitones in valid_semitones]) + is_valid = np.array( + [ + np.all(np.equal(ref_semitones, semitones), axis=1) + for semitones in valid_semitones + ] + ) comparison_scores[np.sum(is_valid, axis=0) == 0] = -1 # Disable inversions that are not part of the quality @@ -1384,12 +1389,14 @@ def directional_hamming_distance(reference_intervals, estimated_intervals): util.validate_intervals(reference_intervals) # make sure chord intervals do not overlap - if len(reference_intervals) > 1 and (reference_intervals[:-1, 1] > - reference_intervals[1:, 0]).any(): - raise ValueError('Chord Intervals must not overlap') + if ( + len(reference_intervals) > 1 + and (reference_intervals[:-1, 1] > reference_intervals[1:, 0]).any() + ): + raise ValueError("Chord Intervals must not overlap") est_ts = np.unique(estimated_intervals.flatten()) - seg = 0. + seg = 0.0 for start, end in reference_intervals: dur = end - start between_start_end = est_ts[(est_ts >= start) & (est_ts < end)] @@ -1421,8 +1428,7 @@ def overseg(reference_intervals, estimated_intervals): oversegmentation score : float Comparison score, in [0.0, 1.0], where 1.0 means no oversegmentation. """ - return 1 - directional_hamming_distance(reference_intervals, - estimated_intervals) + return 1 - directional_hamming_distance(reference_intervals, estimated_intervals) def underseg(reference_intervals, estimated_intervals): @@ -1448,8 +1454,7 @@ def underseg(reference_intervals, estimated_intervals): undersegmentation score : float Comparison score, in [0.0, 1.0], where 1.0 means no undersegmentation. """ - return 1 - directional_hamming_distance(estimated_intervals, - reference_intervals) + return 1 - directional_hamming_distance(estimated_intervals, reference_intervals) def seg(reference_intervals, estimated_intervals): @@ -1475,9 +1480,10 @@ def seg(reference_intervals, estimated_intervals): segmentation score : float Comparison score, in [0.0, 1.0], where 1.0 means perfect segmentation. """ - - return min(underseg(reference_intervals, estimated_intervals), - overseg(reference_intervals, estimated_intervals)) + return min( + underseg(reference_intervals, estimated_intervals), + overseg(reference_intervals, estimated_intervals), + ) def merge_chord_intervals(intervals, labels): @@ -1504,8 +1510,9 @@ def merge_chord_intervals(intervals, labels): prev_rt = None prev_st = None prev_ba = None - for s, e, rt, st, ba in zip(intervals[:, 0], intervals[:, 1], - roots, semitones, basses): + for s, e, rt, st, ba in zip( + intervals[:, 0], intervals[:, 1], roots, semitones, basses + ): if rt != prev_rt or (st != prev_st).any() or ba != prev_ba: prev_rt, prev_st, prev_ba = rt, st, ba merged_ivs.append([s, e]) @@ -1515,7 +1522,7 @@ def merge_chord_intervals(intervals, labels): def evaluate(ref_intervals, ref_labels, est_intervals, est_labels, **kwargs): - """Computes weighted accuracy for all comparison functions for the given + """Compute weighted accuracy for all comparison functions for the given reference and estimated annotations. Examples @@ -1532,20 +1539,16 @@ def evaluate(ref_intervals, ref_labels, est_intervals, est_labels, **kwargs): ref_intervals : np.ndarray, shape=(n, 2) Reference chord intervals, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - ref_labels : list, shape=(n,) reference chord labels, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - est_intervals : np.ndarray, shape=(m, 2) estimated chord intervals, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - est_labels : list, shape=(m,) estimated chord labels, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -1558,47 +1561,50 @@ def evaluate(ref_intervals, ref_labels, est_intervals, est_labels, **kwargs): """ # Append or crop estimated intervals so their span is the same as reference est_intervals, est_labels = util.adjust_intervals( - est_intervals, est_labels, ref_intervals.min(), ref_intervals.max(), - NO_CHORD, NO_CHORD) + est_intervals, + est_labels, + ref_intervals.min(), + ref_intervals.max(), + NO_CHORD, + NO_CHORD, + ) # use merged intervals for segmentation evaluation merged_ref_intervals = merge_chord_intervals(ref_intervals, ref_labels) merged_est_intervals = merge_chord_intervals(est_intervals, est_labels) # Adjust the labels so that they span the same intervals intervals, ref_labels, est_labels = util.merge_labeled_intervals( - ref_intervals, ref_labels, est_intervals, est_labels) + ref_intervals, ref_labels, est_intervals, est_labels + ) # Convert intervals to durations (used as weights) durations = util.intervals_to_durations(intervals) # Store scores for each comparison function scores = collections.OrderedDict() - scores['thirds'] = weighted_accuracy(thirds(ref_labels, est_labels), - durations) - scores['thirds_inv'] = weighted_accuracy(thirds_inv(ref_labels, - est_labels), durations) - scores['triads'] = weighted_accuracy(triads(ref_labels, est_labels), - durations) - scores['triads_inv'] = weighted_accuracy(triads_inv(ref_labels, - est_labels), durations) - scores['tetrads'] = weighted_accuracy(tetrads(ref_labels, est_labels), - durations) - scores['tetrads_inv'] = weighted_accuracy(tetrads_inv(ref_labels, - est_labels), - durations) - scores['root'] = weighted_accuracy(root(ref_labels, est_labels), durations) - scores['mirex'] = weighted_accuracy(mirex(ref_labels, est_labels), - durations) - scores['majmin'] = weighted_accuracy(majmin(ref_labels, est_labels), - durations) - scores['majmin_inv'] = weighted_accuracy(majmin_inv(ref_labels, - est_labels), durations) - scores['sevenths'] = weighted_accuracy(sevenths(ref_labels, est_labels), - durations) - scores['sevenths_inv'] = weighted_accuracy(sevenths_inv(ref_labels, - est_labels), - durations) - scores['underseg'] = underseg(merged_ref_intervals, merged_est_intervals) - scores['overseg'] = overseg(merged_ref_intervals, merged_est_intervals) - scores['seg'] = min(scores['overseg'], scores['underseg']) + scores["thirds"] = weighted_accuracy(thirds(ref_labels, est_labels), durations) + scores["thirds_inv"] = weighted_accuracy( + thirds_inv(ref_labels, est_labels), durations + ) + scores["triads"] = weighted_accuracy(triads(ref_labels, est_labels), durations) + scores["triads_inv"] = weighted_accuracy( + triads_inv(ref_labels, est_labels), durations + ) + scores["tetrads"] = weighted_accuracy(tetrads(ref_labels, est_labels), durations) + scores["tetrads_inv"] = weighted_accuracy( + tetrads_inv(ref_labels, est_labels), durations + ) + scores["root"] = weighted_accuracy(root(ref_labels, est_labels), durations) + scores["mirex"] = weighted_accuracy(mirex(ref_labels, est_labels), durations) + scores["majmin"] = weighted_accuracy(majmin(ref_labels, est_labels), durations) + scores["majmin_inv"] = weighted_accuracy( + majmin_inv(ref_labels, est_labels), durations + ) + scores["sevenths"] = weighted_accuracy(sevenths(ref_labels, est_labels), durations) + scores["sevenths_inv"] = weighted_accuracy( + sevenths_inv(ref_labels, est_labels), durations + ) + scores["underseg"] = underseg(merged_ref_intervals, merged_est_intervals) + scores["overseg"] = overseg(merged_ref_intervals, merged_est_intervals) + scores["seg"] = min(scores["overseg"], scores["underseg"]) return scores diff --git a/mir_eval/display.py b/mir_eval/display.py index 3ede60e8..30156ea3 100644 --- a/mir_eval/display.py +++ b/mir_eval/display.py @@ -1,46 +1,29 @@ -# -*- encoding: utf-8 -*- -'''Display functions''' +"""Display functions""" from collections import defaultdict +from weakref import WeakKeyDictionary import numpy as np from scipy.signal import spectrogram +import matplotlib as mpl from matplotlib.patches import Rectangle from matplotlib.ticker import FuncFormatter, MultipleLocator from matplotlib.ticker import Formatter from matplotlib.colors import LinearSegmentedColormap, LogNorm, ColorConverter -from matplotlib.collections import BrokenBarHCollection +from matplotlib.transforms import Bbox, TransformedBbox from .melody import freq_to_voicing from .util import midi_to_hz, hz_to_midi -def __expand_limits(ax, limits, which='x'): - '''Helper function to expand axis limits''' - - if which == 'x': - getter, setter = ax.get_xlim, ax.set_xlim - elif which == 'y': - getter, setter = ax.get_ylim, ax.set_ylim - else: - raise ValueError('invalid axis: {}'.format(which)) - - old_lims = getter() - new_lims = list(limits) - - # infinite limits occur on new axis objects with no data - if np.isfinite(old_lims[0]): - new_lims[0] = min(old_lims[0], limits[0]) - - if np.isfinite(old_lims[1]): - new_lims[1] = max(old_lims[1], limits[1]) - - setter(new_lims) +# This dictionary is used to track mir_eval-specific attributes +# attached to matplotlib axes +__AXMAP = WeakKeyDictionary() def __get_axes(ax=None, fig=None): - '''Get or construct the target axes object for a new plot. + """Get or construct the target axes object for a new plot. Parameters ---------- @@ -57,31 +40,41 @@ def __get_axes(ax=None, fig=None): ax : matplotlib.pyplot.axes An axis handle on which to draw the segmentation. If none is provided, a new set of axes is created. - new_axes : bool If `True`, the axis object was newly constructed. If `False`, the axis object already existed. - - ''' - + """ new_axes = False - if ax is not None: - return ax, new_axes + if ax is None: + if fig is None: + import matplotlib.pyplot as plt - if fig is None: - import matplotlib.pyplot as plt - fig = plt.gcf() + fig = plt.gcf() - if not fig.get_axes(): - new_axes = True + if not fig.get_axes(): + new_axes = True + ax = fig.gca() - return fig.gca(), new_axes + # Create a storage bucket for this axes in case we need it + if ax not in __AXMAP: + __AXMAP[ax] = dict() + return ax, new_axes -def segments(intervals, labels, base=None, height=None, text=False, - text_kw=None, ax=None, **kwargs): - '''Plot a segmentation as a set of disjoint rectangles. + +def segments( + intervals, + labels, + base=None, + height=None, + text=False, + text_kw=None, + ax=None, + prop_cycle=None, + **kwargs, +): + """Plot a segmentation as a set of disjoint rectangles. Parameters ---------- @@ -89,33 +82,30 @@ def segments(intervals, labels, base=None, height=None, text=False, segment intervals, in the format returned by :func:`mir_eval.io.load_intervals` or :func:`mir_eval.io.load_labeled_intervals`. - labels : list, shape=(n,) reference segment labels, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - base : number The vertical position of the base of the rectangles. By default, this will be the bottom of the plot. - height : number The height of the rectangles. By default, this will be the top of the plot (minus ``base``). - + .. note:: If either `base` or `height` are provided, both must be provided. text : bool If true, each segment's label is displayed in its upper-left corner - text_kw : dict If ``text == True``, the properties of the text object can be specified here. See ``matplotlib.pyplot.Text`` for valid parameters - ax : matplotlib.pyplot.axes An axis handle on which to draw the segmentation. If none is provided, a new set of axes is created. - - kwargs + prop_cycle : cycle.Cycler + An optional property cycle object to specify style properties. + If not provided, the default property cycler will be retrieved from matplotlib. + **kwargs Additional keyword arguments to pass to ``matplotlib.patches.Rectangle``. @@ -123,12 +113,12 @@ def segments(intervals, labels, base=None, height=None, text=False, ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' + """ if text_kw is None: text_kw = dict() - text_kw.setdefault('va', 'top') - text_kw.setdefault('clip_on', True) - text_kw.setdefault('bbox', dict(boxstyle='round', facecolor='white')) + text_kw.setdefault("va", "top") + text_kw.setdefault("clip_on", True) + text_kw.setdefault("bbox", dict(boxstyle="round", facecolor="white")) # Make sure we have a numpy array intervals = np.atleast_2d(intervals) @@ -137,17 +127,29 @@ def segments(intervals, labels, base=None, height=None, text=False, ax, new_axes = __get_axes(ax=ax) - if new_axes: - ax.set_ylim([0, 1]) + if prop_cycle is None: + __AXMAP[ax].setdefault("prop_cycle", mpl.rcParams["axes.prop_cycle"]) + __AXMAP[ax].setdefault("prop_iter", iter(mpl.rcParams["axes.prop_cycle"])) + elif "prop_iter" not in __AXMAP[ax]: + __AXMAP[ax]["prop_cycle"] = prop_cycle + __AXMAP[ax]["prop_iter"] = iter(prop_cycle) - # Infer height - if base is None: - base = ax.get_ylim()[0] + prop_cycle = __AXMAP[ax]["prop_cycle"] + prop_iter = __AXMAP[ax]["prop_iter"] - if height is None: - height = ax.get_ylim()[1] + if new_axes: + ax.set_yticks([]) - cycler = ax._get_patches_for_fill.prop_cycler + if base is None and height is None: + # If neither are provided, we'll use axes coordinates to span the figure + base, height = 0, 1 + transform = ax.get_xaxis_transform() + + elif base is not None and height is not None: + # If both are provided, we'll use data coordinates + transform = None + else: + raise ValueError("When specifying base or height, both must be provided.") seg_map = dict() @@ -155,41 +157,57 @@ def segments(intervals, labels, base=None, height=None, text=False, if lab in seg_map: continue - style = next(cycler) + try: + properties = next(prop_iter) + except StopIteration: + prop_iter = iter(prop_cycle) + __AXMAP[ax]["prop_iter"] = prop_iter + properties = next(prop_iter) + + style = { + k: v + for k, v in properties.items() + if k in ["color", "facecolor", "edgecolor", "linewidth"] + } + # Swap color -> facecolor here so we preserve edgecolor on rects + style.setdefault("facecolor", style["color"]) + style.pop("color", None) seg_map[lab] = seg_def_style.copy() seg_map[lab].update(style) - # Swap color -> facecolor here so we preserve edgecolor on rects - seg_map[lab]['facecolor'] = seg_map[lab].pop('color') seg_map[lab].update(kwargs) - seg_map[lab]['label'] = lab + seg_map[lab]["label"] = lab for ival, lab in zip(intervals, labels): - rect = Rectangle((ival[0], base), ival[1] - ival[0], height, - **seg_map[lab]) - ax.add_patch(rect) - seg_map[lab].pop('label', None) + rect = ax.axvspan(ival[0], ival[1], ymin=base, ymax=height, **seg_map[lab]) + seg_map[lab].pop("label", None) if text: - ann = ax.annotate(lab, - xy=(ival[0], height), xycoords='data', - xytext=(8, -10), textcoords='offset points', - **text_kw) + ann = ax.annotate( + lab, + xy=(ival[0], height), + xycoords=transform, + xytext=(8, -10), + textcoords="offset points", + **text_kw, + ) ann.set_clip_path(rect) - if new_axes: - ax.set_yticks([]) - - # Only expand if we have data - if intervals.size: - __expand_limits(ax, [intervals.min(), intervals.max()], which='x') - return ax -def labeled_intervals(intervals, labels, label_set=None, - base=None, height=None, extend_labels=True, - ax=None, tick=True, **kwargs): - '''Plot labeled intervals with each label on its own row. +def labeled_intervals( + intervals, + labels, + label_set=None, + base=None, + height=None, + extend_labels=True, + ax=None, + tick=True, + prop_cycle=None, + **kwargs, +): + """Plot labeled intervals with each label on its own row. Parameters ---------- @@ -235,44 +253,73 @@ def labeled_intervals(intervals, labels, label_set=None, tick : bool If ``True``, sets tick positions and labels on the y-axis. - kwargs + prop_cycle : cycle.Cycler + An optional property cycle object to specify style properties. + If not provided, the default property cycler will be retrieved from matplotlib. + + **kwargs Additional keyword arguments to pass to - `matplotlib.collection.BrokenBarHCollection`. + `matplotlib.collection.PolyCollection`. Returns ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' - + """ # Get the axes handle - ax, _ = __get_axes(ax=ax) + ax, new_axes = __get_axes(ax=ax) + + if prop_cycle is None: + __AXMAP[ax].setdefault("prop_cycle", mpl.rcParams["axes.prop_cycle"]) + __AXMAP[ax].setdefault("prop_iter", iter(mpl.rcParams["axes.prop_cycle"])) + elif "prop_iter" not in __AXMAP[ax]: + __AXMAP[ax]["prop_cycle"] = prop_cycle + __AXMAP[ax]["prop_iter"] = iter(prop_cycle) + + prop_cycle = __AXMAP[ax]["prop_cycle"] + prop_iter = __AXMAP[ax]["prop_iter"] # Make sure we have a numpy array intervals = np.atleast_2d(intervals) if label_set is None: # If we have non-empty pre-existing tick labels, use them - label_set = [_.get_text() for _ in ax.get_yticklabels()] # If none of the label strings have content, treat it as empty - if not any(label_set): - label_set = [] + label_set = __AXMAP[ax].get("labels", []) else: label_set = list(label_set) # Put additional labels at the end, in order + extended = False if extend_labels: ticks = label_set + sorted(set(labels) - set(label_set)) + if ticks != label_set and len(label_set) > 0: + extended = True elif label_set: ticks = label_set else: ticks = sorted(set(labels)) + # Push the ticks up into the axmap + __AXMAP[ax]["labels"] = ticks + style = dict(linewidth=1) - style.update(next(ax._get_patches_for_fill.prop_cycler)) + try: + properties = next(prop_iter) + except StopIteration: + prop_iter = iter(prop_cycle) + __AXMAP[ax]["prop_iter"] = prop_iter + properties = next(prop_iter) + + style = { + k: v + for k, v in properties.items() + if k in ["color", "facecolor", "edgecolor", "linewidth"] + } # Swap color -> facecolor here so we preserve edgecolor on rects - style['facecolor'] = style.pop('color') + style.setdefault("facecolor", style["color"]) + style.pop("color", None) style.update(kwargs) if base is None: @@ -295,33 +342,27 @@ def labeled_intervals(intervals, labels, label_set=None, xvals[lab].append((ival[0], ival[1] - ival[0])) for lab in seg_y: - ax.add_collection(BrokenBarHCollection(xvals[lab], seg_y[lab], - **style)) + ax.broken_barh(xvals[lab], seg_y[lab], **style) # Pop the label after the first time we see it, so we only get # one legend entry - style.pop('label', None) + style.pop("label", None) # Draw a line separating the new labels from pre-existing labels - if label_set != ticks: - ax.axhline(len(label_set), color='k', alpha=0.5) + if extended: + ax.axhline(len(label_set), color="k", alpha=0.5) if tick: - ax.grid(True, axis='y') + ax.grid(True, axis="y") ax.set_yticks([]) ax.set_yticks(base) - ax.set_yticklabels(ticks, va='bottom') + ax.set_yticklabels(ticks, va="bottom") ax.yaxis.set_major_formatter(IntervalFormatter(base, ticks)) - if base.size: - __expand_limits(ax, [base.min(), (base + height).max()], which='y') - if intervals.size: - __expand_limits(ax, [intervals.min(), intervals.max()], which='x') - return ax class IntervalFormatter(Formatter): - '''Ticker formatter for labeled interval plots. + """Ticker formatter for labeled interval plots. Parameters ---------- @@ -330,18 +371,18 @@ class IntervalFormatter(Formatter): ticks : array-like of string The labels for the ticks - ''' - def __init__(self, base, ticks): + """ + def __init__(self, base, ticks): self._map = {int(k): v for k, v in zip(base, ticks)} def __call__(self, x, pos=None): - - return self._map.get(int(x), '') + """Map the input position to its corresponding interval label""" + return self._map.get(int(x), "") def hierarchy(intervals_hier, labels_hier, levels=None, ax=None, **kwargs): - '''Plot a hierarchical segmentation + """Plot a hierarchical segmentation Parameters ---------- @@ -351,25 +392,24 @@ def hierarchy(intervals_hier, labels_hier, levels=None, ax=None, **kwargs): :func:`mir_eval.io.load_intervals` or :func:`mir_eval.io.load_labeled_intervals`. Segmentations should be ordered by increasing specificity. - labels_hier : list of list-like A list of segmentation labels. Each element should be a list of labels for the corresponding element in `intervals_hier`. - levels : list of string Each element ``levels[i]`` is a label for the ```i`` th segmentation. This is used in the legend to denote the levels in a segment hierarchy. - - kwargs + ax : matplotlib.pyplot.axes + An axis handle on which to draw the intervals. + If none is provided, a new set of axes is created. + **kwargs Additional keyword arguments to `labeled_intervals`. Returns ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' - + """ # This will break if a segment label exists in multiple levels if levels is None: levels = list(range(len(intervals_hier))) @@ -380,20 +420,23 @@ def hierarchy(intervals_hier, labels_hier, levels=None, ax=None, **kwargs): # Count the pre-existing patches n_patches = len(ax.patches) - for ints, labs, key in zip(intervals_hier[::-1], - labels_hier[::-1], - levels[::-1]): + for ints, labs, key in zip(intervals_hier[::-1], labels_hier[::-1], levels[::-1]): labeled_intervals(ints, labs, label=key, ax=ax, **kwargs) - # Reverse the patch ordering for anything we've added. - # This way, intervals are listed in the legend from top to bottom - ax.patches[n_patches:] = ax.patches[n_patches:][::-1] return ax -def events(times, labels=None, base=None, height=None, ax=None, text_kw=None, - **kwargs): - '''Plot event times as a set of vertical lines +def events( + times, + labels=None, + base=None, + height=None, + ax=None, + text_kw=None, + prop_cycle=None, + **kwargs, +): + """Plot event times as a set of vertical lines Parameters ---------- @@ -401,29 +444,27 @@ def events(times, labels=None, base=None, height=None, ax=None, text_kw=None, event times, in the format returned by :func:`mir_eval.io.load_events` or :func:`mir_eval.io.load_labeled_events`. - labels : list, shape=(n,), optional event labels, in the format returned by :func:`mir_eval.io.load_labeled_events`. - base : number The vertical position of the base of the line. By default, this will be the bottom of the plot. - height : number The height of the lines. By default, this will be the top of the plot (minus `base`). - + .. note:: If either `base` or `height` are provided, both must be provided. ax : matplotlib.pyplot.axes An axis handle on which to draw the segmentation. If none is provided, a new set of axes is created. - text_kw : dict If `labels` is provided, the properties of the text objects can be specified here. See `matplotlib.pyplot.Text` for valid parameters - - kwargs + prop_cycle : cycle.Cycler + An optional property cycle object to specify style properties. + If not provided, the default property cycler will be retrieved from matplotlib. + **kwargs Additional keyword arguments to pass to `matplotlib.pyplot.vlines`. @@ -431,12 +472,12 @@ def events(times, labels=None, base=None, height=None, ax=None, text_kw=None, ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' + """ if text_kw is None: text_kw = dict() - text_kw.setdefault('va', 'top') - text_kw.setdefault('clip_on', True) - text_kw.setdefault('bbox', dict(boxstyle='round', facecolor='white')) + text_kw.setdefault("va", "top") + text_kw.setdefault("clip_on", True) + text_kw.setdefault("bbox", dict(boxstyle="round", facecolor="white")) # make sure we have an array for times times = np.asarray(times) @@ -444,54 +485,67 @@ def events(times, labels=None, base=None, height=None, ax=None, text_kw=None, # Get the axes handle ax, new_axes = __get_axes(ax=ax) - # If we have fresh axes, set the limits + if prop_cycle is None: + __AXMAP[ax].setdefault("prop_cycle", mpl.rcParams["axes.prop_cycle"]) + __AXMAP[ax].setdefault("prop_iter", iter(mpl.rcParams["axes.prop_cycle"])) + elif "prop_iter" not in __AXMAP[ax]: + __AXMAP[ax]["prop_cycle"] = prop_cycle + __AXMAP[ax]["prop_iter"] = iter(prop_cycle) - if new_axes: - # Infer base and height - if base is None: - base = 0 - if height is None: - height = 1 - - ax.set_ylim([base, height]) - else: - if base is None: - base = ax.get_ylim()[0] + prop_cycle = __AXMAP[ax]["prop_cycle"] + prop_iter = __AXMAP[ax]["prop_iter"] - if height is None: - height = ax.get_ylim()[1] + if base is None and height is None: + # If neither are provided, we'll use axes coordinates to span the figure + base, height = 0, 1 + transform = ax.get_xaxis_transform() - cycler = ax._get_patches_for_fill.prop_cycler - - style = next(cycler).copy() + elif base is not None and height is not None: + # If both are provided, we'll use data coordinates + transform = None + else: + raise ValueError("When specifying base or height, both must be provided.") + + # Advance the property iterator if we can, restart it if we must + try: + properties = next(prop_iter) + except StopIteration: + prop_iter = iter(prop_cycle) + __AXMAP[ax]["prop_iter"] = prop_iter + properties = next(prop_iter) + + style = { + k: v for k, v in properties.items() if k in ["color", "linestyle", "linewidth"] + } style.update(kwargs) + # If the user provided 'colors', don't override it with 'color' - if 'colors' in style: - style.pop('color', None) + if "colors" in style: + style.pop("color", None) - lines = ax.vlines(times, base, base + height, **style) + lines = ax.vlines(times, base, base + height, transform=transform, **style) if labels: for path, lab in zip(lines.get_paths(), labels): - ax.annotate(lab, - xy=(path.vertices[0][0], height), - xycoords='data', - xytext=(8, -10), textcoords='offset points', - **text_kw) + ax.annotate( + lab, + xy=(path.vertices[0][0], height), + xycoords=transform, + xytext=(8, -10), + textcoords="offset points", + **text_kw, + ) if new_axes: ax.set_yticks([]) - __expand_limits(ax, [base, base + height], which='y') - - if times.size: - __expand_limits(ax, [times.min(), times.max()], which='x') - return ax -def pitch(times, frequencies, midi=False, unvoiced=False, ax=None, **kwargs): - '''Visualize pitch contours +def pitch( + times, frequencies, midi=False, unvoiced=False, ax=None, prop_cycle=None, **kwargs +): + """Visualize pitch contours Parameters ---------- @@ -517,22 +571,34 @@ def pitch(times, frequencies, midi=False, unvoiced=False, ax=None, **kwargs): An axis handle on which to draw the pitch contours. If none is provided, a new set of axes is created. - kwargs + prop_cycle : cycle.Cycler + An optional property cycle object to specify style properties. + If not provided, the default property cycler will be retrieved from matplotlib. + + **kwargs Additional keyword arguments to `matplotlib.pyplot.plot`. Returns ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' - + """ ax, _ = __get_axes(ax=ax) + if prop_cycle is None: + __AXMAP[ax].setdefault("prop_cycle", mpl.rcParams["axes.prop_cycle"]) + __AXMAP[ax].setdefault("prop_iter", iter(mpl.rcParams["axes.prop_cycle"])) + elif "prop_iter" not in __AXMAP[ax]: + __AXMAP[ax]["prop_cycle"] = prop_cycle + __AXMAP[ax]["prop_iter"] = iter(prop_cycle) + + prop_cycle = __AXMAP[ax]["prop_cycle"] + prop_iter = __AXMAP[ax]["prop_iter"] + times = np.asarray(times) # First, segment into contiguously voiced contours - frequencies, voicings = freq_to_voicing(np.asarray(frequencies, - dtype=np.float)) + frequencies, voicings = freq_to_voicing(np.asarray(frequencies, dtype=np.float64)) voicings = voicings.astype(bool) # Here are all the change-points @@ -551,8 +617,12 @@ def pitch(times, frequencies, midi=False, unvoiced=False, ax=None, **kwargs): u_slices.append(idx) # Now we just need to plot the contour - style = dict() - style.update(next(ax._get_lines.prop_cycler)) + try: + style = next(prop_iter) + except StopIteration: + prop_iter = iter(prop_cycle) + __AXMAP[ax]["prop_iter"] = prop_iter + style = next(prop_iter) style.update(kwargs) if midi: @@ -564,20 +634,21 @@ def pitch(times, frequencies, midi=False, unvoiced=False, ax=None, **kwargs): for idx in v_slices: ax.plot(times[idx], frequencies[idx], **style) - style.pop('label', None) + style.pop("label", None) # Plot the unvoiced portions if unvoiced: - style['alpha'] = style.get('alpha', 1.0) * 0.5 + style["alpha"] = style.get("alpha", 1.0) * 0.5 for idx in u_slices: ax.plot(times[idx], frequencies[idx], **style) return ax -def multipitch(times, frequencies, midi=False, unvoiced=False, ax=None, - **kwargs): - '''Visualize multiple f0 measurements +def multipitch( + times, frequencies, midi=False, unvoiced=False, ax=None, prop_cycle=None, **kwargs +): + """Visualize multiple f0 measurements Parameters ---------- @@ -606,26 +677,44 @@ def multipitch(times, frequencies, midi=False, unvoiced=False, ax=None, An axis handle on which to draw the pitch contours. If none is provided, a new set of axes is created. - kwargs + prop_cycle : cycle.Cycler + An optional property cycle object to specify style properties. + If not provided, the default property cycler will be retrieved from matplotlib. + + **kwargs Additional keyword arguments to `plt.scatter`. Returns ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' - + """ # Get the axes handle ax, _ = __get_axes(ax=ax) + if prop_cycle is None: + __AXMAP[ax].setdefault("prop_cycle", mpl.rcParams["axes.prop_cycle"]) + __AXMAP[ax].setdefault("prop_iter", iter(mpl.rcParams["axes.prop_cycle"])) + elif "prop_iter" not in __AXMAP[ax]: + __AXMAP[ax]["prop_cycle"] = prop_cycle + __AXMAP[ax]["prop_iter"] = iter(prop_cycle) + + prop_cycle = __AXMAP[ax]["prop_cycle"] + prop_iter = __AXMAP[ax]["prop_iter"] + # Set up a style for the plot - style_voiced = dict() - style_voiced.update(next(ax._get_lines.prop_cycler)) + try: + style_voiced = next(prop_iter) + except StopIteration: + prop_iter = iter(prop_cycle) + __AXMAP[ax]["prop_iter"] = prop_iter + style_voiced = next(prop_iter) + style_voiced.update(kwargs) style_unvoiced = style_voiced.copy() - style_unvoiced.pop('label', None) - style_unvoiced['alpha'] = style_unvoiced.get('alpha', 1.0) * 0.5 + style_unvoiced.pop("label", None) + style_unvoiced["alpha"] = style_unvoiced.get("alpha", 1.0) * 0.5 # We'll collect all times and frequencies first, then plot them voiced_times = [] @@ -638,7 +727,7 @@ def multipitch(times, frequencies, midi=False, unvoiced=False, ax=None, if not len(freqs): continue - freqs, voicings = freq_to_voicing(np.asarray(freqs, dtype=np.float)) + freqs, voicings = freq_to_voicing(np.asarray(freqs, dtype=np.float64)) # Discard all 0-frequency measurements idx = freqs > 0 @@ -668,7 +757,7 @@ def multipitch(times, frequencies, midi=False, unvoiced=False, ax=None, def piano_roll(intervals, pitches=None, midi=None, ax=None, **kwargs): - '''Plot a quantized piano roll as intervals + """Plot a quantized piano roll as intervals Parameters ---------- @@ -687,66 +776,82 @@ def piano_roll(intervals, pitches=None, midi=None, ax=None, **kwargs): An axis handle on which to draw the intervals. If none is provided, a new set of axes is created. - kwargs + **kwargs Additional keyword arguments to :func:`labeled_intervals`. Returns ------- ax : matplotlib.pyplot.axes._subplots.AxesSubplot A handle to the (possibly constructed) plot axes - ''' - + """ if midi is None: if pitches is None: - raise ValueError('At least one of `midi` or `pitches` ' - 'must be provided.') + raise ValueError("At least one of `midi` or `pitches` " "must be provided.") midi = hz_to_midi(pitches) scale = np.arange(128) - ax = labeled_intervals(intervals, np.round(midi).astype(int), - label_set=scale, - tick=False, - ax=ax, - **kwargs) + ax = labeled_intervals( + intervals, + np.round(midi).astype(int), + label_set=scale, + tick=False, + ax=ax, + **kwargs, + ) # Minor tick at each semitone ax.yaxis.set_minor_locator(MultipleLocator(1)) - ax.axis('auto') return ax -def separation(sources, fs=22050, labels=None, alpha=0.75, ax=None, **kwargs): - '''Source-separation visualization +def separation( + sources, + fs=22050, + labels=None, + alpha=0.75, + ax=None, + rasterized=True, + edgecolors="None", + shading="gouraud", + prop_cycle=None, + **kwargs, +): + """Source-separation visualization Parameters ---------- sources : np.ndarray, shape=(nsrc, nsampl) A list of waveform buffers corresponding to each source - fs : number > 0 The sampling rate - labels : list of strings An optional list of descriptors corresponding to each source - alpha : float in [0, 1] Maximum alpha (opacity) of spectrogram values. - ax : matplotlib.pyplot.axes An axis handle on which to draw the spectrograms. If none is provided, a new set of axes is created. - - kwargs + rasterized : bool + If `True`, the spectrogram is rasterized. + edgecolors : str or None + The color of the edges of the spectrogram patches. + Set to "None" (default) to disable edge coloring. + shading : str + The shading method to use for the spectrogram. + See `matplotlib.pyplot.pcolormesh` for valid options. + prop_cycle : cycle.Cycler + An optional property cycle object to specify colors for each signal. + If not provided, the default property cycler will be retrieved from matplotlib. + **kwargs Additional keyword arguments to ``scipy.signal.spectrogram`` Returns ------- ax The axis handle for this plot - ''' - + """ # Get the axes handle ax, new_axes = __get_axes(ax=ax) @@ -754,9 +859,9 @@ def separation(sources, fs=22050, labels=None, alpha=0.75, ax=None, **kwargs): sources = np.atleast_2d(sources) if labels is None: - labels = ['Source {:d}'.format(_) for _ in range(len(sources))] + labels = [f"Source {_:d}" for _ in range(len(sources))] - kwargs.setdefault('scaling', 'spectrum') + kwargs.setdefault("scaling", "spectrum") # The cumulative spectrogram across sources # is used to establish the reference power @@ -777,41 +882,60 @@ def separation(sources, fs=22050, labels=None, alpha=0.75, ax=None, **kwargs): color_conv = ColorConverter() - for i, spec in enumerate(specs): + if prop_cycle is None: + __AXMAP[ax].setdefault("prop_cycle", mpl.rcParams["axes.prop_cycle"]) + __AXMAP[ax].setdefault("prop_iter", iter(mpl.rcParams["axes.prop_cycle"])) + elif "prop_iter" not in __AXMAP[ax]: + __AXMAP[ax]["prop_cycle"] = prop_cycle + __AXMAP[ax]["prop_iter"] = iter(prop_cycle) + + prop_cycle = __AXMAP[ax]["prop_cycle"] + prop_iter = __AXMAP[ax]["prop_iter"] + for i, spec in enumerate(specs): # For each source, grab a new color from the cycler # Then construct a colormap that interpolates from # [transparent white -> new color] - color = next(ax._get_lines.prop_cycler)['color'] - color = color_conv.to_rgba(color, alpha=alpha) - cmap = LinearSegmentedColormap.from_list(labels[i], - [(1.0, 1.0, 1.0, 0.0), - color]) - - ax.pcolormesh(times, freqs, spec, - cmap=cmap, - norm=LogNorm(vmin=ref_min, vmax=ref_max), - shading='gouraud', - label=labels[i]) + # Advance the property iterator if we can, restart it if we must + try: + properties = next(prop_iter) + except StopIteration: + prop_iter = iter(prop_cycle) + __AXMAP[ax]["prop_iter"] = prop_iter + properties = next(prop_iter) + + color = color_conv.to_rgba(properties["color"], alpha=alpha) + cmap = LinearSegmentedColormap.from_list( + labels[i], [(1.0, 1.0, 1.0, 0.0), color] + ) + + ax.pcolormesh( + times, + freqs, + spec, + cmap=cmap, + norm=LogNorm(vmin=ref_min, vmax=ref_max), + rasterized=rasterized, + edgecolors=edgecolors, + shading=shading, + ) # Attach a 0x0 rect to the axis with the corresponding label # This way, it will show up in the legend - ax.add_patch(Rectangle((0, 0), 0, 0, color=color, label=labels[i])) - - if new_axes: - ax.axis('tight') + ax.add_patch( + Rectangle((times.min(), freqs.min()), 0, 0, color=color, label=labels[i]) + ) return ax def __ticker_midi_note(x, pos): - '''A ticker function for midi notes. + """Format midi notes for ticker decoration. Inputs x are interpreted as midi numbers, and converted to [NOTE][OCTAVE]+[cents]. - ''' - - NOTES = ['C', 'C#', 'D', 'D#', 'E', 'F', 'F#', 'G', 'G#', 'A', 'A#', 'B'] + """ + NOTES = ["C", "C#", "D", "D#", "E", "F", "F#", "G", "G#", "A", "A#", "B"] cents = float(np.mod(x, 1.0)) if cents >= 0.5: @@ -823,22 +947,21 @@ def __ticker_midi_note(x, pos): octave = int(x / 12) - 1 if cents == 0: - return '{:s}{:2d}'.format(NOTES[idx], octave) - return '{:s}{:2d}{:+02d}'.format(NOTES[idx], octave, int(cents * 100)) + return f"{NOTES[idx]:s}{octave:2d}" + return f"{NOTES[idx]:s}{octave:2d}{int(cents * 100):+02d}" def __ticker_midi_hz(x, pos): - '''A ticker function for midi pitches. + """Format midi pitches for ticker decoration. Inputs x are interpreted as midi numbers, and converted to Hz. - ''' - - return '{:g}'.format(midi_to_hz(x)) + """ + return f"{midi_to_hz(x):g}" def ticker_notes(ax=None): - '''Set the y-axis of the given axes to MIDI notes + """Set the y-axis of the given axes to MIDI notes Parameters ---------- @@ -846,24 +969,24 @@ def ticker_notes(ax=None): The axes handle to apply the ticker. By default, uses the current axes handle. - ''' + """ ax, _ = __get_axes(ax=ax) ax.yaxis.set_major_formatter(FMT_MIDI_NOTE) # Get the tick labels and reset the vertical alignment for tick in ax.yaxis.get_ticklabels(): - tick.set_verticalalignment('baseline') + tick.set_verticalalignment("baseline") def ticker_pitch(ax=None): - '''Set the y-axis of the given axes to MIDI frequencies + """Set the y-axis of the given axes to MIDI frequencies Parameters ---------- ax : matplotlib.pyplot.axes The axes handle to apply the ticker. By default, uses the current axes handle. - ''' + """ ax, _ = __get_axes(ax=ax) ax.yaxis.set_major_formatter(FMT_MIDI_HZ) diff --git a/mir_eval/hierarchy.py b/mir_eval/hierarchy.py index c5015970..255a6e7a 100644 --- a/mir_eval/hierarchy.py +++ b/mir_eval/hierarchy.py @@ -1,6 +1,5 @@ # CREATED:2015-09-16 14:46:47 by Brian McFee -# -*- encoding: utf-8 -*- -'''Evaluation criteria for hierarchical structure analysis. +"""Evaluation criteria for hierarchical structure analysis. Hierarchical structure analysis seeks to annotate a track with a nested decomposition of the temporal elements of the piece, effectively providing @@ -39,7 +38,7 @@ Juan P. Bello. "Evaluating hierarchical structure in music annotations", Frontiers in Psychology, 2017. -''' +""" import collections import itertools @@ -53,7 +52,7 @@ def _round(t, frame_size): - '''Round a time-stamp to a specified resolution. + """Round a time-stamp to a specified resolution. Equivalent to ``t - np.mod(t, frame_size)``. @@ -68,7 +67,6 @@ def _round(t, frame_size): ---------- t : number or ndarray The time-stamp to round - frame_size : number > 0 The resolution to round to @@ -76,12 +74,12 @@ def _round(t, frame_size): ------- t_round : number The rounded time-stamp - ''' + """ return t - np.mod(t, float(frame_size)) def _hierarchy_bounds(intervals_hier): - '''Compute the covered time range of a hierarchical segmentation. + """Compute the covered time range of a hierarchical segmentation. Parameters ---------- @@ -94,14 +92,14 @@ def _hierarchy_bounds(intervals_hier): t_min : float t_max : float The minimum and maximum times spanned by the annotation - ''' + """ boundaries = list(itertools.chain(*list(itertools.chain(*intervals_hier)))) return min(boundaries), max(boundaries) def _align_intervals(int_hier, lab_hier, t_min=0.0, t_max=None): - '''Align a hierarchical annotation to span a fixed start and end time. + """Align a hierarchical annotation to span a fixed start and end time. Parameters ---------- @@ -110,10 +108,8 @@ def _align_intervals(int_hier, lab_hier, t_min=0.0, t_max=None): Hierarchical segment annotations, encoded as a list of list of intervals (int_hier) and list of list of strings (lab_hier) - t_min : None or number >= 0 The minimum time value for the segmentation - t_max : None or number >= t_min The maximum time value for the segmentation @@ -122,16 +118,22 @@ def _align_intervals(int_hier, lab_hier, t_min=0.0, t_max=None): intervals_hier : list of list of intervals labels_hier : list of list of str `int_hier` `lab_hier` aligned to span `[t_min, t_max]`. - ''' - return [list(_) for _ in zip(*[util.adjust_intervals(np.asarray(ival), - labels=lab, - t_min=t_min, - t_max=t_max) - for ival, lab in zip(int_hier, lab_hier)])] + """ + return [ + list(_) + for _ in zip( + *[ + util.adjust_intervals( + np.asarray(ival), labels=lab, t_min=t_min, t_max=t_max + ) + for ival, lab in zip(int_hier, lab_hier) + ] + ) + ] def _lca(intervals_hier, frame_size): - '''Compute the (sparse) least-common-ancestor (LCA) matrix for a + """Compute the (sparse) least-common-ancestor (LCA) matrix for a hierarchical segmentation. For any pair of frames ``(s, t)``, the LCA is the deepest level in @@ -143,7 +145,6 @@ def _lca(intervals_hier, frame_size): intervals_hier : list of ndarray An ordered list of segment interval arrays. The list is assumed to be ordered by increasing specificity (depth). - frame_size : number The length of the sample frames (in seconds) @@ -152,23 +153,22 @@ def _lca(intervals_hier, frame_size): lca_matrix : scipy.sparse.csr_matrix A sparse matrix such that ``lca_matrix[i, j]`` contains the depth of the deepest segment containing frames ``i`` and ``j``. - ''' - + """ frame_size = float(frame_size) # Figure out how many frames we need n_start, n_end = _hierarchy_bounds(intervals_hier) - n = int((_round(n_end, frame_size) - - _round(n_start, frame_size)) / frame_size) + n = int((_round(n_end, frame_size) - _round(n_start, frame_size)) / frame_size) # Initialize the LCA matrix lca_matrix = scipy.sparse.lil_matrix((n, n), dtype=np.uint8) for level, intervals in enumerate(intervals_hier, 1): - for ival in (_round(np.asarray(intervals), - frame_size) / frame_size).astype(int): + for ival in (_round(np.asarray(intervals), frame_size) / frame_size).astype( + int + ): idx = slice(ival[0], ival[1]) lca_matrix[idx, idx] = level @@ -176,7 +176,7 @@ def _lca(intervals_hier, frame_size): def _meet(intervals_hier, labels_hier, frame_size): - '''Compute the (sparse) least-common-ancestor (LCA) matrix for a + """Compute the (sparse) least-common-ancestor (LCA) matrix for a hierarchical segmentation. For any pair of frames ``(s, t)``, the LCA is the deepest level in @@ -188,11 +188,9 @@ def _meet(intervals_hier, labels_hier, frame_size): intervals_hier : list of ndarray An ordered list of segment interval arrays. The list is assumed to be ordered by increasing specificity (depth). - labels_hier : list of list of str ``labels_hier[i]`` contains the segment labels for the ``i``th layer of the annotations - frame_size : number The length of the sample frames (in seconds) @@ -201,23 +199,19 @@ def _meet(intervals_hier, labels_hier, frame_size): meet_matrix : scipy.sparse.csr_matrix A sparse matrix such that ``meet_matrix[i, j]`` contains the depth of the deepest segment label containing both ``i`` and ``j``. - ''' - + """ frame_size = float(frame_size) # Figure out how many frames we need n_start, n_end = _hierarchy_bounds(intervals_hier) - n = int((_round(n_end, frame_size) - - _round(n_start, frame_size)) / frame_size) + n = int((_round(n_end, frame_size) - _round(n_start, frame_size)) / frame_size) # Initialize the meet matrix meet_matrix = scipy.sparse.lil_matrix((n, n), dtype=np.uint8) - for level, (intervals, labels) in enumerate(zip(intervals_hier, - labels_hier), 1): - + for level, (intervals, labels) in enumerate(zip(intervals_hier, labels_hier), 1): # Encode the labels at this level lab_enc = util.index_labels(labels)[0] @@ -228,7 +222,7 @@ def _meet(intervals_hier, labels_hier, frame_size): int_frames = (_round(intervals, frame_size) / frame_size).astype(int) # For each intervals i, j where labels agree, update the meet matrix - for (seg_i, seg_j) in zip(*np.where(int_agree)): + for seg_i, seg_j in zip(*np.where(int_agree)): idx_i = slice(*list(int_frames[seg_i])) idx_j = slice(*list(int_frames[seg_j])) meet_matrix[idx_i, idx_j] = level @@ -239,7 +233,7 @@ def _meet(intervals_hier, labels_hier, frame_size): def _gauc(ref_lca, est_lca, transitive, window): - '''Generalized area under the curve (GAUC) + """Generalized area under the curve (GAUC) This function computes the normalized recall score for correctly ordering triples ``(q, i, j)`` where frames ``(q, i)`` are closer than @@ -248,6 +242,7 @@ def _gauc(ref_lca, est_lca, transitive, window): Parameters ---------- ref_lca : scipy.sparse + est_lca : scipy.sparse The least common ancestor matrices for the reference and estimated annotations @@ -273,12 +268,13 @@ def _gauc(ref_lca, est_lca, transitive, window): ------ ValueError If ``ref_lca`` and ``est_lca`` have different shapes - ''' + """ # Make sure we have the right number of frames if ref_lca.shape != est_lca.shape: - raise ValueError('Estimated and reference hierarchies ' - 'must have the same shape.') + raise ValueError( + "Estimated and reference hierarchies " "must have the same shape." + ) # How many frames? n = ref_lca.shape[0] @@ -294,7 +290,6 @@ def _gauc(ref_lca, est_lca, transitive, window): num_frames = 0 for query in range(n): - # Find all pairs i,j such that ref_lca[q, i] > ref_lca[q, j] results = slice(max(0, query - window), min(n, query + window)) @@ -311,11 +306,12 @@ def _gauc(ref_lca, est_lca, transitive, window): # (this also holds when the slice goes off the end of the array.) idx = min(query, window) - ref_score = np.concatenate((ref_score[:idx], ref_score[idx+1:])) - est_score = np.concatenate((est_score[:idx], est_score[idx+1:])) + ref_score = np.concatenate((ref_score[:idx], ref_score[idx + 1 :])) + est_score = np.concatenate((est_score[:idx], est_score[idx + 1 :])) - inversions, normalizer = _compare_frame_rankings(ref_score, est_score, - transitive=transitive) + inversions, normalizer = _compare_frame_rankings( + ref_score, est_score, transitive=transitive + ) if normalizer: score += 1.0 - inversions / float(normalizer) @@ -332,7 +328,7 @@ def _gauc(ref_lca, est_lca, transitive, window): def _count_inversions(a, b): - '''Count the number of inversions in two numpy arrays: + """Count the number of inversions in two numpy arrays: # points i, j where a[i] >= b[j] @@ -348,8 +344,7 @@ def _count_inversions(a, b): ------- inversions : int The number of detected inversions - ''' - + """ a, a_counts = np.unique(a, return_counts=True) b, b_counts = np.unique(b, return_counts=True) @@ -368,7 +363,7 @@ def _count_inversions(a, b): def _compare_frame_rankings(ref, est, transitive=False): - '''Compute the number of ranking disagreements in two lists. + """Compute the number of ranking disagreements in two lists. Parameters ---------- @@ -376,7 +371,6 @@ def _compare_frame_rankings(ref, est, transitive=False): est : np.ndarray, shape=(n,) Reference and estimate ranked lists. `ref[i]` is the relevance score for point `i`. - transitive : bool If true, all pairs of reference levels are compared. If false, only adjacent pairs of reference levels are compared. @@ -386,21 +380,19 @@ def _compare_frame_rankings(ref, est, transitive=False): inversions : int The number of pairs of indices `i, j` where `ref[i] < ref[j]` but `est[i] >= est[j]`. - normalizer : float The total number of pairs (i, j) under consideration. If transitive=True, then this is |{(i,j) : ref[i] < ref[j]}| If transitive=False, then this is |{i,j) : ref[i] +1 = ref[j]}| - ''' - + """ idx = np.argsort(ref) ref_sorted = ref[idx] est_sorted = est[idx] # Find the break-points in ref_sorted - levels, positions, counts = np.unique(ref_sorted, - return_index=True, - return_counts=True) + levels, positions, counts = np.unique( + ref_sorted, return_index=True, return_counts=True + ) positions = list(positions) positions.append(len(ref_sorted)) @@ -408,8 +400,7 @@ def _compare_frame_rankings(ref, est, transitive=False): index = collections.defaultdict(lambda: slice(0)) ref_map = collections.defaultdict(lambda: 0) - for level, cnt, start, end in zip(levels, counts, - positions[:-1], positions[1:]): + for level, cnt, start, end in zip(levels, counts, positions[:-1], positions[1:]): index[level] = slice(start, end) ref_map[level] = cnt @@ -418,7 +409,7 @@ def _compare_frame_rankings(ref, est, transitive=False): if transitive: level_pairs = itertools.combinations(levels, 2) else: - level_pairs = [(i, i+1) for i in levels] + level_pairs = [(i, i + 1) for i in levels] level_pairs, lcounter = itertools.tee(level_pairs) @@ -430,14 +421,15 @@ def _compare_frame_rankings(ref, est, transitive=False): inversions = 0 for level_1, level_2 in level_pairs: - inversions += _count_inversions(est_sorted[index[level_1]], - est_sorted[index[level_2]]) + inversions += _count_inversions( + est_sorted[index[level_1]], est_sorted[index[level_2]] + ) return inversions, float(normalizer) def validate_hier_intervals(intervals_hier): - '''Validate a hierarchical segment annotation. + """Validate a hierarchical segment annotation. Parameters ---------- @@ -450,8 +442,7 @@ def validate_hier_intervals(intervals_hier): segmentation. If any segmentation does not start at 0. - ''' - + """ # Synthesize a label array for the top layer. label_top = util.generate_labels(intervals_hier[0]) @@ -460,21 +451,27 @@ def validate_hier_intervals(intervals_hier): for level, intervals in enumerate(intervals_hier[1:], 1): # Make sure this level is consistent with the root label_current = util.generate_labels(intervals) - validate_structure(intervals_hier[0], label_top, - intervals, label_current) + validate_structure(intervals_hier[0], label_top, intervals, label_current) # Make sure all previous boundaries are accounted for new_bounds = set(util.intervals_to_boundaries(intervals)) if boundaries - new_bounds: - warnings.warn('Segment hierarchy is inconsistent ' - 'at level {:d}'.format(level)) + warnings.warn( + "Segment hierarchy is inconsistent " "at level {:d}".format(level) + ) boundaries |= new_bounds -def tmeasure(reference_intervals_hier, estimated_intervals_hier, - transitive=False, window=15.0, frame_size=0.1, beta=1.0): - '''Computes the tree measures for hierarchical segment annotations. +def tmeasure( + reference_intervals_hier, + estimated_intervals_hier, + transitive=False, + window=15.0, + frame_size=0.1, + beta=1.0, +): + """Compute the tree measures for hierarchical segment annotations. Parameters ---------- @@ -483,21 +480,16 @@ def tmeasure(reference_intervals_hier, estimated_intervals_hier, (in seconds) for the ``i`` th layer of the annotations. Layers are ordered from top to bottom, so that the last list of intervals should be the most specific. - estimated_intervals_hier : list of ndarray Like ``reference_intervals_hier`` but for the estimated annotation - transitive : bool whether to compute the t-measures using transitivity or not. - window : float > 0 size of the window (in seconds). For each query frame q, result frames are only counted within q +- window. - frame_size : float > 0 length (in seconds) of frames. The frame size cannot be longer than the window. - beta : float > 0 beta parameter for the F-measure. @@ -505,10 +497,8 @@ def tmeasure(reference_intervals_hier, estimated_intervals_hier, ------- t_precision : number [0, 1] T-measure Precision - t_recall : number [0, 1] T-measure Recall - t_measure : number [0, 1] F-beta measure for ``(t_precision, t_recall)`` @@ -520,19 +510,21 @@ def tmeasure(reference_intervals_hier, estimated_intervals_hier, If the input hierarchies have different time durations If ``frame_size > window`` or ``frame_size <= 0`` - ''' - + """ # Compute the number of frames in the window if frame_size <= 0: - raise ValueError('frame_size ({:.2f}) must be a positive ' - 'number.'.format(frame_size)) + raise ValueError( + "frame_size ({:.2f}) must be a positive " "number.".format(frame_size) + ) if window is None: window_frames = None else: if frame_size > window: - raise ValueError('frame_size ({:.2f}) cannot exceed ' - 'window ({:.2f})'.format(frame_size, window)) + raise ValueError( + "frame_size ({:.2f}) cannot exceed " + "window ({:.2f})".format(frame_size, window) + ) window_frames = int(_round(window, frame_size) / frame_size) @@ -553,10 +545,15 @@ def tmeasure(reference_intervals_hier, estimated_intervals_hier, return t_precision, t_recall, t_measure -def lmeasure(reference_intervals_hier, reference_labels_hier, - estimated_intervals_hier, estimated_labels_hier, - frame_size=0.1, beta=1.0): - '''Computes the tree measures for hierarchical segment annotations. +def lmeasure( + reference_intervals_hier, + reference_labels_hier, + estimated_intervals_hier, + estimated_labels_hier, + frame_size=0.1, + beta=1.0, +): + """Compute the tree measures for hierarchical segment annotations. Parameters ---------- @@ -565,20 +562,16 @@ def lmeasure(reference_intervals_hier, reference_labels_hier, (in seconds) for the ``i`` th layer of the annotations. Layers are ordered from top to bottom, so that the last list of intervals should be the most specific. - reference_labels_hier : list of list of str ``reference_labels_hier[i]`` contains the segment labels for the - ``i``th layer of the annotations - + ``i`` th layer of the annotations estimated_intervals_hier : list of ndarray estimated_labels_hier : list of ndarray Like ``reference_intervals_hier`` and ``reference_labels_hier`` but for the estimated annotation - frame_size : float > 0 length (in seconds) of frames. The frame size cannot be longer than the window. - beta : float > 0 beta parameter for the F-measure. @@ -586,10 +579,8 @@ def lmeasure(reference_intervals_hier, reference_labels_hier, ------- l_precision : number [0, 1] L-measure Precision - l_recall : number [0, 1] L-measure Recall - l_measure : number [0, 1] F-beta measure for ``(l_precision, l_recall)`` @@ -601,22 +592,20 @@ def lmeasure(reference_intervals_hier, reference_labels_hier, If the input hierarchies have different time durations If ``frame_size > window`` or ``frame_size <= 0`` - ''' - + """ # Compute the number of frames in the window if frame_size <= 0: - raise ValueError('frame_size ({:.2f}) must be a positive ' - 'number.'.format(frame_size)) + raise ValueError( + "frame_size ({:.2f}) must be a positive " "number.".format(frame_size) + ) # Validate the hierarchical segmentations validate_hier_intervals(reference_intervals_hier) validate_hier_intervals(estimated_intervals_hier) # Build the least common ancestor matrices - ref_meet = _meet(reference_intervals_hier, reference_labels_hier, - frame_size) - est_meet = _meet(estimated_intervals_hier, estimated_labels_hier, - frame_size) + ref_meet = _meet(reference_intervals_hier, reference_labels_hier, frame_size) + est_meet = _meet(estimated_intervals_hier, estimated_labels_hier, frame_size) # Compute precision and recall l_recall = _gauc(ref_meet, est_meet, True, None) @@ -627,9 +616,10 @@ def lmeasure(reference_intervals_hier, reference_labels_hier, return l_precision, l_recall, l_measure -def evaluate(ref_intervals_hier, ref_labels_hier, - est_intervals_hier, est_labels_hier, **kwargs): - '''Compute all hierarchical structure metrics for the given reference and +def evaluate( + ref_intervals_hier, ref_labels_hier, est_intervals_hier, est_labels_hier, **kwargs +): + r"""Compute all hierarchical structure metrics for the given reference and estimated annotations. Examples @@ -676,7 +666,6 @@ def evaluate(ref_intervals_hier, ref_labels_hier, 'T-Recall full': 0.6523334654992341, 'T-Recall reduced': 0.60799919710921635} - Parameters ---------- ref_intervals_hier : list of list-like @@ -687,13 +676,12 @@ def evaluate(ref_intervals_hier, ref_labels_hier, of segmentations. Each segmentation itself is a list (or list-like) of intervals (\*_intervals_hier) and a list of lists of labels (\*_labels_hier). - - kwargs + **kwargs additional keyword arguments to the evaluation metrics. Returns ------- - scores : OrderedDict + scores : OrderedDict Dictionary of scores, where the key is the metric name (str) and the value is the (float) score achieved. @@ -704,48 +692,47 @@ def evaluate(ref_intervals_hier, ref_labels_hier, ------ ValueError Thrown when the provided annotations are not valid. - ''' - + """ # First, find the maximum length of the reference _, t_end = _hierarchy_bounds(ref_intervals_hier) # Pre-process the intervals to match the range of the reference, # and start at 0 - ref_intervals_hier, ref_labels_hier = _align_intervals(ref_intervals_hier, - ref_labels_hier, - t_min=0.0, - t_max=None) + ref_intervals_hier, ref_labels_hier = _align_intervals( + ref_intervals_hier, ref_labels_hier, t_min=0.0, t_max=None + ) - est_intervals_hier, est_labels_hier = _align_intervals(est_intervals_hier, - est_labels_hier, - t_min=0.0, - t_max=t_end) + est_intervals_hier, est_labels_hier = _align_intervals( + est_intervals_hier, est_labels_hier, t_min=0.0, t_max=t_end + ) scores = collections.OrderedDict() # Force the transitivity setting - kwargs['transitive'] = False - (scores['T-Precision reduced'], - scores['T-Recall reduced'], - scores['T-Measure reduced']) = util.filter_kwargs(tmeasure, - ref_intervals_hier, - est_intervals_hier, - **kwargs) - - kwargs['transitive'] = True - (scores['T-Precision full'], - scores['T-Recall full'], - scores['T-Measure full']) = util.filter_kwargs(tmeasure, - ref_intervals_hier, - est_intervals_hier, - **kwargs) - - (scores['L-Precision'], - scores['L-Recall'], - scores['L-Measure']) = util.filter_kwargs(lmeasure, - ref_intervals_hier, - ref_labels_hier, - est_intervals_hier, - est_labels_hier, - **kwargs) + kwargs["transitive"] = False + ( + scores["T-Precision reduced"], + scores["T-Recall reduced"], + scores["T-Measure reduced"], + ) = util.filter_kwargs(tmeasure, ref_intervals_hier, est_intervals_hier, **kwargs) + + kwargs["transitive"] = True + ( + scores["T-Precision full"], + scores["T-Recall full"], + scores["T-Measure full"], + ) = util.filter_kwargs(tmeasure, ref_intervals_hier, est_intervals_hier, **kwargs) + + ( + scores["L-Precision"], + scores["L-Recall"], + scores["L-Measure"], + ) = util.filter_kwargs( + lmeasure, + ref_intervals_hier, + ref_labels_hier, + est_intervals_hier, + est_labels_hier, + **kwargs + ) return scores diff --git a/mir_eval/io.py b/mir_eval/io.py index 6529e571..1f508691 100644 --- a/mir_eval/io.py +++ b/mir_eval/io.py @@ -1,13 +1,10 @@ -""" -Functions for loading in annotations from files in different formats. -""" +"""Functions for loading annotations from files in different formats.""" import contextlib import numpy as np import re import warnings import scipy.io.wavfile -import six from . import util from . import key @@ -15,28 +12,26 @@ @contextlib.contextmanager -def _open(file_or_str, **kwargs): - '''Either open a file handle, or use an existing file-like object. +def _open(file_or_path, **kwargs): + """Either open a file handle, or use an existing file-like object. - This will behave as the `open` function if `file_or_str` is a string. + If `file_or_path` has the `read` attribute, it will return `file_or_path`. - If `file_or_str` has the `read` attribute, it will return `file_or_str`. - - Otherwise, an `IOError` is raised. - ''' - if hasattr(file_or_str, 'read'): - yield file_or_str - elif isinstance(file_or_str, six.string_types): - with open(file_or_str, **kwargs) as file_desc: - yield file_desc + Otherwise, it will attempt to open the file at the specified location. + """ + if hasattr(file_or_path, "read"): + yield file_or_path else: - raise IOError('Invalid file-or-str object: {}'.format(file_or_str)) + try: + with open(file_or_path, **kwargs) as file_desc: + yield file_desc + except TypeError as exc: + raise IOError(f"Invalid file-or-path object: {file_or_path}") from exc -def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): - r"""Utility function for loading in data from an annotation file where columns - are delimited. The number of columns is inferred from the length of - the provided converters list. +def load_delimited(filename, converters, delimiter=r"\s+", comment="#"): + r"""Load data from an annotation file where columns are delimited. + The number of columns is inferred from the length of the provided converters list. Examples -------- @@ -47,14 +42,17 @@ def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + converters : list of functions Each entry in column ``n`` of the file will be cast by the function ``converters[n]``. + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -66,7 +64,6 @@ def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): columns : tuple of lists Each list in this tuple corresponds to values in one of the columns in the file. - """ # Initialize list of empty lists n_columns = len(converters) @@ -79,7 +76,7 @@ def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): if comment is None: commenter = None else: - commenter = re.compile('^{}'.format(comment)) + commenter = re.compile(f"^{comment}") # Note: we do io manually here for two reasons. # 1. The csv module has difficulties with unicode, which may lead @@ -87,7 +84,7 @@ def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): # # 2. numpy's text loader does not handle non-numeric data # - with _open(filename, mode='r') as input_file: + with _open(filename, mode="r") as input_file: for row, line in enumerate(input_file, 1): # Skip commented lines if comment is not None and commenter.match(line): @@ -98,19 +95,22 @@ def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): # Throw a helpful error if we got an unexpected # of columns if n_columns != len(data): - raise ValueError('Expected {} columns, got {} at ' - '{}:{:d}:\n\t{}'.format(n_columns, len(data), - filename, row, line)) + raise ValueError( + "Expected {} columns, got {} at " + "{}:{:d}:\n\t{}".format(n_columns, len(data), filename, row, line) + ) for value, column, converter in zip(data, columns, converters): # Try converting the value, throw a helpful error on failure try: converted_value = converter(value) except: - raise ValueError("Couldn't convert value {} using {} " - "found at {}:{:d}:\n\t{}".format( - value, converter.__name__, filename, - row, line)) + raise ValueError( + "Couldn't convert value {} using {} " + "found at {}:{:d}:\n\t{}".format( + value, converter.__name__, filename, row, line + ) + ) column.append(converted_value) # Sane output @@ -120,7 +120,7 @@ def load_delimited(filename, converters, delimiter=r'\s+', comment='#'): return columns -def load_events(filename, delimiter=r'\s+', comment='#'): +def load_events(filename, delimiter=r"\s+", comment="#"): r"""Import time-stamp events from an annotation file. The file should consist of a single column of numeric values corresponding to the event times. This is primarily useful for processing events which lack duration, @@ -128,11 +128,13 @@ def load_events(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -146,8 +148,7 @@ def load_events(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load in the events - events = load_delimited(filename, [float], - delimiter=delimiter, comment=comment) + events = load_delimited(filename, [float], delimiter=delimiter, comment=comment) events = np.array(events) # Validate them, but throw a warning in place of an error try: @@ -158,7 +159,7 @@ def load_events(filename, delimiter=r'\s+', comment='#'): return events -def load_labeled_events(filename, delimiter=r'\s+', comment='#'): +def load_labeled_events(filename, delimiter=r"\s+", comment="#"): r"""Import labeled time-stamp events from an annotation file. The file should consist of two columns; the first having numeric values corresponding to the event times and the second having string labels for each event. This @@ -167,11 +168,13 @@ def load_labeled_events(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -187,9 +190,9 @@ def load_labeled_events(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load in the events - events, labels = load_delimited(filename, [float, str], - delimiter=delimiter, - comment=comment) + events, labels = load_delimited( + filename, [float, str], delimiter=delimiter, comment=comment + ) events = np.array(events) # Validate them, but throw a warning in place of an error try: @@ -200,7 +203,7 @@ def load_labeled_events(filename, delimiter=r'\s+', comment='#'): return events, labels -def load_intervals(filename, delimiter=r'\s+', comment='#'): +def load_intervals(filename, delimiter=r"\s+", comment="#"): r"""Import intervals from an annotation file. The file should consist of two columns of numeric values corresponding to start and end time of each interval. This is primarily useful for processing events which span a @@ -208,11 +211,13 @@ def load_intervals(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -226,9 +231,9 @@ def load_intervals(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load in the events - starts, ends = load_delimited(filename, [float, float], - delimiter=delimiter, - comment=comment) + starts, ends = load_delimited( + filename, [float, float], delimiter=delimiter, comment=comment + ) # Stack into an interval matrix intervals = np.array([starts, ends]).T # Validate them, but throw a warning in place of an error @@ -240,7 +245,7 @@ def load_intervals(filename, delimiter=r'\s+', comment='#'): return intervals -def load_labeled_intervals(filename, delimiter=r'\s+', comment='#'): +def load_labeled_intervals(filename, delimiter=r"\s+", comment="#"): r"""Import labeled intervals from an annotation file. The file should consist of three columns: Two consisting of numeric values corresponding to start and end time of each interval and a third corresponding to the label of @@ -249,11 +254,13 @@ def load_labeled_intervals(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -269,9 +276,9 @@ def load_labeled_intervals(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load in the events - starts, ends, labels = load_delimited(filename, [float, float, str], - delimiter=delimiter, - comment=comment) + starts, ends, labels = load_delimited( + filename, [float, float, str], delimiter=delimiter, comment=comment + ) # Stack into an interval matrix intervals = np.array([starts, ends]).T # Validate them, but throw a warning in place of an error @@ -283,18 +290,20 @@ def load_labeled_intervals(filename, delimiter=r'\s+', comment='#'): return intervals, labels -def load_time_series(filename, delimiter=r'\s+', comment='#'): +def load_time_series(filename, delimiter=r"\s+", comment="#"): r"""Import a time series from an annotation file. The file should consist of two columns of numeric values corresponding to the time and value of each sample of the time series. Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -310,9 +319,9 @@ def load_time_series(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load in the events - times, values = load_delimited(filename, [float, float], - delimiter=delimiter, - comment=comment) + times, values = load_delimited( + filename, [float, float], delimiter=delimiter, comment=comment + ) times = np.array(times) values = np.array(values) @@ -320,7 +329,7 @@ def load_time_series(filename, delimiter=r'\s+', comment='#'): def load_patterns(filename): - """Loads the patters contained in the filename and puts them into a list + """Load the patterns contained in the filename and puts them into a list of patterns, each pattern being a list of occurrence, and each occurrence being a list of (onset, midi) pairs. @@ -329,7 +338,7 @@ def load_patterns(filename): Parameters ---------- - filename : str + filename : str or `os.Pathlike` The input file path containing the patterns of a given piece using the MIREX 2013 format. @@ -361,16 +370,14 @@ def load_patterns(filename): pattern2 = [occ1, occ2] pattern_list = [pattern1, pattern2] - """ - # List with all the patterns pattern_list = [] # Current pattern, which will contain all occs pattern = [] # Current occurrence, containing (onset, midi) occurrence = [] - with _open(filename, mode='r') as input_file: + with _open(filename, mode="r") as input_file: for line in input_file.readlines(): if "pattern" in line: if occurrence != []: @@ -398,12 +405,18 @@ def load_patterns(filename): return pattern_list +@util.deprecated(version="0.8.1", version_removed="0.9.0") def load_wav(path, mono=True): - """Loads a .wav file as a numpy array using ``scipy.io.wavfile``. + """Load a .wav file as a numpy array using ``scipy.io.wavfile``. + + .. warning:: This function is deprecatred in mir_eval 0.8.1 + and will be removed in 0.9.0. + We recommend using a dedicated audio IO library such as + `soundfile` instead. Parameters ---------- - path : str + path : str or `os.Pathlike` Path to a .wav file mono : bool If the provided .wav has more than one channel, it will be @@ -415,27 +428,24 @@ def load_wav(path, mono=True): Array of audio samples, normalized to the range [-1., 1.] fs : int Sampling rate of the audio data - """ - fs, audio_data = scipy.io.wavfile.read(path) # Make float in range [-1, 1] - if audio_data.dtype == 'int8': - audio_data = audio_data/float(2**8) - elif audio_data.dtype == 'int16': - audio_data = audio_data/float(2**16) - elif audio_data.dtype == 'int32': - audio_data = audio_data/float(2**24) + if audio_data.dtype == "int8": + audio_data = audio_data / float(2**8) + elif audio_data.dtype == "int16": + audio_data = audio_data / float(2**16) + elif audio_data.dtype == "int32": + audio_data = audio_data / float(2**24) else: - raise ValueError('Got unexpected .wav data type ' - '{}'.format(audio_data.dtype)) + raise ValueError("Got unexpected .wav data type " "{}".format(audio_data.dtype)) # Optionally convert to mono if mono and audio_data.ndim != 1: audio_data = audio_data.mean(axis=1) return audio_data, fs -def load_valued_intervals(filename, delimiter=r'\s+', comment='#'): +def load_valued_intervals(filename, delimiter=r"\s+", comment="#"): r"""Import valued intervals from an annotation file. The file should consist of three columns: Two consisting of numeric values corresponding to start and end time of each interval and a third, also of numeric values, @@ -445,11 +455,13 @@ def load_valued_intervals(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -465,9 +477,9 @@ def load_valued_intervals(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load in the events - starts, ends, values = load_delimited(filename, [float, float, float], - delimiter=delimiter, - comment=comment) + starts, ends, values = load_delimited( + filename, [float, float, float], delimiter=delimiter, comment=comment + ) # Stack into an interval matrix intervals = np.array([starts, ends]).T # Validate them, but throw a warning in place of an error @@ -482,7 +494,7 @@ def load_valued_intervals(filename, delimiter=r'\s+', comment='#'): return intervals, values -def load_key(filename, delimiter=r'\s+', comment='#'): +def load_key(filename, delimiter=r"\s+", comment="#"): r"""Load key labels from an annotation file. The file should consist of two string columns: One denoting the key scale degree (semitone), and the other denoting the mode (major or minor). The file @@ -490,11 +502,13 @@ def load_key(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -508,14 +522,14 @@ def load_key(filename, delimiter=r'\s+', comment='#'): """ # Use our universal function to load the key and mode strings - scale, mode = load_delimited(filename, [str, str], - delimiter=delimiter, - comment=comment) + scale, mode = load_delimited( + filename, [str, str], delimiter=delimiter, comment=comment + ) if len(scale) != 1: - raise ValueError('Key file should contain only one line.') + raise ValueError("Key file should contain only one line.") scale, mode = scale[0], mode[0] # Join with a space - key_string = '{} {}'.format(scale, mode) + key_string = f"{scale} {mode}" # Validate them, but throw a warning in place of an error try: key.validate_key(key_string) @@ -525,7 +539,7 @@ def load_key(filename, delimiter=r'\s+', comment='#'): return key_string -def load_tempo(filename, delimiter=r'\s+', comment='#'): +def load_tempo(filename, delimiter=r"\s+", comment="#"): r"""Load tempo estimates from an annotation file in MIREX format. The file should consist of three numeric columns: the first two correspond to tempo estimates (in beats-per-minute), and the third @@ -534,11 +548,13 @@ def load_tempo(filename, delimiter=r'\s+', comment='#'): Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -549,20 +565,19 @@ def load_tempo(filename, delimiter=r'\s+', comment='#'): ------- tempi : np.ndarray, non-negative The two tempo estimates - weight : float [0, 1] The relative importance of ``tempi[0]`` compared to ``tempi[1]`` """ # Use our universal function to load the key and mode strings - t1, t2, weight = load_delimited(filename, [float, float, float], - delimiter=delimiter, - comment=comment) + t1, t2, weight = load_delimited( + filename, [float, float, float], delimiter=delimiter, comment=comment + ) weight = weight[0] tempi = np.concatenate([t1, t2]) if len(t1) != 1: - raise ValueError('Tempo file should contain only one line.') + raise ValueError("Tempo file should contain only one line.") # Validate them, but throw a warning in place of an error try: @@ -571,17 +586,20 @@ def load_tempo(filename, delimiter=r'\s+', comment='#'): warnings.warn(error.args[0]) if not 0 <= weight <= 1: - raise ValueError('Invalid weight: {}'.format(weight)) + raise ValueError(f"Invalid weight: {weight}") return tempi, weight -def load_ragged_time_series(filename, dtype=float, delimiter=r'\s+', - header=False, comment='#'): - r"""Utility function for loading in data from a delimited time series - annotation file with a variable number of columns. - Assumes that column 0 contains time stamps and columns 1 through n contain - values. n may be variable from time stamp to time stamp. +def load_ragged_time_series( + filename, dtype=float, delimiter=r"\s+", header=False, comment="#" +): + r"""Load data from a delimited time series annotation file with + a variable number of columns. + + This function assumes that column 0 contains time stamps and + columns 1 through n contain values. + n may be variable from time stamp to time stamp. Examples -------- @@ -594,16 +612,20 @@ def load_ragged_time_series(filename, dtype=float, delimiter=r'\s+', Parameters ---------- - filename : str + filename : str or `os.Pathlike` Path to the annotation file + dtype : function Data type to apply to values columns. + delimiter : str Separator regular expression. By default, lines will be split by any amount of whitespace. + header : bool Indicates whether a header row is present or not. By default, assumes no header is present. + comment : str or None Comment regular expression. Any lines beginning with this string or pattern will be ignored. @@ -629,13 +651,13 @@ def load_ragged_time_series(filename, dtype=float, delimiter=r'\s+', if comment is None: commenter = None else: - commenter = re.compile('^{}'.format(comment)) + commenter = re.compile(f"^{comment}") if header: start_row = 1 else: start_row = 0 - with _open(filename, mode='r') as input_file: + with _open(filename, mode="r") as input_file: for row, line in enumerate(input_file, start_row): # If this is a comment line, skip it if comment is not None and commenter.match(line): @@ -646,10 +668,12 @@ def load_ragged_time_series(filename, dtype=float, delimiter=r'\s+', try: converted_time = float(data[0]) except (TypeError, ValueError) as exe: - six.raise_from(ValueError("Couldn't convert value {} using {} " - "found at {}:{:d}:\n\t{}".format( - data[0], float.__name__, - filename, row, line)), exe) + raise ValueError( + "Couldn't convert value {} using {} " + "found at {}:{:d}:\n\t{}".format( + data[0], float.__name__, filename, row, line + ) + ) from exe times.append(converted_time) # cast values to a numpy array. time stamps with no values are cast @@ -657,10 +681,12 @@ def load_ragged_time_series(filename, dtype=float, delimiter=r'\s+', try: converted_value = np.array(data[1:], dtype=dtype) except (TypeError, ValueError) as exe: - six.raise_from(ValueError("Couldn't convert value {} using {} " - "found at {}:{:d}:\n\t{}".format( - data[1:], dtype.__name__, - filename, row, line)), exe) + raise ValueError( + "Couldn't convert value {} using {} " + "found at {}:{:d}:\n\t{}".format( + data[1:], dtype.__name__, filename, row, line + ) + ) from exe values.append(converted_value) return np.array(times), values diff --git a/mir_eval/key.py b/mir_eval/key.py index 421481d7..d8bc1a59 100644 --- a/mir_eval/key.py +++ b/mir_eval/key.py @@ -1,4 +1,4 @@ -''' +""" Key Detection involves determining the underlying key (distribution of notes and note transitions) in a piece of music. Key detection algorithms are evaluated by comparing their estimated key to a ground-truth reference key and @@ -16,19 +16,36 @@ ------- * :func:`mir_eval.key.weighted_score`: Heuristic scoring of the relation of two keys. -''' +""" import collections import warnings from . import util -KEY_TO_SEMITONE = {'c': 0, 'c#': 1, 'db': 1, 'd': 2, 'd#': 3, 'eb': 3, 'e': 4, - 'f': 5, 'f#': 6, 'gb': 6, 'g': 7, 'g#': 8, 'ab': 8, 'a': 9, - 'a#': 10, 'bb': 10, 'b': 11, 'x': None} +KEY_TO_SEMITONE = { + "c": 0, + "c#": 1, + "db": 1, + "d": 2, + "d#": 3, + "eb": 3, + "e": 4, + "f": 5, + "f#": 6, + "gb": 6, + "g": 7, + "g#": 8, + "ab": 8, + "a": 9, + "a#": 10, + "bb": 10, + "b": 11, + "x": None, +} def validate_key(key): - """Checks that a key is well-formatted, e.g. in the form ``'C# major'``. + """Check that a key is well-formatted, e.g. in the form ``'C# major'``. The Key can be 'X' if it is not possible to categorize the Key and mode can be 'other' if it can't be categorized as major or minor. @@ -37,29 +54,29 @@ def validate_key(key): key : str Key to verify """ - if len(key.split()) != 2 \ - and not (len(key.split()) and key.lower() == 'x'): - raise ValueError("'{}' is not in the form '(key) (mode)' " - "or 'X'".format(key)) - if key.lower() != 'x': + if len(key.split()) != 2 and not (len(key.split()) and key.lower() == "x"): + raise ValueError("'{}' is not in the form '(key) (mode)' " "or 'X'".format(key)) + if key.lower() != "x": key, mode = key.split() - if key.lower() == 'x': + if key.lower() == "x": raise ValueError( "Mode {} is invalid; 'X' (Uncategorized) " - "doesn't have mode".format(mode)) + "doesn't have mode".format(mode) + ) if key.lower() not in KEY_TO_SEMITONE: raise ValueError( "Key {} is invalid; should be e.g. D or C# or Eb or " - "X (Uncategorized)".format(key)) - if mode not in ['major', 'minor', 'other']: + "X (Uncategorized)".format(key) + ) + if mode not in ["major", "minor", "other"]: raise ValueError( - "Mode '{}' is invalid; must be 'major', 'minor' or 'other'" - .format(mode)) + f"Mode '{mode}' is invalid; must be 'major', 'minor' or 'other'" + ) def validate(reference_key, estimated_key): - """Checks that the input annotations to a metric are valid key strings and + """Check that the input annotations to a metric are valid key strings and throws helpful errors if not. Parameters @@ -74,7 +91,7 @@ def validate(reference_key, estimated_key): def split_key_string(key): - """Splits a key string (of the form, e.g. ``'C# major'``), into a tuple of + """Split a key string (of the form, e.g. ``'C# major'``), into a tuple of ``(key, mode)`` where ``key`` is is an integer representing the semitone distance from C. @@ -90,7 +107,7 @@ def split_key_string(key): mode : str String representing the mode. """ - if key.lower() != 'x': + if key.lower() != "x": key, mode = key.split() else: mode = None @@ -99,7 +116,7 @@ def split_key_string(key): def weighted_score(reference_key, estimated_key, allow_descending_fifths=False): - """Computes a heuristic score which is weighted according to the + """Compute a heuristic score which is weighted according to the relationship of the reference and estimated key, as follows: +------------------------------------------------------+-------+ @@ -156,32 +173,35 @@ def weighted_score(reference_key, estimated_key, estimated_key, estimated_mode = split_key_string(estimated_key) # If keys are the same, return 1. if reference_key == estimated_key and reference_mode == estimated_mode: - return 1. + return 1.0 # If reference or estimated key are x and they are not the same key # then the result is 'Other'. if reference_key is None or estimated_key is None: - return 0. + return 0.0 # If keys are the same mode and a perfect fifth up (7 semitones) - if (estimated_mode == reference_mode and - (estimated_key - reference_key) % 12 == 7): + if estimated_mode == reference_mode and (estimated_key - reference_key) % 12 == 7: return 0.5 # If keys are the same mode and a perfect fifth down (7 semitones) if (allow_descending_fifths and estimated_mode == reference_mode and (reference_key - estimated_key) % 12 == 7): return 0.5 # Estimated key is relative minor of reference key (9 semitones) - if (estimated_mode != reference_mode == 'major' and - (estimated_key - reference_key) % 12 == 9): + if ( + estimated_mode != reference_mode == "major" + and (estimated_key - reference_key) % 12 == 9 + ): return 0.3 # Estimated key is relative major of reference key (3 semitones) - if (estimated_mode != reference_mode == 'minor' and - (estimated_key - reference_key) % 12 == 3): + if ( + estimated_mode != reference_mode == "minor" + and (estimated_key - reference_key) % 12 == 3 + ): return 0.3 # If keys are in different modes and parallel (same key name) if estimated_mode != reference_mode and reference_key == estimated_key: return 0.2 # Otherwise return 0 - return 0. + return 0.0 def evaluate(reference_key, estimated_key, allow_descending_fifths=False, @@ -203,7 +223,7 @@ def evaluate(reference_key, estimated_key, allow_descending_fifths=False, Estimated key string. allow_descending_fifths : bool Specifies whether to score descending fifth errors or not. - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. diff --git a/mir_eval/melody.py b/mir_eval/melody.py index f548d8db..0268db6e 100644 --- a/mir_eval/melody.py +++ b/mir_eval/melody.py @@ -1,5 +1,5 @@ # CREATED:2014-03-07 by Justin Salamon -''' +""" Melody extraction algorithms aim to produce a sequence of frequency values corresponding to the pitch of the dominant melody from a musical recording. For evaluation, an estimated pitch series is evaluated against a @@ -7,10 +7,13 @@ is correct (within some tolerance). For a detailed explanation of the measures please refer to: + J. Salamon, E. Gomez, D. P. W. Ellis and G. Richard, "Melody Extraction from Polyphonic Music Signals: Approaches, Applications and Challenges", IEEE Signal Processing Magazine, 31(2):118-134, Mar. 2014. + and: + G. E. Poliner, D. P. W. Ellis, A. F. Ehmann, E. Gomez, S. Streich, and B. Ong. "Melody transcription from music audio: Approaches and evaluation", IEEE Transactions on Audio, Speech, and @@ -18,6 +21,7 @@ For an explanation of the generalized measures (using non-binary voicings), please refer to: + R. Bittner and J. Bosch, "Generalized Metrics for Single-F0 Estimation Evaluation", International Society for Music Information Retrieval Conference (ISMIR), 2019. @@ -61,7 +65,7 @@ the proportion of all frames correctly estimated by the algorithm, including whether non-melody frames where labeled by the algorithm as non-melody -''' +""" import numpy as np import scipy.interpolate @@ -71,7 +75,7 @@ def validate_voicing(ref_voicing, est_voicing): - """Checks that voicing inputs to a metric are in the correct format. + """Check that voicing inputs to a metric are in the correct format. Parameters ---------- @@ -91,16 +95,17 @@ def validate_voicing(ref_voicing, est_voicing): warnings.warn("Estimated melody has no voiced frames.") # Make sure they're the same length if ref_voicing.shape[0] != est_voicing.shape[0]: - raise ValueError('Reference and estimated voicing arrays should ' - 'be the same length.') + raise ValueError( + "Reference and estimated voicing arrays should " "be the same length." + ) for voicing in [ref_voicing, est_voicing]: # Make sure voicing is between 0 and 1 if np.logical_or(voicing < 0, voicing > 1).any(): - raise ValueError('Voicing arrays must be between 0 and 1.') + raise ValueError("Voicing arrays must be between 0 and 1.") def validate(ref_voicing, ref_cent, est_voicing, est_cent): - """Checks that voicing and frequency arrays are well-formed. To be used in + """Check that voicing and frequency arrays are well-formed. To be used in conjunction with :func:`mir_eval.melody.validate_voicing` Parameters @@ -120,11 +125,14 @@ def validate(ref_voicing, ref_cent, est_voicing, est_cent): if est_cent.size == 0: warnings.warn("Estimated frequency array is empty.") # Make sure they're the same length - if ref_voicing.shape[0] != ref_cent.shape[0] or \ - est_voicing.shape[0] != est_cent.shape[0] or \ - ref_cent.shape[0] != est_cent.shape[0]: - raise ValueError('All voicing and frequency arrays must have the ' - 'same length.') + if ( + ref_voicing.shape[0] != ref_cent.shape[0] + or est_voicing.shape[0] != est_cent.shape[0] + or ref_cent.shape[0] != est_cent.shape[0] + ): + raise ValueError( + "All voicing and frequency arrays must have the " "same length." + ) def hz2cents(freq_hz, base_frequency=10.0): @@ -138,10 +146,6 @@ def hz2cents(freq_hz, base_frequency=10.0): base_frequency : float Base frequency for conversion. (Default value = 10.0) - Returns - ------- - freq_cent : np.ndarray - Array of frequencies in cents, relative to base_frequency """ freq_cent = np.zeros(freq_hz.shape[0]) freq_nonz_ind = np.flatnonzero(freq_hz) @@ -158,15 +162,19 @@ def freq_to_voicing(frequencies, voicing=None): ---------- frequencies : np.ndarray Array of frequencies. A frequency <= 0 indicates "unvoiced". + voicing : np.ndarray Array of voicing values. (Default value = None) Default None, which means the voicing is inferred from `frequencies`: - frames with frequency <= 0.0 are considered "unvoiced" - frames with frequency > 0.0 are considered "voiced" + + - frames with frequency <= 0.0 are considered "unvoiced" + - frames with frequency > 0.0 are considered "voiced" + If specified, `voicing` is used as the voicing array, but frequencies with value 0 are forced to have 0 voicing. - Voicing inferred by negative frequency values is ignored. + + - Voicing inferred by negative frequency values is ignored. Returns ------- @@ -185,8 +193,7 @@ def freq_to_voicing(frequencies, voicing=None): def constant_hop_timebase(hop, end_time): - """Generates a time series from 0 to ``end_time`` with times spaced ``hop`` - apart + """Generate a time series from 0 to ``end_time`` with times spaced ``hop`` apart Parameters ---------- @@ -203,14 +210,14 @@ def constant_hop_timebase(hop, end_time): """ # Compute new timebase. Rounding/linspace is to avoid float problems. end_time = np.round(end_time, 10) - times = np.linspace(0, hop * int(np.floor(end_time / hop)), - int(np.floor(end_time / hop)) + 1) + times = np.linspace( + 0, hop * int(np.floor(end_time / hop)), int(np.floor(end_time / hop)) + 1 + ) times = np.round(times, 10) return times -def resample_melody_series(times, frequencies, voicing, - times_new, kind='linear'): +def resample_melody_series(times, frequencies, voicing, times_new, kind="linear"): """Resamples frequency and voicing time series to a new timescale. Maintains any zero ("unvoiced") values in frequencies. @@ -247,14 +254,19 @@ def resample_melody_series(times, frequencies, voicing, # Warn when the delta between the original times is not constant, # unless times[0] == 0. and frequencies[0] == frequencies[1] (see logic at # the beginning of to_cent_voicing) - if not (np.allclose(np.diff(times), np.diff(times).mean()) or - (np.allclose(np.diff(times[1:]), np.diff(times[1:]).mean()) and - frequencies[0] == frequencies[1])): + if not ( + np.allclose(np.diff(times), np.diff(times).mean()) + or ( + np.allclose(np.diff(times[1:]), np.diff(times[1:]).mean()) + and frequencies[0] == frequencies[1] + ) + ): warnings.warn( "Non-uniform timescale passed to resample_melody_series. Pitch " "will be linearly interpolated, which will result in undesirable " "behavior if silences are indicated by missing values. Silences " - "should be indicated by nonpositive frequency values.") + "should be indicated by nonpositive frequency values." + ) # Round to avoid floating point problems times = np.round(times, 10) times_new = np.round(times_new, 10) @@ -264,7 +276,7 @@ def resample_melody_series(times, frequencies, voicing, frequencies = np.append(frequencies, 0) voicing = np.append(voicing, 0) # We need to fix zero transitions if interpolation is not zero or nearest - if kind != 'zero' and kind != 'nearest': + if kind != "zero" and kind != "nearest": # Fill in zero values with the last reported frequency # to avoid erroneous values when resampling frequencies_held = np.array(frequencies) @@ -272,74 +284,91 @@ def resample_melody_series(times, frequencies, voicing, if frequency == 0: frequencies_held[n + 1] = frequencies_held[n] # Linearly interpolate frequencies - frequencies_resampled = scipy.interpolate.interp1d(times, - frequencies_held, - kind)(times_new) + frequencies_resampled = scipy.interpolate.interp1d( + times, frequencies_held, kind + )(times_new) # Retain zeros - frequency_mask = scipy.interpolate.interp1d(times, - frequencies, - 'zero')(times_new) - frequencies_resampled *= (frequency_mask != 0) + frequency_mask = scipy.interpolate.interp1d(times, frequencies, "zero")( + times_new + ) + frequencies_resampled *= frequency_mask != 0 else: - frequencies_resampled = scipy.interpolate.interp1d(times, - frequencies, - kind)(times_new) + frequencies_resampled = scipy.interpolate.interp1d(times, frequencies, kind)( + times_new + ) # Use nearest-neighbor for voicing if it was used for frequencies # if voicing is not binary, use linear interpolation is_binary_voicing = np.all( - np.logical_or(np.equal(voicing, 0), np.equal(voicing, 1))) - if kind == 'nearest' or (kind == 'linear' and not is_binary_voicing): - voicing_resampled = scipy.interpolate.interp1d(times, - voicing, - kind)(times_new) + np.logical_or(np.equal(voicing, 0), np.equal(voicing, 1)) + ) + if kind == "nearest" or (kind == "linear" and not is_binary_voicing): + voicing_resampled = scipy.interpolate.interp1d(times, voicing, kind)(times_new) # otherwise, always use zeroth order else: - voicing_resampled = scipy.interpolate.interp1d(times, - voicing, - 'zero')(times_new) + voicing_resampled = scipy.interpolate.interp1d(times, voicing, "zero")( + times_new + ) return frequencies_resampled, voicing_resampled -def to_cent_voicing(ref_time, ref_freq, est_time, est_freq, - est_voicing=None, ref_reward=None, base_frequency=10., - hop=None, kind='linear'): - """Converts reference and estimated time/frequency (Hz) annotations to sampled +def to_cent_voicing( + ref_time, + ref_freq, + est_time, + est_freq, + est_voicing=None, + ref_reward=None, + base_frequency=10.0, + hop=None, + kind="linear", +): + """Convert reference and estimated time/frequency (Hz) annotations to sampled frequency (cent)/voicing arrays. A zero frequency indicates "unvoiced". If est_voicing is not provided, a negative frequency indicates: "Predicted as unvoiced, but if it's voiced, - this is the frequency estimate". + this is the frequency estimate". + If it is provided, negative frequency values are ignored, and the voicing - from est_voicing is directly used. + from est_voicing is directly used. Parameters ---------- ref_time : np.ndarray Time of each reference frequency value + ref_freq : np.ndarray Array of reference frequency values + est_time : np.ndarray Time of each estimated frequency value + est_freq : np.ndarray Array of estimated frequency values + est_voicing : np.ndarray Estimate voicing confidence. Default None, which means the voicing is inferred from est_freq: - frames with frequency <= 0.0 are considered "unvoiced" - frames with frequency > 0.0 are considered "voiced" + + - frames with frequency <= 0.0 are considered "unvoiced" + - frames with frequency > 0.0 are considered "voiced" + ref_reward : np.ndarray Reference voicing reward. Default None, which means all frames are weighted equally. + base_frequency : float Base frequency in Hz for conversion to cents (Default value = 10.) + hop : float Hop size, in seconds, to resample, default None which means use ref_time + kind : str kind parameter to pass to scipy.interpolate.interp1d. (Default value = 'linear') @@ -379,23 +408,32 @@ def to_cent_voicing(ref_time, ref_freq, est_time, est_freq, if hop is not None: # Resample to common time base ref_cent, ref_voicing = resample_melody_series( - ref_time, ref_cent, ref_voicing, - constant_hop_timebase(hop, ref_time.max()), kind) + ref_time, + ref_cent, + ref_voicing, + constant_hop_timebase(hop, ref_time.max()), + kind, + ) est_cent, est_voicing = resample_melody_series( - est_time, est_cent, est_voicing, - constant_hop_timebase(hop, est_time.max()), kind) + est_time, + est_cent, + est_voicing, + constant_hop_timebase(hop, est_time.max()), + kind, + ) # Otherwise, only resample estimated to the reference time base else: est_cent, est_voicing = resample_melody_series( - est_time, est_cent, est_voicing, ref_time, kind) + est_time, est_cent, est_voicing, ref_time, kind + ) # ensure the estimated sequence is the same length as the reference len_diff = ref_cent.shape[0] - est_cent.shape[0] if len_diff >= 0: est_cent = np.append(est_cent, np.zeros(len_diff)) est_voicing = np.append(est_voicing, np.zeros(len_diff)) else: - est_cent = est_cent[:ref_cent.shape[0]] - est_voicing = est_voicing[:ref_voicing.shape[0]] + est_cent = est_cent[: ref_cent.shape[0]] + est_voicing = est_voicing[: ref_voicing.shape[0]] return (ref_voicing, ref_cent, est_voicing, est_cent) @@ -404,6 +442,7 @@ def voicing_recall(ref_voicing, est_voicing): """Compute the voicing recall given two voicing indicator sequences, one as reference (truth) and the other as the estimate (prediction). The sequences must be of the same length. + Examples -------- >>> ref_time, ref_freq = mir_eval.io.load_time_series('ref.txt') @@ -414,12 +453,14 @@ def voicing_recall(ref_voicing, est_voicing): ... est_time, ... est_freq) >>> recall = mir_eval.melody.voicing_recall(ref_v, est_v) + Parameters ---------- ref_voicing : np.ndarray Reference boolean voicing array est_voicing : np.ndarray Estimated boolean voicing array + Returns ------- vx_recall : float @@ -427,7 +468,7 @@ def voicing_recall(ref_voicing, est_voicing): indicated as voiced in est """ if ref_voicing.size == 0 or est_voicing.size == 0: - return 0. + return 0.0 ref_indicator = (ref_voicing > 0).astype(float) if np.sum(ref_indicator) == 0: return 1 @@ -438,6 +479,7 @@ def voicing_false_alarm(ref_voicing, est_voicing): """Compute the voicing false alarm rates given two voicing indicator sequences, one as reference (truth) and the other as the estimate (prediction). The sequences must be of the same length. + Examples -------- >>> ref_time, ref_freq = mir_eval.io.load_time_series('ref.txt') @@ -448,12 +490,14 @@ def voicing_false_alarm(ref_voicing, est_voicing): ... est_time, ... est_freq) >>> false_alarm = mir_eval.melody.voicing_false_alarm(ref_v, est_v) + Parameters ---------- ref_voicing : np.ndarray Reference boolean voicing array est_voicing : np.ndarray Estimated boolean voicing array + Returns ------- vx_false_alarm : float @@ -461,7 +505,7 @@ def voicing_false_alarm(ref_voicing, est_voicing): indicated as voiced in est """ if ref_voicing.size == 0 or est_voicing.size == 0: - return 0. + return 0.0 ref_indicator = (ref_voicing == 0).astype(float) if np.sum(ref_indicator) == 0: return 0 @@ -472,6 +516,7 @@ def voicing_measures(ref_voicing, est_voicing): """Compute the voicing recall and false alarm rates given two voicing indicator sequences, one as reference (truth) and the other as the estimate (prediction). The sequences must be of the same length. + Examples -------- >>> ref_time, ref_freq = mir_eval.io.load_time_series('ref.txt') @@ -483,12 +528,14 @@ def voicing_measures(ref_voicing, est_voicing): ... est_freq) >>> recall, false_alarm = mir_eval.melody.voicing_measures(ref_v, ... est_v) + Parameters ---------- ref_voicing : np.ndarray Reference boolean voicing array est_voicing : np.ndarray Estimated boolean voicing array + Returns ------- vx_recall : float @@ -504,8 +551,7 @@ def voicing_measures(ref_voicing, est_voicing): return vx_recall, vx_false_alm -def raw_pitch_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, - cent_tolerance=50): +def raw_pitch_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, cent_tolerance=50): """Compute the raw pitch accuracy given two pitch (frequency) sequences in cents and matching voicing indicator sequences. The first pitch and voicing arrays are treated as the reference (truth), and the second two as the @@ -545,16 +591,18 @@ def raw_pitch_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, Raw pitch accuracy, the fraction of voiced frames in ref_cent for which est_cent provides a correct frequency values (within cent_tolerance cents). - """ - validate_voicing(ref_voicing, est_voicing) validate(ref_voicing, ref_cent, est_voicing, est_cent) # When input arrays are empty, return 0 by special case # If there are no voiced frames in reference, metric is 0 - if ref_voicing.size == 0 or ref_voicing.sum() == 0 \ - or ref_cent.size == 0 or est_cent.size == 0: - return 0. + if ( + ref_voicing.size == 0 + or ref_voicing.sum() == 0 + or ref_cent.size == 0 + or est_cent.size == 0 + ): + return 0.0 # Raw pitch = the number of voiced frames in the reference for which the # estimate provides a correct frequency value (within cent_tolerance cents) @@ -563,19 +611,17 @@ def raw_pitch_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, nonzero_freqs = np.logical_and(est_cent != 0, ref_cent != 0) if sum(nonzero_freqs) == 0: - return 0. + return 0.0 freq_diff_cents = np.abs(ref_cent - est_cent)[nonzero_freqs] correct_frequencies = freq_diff_cents < cent_tolerance - rpa = ( - np.sum(ref_voicing[nonzero_freqs] * correct_frequencies) / - np.sum(ref_voicing) - ) + rpa = np.sum(ref_voicing[nonzero_freqs] * correct_frequencies) / np.sum(ref_voicing) return rpa -def raw_chroma_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, - cent_tolerance=50): +def raw_chroma_accuracy( + ref_voicing, ref_cent, est_voicing, est_cent, cent_tolerance=50 +): """Compute the raw chroma accuracy given two pitch (frequency) sequences in cents and matching voicing indicator sequences. The first pitch and voicing arrays are treated as the reference (truth), and the second two as @@ -593,7 +639,6 @@ def raw_chroma_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, >>> raw_chroma = mir_eval.melody.raw_chroma_accuracy(ref_v, ref_c, ... est_v, est_c) - Parameters ---------- ref_voicing : np.ndarray @@ -622,28 +667,28 @@ def raw_chroma_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, validate(ref_voicing, ref_cent, est_voicing, est_cent) # When input arrays are empty, return 0 by special case # If there are no voiced frames in reference, metric is 0 - if ref_voicing.size == 0 or ref_voicing.sum() == 0 \ - or ref_cent.size == 0 or est_cent.size == 0: - return 0. + if ( + ref_voicing.size == 0 + or ref_voicing.sum() == 0 + or ref_cent.size == 0 + or est_cent.size == 0 + ): + return 0.0 # # Raw chroma = same as raw pitch except that octave errors are ignored. nonzero_freqs = np.logical_and(est_cent != 0, ref_cent != 0) if sum(nonzero_freqs) == 0: - return 0. + return 0.0 freq_diff_cents = np.abs(ref_cent - est_cent)[nonzero_freqs] octave = 1200.0 * np.floor(freq_diff_cents / 1200 + 0.5) correct_chroma = np.abs(freq_diff_cents - octave) < cent_tolerance - rca = ( - np.sum(ref_voicing[nonzero_freqs] * correct_chroma) / - np.sum(ref_voicing) - ) + rca = np.sum(ref_voicing[nonzero_freqs] * correct_chroma) / np.sum(ref_voicing) return rca -def overall_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, - cent_tolerance=50): +def overall_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, cent_tolerance=50): """Compute the overall accuracy given two pitch (frequency) sequences in cents and matching voicing indicator sequences. The first pitch and voicing arrays are treated as the reference (truth), and the second two @@ -688,9 +733,13 @@ def overall_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, validate(ref_voicing, ref_cent, est_voicing, est_cent) # When input arrays are empty, return 0 by special case - if ref_voicing.size == 0 or est_voicing.size == 0 \ - or ref_cent.size == 0 or est_cent.size == 0: - return 0. + if ( + ref_voicing.size == 0 + or est_voicing.size == 0 + or ref_cent.size == 0 + or est_cent.size == 0 + ): + return 0.0 nonzero_freqs = np.logical_and(est_cent != 0, ref_cent != 0) freq_diff_cents = np.abs(ref_cent - est_cent)[nonzero_freqs] @@ -701,22 +750,26 @@ def overall_accuracy(ref_voicing, ref_cent, est_voicing, est_cent, if np.sum(ref_voicing) == 0: ratio = 0.0 else: - ratio = (np.sum(ref_binary) / np.sum(ref_voicing)) + ratio = np.sum(ref_binary) / np.sum(ref_voicing) accuracy = ( ( - ratio * np.sum(ref_voicing[nonzero_freqs] * - est_voicing[nonzero_freqs] * - correct_frequencies) - ) + - np.sum((1.0 - ref_binary) * (1.0 - est_voicing)) + ratio + * np.sum( + ref_voicing[nonzero_freqs] + * est_voicing[nonzero_freqs] + * correct_frequencies + ) + ) + + np.sum((1.0 - ref_binary) * (1.0 - est_voicing)) ) / n_frames return accuracy -def evaluate(ref_time, ref_freq, est_time, est_freq, - est_voicing=None, ref_reward=None, **kwargs): +def evaluate( + ref_time, ref_freq, est_time, est_freq, est_voicing=None, ref_reward=None, **kwargs +): """Evaluate two melody (predominant f0) transcriptions, where the first is treated as the reference (ground truth) and the second as the estimate to be evaluated (prediction). @@ -732,21 +785,28 @@ def evaluate(ref_time, ref_freq, est_time, est_freq, ---------- ref_time : np.ndarray Time of each reference frequency value + ref_freq : np.ndarray Array of reference frequency values + est_time : np.ndarray Time of each estimated frequency value + est_freq : np.ndarray Array of estimated frequency values + est_voicing : np.ndarray Estimate voicing confidence. Default None, which means the voicing is inferred from est_freq: - frames with frequency <= 0.0 are considered "unvoiced" - frames with frequency > 0.0 are considered "voiced" + + - frames with frequency <= 0.0 are considered "unvoiced" + - frames with frequency > 0.0 are considered "voiced" + ref_reward : np.ndarray Reference pitch estimation reward. Default None, which means all frames are weighted equally. - kwargs + + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -756,7 +816,6 @@ def evaluate(ref_time, ref_freq, est_time, est_freq, Dictionary of scores, where the key is the metric name (str) and the value is the (float) score achieved. - References ---------- .. [#] J. Salamon, E. Gomez, D. P. W. Ellis and G. Richard, "Melody @@ -764,7 +823,6 @@ def evaluate(ref_time, ref_freq, est_time, est_freq, and Challenges", IEEE Signal Processing Magazine, 31(2):118-134, Mar. 2014. - .. [#] G. E. Poliner, D. P. W. Ellis, A. F. Ehmann, E. Gomez, S. Streich, and B. Ong. "Melody transcription from music audio: Approaches and evaluation", IEEE Transactions on Audio, Speech, and @@ -776,34 +834,37 @@ def evaluate(ref_time, ref_freq, est_time, est_freq, """ # Convert to reference/estimated voicing/frequency (cent) arrays - (ref_voicing, ref_cent, - est_voicing, est_cent) = util.filter_kwargs( - to_cent_voicing, ref_time, ref_freq, est_time, est_freq, - est_voicing, ref_reward, **kwargs) + (ref_voicing, ref_cent, est_voicing, est_cent) = util.filter_kwargs( + to_cent_voicing, + ref_time, + ref_freq, + est_time, + est_freq, + est_voicing, + ref_reward, + **kwargs + ) # Compute metrics scores = collections.OrderedDict() - scores['Voicing Recall'] = util.filter_kwargs(voicing_recall, - ref_voicing, - est_voicing, **kwargs) + scores["Voicing Recall"] = util.filter_kwargs( + voicing_recall, ref_voicing, est_voicing, **kwargs + ) - scores['Voicing False Alarm'] = util.filter_kwargs(voicing_false_alarm, - ref_voicing, - est_voicing, **kwargs) + scores["Voicing False Alarm"] = util.filter_kwargs( + voicing_false_alarm, ref_voicing, est_voicing, **kwargs + ) - scores['Raw Pitch Accuracy'] = util.filter_kwargs(raw_pitch_accuracy, - ref_voicing, ref_cent, - est_voicing, est_cent, - **kwargs) + scores["Raw Pitch Accuracy"] = util.filter_kwargs( + raw_pitch_accuracy, ref_voicing, ref_cent, est_voicing, est_cent, **kwargs + ) - scores['Raw Chroma Accuracy'] = util.filter_kwargs(raw_chroma_accuracy, - ref_voicing, ref_cent, - est_voicing, est_cent, - **kwargs) + scores["Raw Chroma Accuracy"] = util.filter_kwargs( + raw_chroma_accuracy, ref_voicing, ref_cent, est_voicing, est_cent, **kwargs + ) - scores['Overall Accuracy'] = util.filter_kwargs(overall_accuracy, - ref_voicing, ref_cent, - est_voicing, est_cent, - **kwargs) + scores["Overall Accuracy"] = util.filter_kwargs( + overall_accuracy, ref_voicing, ref_cent, est_voicing, est_cent, **kwargs + ) return scores diff --git a/mir_eval/multipitch.py b/mir_eval/multipitch.py index 9430d60a..bc74f3de 100644 --- a/mir_eval/multipitch.py +++ b/mir_eval/multipitch.py @@ -1,4 +1,4 @@ -''' +""" The goal of multiple f0 (multipitch) estimation and tracking is to identify all of the active fundamental frequencies in each time frame in a complex music signal. @@ -40,7 +40,7 @@ Signal Processing, 2007(1):154-163, Jan. 2007. .. [#bay2009] Bay, M., Ehmann, A. F., & Downie, J. S. (2009). Evaluation of Multiple-F0 Estimation and Tracking Systems. In ISMIR (pp. 315-320). -''' +""" import numpy as np import collections @@ -49,13 +49,13 @@ import warnings -MAX_TIME = 30000. # The maximum allowable time stamp (seconds) -MAX_FREQ = 5000. # The maximum allowable frequency (Hz) -MIN_FREQ = 20. # The minimum allowable frequency (Hz) +MAX_TIME = 30000.0 # The maximum allowable time stamp (seconds) +MAX_FREQ = 5000.0 # The maximum allowable frequency (Hz) +MIN_FREQ = 20.0 # The minimum allowable frequency (Hz) def validate(ref_time, ref_freqs, est_time, est_freqs): - """Checks that the time and frequency inputs are well-formed. + """Check that the time and frequency inputs are well-formed. Parameters ---------- @@ -67,9 +67,7 @@ def validate(ref_time, ref_freqs, est_time, est_freqs): estimate time stamps in seconds est_freqs : list of np.ndarray estimated frequencies in Hz - """ - util.validate_events(ref_time, max_time=MAX_TIME) util.validate_events(est_time, max_time=MAX_TIME) @@ -86,24 +84,25 @@ def validate(ref_time, ref_freqs, est_time, est_freqs): if len(est_freqs) == 0: warnings.warn("Estimated frequencies are empty.") if ref_time.size != len(ref_freqs): - raise ValueError('Reference times and frequencies have unequal ' - 'lengths.') + raise ValueError("Reference times and frequencies have unequal " "lengths.") if est_time.size != len(est_freqs): - raise ValueError('Estimate times and frequencies have unequal ' - 'lengths.') + raise ValueError("Estimate times and frequencies have unequal " "lengths.") for freq in ref_freqs: - util.validate_frequencies(freq, max_freq=MAX_FREQ, min_freq=MIN_FREQ, - allow_negatives=False) + util.validate_frequencies( + freq, max_freq=MAX_FREQ, min_freq=MIN_FREQ, allow_negatives=False + ) for freq in est_freqs: - util.validate_frequencies(freq, max_freq=MAX_FREQ, min_freq=MIN_FREQ, - allow_negatives=False) + util.validate_frequencies( + freq, max_freq=MAX_FREQ, min_freq=MIN_FREQ, allow_negatives=False + ) def resample_multipitch(times, frequencies, target_times): - """Resamples multipitch time series to a new timescale. Values in - ``target_times`` outside the range of ``times`` return no pitch estimate. + """Resamples multipitch time series to a new timescale using nearest + neighbor interpolation. Values in ``target_times`` outside the range + of ``times`` return no pitch estimate. Parameters ---------- @@ -123,7 +122,7 @@ def resample_multipitch(times, frequencies, target_times): return [] if times.size == 0: - return [np.array([])]*len(target_times) + return [np.array([])] * len(target_times) n_times = len(frequencies) @@ -136,22 +135,26 @@ def resample_multipitch(times, frequencies, target_times): # since we're interpolating the index, fill_value is set to the first index # that is out of range. We handle this in the next line. new_frequency_index = scipy.interpolate.interp1d( - times, frequency_index, kind='nearest', bounds_error=False, - assume_sorted=True, fill_value=n_times)(target_times) + times, + frequency_index, + kind="nearest", + bounds_error=False, + assume_sorted=True, + fill_value=n_times, + )(target_times) # create array of frequencies plus additional empty element at the end for # target time stamps that are out of the interpolation range freq_vals = frequencies + [np.array([])] # map interpolated indices back to frequency values - frequencies_resampled = [ - freq_vals[i] for i in new_frequency_index.astype(int)] + frequencies_resampled = [freq_vals[i] for i in new_frequency_index.astype(int)] return frequencies_resampled def frequencies_to_midi(frequencies, ref_frequency=440.0): - """Converts frequencies to continuous MIDI values. + """Convert frequencies to continuous MIDI values. Parameters ---------- @@ -165,7 +168,7 @@ def frequencies_to_midi(frequencies, ref_frequency=440.0): frequencies_midi : list of np.ndarray Continuous MIDI frequency values. """ - return [69.0 + 12.0*np.log2(freqs/ref_frequency) for freqs in frequencies] + return [69.0 + 12.0 * np.log2(freqs / ref_frequency) for freqs in frequencies] def midi_to_chroma(frequencies_midi): @@ -186,7 +189,7 @@ def midi_to_chroma(frequencies_midi): def compute_num_freqs(frequencies): - """Computes the number of frequencies for each time point. + """Compute the number of frequencies for each time point. Parameters ---------- @@ -226,14 +229,14 @@ def compute_num_true_positives(ref_freqs, est_freqs, window=0.5, chroma=False): """ n_frames = len(ref_freqs) - true_positives = np.zeros((n_frames, )) + true_positives = np.zeros((n_frames,)) for i, (ref_frame, est_frame) in enumerate(zip(ref_freqs, est_freqs)): if chroma: # match chroma-wrapped frequency events matching = util.match_events( - ref_frame, est_frame, window, - distance=util._outer_distance_mod_n) + ref_frame, est_frame, window, distance=util._outer_distance_mod_n + ) else: # match frequency events within tolerance window in semitones matching = util.match_events(ref_frame, est_frame, window) @@ -270,21 +273,21 @@ def compute_accuracy(true_positives, n_ref, n_est): n_est_sum = n_est.sum() if n_est_sum > 0: - precision = true_positive_sum/n_est.sum() + precision = true_positive_sum / n_est.sum() else: warnings.warn("Estimate frequencies are all empty.") precision = 0.0 n_ref_sum = n_ref.sum() if n_ref_sum > 0: - recall = true_positive_sum/n_ref.sum() + recall = true_positive_sum / n_ref.sum() else: warnings.warn("Reference frequencies are all empty.") recall = 0.0 acc_denom = (n_est + n_ref - true_positives).sum() if acc_denom > 0: - acc = true_positive_sum/acc_denom + acc = true_positive_sum / acc_denom else: acc = 0.0 @@ -320,25 +323,25 @@ def compute_err_score(true_positives, n_ref, n_est): if n_ref_sum == 0: warnings.warn("Reference frequencies are all empty.") - return 0., 0., 0., 0. + return 0.0, 0.0, 0.0, 0.0 # Substitution error - e_sub = (np.min([n_ref, n_est], axis=0) - true_positives).sum()/n_ref_sum + e_sub = (np.min([n_ref, n_est], axis=0) - true_positives).sum() / n_ref_sum # compute the max of (n_ref - n_est) and 0 e_miss_numerator = n_ref - n_est e_miss_numerator[e_miss_numerator < 0] = 0 # Miss error - e_miss = e_miss_numerator.sum()/n_ref_sum + e_miss = e_miss_numerator.sum() / n_ref_sum # compute the max of (n_est - n_ref) and 0 e_fa_numerator = n_est - n_ref e_fa_numerator[e_fa_numerator < 0] = 0 # False alarm error - e_fa = e_fa_numerator.sum()/n_ref_sum + e_fa = e_fa_numerator.sum() / n_ref_sum # total error - e_tot = (np.max([n_ref, n_est], axis=0) - true_positives).sum()/n_ref_sum + e_tot = (np.max([n_ref, n_est], axis=0) - true_positives).sum() / n_ref_sum return e_sub, e_miss, e_fa, e_tot @@ -367,7 +370,7 @@ def metrics(ref_time, ref_freqs, est_time, est_freqs, **kwargs): Time of each estimated frequency value est_freqs : list of np.ndarray List of np.ndarrays of estimate frequency values - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -407,8 +410,10 @@ def metrics(ref_time, ref_freqs, est_time, est_freqs, **kwargs): # resample est_freqs if est_times is different from ref_times if est_time.size != ref_time.size or not np.allclose(est_time, ref_time): - warnings.warn("Estimate times not equal to reference times. " - "Resampling to common time base.") + warnings.warn( + "Estimate times not equal to reference times. " + "Resampling to common time base." + ) est_freqs = resample_multipitch(est_time, est_freqs, ref_time) # convert frequencies from Hz to continuous midi note number @@ -419,38 +424,56 @@ def metrics(ref_time, ref_freqs, est_time, est_freqs, **kwargs): ref_freqs_chroma = midi_to_chroma(ref_freqs_midi) est_freqs_chroma = midi_to_chroma(est_freqs_midi) - # count number of occurences + # count number of occurrences n_ref = compute_num_freqs(ref_freqs_midi) n_est = compute_num_freqs(est_freqs_midi) # compute the number of true positives true_positives = util.filter_kwargs( - compute_num_true_positives, ref_freqs_midi, est_freqs_midi, **kwargs) + compute_num_true_positives, ref_freqs_midi, est_freqs_midi, **kwargs + ) # compute the number of true positives ignoring octave mistakes true_positives_chroma = util.filter_kwargs( - compute_num_true_positives, ref_freqs_chroma, - est_freqs_chroma, chroma=True, **kwargs) + compute_num_true_positives, + ref_freqs_chroma, + est_freqs_chroma, + chroma=True, + **kwargs + ) # compute accuracy metrics - precision, recall, accuracy = compute_accuracy( - true_positives, n_ref, n_est) + precision, recall, accuracy = compute_accuracy(true_positives, n_ref, n_est) # compute error metrics - e_sub, e_miss, e_fa, e_tot = compute_err_score( - true_positives, n_ref, n_est) + e_sub, e_miss, e_fa, e_tot = compute_err_score(true_positives, n_ref, n_est) # compute accuracy metrics ignoring octave mistakes precision_chroma, recall_chroma, accuracy_chroma = compute_accuracy( - true_positives_chroma, n_ref, n_est) + true_positives_chroma, n_ref, n_est + ) # compute error metrics ignoring octave mistakes e_sub_chroma, e_miss_chroma, e_fa_chroma, e_tot_chroma = compute_err_score( - true_positives_chroma, n_ref, n_est) - - return (precision, recall, accuracy, e_sub, e_miss, e_fa, e_tot, - precision_chroma, recall_chroma, accuracy_chroma, e_sub_chroma, - e_miss_chroma, e_fa_chroma, e_tot_chroma) + true_positives_chroma, n_ref, n_est + ) + + return ( + precision, + recall, + accuracy, + e_sub, + e_miss, + e_fa, + e_tot, + precision_chroma, + recall_chroma, + accuracy_chroma, + e_sub_chroma, + e_miss_chroma, + e_fa_chroma, + e_tot_chroma, + ) def evaluate(ref_time, ref_freqs, est_time, est_freqs, **kwargs): @@ -475,7 +498,7 @@ def evaluate(ref_time, ref_freqs, est_time, est_freqs, **kwargs): Time of each estimated frequency value est_freqs : list of np.ndarray List of np.ndarrays of estimate frequency values - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -488,20 +511,21 @@ def evaluate(ref_time, ref_freqs, est_time, est_freqs, **kwargs): """ scores = collections.OrderedDict() - (scores['Precision'], - scores['Recall'], - scores['Accuracy'], - scores['Substitution Error'], - scores['Miss Error'], - scores['False Alarm Error'], - scores['Total Error'], - scores['Chroma Precision'], - scores['Chroma Recall'], - scores['Chroma Accuracy'], - scores['Chroma Substitution Error'], - scores['Chroma Miss Error'], - scores['Chroma False Alarm Error'], - scores['Chroma Total Error']) = util.filter_kwargs( - metrics, ref_time, ref_freqs, est_time, est_freqs, **kwargs) + ( + scores["Precision"], + scores["Recall"], + scores["Accuracy"], + scores["Substitution Error"], + scores["Miss Error"], + scores["False Alarm Error"], + scores["Total Error"], + scores["Chroma Precision"], + scores["Chroma Recall"], + scores["Chroma Accuracy"], + scores["Chroma Substitution Error"], + scores["Chroma Miss Error"], + scores["Chroma False Alarm Error"], + scores["Chroma Total Error"], + ) = util.filter_kwargs(metrics, ref_time, ref_freqs, est_time, est_freqs, **kwargs) return scores diff --git a/mir_eval/onset.py b/mir_eval/onset.py index 606ad0d7..d3437a33 100644 --- a/mir_eval/onset.py +++ b/mir_eval/onset.py @@ -1,4 +1,4 @@ -''' +""" The goal of an onset detection algorithm is to automatically determine when notes are played in a piece of music. The primary method used to evaluate onset detectors is to first determine which estimated onsets are "correct", @@ -19,9 +19,9 @@ ------- * :func:`mir_eval.onset.f_measure`: Precision, Recall, and F-measure scores - based on the number of esimated onsets which are sufficiently close to + based on the number of estimated onsets which are sufficiently close to reference onsets. -''' +""" import collections from . import util @@ -29,11 +29,11 @@ # The maximum allowable beat time -MAX_TIME = 30000. +MAX_TIME = 30000.0 def validate(reference_onsets, estimated_onsets): - """Checks that the input annotations to a metric look like valid onset time + """Check that the input annotations to a metric look like valid onset time arrays, and throws helpful errors if not. Parameters @@ -53,9 +53,9 @@ def validate(reference_onsets, estimated_onsets): util.validate_events(onsets, MAX_TIME) -def f_measure(reference_onsets, estimated_onsets, window=.05): +def f_measure(reference_onsets, estimated_onsets, window=0.05): """Compute the F-measure of correct vs incorrectly predicted onsets. - "Corectness" is determined over a small window. + "Correctness" is determined over a small window. Examples -------- @@ -87,13 +87,13 @@ def f_measure(reference_onsets, estimated_onsets, window=.05): validate(reference_onsets, estimated_onsets) # If either list is empty, return 0s if reference_onsets.size == 0 or estimated_onsets.size == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 # Compute the best-case matching between reference and estimated onset # locations matching = util.match_events(reference_onsets, estimated_onsets, window) - precision = float(len(matching))/len(estimated_onsets) - recall = float(len(matching))/len(reference_onsets) + precision = float(len(matching)) / len(estimated_onsets) + recall = float(len(matching)) / len(reference_onsets) # Compute F-measure and return all statistics return util.f_measure(precision, recall), precision, recall @@ -114,7 +114,7 @@ def evaluate(reference_onsets, estimated_onsets, **kwargs): reference onset locations, in seconds estimated_onsets : np.ndarray estimated onset locations, in seconds - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -128,9 +128,8 @@ def evaluate(reference_onsets, estimated_onsets, **kwargs): # Compute all metrics scores = collections.OrderedDict() - (scores['F-measure'], - scores['Precision'], - scores['Recall']) = util.filter_kwargs(f_measure, reference_onsets, - estimated_onsets, **kwargs) + (scores["F-measure"], scores["Precision"], scores["Recall"]) = util.filter_kwargs( + f_measure, reference_onsets, estimated_onsets, **kwargs + ) return scores diff --git a/mir_eval/pattern.py b/mir_eval/pattern.py index b5c016d5..56aa851f 100644 --- a/mir_eval/pattern.py +++ b/mir_eval/pattern.py @@ -16,7 +16,7 @@ The input format can be automatically generated by calling :func:`mir_eval.io.load_patterns`. This format is a list of a list of tuples. The first list collections patterns, each of which is a list of -occurences, and each occurrence is a list of MIDI onset tuples of +occurrences, and each occurrence is a list of MIDI onset tuples of ``(onset_time, mid_note)`` A pattern is a list of occurrences. The first occurrence must be the prototype @@ -54,7 +54,6 @@ relevance. """ - import numpy as np from . import util import warnings @@ -62,11 +61,11 @@ def _n_onset_midi(patterns): - """Computes the number of onset_midi objects in a pattern + """Compute the number of onset_midi objects in a pattern Parameters ---------- - patterns : + patterns A list of patterns using the format returned by :func:`mir_eval.io.load_patterns()` @@ -80,7 +79,7 @@ def _n_onset_midi(patterns): def validate(reference_patterns, estimated_patterns): - """Checks that the input annotations to a metric look like valid pattern + """Check that the input annotations to a metric look like valid pattern lists, and throws helpful errors if not. Parameters @@ -90,30 +89,29 @@ def validate(reference_patterns, estimated_patterns): :func:`mir_eval.io.load_patterns()` estimated_patterns : list The estimated patterns in the same format - - Returns - ------- - """ # Warn if pattern lists are empty if _n_onset_midi(reference_patterns) == 0: - warnings.warn('Reference patterns are empty.') + warnings.warn("Reference patterns are empty.") if _n_onset_midi(estimated_patterns) == 0: - warnings.warn('Estimated patterns are empty.') + warnings.warn("Estimated patterns are empty.") for patterns in [reference_patterns, estimated_patterns]: for pattern in patterns: if len(pattern) <= 0: - raise ValueError("Each pattern must contain at least one " - "occurrence.") + raise ValueError( + "Each pattern must contain at least one " "occurrence." + ) for occurrence in pattern: for onset_midi in occurrence: if len(onset_midi) != 2: - raise ValueError("The (onset, midi) tuple must " - "contain exactly 2 elements.") + raise ValueError( + "The (onset, midi) tuple must " + "contain exactly 2 elements." + ) def _occurrence_intersection(occ_P, occ_Q): - """Computes the intersection between two occurrences. + """Compute the intersection between two occurrences. Parameters ---------- @@ -128,13 +126,13 @@ def _occurrence_intersection(occ_P, occ_Q): Set of the intersection between occ_P and occ_Q. """ - set_P = set([tuple(onset_midi) for onset_midi in occ_P]) - set_Q = set([tuple(onset_midi) for onset_midi in occ_Q]) - return set_P & set_Q # Return the intersection + set_P = {tuple(onset_midi) for onset_midi in occ_P} + set_Q = {tuple(onset_midi) for onset_midi in occ_Q} + return set_P & set_Q # Return the intersection def _compute_score_matrix(P, Q, similarity_metric="cardinality_score"): - """Computes the score matrix between the patterns P and Q. + """Compute the score matrix between the patterns P and Q. Parameters ---------- @@ -155,28 +153,28 @@ def _compute_score_matrix(P, Q, similarity_metric="cardinality_score"): The score matrix between P and Q using the similarity_metric. """ - sm = np.zeros((len(P), len(Q))) # The score matrix + sm = np.zeros((len(P), len(Q))) # The score matrix for iP, occ_P in enumerate(P): for iQ, occ_Q in enumerate(Q): if similarity_metric == "cardinality_score": denom = float(np.max([len(occ_P), len(occ_Q)])) # Compute the score - sm[iP, iQ] = len(_occurrence_intersection(occ_P, occ_Q)) / \ - denom + sm[iP, iQ] = len(_occurrence_intersection(occ_P, occ_Q)) / denom # TODO: More scores: 'normalised matching socre' else: - raise ValueError("The similarity metric (%s) can only be: " - "'cardinality_score'.") + raise ValueError( + "The similarity metric (%s) can only be: " "'cardinality_score'." + ) return sm def standard_FPR(reference_patterns, estimated_patterns, tol=1e-5): - """Standard F1 Score, Precision and Recall. + """Compute the standard F1 Score, Precision and Recall. This metric checks if the prototype patterns of the reference match possible translated patterns in the prototype patterns of the estimations. Since the sizes of these prototypes must be equal, this metric is quite - restictive and it tends to be 0 in most of 2013 MIREX results. + restrictive and it tends to be 0 in most of 2013 MIREX results. Examples -------- @@ -208,18 +206,17 @@ def standard_FPR(reference_patterns, estimated_patterns, tol=1e-5): """ validate(reference_patterns, estimated_patterns) - nP = len(reference_patterns) # Number of patterns in the reference - nQ = len(estimated_patterns) # Number of patterns in the estimation - k = 0 # Number of patterns that match + nP = len(reference_patterns) # Number of patterns in the reference + nQ = len(estimated_patterns) # Number of patterns in the estimation + k = 0 # Number of patterns that match # If no patterns were provided, metric is zero - if _n_onset_midi(reference_patterns) == 0 or \ - _n_onset_midi(estimated_patterns) == 0: - return 0., 0., 0. + if _n_onset_midi(reference_patterns) == 0 or _n_onset_midi(estimated_patterns) == 0: + return 0.0, 0.0, 0.0 # Find matches of the prototype patterns for ref_pattern in reference_patterns: - P = np.asarray(ref_pattern[0]) # Get reference prototype + P = np.asarray(ref_pattern[0]) # Get reference prototype for est_pattern in estimated_patterns: Q = np.asarray(est_pattern[0]) # Get estimation prototype @@ -227,8 +224,7 @@ def standard_FPR(reference_patterns, estimated_patterns, tol=1e-5): continue # Check transposition given a certain tolerance - if (len(P) == len(Q) == 1 or - np.max(np.abs(np.diff(P - Q, axis=0))) < tol): + if len(P) == len(Q) == 1 or np.max(np.abs(np.diff(P - Q, axis=0))) < tol: k += 1 break @@ -239,9 +235,10 @@ def standard_FPR(reference_patterns, estimated_patterns, tol=1e-5): return f_measure, precision, recall -def establishment_FPR(reference_patterns, estimated_patterns, - similarity_metric="cardinality_score"): - """Establishment F1 Score, Precision and Recall. +def establishment_FPR( + reference_patterns, estimated_patterns, similarity_metric="cardinality_score" +): + """Compute the establishment F1 Score, Precision and Recall. Examples -------- @@ -250,7 +247,6 @@ def establishment_FPR(reference_patterns, estimated_patterns, >>> F, P, R = mir_eval.pattern.establishment_FPR(ref_patterns, ... est_patterns) - Parameters ---------- reference_patterns : list @@ -269,7 +265,6 @@ def establishment_FPR(reference_patterns, estimated_patterns, (Default value = "cardinality_score") - Returns ------- f_measure : float @@ -281,19 +276,17 @@ def establishment_FPR(reference_patterns, estimated_patterns, """ validate(reference_patterns, estimated_patterns) - nP = len(reference_patterns) # Number of elements in reference - nQ = len(estimated_patterns) # Number of elements in estimation - S = np.zeros((nP, nQ)) # Establishment matrix + nP = len(reference_patterns) # Number of elements in reference + nQ = len(estimated_patterns) # Number of elements in estimation + S = np.zeros((nP, nQ)) # Establishment matrix # If no patterns were provided, metric is zero - if _n_onset_midi(reference_patterns) == 0 or \ - _n_onset_midi(estimated_patterns) == 0: - return 0., 0., 0. + if _n_onset_midi(reference_patterns) == 0 or _n_onset_midi(estimated_patterns) == 0: + return 0.0, 0.0, 0.0 for iP, ref_pattern in enumerate(reference_patterns): for iQ, est_pattern in enumerate(estimated_patterns): - s = _compute_score_matrix(ref_pattern, est_pattern, - similarity_metric) + s = _compute_score_matrix(ref_pattern, est_pattern, similarity_metric) S[iP, iQ] = np.max(s) # Compute scores @@ -303,10 +296,13 @@ def establishment_FPR(reference_patterns, estimated_patterns, return f_measure, precision, recall -def occurrence_FPR(reference_patterns, estimated_patterns, thres=.75, - similarity_metric="cardinality_score"): - """Establishment F1 Score, Precision and Recall. - +def occurrence_FPR( + reference_patterns, + estimated_patterns, + thres=0.75, + similarity_metric="cardinality_score", +): + """Compute the occurrence F1 Score, Precision and Recall. Examples -------- @@ -315,18 +311,20 @@ def occurrence_FPR(reference_patterns, estimated_patterns, thres=.75, >>> F, P, R = mir_eval.pattern.occurrence_FPR(ref_patterns, ... est_patterns) - Parameters ---------- reference_patterns : list The reference patterns in the format returned by :func:`mir_eval.io.load_patterns()` + estimated_patterns : list The estimated patterns in the same format + thres : float - How similar two occcurrences must be in order to be considered + How similar two occurrences must be in order to be considered equal (Default value = .75) + similarity_metric : str A string representing the metric to be used when computing the similarity matrix. Accepted values: @@ -336,16 +334,14 @@ def occurrence_FPR(reference_patterns, estimated_patterns, thres=.75, (Default value = "cardinality_score") - Returns ------- f_measure : float - The establishment F1 Score + The occurrence F1 Score precision : float - The establishment Precision + The occurrence Precision recall : float - The establishment Recall - + The occurrence Recall """ validate(reference_patterns, estimated_patterns) # Number of elements in reference @@ -359,14 +355,12 @@ def occurrence_FPR(reference_patterns, estimated_patterns, thres=.75, rel_idx = np.empty((0, 2), dtype=int) # If no patterns were provided, metric is zero - if _n_onset_midi(reference_patterns) == 0 or \ - _n_onset_midi(estimated_patterns) == 0: - return 0., 0., 0. + if _n_onset_midi(reference_patterns) == 0 or _n_onset_midi(estimated_patterns) == 0: + return 0.0, 0.0, 0.0 for iP, ref_pattern in enumerate(reference_patterns): for iQ, est_pattern in enumerate(estimated_patterns): - s = _compute_score_matrix(ref_pattern, est_pattern, - similarity_metric) + s = _compute_score_matrix(ref_pattern, est_pattern, similarity_metric) if np.max(s) >= thres: O_PR[iP, iQ, 0] = np.mean(np.max(s, axis=0)) O_PR[iP, iQ, 1] = np.mean(np.max(s, axis=1)) @@ -378,11 +372,9 @@ def occurrence_FPR(reference_patterns, estimated_patterns, thres=.75, recall = 0 else: P = O_PR[:, :, 0] - precision = np.mean(np.max(P[np.ix_(rel_idx[:, 0], rel_idx[:, 1])], - axis=0)) + precision = np.mean(np.max(P[np.ix_(rel_idx[:, 0], rel_idx[:, 1])], axis=0)) R = O_PR[:, :, 1] - recall = np.mean(np.max(R[np.ix_(rel_idx[:, 0], rel_idx[:, 1])], - axis=1)) + recall = np.mean(np.max(R[np.ix_(rel_idx[:, 0], rel_idx[:, 1])], axis=1)) f_measure = util.f_measure(precision, recall) return f_measure, precision, recall @@ -418,20 +410,19 @@ def three_layer_FPR(reference_patterns, estimated_patterns): validate(reference_patterns, estimated_patterns) def compute_first_layer_PR(ref_occs, est_occs): - """Computes the first layer Precision and Recall values given the + """Compute the first layer Precision and Recall values given the set of occurrences in the reference and the set of occurrences in the estimation. Parameters ---------- - ref_occs : - - est_occs : - + ref_occs + est_occs Returns ------- - + precision + recall """ # Find the length of the intersection between reference and estimation s = len(_occurrence_intersection(ref_occs, est_occs)) @@ -442,20 +433,19 @@ def compute_first_layer_PR(ref_occs, est_occs): return precision, recall def compute_second_layer_PR(ref_pattern, est_pattern): - """Computes the second layer Precision and Recall values given the + """Compute the second layer Precision and Recall values given the set of occurrences in the reference and the set of occurrences in the estimation. Parameters ---------- - ref_pattern : - - est_pattern : - + ref_pattern + est_pattern Returns ------- - + precision + recall """ # Compute the first layer scores F_1 = compute_layer(ref_pattern, est_pattern) @@ -466,8 +456,8 @@ def compute_second_layer_PR(ref_pattern, est_pattern): return precision, recall def compute_layer(ref_elements, est_elements, layer=1): - """Computes the F-measure matrix for a given layer. The reference and - estimated elements can be either patters or occurrences, depending + """Compute the F-measure matrix for a given layer. The reference and + estimated elements can be either patterns or occurrences, depending on the layer. For layer 1, the elements must be occurrences. @@ -475,24 +465,21 @@ def compute_layer(ref_elements, est_elements, layer=1): Parameters ---------- - ref_elements : - - est_elements : - - layer : - (Default value = 1) + ref_elements + est_elements + layer + (Default value = 1) Returns ------- - + F : F-measure for the given layer """ if layer != 1 and layer != 2: - raise ValueError("Layer (%d) must be an integer between 1 and 2" - % layer) + raise ValueError("Layer (%d) must be an integer between 1 and 2" % layer) - nP = len(ref_elements) # Number of elements in reference - nQ = len(est_elements) # Number of elements in estimation - F = np.zeros((nP, nQ)) # F-measure matrix for the given layer + nP = len(ref_elements) # Number of elements in reference + nQ = len(est_elements) # Number of elements in estimation + F = np.zeros((nP, nQ)) # F-measure matrix for the given layer for iP in range(nP): for iQ in range(nQ): if layer == 1: @@ -506,9 +493,8 @@ def compute_layer(ref_elements, est_elements, layer=1): return F # If no patterns were provided, metric is zero - if _n_onset_midi(reference_patterns) == 0 or \ - _n_onset_midi(estimated_patterns) == 0: - return 0., 0., 0. + if _n_onset_midi(reference_patterns) == 0 or _n_onset_midi(estimated_patterns) == 0: + return 0.0, 0.0, 0.0 # Compute the second layer (it includes the first layer) F_2 = compute_layer(reference_patterns, estimated_patterns, layer=2) @@ -550,22 +536,19 @@ def first_n_three_layer_P(reference_patterns, estimated_patterns, n=5): ------- precision : float The first n three-layer Precision - """ - validate(reference_patterns, estimated_patterns) # If no patterns were provided, metric is zero - if _n_onset_midi(reference_patterns) == 0 or \ - _n_onset_midi(estimated_patterns) == 0: - return 0., 0., 0. + if _n_onset_midi(reference_patterns) == 0 or _n_onset_midi(estimated_patterns) == 0: + return 0.0, 0.0, 0.0 # Get only the first n patterns from the estimated results - fn_est_patterns = estimated_patterns[:min(len(estimated_patterns), n)] + fn_est_patterns = estimated_patterns[: min(len(estimated_patterns), n)] # Compute the three-layer scores for the first n estimated patterns F, P, R = three_layer_FPR(reference_patterns, fn_est_patterns) - return P # Return the precision only + return P # Return the precision only def first_n_target_proportion_R(reference_patterns, estimated_patterns, n=5): @@ -598,17 +581,14 @@ def first_n_target_proportion_R(reference_patterns, estimated_patterns, n=5): ------- recall : float The first n target proportion Recall. - """ - validate(reference_patterns, estimated_patterns) # If no patterns were provided, metric is zero - if _n_onset_midi(reference_patterns) == 0 or \ - _n_onset_midi(estimated_patterns) == 0: - return 0., 0., 0. + if _n_onset_midi(reference_patterns) == 0 or _n_onset_midi(estimated_patterns) == 0: + return 0.0, 0.0, 0.0 # Get only the first n patterns from the estimated results - fn_est_patterns = estimated_patterns[:min(len(estimated_patterns), n)] + fn_est_patterns = estimated_patterns[: min(len(estimated_patterns), n)] F, P, R = establishment_FPR(reference_patterns, fn_est_patterns) return R @@ -630,7 +610,7 @@ def evaluate(ref_patterns, est_patterns, **kwargs): :func:`mir_eval.io.load_patterns()` est_patterns : list The estimated patterns in the same format - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -639,45 +619,45 @@ def evaluate(ref_patterns, est_patterns, **kwargs): scores : dict Dictionary of scores, where the key is the metric name (str) and the value is the (float) score achieved. - """ - # Compute all the metrics scores = collections.OrderedDict() # Standard scores - scores['F'], scores['P'], scores['R'] = \ - util.filter_kwargs(standard_FPR, ref_patterns, est_patterns, **kwargs) + scores["F"], scores["P"], scores["R"] = util.filter_kwargs( + standard_FPR, ref_patterns, est_patterns, **kwargs + ) # Establishment scores - scores['F_est'], scores['P_est'], scores['R_est'] = \ - util.filter_kwargs(establishment_FPR, ref_patterns, est_patterns, - **kwargs) + scores["F_est"], scores["P_est"], scores["R_est"] = util.filter_kwargs( + establishment_FPR, ref_patterns, est_patterns, **kwargs + ) # Occurrence scores # Force these values for thresh - kwargs['thresh'] = .5 - scores['F_occ.5'], scores['P_occ.5'], scores['R_occ.5'] = \ - util.filter_kwargs(occurrence_FPR, ref_patterns, est_patterns, - **kwargs) - kwargs['thresh'] = .75 - scores['F_occ.75'], scores['P_occ.75'], scores['R_occ.75'] = \ - util.filter_kwargs(occurrence_FPR, ref_patterns, est_patterns, - **kwargs) + kwargs["thresh"] = 0.5 + scores["F_occ.5"], scores["P_occ.5"], scores["R_occ.5"] = util.filter_kwargs( + occurrence_FPR, ref_patterns, est_patterns, **kwargs + ) + kwargs["thresh"] = 0.75 + scores["F_occ.75"], scores["P_occ.75"], scores["R_occ.75"] = util.filter_kwargs( + occurrence_FPR, ref_patterns, est_patterns, **kwargs + ) # Three-layer scores - scores['F_3'], scores['P_3'], scores['R_3'] = \ - util.filter_kwargs(three_layer_FPR, ref_patterns, est_patterns, - **kwargs) + scores["F_3"], scores["P_3"], scores["R_3"] = util.filter_kwargs( + three_layer_FPR, ref_patterns, est_patterns, **kwargs + ) # First Five Patterns scores # Set default value of n - if 'n' not in kwargs: - kwargs['n'] = 5 - scores['FFP'] = util.filter_kwargs(first_n_three_layer_P, ref_patterns, - est_patterns, **kwargs) - scores['FFTP_est'] = \ - util.filter_kwargs(first_n_target_proportion_R, ref_patterns, - est_patterns, **kwargs) + if "n" not in kwargs: + kwargs["n"] = 5 + scores["FFP"] = util.filter_kwargs( + first_n_three_layer_P, ref_patterns, est_patterns, **kwargs + ) + scores["FFTP_est"] = util.filter_kwargs( + first_n_target_proportion_R, ref_patterns, est_patterns, **kwargs + ) return scores diff --git a/mir_eval/segment.py b/mir_eval/segment.py index 9234bc7e..5eea5b22 100644 --- a/mir_eval/segment.py +++ b/mir_eval/segment.py @@ -1,5 +1,5 @@ # CREATED:2013-08-13 12:02:42 by Brian McFee -''' +""" Evaluation criteria for structural segmentation fall into two categories: boundary annotation and structural annotation. Boundary annotation is the task of predicting the times at which structural changes occur, such as when a verse @@ -70,7 +70,7 @@ V-Measure: A Conditional Entropy-Based External Cluster Evaluation Measure. In EMNLP-CoNLL (Vol. 7, pp. 410-420). -''' +""" import collections import warnings @@ -78,14 +78,13 @@ import numpy as np import scipy.stats import scipy.sparse -import scipy.misc import scipy.special from . import util def validate_boundary(reference_intervals, estimated_intervals, trim): - """Checks that the input annotations to a segment boundary estimation + """Check that the input annotations to a segment boundary estimation metric (i.e. one that only takes in segment intervals) look like valid segment times, and throws helpful errors if not. @@ -95,17 +94,13 @@ def validate_boundary(reference_intervals, estimated_intervals, trim): reference segment intervals, in the format returned by :func:`mir_eval.io.load_intervals` or :func:`mir_eval.io.load_labeled_intervals`. - estimated_intervals : np.ndarray, shape=(m, 2) estimated segment intervals, in the format returned by :func:`mir_eval.io.load_intervals` or :func:`mir_eval.io.load_labeled_intervals`. - trim : bool will the start and end events be trimmed? - """ - if trim: # If we're trimming, then we need at least 2 intervals min_size = 2 @@ -123,9 +118,10 @@ def validate_boundary(reference_intervals, estimated_intervals, trim): util.validate_intervals(intervals) -def validate_structure(reference_intervals, reference_labels, - estimated_intervals, estimated_labels): - """Checks that the input annotations to a structure estimation metric (i.e. +def validate_structure( + reference_intervals, reference_labels, estimated_intervals, estimated_labels +): + """Check that the input annotations to a structure estimation metric (i.e. one that takes in both segment boundaries and their labels) look like valid segment times and labels, and throws helpful errors if not. @@ -134,33 +130,30 @@ def validate_structure(reference_intervals, reference_labels, reference_intervals : np.ndarray, shape=(n, 2) reference segment intervals, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - reference_labels : list, shape=(n,) reference segment labels, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - estimated_intervals : np.ndarray, shape=(m, 2) estimated segment intervals, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - estimated_labels : list, shape=(m,) estimated segment labels, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. """ - for (intervals, labels) in [(reference_intervals, reference_labels), - (estimated_intervals, estimated_labels)]: - + for intervals, labels in [ + (reference_intervals, reference_labels), + (estimated_intervals, estimated_labels), + ]: util.validate_intervals(intervals) if intervals.shape[0] != len(labels): - raise ValueError('Number of intervals does not match number ' - 'of labels') + raise ValueError("Number of intervals does not match number " "of labels") # Check only when intervals are non-empty if intervals.size > 0: # Make sure intervals start at 0 if not np.allclose(intervals.min(), 0.0): - raise ValueError('Segment intervals do not start at 0') + raise ValueError("Segment intervals do not start at 0") if reference_intervals.size == 0: warnings.warn("Reference intervals are empty.") @@ -168,13 +161,13 @@ def validate_structure(reference_intervals, reference_labels, warnings.warn("Estimated intervals are empty.") # Check only when intervals are non-empty if reference_intervals.size > 0 and estimated_intervals.size > 0: - if not np.allclose(reference_intervals.max(), - estimated_intervals.max()): - raise ValueError('End times do not match') + if not np.allclose(reference_intervals.max(), estimated_intervals.max()): + raise ValueError("End times do not match") -def detection(reference_intervals, estimated_intervals, - window=0.5, beta=1.0, trim=False): +def detection( + reference_intervals, estimated_intervals, window=0.5, beta=1.0, trim=False +): """Boundary detection hit-rate. A hit is counted whenever an reference boundary is within ``window`` of a @@ -230,9 +223,7 @@ def detection(reference_intervals, estimated_intervals, recall of reference reference boundaries f_measure : float F-measure (weighted harmonic mean of ``precision`` and ``recall``) - """ - validate_boundary(reference_intervals, estimated_intervals, trim) # Convert intervals to boundaries @@ -248,9 +239,7 @@ def detection(reference_intervals, estimated_intervals, if len(reference_boundaries) == 0 or len(estimated_boundaries) == 0: return 0.0, 0.0, 0.0 - matching = util.match_events(reference_boundaries, - estimated_boundaries, - window) + matching = util.match_events(reference_boundaries, estimated_boundaries, window) precision = float(len(matching)) / len(estimated_boundaries) recall = float(len(matching)) / len(reference_boundaries) @@ -294,9 +283,7 @@ def deviation(reference_intervals, estimated_intervals, trim=False): estimated_to_reference : float median time from each estimated boundary to the closest reference boundary - """ - validate_boundary(reference_intervals, estimated_intervals, trim) # Convert intervals to boundaries @@ -312,8 +299,7 @@ def deviation(reference_intervals, estimated_intervals, trim=False): if len(reference_boundaries) == 0 or len(estimated_boundaries) == 0: return np.nan, np.nan - dist = np.abs(np.subtract.outer(reference_boundaries, - estimated_boundaries)) + dist = np.abs(np.subtract.outer(reference_boundaries, estimated_boundaries)) estimated_to_reference = np.median(dist.min(axis=0)) reference_to_estimated = np.median(dist.min(axis=1)) @@ -321,9 +307,14 @@ def deviation(reference_intervals, estimated_intervals, trim=False): return reference_to_estimated, estimated_to_reference -def pairwise(reference_intervals, reference_labels, - estimated_intervals, estimated_labels, - frame_size=0.1, beta=1.0): +def pairwise( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=0.1, + beta=1.0, +): """Frame-clustering segmentation evaluation by pair-wise agreement. Examples @@ -376,25 +367,26 @@ def pairwise(reference_intervals, reference_labels, F-measure of detecting whether frames belong in the same cluster """ - validate_structure(reference_intervals, reference_labels, - estimated_intervals, estimated_labels) + validate_structure( + reference_intervals, reference_labels, estimated_intervals, estimated_labels + ) # Check for empty annotations. Don't need to check labels because # validate_structure makes sure they're the same size as intervals if reference_intervals.size == 0 or estimated_intervals.size == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 # Generate the cluster labels - y_ref = util.intervals_to_samples(reference_intervals, - reference_labels, - sample_size=frame_size)[-1] + y_ref = util.intervals_to_samples( + reference_intervals, reference_labels, sample_size=frame_size + )[-1] y_ref = util.index_labels(y_ref)[0] # Map to index space - y_est = util.intervals_to_samples(estimated_intervals, - estimated_labels, - sample_size=frame_size)[-1] + y_est = util.intervals_to_samples( + estimated_intervals, estimated_labels, sample_size=frame_size + )[-1] y_est = util.index_labels(y_est)[0] @@ -418,9 +410,14 @@ def pairwise(reference_intervals, reference_labels, return precision, recall, f_measure -def rand_index(reference_intervals, reference_labels, - estimated_intervals, estimated_labels, - frame_size=0.1, beta=1.0): +def rand_index( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=0.1, + beta=1.0, +): """(Non-adjusted) Rand index. Examples @@ -467,28 +464,27 @@ def rand_index(reference_intervals, reference_labels, ------- rand_index : float > 0 Rand index - """ - - validate_structure(reference_intervals, reference_labels, - estimated_intervals, estimated_labels) + validate_structure( + reference_intervals, reference_labels, estimated_intervals, estimated_labels + ) # Check for empty annotations. Don't need to check labels because # validate_structure makes sure they're the same size as intervals if reference_intervals.size == 0 or estimated_intervals.size == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 # Generate the cluster labels - y_ref = util.intervals_to_samples(reference_intervals, - reference_labels, - sample_size=frame_size)[-1] + y_ref = util.intervals_to_samples( + reference_intervals, reference_labels, sample_size=frame_size + )[-1] y_ref = util.index_labels(y_ref)[0] # Map to index space - y_est = util.intervals_to_samples(estimated_intervals, - estimated_labels, - sample_size=frame_size)[-1] + y_est = util.intervals_to_samples( + estimated_intervals, estimated_labels, sample_size=frame_size + )[-1] y_est = util.index_labels(y_est)[0] @@ -514,7 +510,7 @@ def rand_index(reference_intervals, reference_labels, def _contingency_matrix(reference_indices, estimated_indices): - """Computes the contingency matrix of a true labeling vs an estimated one. + """Compute the contingency matrix of a true labeling vs an estimated one. Parameters ---------- @@ -530,17 +526,16 @@ def _contingency_matrix(reference_indices, estimated_indices): .. note:: Based on sklearn.metrics.cluster.contingency_matrix """ - ref_classes, ref_class_idx = np.unique(reference_indices, - return_inverse=True) - est_classes, est_class_idx = np.unique(estimated_indices, - return_inverse=True) + ref_classes, ref_class_idx = np.unique(reference_indices, return_inverse=True) + est_classes, est_class_idx = np.unique(estimated_indices, return_inverse=True) n_ref_classes = ref_classes.shape[0] n_est_classes = est_classes.shape[0] # Using coo_matrix is faster than histogram2d - return scipy.sparse.coo_matrix((np.ones(ref_class_idx.shape[0]), - (ref_class_idx, est_class_idx)), - shape=(n_ref_classes, n_est_classes), - dtype=np.int).toarray() + return scipy.sparse.coo_matrix( + (np.ones(ref_class_idx.shape[0]), (ref_class_idx, est_class_idx)), + shape=(n_ref_classes, n_est_classes), + dtype=np.int64, + ).toarray() def _adjusted_rand_index(reference_indices, estimated_indices): @@ -557,7 +552,6 @@ def _adjusted_rand_index(reference_indices, estimated_indices): ------- ari : float Adjusted Rand index - .. note:: Based on sklearn.metrics.cluster.adjusted_rand_score """ @@ -567,32 +561,39 @@ def _adjusted_rand_index(reference_indices, estimated_indices): # Special limit cases: no clustering since the data is not split; # or trivial clustering where each document is assigned a unique cluster. # These are perfect matches hence return 1.0. - if (ref_classes.shape[0] == est_classes.shape[0] == 1 or - ref_classes.shape[0] == est_classes.shape[0] == 0 or - (ref_classes.shape[0] == est_classes.shape[0] == - len(reference_indices))): + if ( + ref_classes.shape[0] == est_classes.shape[0] == 1 + or ref_classes.shape[0] == est_classes.shape[0] == 0 + or (ref_classes.shape[0] == est_classes.shape[0] == len(reference_indices)) + ): return 1.0 contingency = _contingency_matrix(reference_indices, estimated_indices) # Compute the ARI using the contingency data - sum_comb_c = sum(scipy.special.comb(n_c, 2, exact=1) for n_c in - contingency.sum(axis=1)) - sum_comb_k = sum(scipy.special.comb(n_k, 2, exact=1) for n_k in - contingency.sum(axis=0)) - - sum_comb = sum((scipy.special.comb(n_ij, 2, exact=1) for n_ij in - contingency.flatten())) - prod_comb = (sum_comb_c * sum_comb_k)/float(scipy.special.comb(n_samples, - 2)) - mean_comb = (sum_comb_k + sum_comb_c)/2. - return (sum_comb - prod_comb)/(mean_comb - prod_comb) - - -def ari(reference_intervals, reference_labels, - estimated_intervals, estimated_labels, - frame_size=0.1): - """Adjusted Rand Index (ARI) for frame clustering segmentation evaluation. + sum_comb_c = sum( + scipy.special.comb(n_c, 2, exact=1) for n_c in contingency.sum(axis=1) + ) + sum_comb_k = sum( + scipy.special.comb(n_k, 2, exact=1) for n_k in contingency.sum(axis=0) + ) + + sum_comb = sum( + scipy.special.comb(n_ij, 2, exact=1) for n_ij in contingency.flatten() + ) + prod_comb = (sum_comb_c * sum_comb_k) / float(scipy.special.comb(n_samples, 2)) + mean_comb = (sum_comb_k + sum_comb_c) / 2.0 + return (sum_comb - prod_comb) / (mean_comb - prod_comb) + + +def ari( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=0.1, +): + """Compute the Adjusted Rand Index (ARI) for frame clustering segmentation evaluation. Examples -------- @@ -635,25 +636,26 @@ def ari(reference_intervals, reference_labels, Adjusted Rand index between segmentations. """ - validate_structure(reference_intervals, reference_labels, - estimated_intervals, estimated_labels) + validate_structure( + reference_intervals, reference_labels, estimated_intervals, estimated_labels + ) # Check for empty annotations. Don't need to check labels because # validate_structure makes sure they're the same size as intervals if reference_intervals.size == 0 or estimated_intervals.size == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 # Generate the cluster labels - y_ref = util.intervals_to_samples(reference_intervals, - reference_labels, - sample_size=frame_size)[-1] + y_ref = util.intervals_to_samples( + reference_intervals, reference_labels, sample_size=frame_size + )[-1] y_ref = util.index_labels(y_ref)[0] # Map to index space - y_est = util.intervals_to_samples(estimated_intervals, - estimated_labels, - sample_size=frame_size)[-1] + y_est = util.intervals_to_samples( + estimated_intervals, estimated_labels, sample_size=frame_size + )[-1] y_est = util.index_labels(y_est)[0] @@ -677,13 +679,13 @@ def _mutual_info_score(reference_indices, estimated_indices, contingency=None): ------- mi : float Mutual information - .. note:: Based on sklearn.metrics.cluster.mutual_info_score """ if contingency is None: - contingency = _contingency_matrix(reference_indices, - estimated_indices).astype(float) + contingency = _contingency_matrix(reference_indices, estimated_indices).astype( + float + ) contingency_sum = np.sum(contingency) pi = np.sum(contingency, axis=1) pj = np.sum(contingency, axis=0) @@ -696,13 +698,15 @@ def _mutual_info_score(reference_indices, estimated_indices, contingency=None): # log(a / b) should be calculated as log(a) - log(b) for # possible loss of precision log_outer = -np.log(outer[nnz]) + np.log(pi.sum()) + np.log(pj.sum()) - mi = (contingency_nm * (log_contingency_nm - np.log(contingency_sum)) + - contingency_nm * log_outer) + mi = ( + contingency_nm * (log_contingency_nm - np.log(contingency_sum)) + + contingency_nm * log_outer + ) return mi.sum() def _entropy(labels): - """Calculates the entropy for a labeling. + """Calculate the entropy for a labeling. Parameters ---------- @@ -713,14 +717,13 @@ def _entropy(labels): ------- entropy : float Entropy of the labeling. - .. note:: Based on sklearn.metrics.cluster.entropy """ if len(labels) == 0: return 1.0 label_idx = np.unique(labels, return_inverse=True)[1] - pi = np.bincount(label_idx).astype(np.float) + pi = np.bincount(label_idx).astype(np.float64) pi = pi[pi > 0] pi_sum = np.sum(pi) # log(a / b) should be calculated as log(a) - log(b) for @@ -736,7 +739,6 @@ def _adjusted_mutual_info_score(reference_indices, estimated_indices): ---------- reference_indices : np.ndarray Array of reference indices - estimated_indices : np.ndarray Array of estimated indices @@ -744,7 +746,6 @@ def _adjusted_mutual_info_score(reference_indices, estimated_indices): ------- ami : float <= 1.0 Mutual information - .. note:: Based on sklearn.metrics.cluster.adjusted_mutual_info_score and sklearn.metrics.cluster.expected_mutual_info_score @@ -754,14 +755,18 @@ def _adjusted_mutual_info_score(reference_indices, estimated_indices): est_classes = np.unique(estimated_indices) # Special limit cases: no clustering since the data is not split. # This is a perfect match hence return 1.0. - if (ref_classes.shape[0] == est_classes.shape[0] == 1 or - ref_classes.shape[0] == est_classes.shape[0] == 0): + if ( + ref_classes.shape[0] == est_classes.shape[0] == 1 + or ref_classes.shape[0] == est_classes.shape[0] == 0 + ): return 1.0 - contingency = _contingency_matrix(reference_indices, - estimated_indices).astype(float) + contingency = _contingency_matrix(reference_indices, estimated_indices).astype( + float + ) # Calculate the MI for the two clusterings - mi = _mutual_info_score(reference_indices, estimated_indices, - contingency=contingency) + mi = _mutual_info_score( + reference_indices, estimated_indices, contingency=contingency + ) # The following code is based on # sklearn.metrics.cluster.expected_mutual_information R, C = contingency.shape @@ -771,7 +776,7 @@ def _adjusted_mutual_info_score(reference_indices, estimated_indices): # There are three major terms to the EMI equation, which are multiplied to # and then summed over varying nij values. # While nijs[0] will never be used, having it simplifies the indexing. - nijs = np.arange(0, max(np.max(a), np.max(b)) + 1, dtype='float') + nijs = np.arange(0, max(np.max(a), np.max(b)) + 1, dtype="float") # Stops divide by zero warnings. As its not used, no issue. nijs[0] = 1 # term1 is nij / N @@ -790,7 +795,7 @@ def _adjusted_mutual_info_score(reference_indices, estimated_indices): gln_N = scipy.special.gammaln(N + 1) gln_nij = scipy.special.gammaln(nijs + 1) # start and end values for nij terms for each summation. - start = np.array([[v - N + w for w in b] for v in a], dtype='int') + start = np.array([[v - N + w for w in b] for v in a], dtype="int") start = np.maximum(start, 1) end = np.minimum(np.resize(a, (C, R)).T, np.resize(b, (R, C))) + 1 # emi itself is a summation over the various values. @@ -800,13 +805,19 @@ def _adjusted_mutual_info_score(reference_indices, estimated_indices): for nij in range(start[i, j], end[i, j]): term2 = log_Nnij[nij] - log_ab_outer[i, j] # Numerators are positive, denominators are negative. - gln = (gln_a[i] + gln_b[j] + gln_Na[i] + gln_Nb[j] - - gln_N - gln_nij[nij] - - scipy.special.gammaln(a[i] - nij + 1) - - scipy.special.gammaln(b[j] - nij + 1) - - scipy.special.gammaln(N - a[i] - b[j] + nij + 1)) + gln = ( + gln_a[i] + + gln_b[j] + + gln_Na[i] + + gln_Nb[j] + - gln_N + - gln_nij[nij] + - scipy.special.gammaln(a[i] - nij + 1) + - scipy.special.gammaln(b[j] - nij + 1) + - scipy.special.gammaln(N - a[i] - b[j] + nij + 1) + ) term3 = np.exp(gln) - emi += (term1[nij] * term2 * term3) + emi += term1[nij] * term2 * term3 # Calculate entropy for each labeling h_true, h_pred = _entropy(reference_indices), _entropy(estimated_indices) ami = (mi - emi) / (max(h_true, h_pred) - emi) @@ -821,7 +832,6 @@ def _normalized_mutual_info_score(reference_indices, estimated_indices): ---------- reference_indices : np.ndarray Array of reference indices - estimated_indices : np.ndarray Array of estimated indices @@ -829,7 +839,6 @@ def _normalized_mutual_info_score(reference_indices, estimated_indices): ------- nmi : float <= 1.0 Normalized mutual information - .. note:: Based on sklearn.metrics.cluster.normalized_mutual_info_score """ @@ -837,15 +846,19 @@ def _normalized_mutual_info_score(reference_indices, estimated_indices): est_classes = np.unique(estimated_indices) # Special limit cases: no clustering since the data is not split. # This is a perfect match hence return 1.0. - if (ref_classes.shape[0] == est_classes.shape[0] == 1 or - ref_classes.shape[0] == est_classes.shape[0] == 0): + if ( + ref_classes.shape[0] == est_classes.shape[0] == 1 + or ref_classes.shape[0] == est_classes.shape[0] == 0 + ): return 1.0 - contingency = _contingency_matrix(reference_indices, - estimated_indices).astype(float) - contingency = np.array(contingency, dtype='float') + contingency = _contingency_matrix(reference_indices, estimated_indices).astype( + float + ) + contingency = np.array(contingency, dtype="float") # Calculate the MI for the two clusterings - mi = _mutual_info_score(reference_indices, estimated_indices, - contingency=contingency) + mi = _mutual_info_score( + reference_indices, estimated_indices, contingency=contingency + ) # Calculate the expected value for the mutual information # Calculate entropy for each labeling h_true, h_pred = _entropy(reference_indices), _entropy(estimated_indices) @@ -853,9 +866,13 @@ def _normalized_mutual_info_score(reference_indices, estimated_indices): return nmi -def mutual_information(reference_intervals, reference_labels, - estimated_intervals, estimated_labels, - frame_size=0.1): +def mutual_information( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=0.1, +): """Frame-clustering segmentation: mutual information metrics. Examples @@ -905,25 +922,26 @@ def mutual_information(reference_intervals, reference_labels, Normalize mutual information between segmentations """ - validate_structure(reference_intervals, reference_labels, - estimated_intervals, estimated_labels) + validate_structure( + reference_intervals, reference_labels, estimated_intervals, estimated_labels + ) # Check for empty annotations. Don't need to check labels because # validate_structure makes sure they're the same size as intervals if reference_intervals.size == 0 or estimated_intervals.size == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 # Generate the cluster labels - y_ref = util.intervals_to_samples(reference_intervals, - reference_labels, - sample_size=frame_size)[-1] + y_ref = util.intervals_to_samples( + reference_intervals, reference_labels, sample_size=frame_size + )[-1] y_ref = util.index_labels(y_ref)[0] # Map to index space - y_est = util.intervals_to_samples(estimated_intervals, - estimated_labels, - sample_size=frame_size)[-1] + y_est = util.intervals_to_samples( + estimated_intervals, estimated_labels, sample_size=frame_size + )[-1] y_est = util.index_labels(y_est)[0] @@ -939,8 +957,15 @@ def mutual_information(reference_intervals, reference_labels, return mutual_info, adj_mutual_info, norm_mutual_info -def nce(reference_intervals, reference_labels, estimated_intervals, - estimated_labels, frame_size=0.1, beta=1.0, marginal=False): +def nce( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=0.1, + beta=1.0, + marginal=False, +): """Frame-clustering segmentation: normalized conditional entropy Computes cross-entropy of cluster assignment, normalized by the @@ -985,7 +1010,6 @@ def nce(reference_intervals, reference_labels, estimated_intervals, beta : float > 0 beta for F-measure (Default value = 1.0) - marginal : bool If `False`, normalize conditional entropy by uniform entropy. If `True`, normalize conditional entropy by the marginal entropy. @@ -1001,7 +1025,6 @@ def nce(reference_intervals, reference_labels, estimated_intervals, - For `marginal=True`, ``1 - H(y_est | y_ref) / H(y_est)`` If `|y_est|==1`, then `S_over` will be 0. - S_under Under-clustering score: @@ -1010,31 +1033,29 @@ def nce(reference_intervals, reference_labels, estimated_intervals, - For `marginal=True`, ``1 - H(y_ref | y_est) / H(y_ref)`` If `|y_ref|==1`, then `S_under` will be 0. - S_F F-measure for (S_over, S_under) - """ - - validate_structure(reference_intervals, reference_labels, - estimated_intervals, estimated_labels) + validate_structure( + reference_intervals, reference_labels, estimated_intervals, estimated_labels + ) # Check for empty annotations. Don't need to check labels because # validate_structure makes sure they're the same size as intervals if reference_intervals.size == 0 or estimated_intervals.size == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 # Generate the cluster labels - y_ref = util.intervals_to_samples(reference_intervals, - reference_labels, - sample_size=frame_size)[-1] + y_ref = util.intervals_to_samples( + reference_intervals, reference_labels, sample_size=frame_size + )[-1] y_ref = util.index_labels(y_ref)[0] # Map to index space - y_est = util.intervals_to_samples(estimated_intervals, - estimated_labels, - sample_size=frame_size)[-1] + y_est = util.intervals_to_samples( + estimated_intervals, estimated_labels, sample_size=frame_size + )[-1] y_est = util.index_labels(y_est)[0] @@ -1065,19 +1086,25 @@ def nce(reference_intervals, reference_labels, estimated_intervals, score_under = 0.0 if z_ref > 0: - score_under = 1. - true_given_est / z_ref + score_under = 1.0 - true_given_est / z_ref score_over = 0.0 if z_est > 0: - score_over = 1. - pred_given_ref / z_est + score_over = 1.0 - pred_given_ref / z_est f_measure = util.f_measure(score_over, score_under, beta=beta) return score_over, score_under, f_measure -def vmeasure(reference_intervals, reference_labels, estimated_intervals, - estimated_labels, frame_size=0.1, beta=1.0): +def vmeasure( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=0.1, + beta=1.0, +): """Frame-clustering segmentation: v-measure Computes cross-entropy of cluster assignment, normalized by the @@ -1132,22 +1159,23 @@ def vmeasure(reference_intervals, reference_labels, estimated_intervals, ``1 - H(y_est | y_ref) / H(y_est)`` If `|y_est|==1`, then `V_precision` will be 0. - V_recall Under-clustering score: ``1 - H(y_ref | y_est) / H(y_ref)`` If `|y_ref|==1`, then `V_recall` will be 0. - V_F F-measure for (V_precision, V_recall) - """ - - return nce(reference_intervals, reference_labels, - estimated_intervals, estimated_labels, - frame_size=frame_size, beta=beta, - marginal=True) + return nce( + reference_intervals, + reference_labels, + estimated_intervals, + estimated_labels, + frame_size=frame_size, + beta=beta, + marginal=True, + ) def evaluate(ref_intervals, ref_labels, est_intervals, est_labels, **kwargs): @@ -1176,7 +1204,7 @@ def evaluate(ref_intervals, ref_labels, est_intervals, est_labels, **kwargs): est_labels : list, shape=(m,) estimated segment labels, in the format returned by :func:`mir_eval.io.load_labeled_intervals`. - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -1185,68 +1213,84 @@ def evaluate(ref_intervals, ref_labels, est_intervals, est_labels, **kwargs): scores : dict Dictionary of scores, where the key is the metric name (str) and the value is the (float) score achieved. - """ - # Adjust timespan of estimations relative to ground truth - ref_intervals, ref_labels = \ - util.adjust_intervals(ref_intervals, labels=ref_labels, t_min=0.0) + ref_intervals, ref_labels = util.adjust_intervals( + ref_intervals, labels=ref_labels, t_min=0.0 + ) - est_intervals, est_labels = \ - util.adjust_intervals(est_intervals, labels=est_labels, t_min=0.0, - t_max=ref_intervals.max()) + est_intervals, est_labels = util.adjust_intervals( + est_intervals, labels=est_labels, t_min=0.0, t_max=ref_intervals.max() + ) # Now compute all the metrics scores = collections.OrderedDict() # Boundary detection # Force these values for window - kwargs['window'] = .5 - scores['Precision@0.5'], scores['Recall@0.5'], scores['F-measure@0.5'] = \ - util.filter_kwargs(detection, ref_intervals, est_intervals, **kwargs) - - kwargs['window'] = 3.0 - scores['Precision@3.0'], scores['Recall@3.0'], scores['F-measure@3.0'] = \ - util.filter_kwargs(detection, ref_intervals, est_intervals, **kwargs) + kwargs["window"] = 0.5 + ( + scores["Precision@0.5"], + scores["Recall@0.5"], + scores["F-measure@0.5"], + ) = util.filter_kwargs(detection, ref_intervals, est_intervals, **kwargs) + + kwargs["window"] = 3.0 + ( + scores["Precision@3.0"], + scores["Recall@3.0"], + scores["F-measure@3.0"], + ) = util.filter_kwargs(detection, ref_intervals, est_intervals, **kwargs) # Boundary deviation - scores['Ref-to-est deviation'], scores['Est-to-ref deviation'] = \ - util.filter_kwargs(deviation, ref_intervals, est_intervals, **kwargs) + scores["Ref-to-est deviation"], scores["Est-to-ref deviation"] = util.filter_kwargs( + deviation, ref_intervals, est_intervals, **kwargs + ) # Pairwise clustering - (scores['Pairwise Precision'], - scores['Pairwise Recall'], - scores['Pairwise F-measure']) = util.filter_kwargs(pairwise, - ref_intervals, - ref_labels, - est_intervals, - est_labels, **kwargs) + ( + scores["Pairwise Precision"], + scores["Pairwise Recall"], + scores["Pairwise F-measure"], + ) = util.filter_kwargs( + pairwise, ref_intervals, ref_labels, est_intervals, est_labels, **kwargs + ) # Rand index - scores['Rand Index'] = util.filter_kwargs(rand_index, ref_intervals, - ref_labels, est_intervals, - est_labels, **kwargs) + scores["Rand Index"] = util.filter_kwargs( + rand_index, ref_intervals, ref_labels, est_intervals, est_labels, **kwargs + ) # Adjusted rand index - scores['Adjusted Rand Index'] = util.filter_kwargs(ari, ref_intervals, - ref_labels, - est_intervals, - est_labels, **kwargs) + scores["Adjusted Rand Index"] = util.filter_kwargs( + ari, ref_intervals, ref_labels, est_intervals, est_labels, **kwargs + ) # Mutual information metrics - (scores['Mutual Information'], - scores['Adjusted Mutual Information'], - scores['Normalized Mutual Information']) = \ - util.filter_kwargs(mutual_information, ref_intervals, ref_labels, - est_intervals, est_labels, **kwargs) + ( + scores["Mutual Information"], + scores["Adjusted Mutual Information"], + scores["Normalized Mutual Information"], + ) = util.filter_kwargs( + mutual_information, + ref_intervals, + ref_labels, + est_intervals, + est_labels, + **kwargs + ) # Conditional entropy metrics - scores['NCE Over'], scores['NCE Under'], scores['NCE F-measure'] = \ - util.filter_kwargs(nce, ref_intervals, ref_labels, est_intervals, - est_labels, **kwargs) + ( + scores["NCE Over"], + scores["NCE Under"], + scores["NCE F-measure"], + ) = util.filter_kwargs( + nce, ref_intervals, ref_labels, est_intervals, est_labels, **kwargs + ) # V-measure metrics - scores['V Precision'], scores['V Recall'], scores['V-measure'] = \ - util.filter_kwargs(vmeasure, ref_intervals, ref_labels, est_intervals, - est_labels, **kwargs) + scores["V Precision"], scores["V Recall"], scores["V-measure"] = util.filter_kwargs( + vmeasure, ref_intervals, ref_labels, est_intervals, est_labels, **kwargs + ) return scores diff --git a/mir_eval/separation.py b/mir_eval/separation.py index 78305761..50597d5e 100644 --- a/mir_eval/separation.py +++ b/mir_eval/separation.py @@ -1,5 +1,10 @@ -# -*- coding: utf-8 -*- -''' +""" +.. warning:: + The mir_eval.separation module is deprecated in mir_eval version 0.8, and will be removed. + We recommend that you migrate your code to use an alternative package such as sigsep-museval + https://sigsep.github.io/sigsep-mus-eval/ + + Source separation algorithms attempt to extract recordings of individual sources from a recording of a mixture of sources. Evaluation methods for source separation compare the extracted sources from reference sources and @@ -43,7 +48,7 @@ Trans. on Audio, Speech and Language Processing, 14(4):1462-1469, 2006. -''' +""" import numpy as np import scipy.fftpack @@ -60,7 +65,7 @@ def validate(reference_sources, estimated_sources): - """Checks that the input data to a metric are valid, and throws helpful + """Check that the input data to a metric are valid, and throws helpful errors if not. Parameters @@ -71,64 +76,78 @@ def validate(reference_sources, estimated_sources): matrix containing estimated sources """ - if reference_sources.shape != estimated_sources.shape: - raise ValueError('The shape of estimated sources and the true ' - 'sources should match. reference_sources.shape ' - '= {}, estimated_sources.shape ' - '= {}'.format(reference_sources.shape, - estimated_sources.shape)) + raise ValueError( + "The shape of estimated sources and the true " + "sources should match. reference_sources.shape " + "= {}, estimated_sources.shape " + "= {}".format(reference_sources.shape, estimated_sources.shape) + ) if reference_sources.ndim > 3 or estimated_sources.ndim > 3: - raise ValueError('The number of dimensions is too high (must be less ' - 'than 3). reference_sources.ndim = {}, ' - 'estimated_sources.ndim ' - '= {}'.format(reference_sources.ndim, - estimated_sources.ndim)) + raise ValueError( + "The number of dimensions is too high (must be less " + "than 3). reference_sources.ndim = {}, " + "estimated_sources.ndim " + "= {}".format(reference_sources.ndim, estimated_sources.ndim) + ) if reference_sources.size == 0: - warnings.warn("reference_sources is empty, should be of size " - "(nsrc, nsample). sdr, sir, sar, and perm will all " - "be empty np.ndarrays") + warnings.warn( + "reference_sources is empty, should be of size " + "(nsrc, nsample). sdr, sir, sar, and perm will all " + "be empty np.ndarrays" + ) elif _any_source_silent(reference_sources): - raise ValueError('All the reference sources should be non-silent (not ' - 'all-zeros), but at least one of the reference ' - 'sources is all 0s, which introduces ambiguity to the' - ' evaluation. (Otherwise we can add infinitely many ' - 'all-zero sources.)') + raise ValueError( + "All the reference sources should be non-silent (not " + "all-zeros), but at least one of the reference " + "sources is all 0s, which introduces ambiguity to the" + " evaluation. (Otherwise we can add infinitely many " + "all-zero sources.)" + ) if estimated_sources.size == 0: - warnings.warn("estimated_sources is empty, should be of size " - "(nsrc, nsample). sdr, sir, sar, and perm will all " - "be empty np.ndarrays") + warnings.warn( + "estimated_sources is empty, should be of size " + "(nsrc, nsample). sdr, sir, sar, and perm will all " + "be empty np.ndarrays" + ) elif _any_source_silent(estimated_sources): - raise ValueError('All the estimated sources should be non-silent (not ' - 'all-zeros), but at least one of the estimated ' - 'sources is all 0s. Since we require each reference ' - 'source to be non-silent, having a silent estimated ' - 'source will result in an underdetermined system.') - - if (estimated_sources.shape[0] > MAX_SOURCES or - reference_sources.shape[0] > MAX_SOURCES): - raise ValueError('The supplied matrices should be of shape (nsrc,' - ' nsampl) but reference_sources.shape[0] = {} and ' - 'estimated_sources.shape[0] = {} which is greater ' - 'than mir_eval.separation.MAX_SOURCES = {}. To ' - 'override this check, set ' - 'mir_eval.separation.MAX_SOURCES to a ' - 'larger value.'.format(reference_sources.shape[0], - estimated_sources.shape[0], - MAX_SOURCES)) + raise ValueError( + "All the estimated sources should be non-silent (not " + "all-zeros), but at least one of the estimated " + "sources is all 0s. Since we require each reference " + "source to be non-silent, having a silent estimated " + "source will result in an underdetermined system." + ) + + if ( + estimated_sources.shape[0] > MAX_SOURCES + or reference_sources.shape[0] > MAX_SOURCES + ): + raise ValueError( + "The supplied matrices should be of shape (nsrc," + " nsampl) but reference_sources.shape[0] = {} and " + "estimated_sources.shape[0] = {} which is greater " + "than mir_eval.separation.MAX_SOURCES = {}. To " + "override this check, set " + "mir_eval.separation.MAX_SOURCES to a " + "larger value.".format( + reference_sources.shape[0], estimated_sources.shape[0], MAX_SOURCES + ) + ) def _any_source_silent(sources): - """Returns true if the parameter sources has any silent first dimensions""" - return np.any(np.all(np.sum( - sources, axis=tuple(range(2, sources.ndim))) == 0, axis=1)) + """Return true if the parameter sources has any silent first dimensions""" + return np.any( + np.all(np.sum(sources, axis=tuple(range(2, sources.ndim))) == 0, axis=1) + ) -def bss_eval_sources(reference_sources, estimated_sources, - compute_permutation=True): +@util.deprecated(version="0.8", version_removed="0.9") +def bss_eval_sources(reference_sources, estimated_sources, compute_permutation=True): """ Ordering and measurement of the separation quality for estimated source signals in terms of filtered true source, interference and artifacts. @@ -184,7 +203,6 @@ def bss_eval_sources(reference_sources, estimated_sources, 92, pp. 1928-1936, 2012. """ - # make sure the input is of shape (nsrc, nsampl) if estimated_sources.ndim == 1: estimated_sources = estimated_sources[np.newaxis, :] @@ -206,18 +224,18 @@ def bss_eval_sources(reference_sources, estimated_sources, sar = np.empty((nsrc, nsrc)) for jest in range(nsrc): for jtrue in range(nsrc): - s_true, e_spat, e_interf, e_artif = \ - _bss_decomp_mtifilt(reference_sources, - estimated_sources[jest], - jtrue, 512) - sdr[jest, jtrue], sir[jest, jtrue], sar[jest, jtrue] = \ - _bss_source_crit(s_true, e_spat, e_interf, e_artif) + s_true, e_spat, e_interf, e_artif = _bss_decomp_mtifilt( + reference_sources, estimated_sources[jest], jtrue, 512 + ) + sdr[jest, jtrue], sir[jest, jtrue], sar[jest, jtrue] = _bss_source_crit( + s_true, e_spat, e_interf, e_artif + ) # select the best ordering perms = list(itertools.permutations(list(range(nsrc)))) mean_sir = np.empty(len(perms)) dum = np.arange(nsrc) - for (i, perm) in enumerate(perms): + for i, perm in enumerate(perms): mean_sir[i] = np.mean(sir[perm, dum]) popt = perms[np.argmax(mean_sir)] idx = (popt, dum) @@ -229,21 +247,24 @@ def bss_eval_sources(reference_sources, estimated_sources, sir = np.empty(nsrc) sar = np.empty(nsrc) for j in range(nsrc): - s_true, e_spat, e_interf, e_artif = \ - _bss_decomp_mtifilt(reference_sources, - estimated_sources[j], - j, 512) - sdr[j], sir[j], sar[j] = \ - _bss_source_crit(s_true, e_spat, e_interf, e_artif) + s_true, e_spat, e_interf, e_artif = _bss_decomp_mtifilt( + reference_sources, estimated_sources[j], j, 512 + ) + sdr[j], sir[j], sar[j] = _bss_source_crit(s_true, e_spat, e_interf, e_artif) # return the default permutation for compatibility popt = np.arange(nsrc) return (sdr, sir, sar, popt) -def bss_eval_sources_framewise(reference_sources, estimated_sources, - window=30*44100, hop=15*44100, - compute_permutation=False): +@util.deprecated(version="0.8", version_removed="0.9") +def bss_eval_sources_framewise( + reference_sources, + estimated_sources, + window=30 * 44100, + hop=15 * 44100, + compute_permutation=False, +): """Framewise computation of bss_eval_sources Please be aware that this function does not compute permutations (by @@ -303,9 +324,7 @@ def bss_eval_sources_framewise(reference_sources, estimated_sources, the mean SIR sense (estimated source number ``perm[j]`` corresponds to true source number ``j``). Note: ``perm`` will be ``range(nsrc)`` for all windows if ``compute_permutation`` is ``False`` - """ - # make sure the input is of shape (nsrc, nsampl) if estimated_sources.ndim == 1: estimated_sources = estimated_sources[np.newaxis, :] @@ -319,14 +338,12 @@ def bss_eval_sources_framewise(reference_sources, estimated_sources, nsrc = reference_sources.shape[0] - nwin = int( - np.floor((reference_sources.shape[1] - window + hop) / hop) - ) + nwin = int(np.floor((reference_sources.shape[1] - window + hop) / hop)) # if fewer than 2 windows would be evaluated, return the sources result if nwin < 2: - result = bss_eval_sources(reference_sources, - estimated_sources, - compute_permutation) + result = bss_eval_sources( + reference_sources, estimated_sources, compute_permutation + ) return [np.expand_dims(score, -1) for score in result] # compute the criteria across all windows @@ -341,8 +358,7 @@ def bss_eval_sources_framewise(reference_sources, estimated_sources, ref_slice = reference_sources[:, win_slice] est_slice = estimated_sources[:, win_slice] # check for a silent frame - if (not _any_source_silent(ref_slice) and - not _any_source_silent(est_slice)): + if not _any_source_silent(ref_slice) and not _any_source_silent(est_slice): sdr[:, k], sir[:, k], sar[:, k], perm[:, k] = bss_eval_sources( ref_slice, est_slice, compute_permutation ) @@ -353,9 +369,9 @@ def bss_eval_sources_framewise(reference_sources, estimated_sources, return sdr, sir, sar, perm -def bss_eval_images(reference_sources, estimated_sources, - compute_permutation=True): - """Implementation of the bss_eval_images function from the +@util.deprecated(version="0.8", version_removed="0.9") +def bss_eval_images(reference_sources, estimated_sources, compute_permutation=True): + """Compute the bss_eval_images function from the BSS_EVAL Matlab toolbox. Ordering and measurement of the separation quality for estimated source @@ -411,9 +427,7 @@ def bss_eval_images(reference_sources, estimated_sources, Lutter and Ngoc Q.K. Duong, "The Signal Separation Evaluation Campaign (2007-2010): Achievements and remaining challenges", Signal Processing, 92, pp. 1928-1936, 2012. - """ - # make sure the input has 3 dimensions # assuming input is in shape (nsampl) or (nsrc, nsampl) estimated_sources = np.atleast_3d(estimated_sources) @@ -423,8 +437,7 @@ def bss_eval_images(reference_sources, estimated_sources, validate(reference_sources, estimated_sources) # If empty matrices were supplied, return empty lists (special case) if reference_sources.size == 0 or estimated_sources.size == 0: - return np.array([]), np.array([]), np.array([]), \ - np.array([]), np.array([]) + return np.array([]), np.array([]), np.array([]), np.array([]), np.array([]) # determine size parameters nsrc = estimated_sources.shape[0] @@ -440,26 +453,24 @@ def bss_eval_images(reference_sources, estimated_sources, sar = np.empty((nsrc, nsrc)) for jest in range(nsrc): for jtrue in range(nsrc): - s_true, e_spat, e_interf, e_artif = \ - _bss_decomp_mtifilt_images( - reference_sources, - np.reshape( - estimated_sources[jest], - (nsampl, nchan), - order='F' - ), - jtrue, - 512 - ) - sdr[jest, jtrue], isr[jest, jtrue], \ - sir[jest, jtrue], sar[jest, jtrue] = \ - _bss_image_crit(s_true, e_spat, e_interf, e_artif) + s_true, e_spat, e_interf, e_artif = _bss_decomp_mtifilt_images( + reference_sources, + np.reshape(estimated_sources[jest], (nsampl, nchan), order="F"), + jtrue, + 512, + ) + ( + sdr[jest, jtrue], + isr[jest, jtrue], + sir[jest, jtrue], + sar[jest, jtrue], + ) = _bss_image_crit(s_true, e_spat, e_interf, e_artif) # select the best ordering - perms = list(itertools.permutations(range(nsrc))) + perms = list(itertools.permutations(list(range(nsrc)))) mean_sir = np.empty(len(perms)) dum = np.arange(nsrc) - for (i, perm) in enumerate(perms): + for i, perm in enumerate(perms): mean_sir[i] = np.mean(sir[perm, dum]) popt = perms[np.argmax(mean_sir)] idx = (popt, dum) @@ -471,28 +482,36 @@ def bss_eval_images(reference_sources, estimated_sources, isr = np.empty(nsrc) sir = np.empty(nsrc) sar = np.empty(nsrc) - Gj = [0] * nsrc # prepare G matrics with zeroes + Gj = [0] * nsrc # prepare G matrices with zeroes G = np.zeros(1) for j in range(nsrc): # save G matrix to avoid recomputing it every call - s_true, e_spat, e_interf, e_artif, Gj_temp, G = \ - _bss_decomp_mtifilt_images(reference_sources, - np.reshape(estimated_sources[j], - (nsampl, nchan), - order='F'), - j, 512, Gj[j], G) + s_true, e_spat, e_interf, e_artif, Gj_temp, G = _bss_decomp_mtifilt_images( + reference_sources, + np.reshape(estimated_sources[j], (nsampl, nchan), order="F"), + j, + 512, + Gj[j], + G, + ) Gj[j] = Gj_temp - sdr[j], isr[j], sir[j], sar[j] = \ - _bss_image_crit(s_true, e_spat, e_interf, e_artif) + sdr[j], isr[j], sir[j], sar[j] = _bss_image_crit( + s_true, e_spat, e_interf, e_artif + ) # return the default permutation for compatibility popt = np.arange(nsrc) return (sdr, isr, sir, sar, popt) -def bss_eval_images_framewise(reference_sources, estimated_sources, - window=30*44100, hop=15*44100, - compute_permutation=False): +@util.deprecated(version="0.8", version_removed="0.9") +def bss_eval_images_framewise( + reference_sources, + estimated_sources, + window=30 * 44100, + hop=15 * 44100, + compute_permutation=False, +): """Framewise computation of bss_eval_images Please be aware that this function does not compute permutations (by @@ -554,9 +573,7 @@ def bss_eval_images_framewise(reference_sources, estimated_sources, true source number j) Note: perm will be range(nsrc) for all windows if compute_permutation is False - """ - # make sure the input has 3 dimensions # assuming input is in shape (nsampl) or (nsrc, nsampl) estimated_sources = np.atleast_3d(estimated_sources) @@ -570,14 +587,12 @@ def bss_eval_images_framewise(reference_sources, estimated_sources, nsrc = reference_sources.shape[0] - nwin = int( - np.floor((reference_sources.shape[1] - window + hop) / hop) - ) + nwin = int(np.floor((reference_sources.shape[1] - window + hop) / hop)) # if fewer than 2 windows would be evaluated, return the images result if nwin < 2: - result = bss_eval_images(reference_sources, - estimated_sources, - compute_permutation) + result = bss_eval_images( + reference_sources, estimated_sources, compute_permutation + ) return [np.expand_dims(score, -1) for score in result] # compute the criteria across all windows @@ -593,12 +608,10 @@ def bss_eval_images_framewise(reference_sources, estimated_sources, ref_slice = reference_sources[:, win_slice, :] est_slice = estimated_sources[:, win_slice, :] # check for a silent frame - if (not _any_source_silent(ref_slice) and - not _any_source_silent(est_slice)): - sdr[:, k], isr[:, k], sir[:, k], sar[:, k], perm[:, k] = \ - bss_eval_images( - ref_slice, est_slice, compute_permutation - ) + if not _any_source_silent(ref_slice) and not _any_source_silent(est_slice): + sdr[:, k], isr[:, k], sir[:, k], sar[:, k], perm[:, k] = bss_eval_images( + ref_slice, est_slice, compute_permutation + ) else: # if we have a silent frame set results as np.nan sdr[:, k] = sir[:, k] = sar[:, k] = perm[:, k] = np.nan @@ -617,19 +630,20 @@ def _bss_decomp_mtifilt(reference_sources, estimated_source, j, flen): # true source image s_true = np.hstack((reference_sources[j], np.zeros(flen - 1))) # spatial (or filtering) distortion - e_spat = _project(reference_sources[j, np.newaxis, :], estimated_source, - flen) - s_true + e_spat = ( + _project(reference_sources[j, np.newaxis, :], estimated_source, flen) - s_true + ) # interference - e_interf = _project(reference_sources, - estimated_source, flen) - s_true - e_spat + e_interf = _project(reference_sources, estimated_source, flen) - s_true - e_spat # artifacts e_artif = -s_true - e_spat - e_interf e_artif[:nsampl] += estimated_source return (s_true, e_spat, e_interf, e_artif) -def _bss_decomp_mtifilt_images(reference_sources, estimated_source, j, flen, - Gj=None, G=None): +def _bss_decomp_mtifilt_images( + reference_sources, estimated_source, j, flen, Gj=None, G=None +): """Decomposition of an estimated source image into four components representing respectively the true source image, spatial (or filtering) distortion, interference and artifacts, derived from the true source @@ -638,7 +652,7 @@ def _bss_decomp_mtifilt_images(reference_sources, estimated_source, j, flen, Improved performance can be gained by passing Gj and G parameters initially as all zeros. These parameters store the results from the computation of the G matrix in _project_images and then return them for subsequent calls - to this function. This only works when not computing permuations. + to this function. This only works when not computing permutations. """ nsampl = np.shape(estimated_source)[0] nchan = np.shape(estimated_source)[1] @@ -646,25 +660,27 @@ def _bss_decomp_mtifilt_images(reference_sources, estimated_source, j, flen, saveg = Gj is not None and G is not None # decomposition # true source image - s_true = np.hstack((np.reshape(reference_sources[j], - (nsampl, nchan), - order="F").transpose(), - np.zeros((nchan, flen - 1)))) + s_true = np.hstack( + ( + np.reshape(reference_sources[j], (nsampl, nchan), order="F").transpose(), + np.zeros((nchan, flen - 1)), + ) + ) # spatial (or filtering) distortion if saveg: - e_spat, Gj = _project_images(reference_sources[j, np.newaxis, :], - estimated_source, flen, Gj) + e_spat, Gj = _project_images( + reference_sources[j, np.newaxis, :], estimated_source, flen, Gj + ) else: - e_spat = _project_images(reference_sources[j, np.newaxis, :], - estimated_source, flen) + e_spat = _project_images( + reference_sources[j, np.newaxis, :], estimated_source, flen + ) e_spat = e_spat - s_true # interference if saveg: - e_interf, G = _project_images(reference_sources, - estimated_source, flen, G) + e_interf, G = _project_images(reference_sources, estimated_source, flen, G) else: - e_interf = _project_images(reference_sources, - estimated_source, flen) + e_interf = _project_images(reference_sources, estimated_source, flen) e_interf = e_interf - s_true - e_spat # artifacts e_artif = -s_true - e_spat - e_interf @@ -685,10 +701,9 @@ def _project(reference_sources, estimated_source, flen): # computing coefficients of least squares problem via FFT ## # zero padding and FFT of input data - reference_sources = np.hstack((reference_sources, - np.zeros((nsrc, flen - 1)))) + reference_sources = np.hstack((reference_sources, np.zeros((nsrc, flen - 1)))) estimated_source = np.hstack((estimated_source, np.zeros(flen - 1))) - n_fft = int(2**np.ceil(np.log2(nsampl + flen - 1.))) + n_fft = int(2 ** np.ceil(np.log2(nsampl + flen - 1.0))) sf = scipy.fftpack.fft(reference_sources, n=n_fft, axis=1) sef = scipy.fftpack.fft(estimated_source, n=n_fft) # inner products between delayed versions of reference_sources @@ -697,28 +712,27 @@ def _project(reference_sources, estimated_source, flen): for j in range(nsrc): ssf = sf[i] * np.conj(sf[j]) ssf = np.real(scipy.fftpack.ifft(ssf)) - ss = toeplitz(np.hstack((ssf[0], ssf[-1:-flen:-1])), - r=ssf[:flen]) - G[i * flen: (i+1) * flen, j * flen: (j+1) * flen] = ss - G[j * flen: (j+1) * flen, i * flen: (i+1) * flen] = ss.T + ss = toeplitz(np.hstack((ssf[0], ssf[-1:-flen:-1])), r=ssf[:flen]) + G[i * flen : (i + 1) * flen, j * flen : (j + 1) * flen] = ss + G[j * flen : (j + 1) * flen, i * flen : (i + 1) * flen] = ss.T # inner products between estimated_source and delayed versions of # reference_sources D = np.zeros(nsrc * flen) for i in range(nsrc): ssef = sf[i] * np.conj(sef) ssef = np.real(scipy.fftpack.ifft(ssef)) - D[i * flen: (i+1) * flen] = np.hstack((ssef[0], ssef[-1:-flen:-1])) + D[i * flen : (i + 1) * flen] = np.hstack((ssef[0], ssef[-1:-flen:-1])) # Computing projection # Distortion filters try: - C = np.linalg.solve(G, D).reshape(flen, nsrc, order='F') + C = np.linalg.solve(G, D).reshape(flen, nsrc, order="F") except np.linalg.linalg.LinAlgError: - C = np.linalg.lstsq(G, D)[0].reshape(flen, nsrc, order='F') + C = np.linalg.lstsq(G, D)[0].reshape(flen, nsrc, order="F") # Filtering sproj = np.zeros(nsampl + flen - 1) for i in range(nsrc): - sproj += fftconvolve(C[:, i], reference_sources[i])[:nsampl + flen - 1] + sproj += fftconvolve(C[:, i], reference_sources[i])[: nsampl + flen - 1] return sproj @@ -732,16 +746,19 @@ def _project_images(reference_sources, estimated_source, flen, G=None): nsrc = reference_sources.shape[0] nsampl = reference_sources.shape[1] nchan = reference_sources.shape[2] - reference_sources = np.reshape(np.transpose(reference_sources, (2, 0, 1)), - (nchan*nsrc, nsampl), order='F') + reference_sources = np.reshape( + np.transpose(reference_sources, (2, 0, 1)), (nchan * nsrc, nsampl), order="F" + ) # computing coefficients of least squares problem via FFT ## # zero padding and FFT of input data - reference_sources = np.hstack((reference_sources, - np.zeros((nchan*nsrc, flen - 1)))) - estimated_source = \ - np.hstack((estimated_source.transpose(), np.zeros((nchan, flen - 1)))) - n_fft = int(2**np.ceil(np.log2(nsampl + flen - 1.))) + reference_sources = np.hstack( + (reference_sources, np.zeros((nchan * nsrc, flen - 1))) + ) + estimated_source = np.hstack( + (estimated_source.transpose(), np.zeros((nchan, flen - 1))) + ) + n_fft = int(2 ** np.ceil(np.log2(nsampl + flen - 1.0))) sf = scipy.fftpack.fft(reference_sources, n=n_fft, axis=1) sef = scipy.fftpack.fft(estimated_source, n=n_fft) @@ -750,25 +767,23 @@ def _project_images(reference_sources, estimated_source, flen, G=None): saveg = False G = np.zeros((nchan * nsrc * flen, nchan * nsrc * flen)) for i in range(nchan * nsrc): - for j in range(i+1): + for j in range(i + 1): ssf = sf[i] * np.conj(sf[j]) ssf = np.real(scipy.fftpack.ifft(ssf)) - ss = toeplitz(np.hstack((ssf[0], ssf[-1:-flen:-1])), - r=ssf[:flen]) - G[i * flen: (i+1) * flen, j * flen: (j+1) * flen] = ss - G[j * flen: (j+1) * flen, i * flen: (i+1) * flen] = ss.T + ss = toeplitz(np.hstack((ssf[0], ssf[-1:-flen:-1])), r=ssf[:flen]) + G[i * flen : (i + 1) * flen, j * flen : (j + 1) * flen] = ss + G[j * flen : (j + 1) * flen, i * flen : (i + 1) * flen] = ss.T else: # avoid recomputing G (only works if no permutation is desired) saveg = True # return G if np.all(G == 0): # only compute G if passed as 0 G = np.zeros((nchan * nsrc * flen, nchan * nsrc * flen)) for i in range(nchan * nsrc): - for j in range(i+1): + for j in range(i + 1): ssf = sf[i] * np.conj(sf[j]) ssf = np.real(scipy.fftpack.ifft(ssf)) - ss = toeplitz(np.hstack((ssf[0], ssf[-1:-flen:-1])), - r=ssf[:flen]) - G[i * flen: (i+1) * flen, j * flen: (j+1) * flen] = ss - G[j * flen: (j+1) * flen, i * flen: (i+1) * flen] = ss.T + ss = toeplitz(np.hstack((ssf[0], ssf[-1:-flen:-1])), r=ssf[:flen]) + G[i * flen : (i + 1) * flen, j * flen : (j + 1) * flen] = ss + G[j * flen : (j + 1) * flen, i * flen : (i + 1) * flen] = ss.T # inner products between estimated_source and delayed versions of # reference_sources @@ -777,22 +792,23 @@ def _project_images(reference_sources, estimated_source, flen, G=None): for i in range(nchan): ssef = sf[k] * np.conj(sef[i]) ssef = np.real(scipy.fftpack.ifft(ssef)) - D[k * flen: (k+1) * flen, i] = \ - np.hstack((ssef[0], ssef[-1:-flen:-1])).transpose() + D[k * flen : (k + 1) * flen, i] = np.hstack( + (ssef[0], ssef[-1:-flen:-1]) + ).transpose() # Computing projection # Distortion filters try: - C = np.linalg.solve(G, D).reshape(flen, nchan*nsrc, nchan, order='F') + C = np.linalg.solve(G, D).reshape(flen, nchan * nsrc, nchan, order="F") except np.linalg.linalg.LinAlgError: - C = np.linalg.lstsq(G, D)[0].reshape(flen, nchan*nsrc, nchan, - order='F') + C = np.linalg.lstsq(G, D)[0].reshape(flen, nchan * nsrc, nchan, order="F") # Filtering sproj = np.zeros((nchan, nsampl + flen - 1)) for k in range(nchan * nsrc): for i in range(nchan): - sproj[i] += fftconvolve(C[:, k, i].transpose(), - reference_sources[k])[:nsampl + flen - 1] + sproj[i] += fftconvolve(C[:, k, i].transpose(), reference_sources[k])[ + : nsampl + flen - 1 + ] # return G only if it was passed in if saveg: return sproj, G @@ -806,9 +822,9 @@ def _bss_source_crit(s_true, e_spat, e_interf, e_artif): """ # energy ratios s_filt = s_true + e_spat - sdr = _safe_db(np.sum(s_filt**2), np.sum((e_interf + e_artif)**2)) + sdr = _safe_db(np.sum(s_filt**2), np.sum((e_interf + e_artif) ** 2)) sir = _safe_db(np.sum(s_filt**2), np.sum(e_interf**2)) - sar = _safe_db(np.sum((s_filt + e_interf)**2), np.sum(e_artif**2)) + sar = _safe_db(np.sum((s_filt + e_interf) ** 2), np.sum(e_artif**2)) return (sdr, sir, sar) @@ -817,10 +833,10 @@ def _bss_image_crit(s_true, e_spat, e_interf, e_artif): filtered true source, spatial error, interference and artifacts. """ # energy ratios - sdr = _safe_db(np.sum(s_true**2), np.sum((e_spat+e_interf+e_artif)**2)) + sdr = _safe_db(np.sum(s_true**2), np.sum((e_spat + e_interf + e_artif) ** 2)) isr = _safe_db(np.sum(s_true**2), np.sum(e_spat**2)) - sir = _safe_db(np.sum((s_true+e_spat)**2), np.sum(e_interf**2)) - sar = _safe_db(np.sum((s_true+e_spat+e_interf)**2), np.sum(e_artif**2)) + sir = _safe_db(np.sum((s_true + e_spat) ** 2), np.sum(e_interf**2)) + sar = _safe_db(np.sum((s_true + e_spat + e_interf) ** 2), np.sum(e_artif**2)) return (sdr, isr, sir, sar) @@ -830,10 +846,11 @@ def _safe_db(num, den): be 0. """ if den == 0: - return np.Inf + return np.inf return 10 * np.log10(num / den) +@util.deprecated(version="0.8", version_removed="0.9") def evaluate(reference_sources, estimated_sources, **kwargs): """Compute all metrics for the given reference and estimated signals. @@ -856,7 +873,7 @@ def evaluate(reference_sources, estimated_sources, **kwargs): matrix containing true sources estimated_sources : np.ndarray, shape=(nsrc, nsampl[, nchan]) matrix containing estimated sources - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -871,51 +888,39 @@ def evaluate(reference_sources, estimated_sources, **kwargs): scores = collections.OrderedDict() sdr, isr, sir, sar, perm = util.filter_kwargs( - bss_eval_images, - reference_sources, - estimated_sources, - **kwargs + bss_eval_images, reference_sources, estimated_sources, **kwargs ) - scores['Images - Source to Distortion'] = sdr.tolist() - scores['Images - Image to Spatial'] = isr.tolist() - scores['Images - Source to Interference'] = sir.tolist() - scores['Images - Source to Artifact'] = sar.tolist() - scores['Images - Source permutation'] = perm.tolist() + scores["Images - Source to Distortion"] = sdr.tolist() + scores["Images - Image to Spatial"] = isr.tolist() + scores["Images - Source to Interference"] = sir.tolist() + scores["Images - Source to Artifact"] = sar.tolist() + scores["Images - Source permutation"] = perm.tolist() sdr, isr, sir, sar, perm = util.filter_kwargs( - bss_eval_images_framewise, - reference_sources, - estimated_sources, - **kwargs + bss_eval_images_framewise, reference_sources, estimated_sources, **kwargs ) - scores['Images Frames - Source to Distortion'] = sdr.tolist() - scores['Images Frames - Image to Spatial'] = isr.tolist() - scores['Images Frames - Source to Interference'] = sir.tolist() - scores['Images Frames - Source to Artifact'] = sar.tolist() - scores['Images Frames - Source permutation'] = perm.tolist() + scores["Images Frames - Source to Distortion"] = sdr.tolist() + scores["Images Frames - Image to Spatial"] = isr.tolist() + scores["Images Frames - Source to Interference"] = sir.tolist() + scores["Images Frames - Source to Artifact"] = sar.tolist() + scores["Images Frames - Source permutation"] = perm.tolist() # Verify we can compute sources on this input if reference_sources.ndim < 3 and estimated_sources.ndim < 3: sdr, sir, sar, perm = util.filter_kwargs( - bss_eval_sources_framewise, - reference_sources, - estimated_sources, - **kwargs + bss_eval_sources_framewise, reference_sources, estimated_sources, **kwargs ) - scores['Sources Frames - Source to Distortion'] = sdr.tolist() - scores['Sources Frames - Source to Interference'] = sir.tolist() - scores['Sources Frames - Source to Artifact'] = sar.tolist() - scores['Sources Frames - Source permutation'] = perm.tolist() + scores["Sources Frames - Source to Distortion"] = sdr.tolist() + scores["Sources Frames - Source to Interference"] = sir.tolist() + scores["Sources Frames - Source to Artifact"] = sar.tolist() + scores["Sources Frames - Source permutation"] = perm.tolist() sdr, sir, sar, perm = util.filter_kwargs( - bss_eval_sources, - reference_sources, - estimated_sources, - **kwargs + bss_eval_sources, reference_sources, estimated_sources, **kwargs ) - scores['Sources - Source to Distortion'] = sdr.tolist() - scores['Sources - Source to Interference'] = sir.tolist() - scores['Sources - Source to Artifact'] = sar.tolist() - scores['Sources - Source permutation'] = perm.tolist() + scores["Sources - Source to Distortion"] = sdr.tolist() + scores["Sources - Source to Interference"] = sir.tolist() + scores["Sources - Source to Artifact"] = sar.tolist() + scores["Sources - Source permutation"] = perm.tolist() return scores diff --git a/mir_eval/sonify.py b/mir_eval/sonify.py index 0e5d3da8..ee1fcfc1 100644 --- a/mir_eval/sonify.py +++ b/mir_eval/sonify.py @@ -1,9 +1,10 @@ -''' +""" Methods which sonify annotations for "evaluation by ear". All functions return a raw signal at the specified sampling rate. -''' +""" import numpy as np +import scipy.signal from numpy.lib.stride_tricks import as_strided from scipy.interpolate import interp1d @@ -12,7 +13,7 @@ def clicks(times, fs, click=None, length=None): - """Returns a signal with the signal 'click' placed at each specified time + """Return a signal with the signal 'click' placed at each specified time Parameters ---------- @@ -35,34 +36,35 @@ def clicks(times, fs, click=None, length=None): # Create default click signal if click is None: # 1 kHz tone, 100ms - click = np.sin(2*np.pi*np.arange(fs*.1)*1000/(1.*fs)) + click = np.sin(2 * np.pi * np.arange(fs * 0.1) * 1000 / (1.0 * fs)) # Exponential decay - click *= np.exp(-np.arange(fs*.1)/(fs*.01)) + click *= np.exp(-np.arange(fs * 0.1) / (fs * 0.01)) # Set default length if length is None: - length = int(times.max()*fs + click.shape[0] + 1) + length = int(times.max() * fs + click.shape[0] + 1) # Pre-allocate click signal click_signal = np.zeros(length) # Place clicks for time in times: # Compute the boundaries of the click - start = int(time*fs) + start = int(time * fs) end = start + click.shape[0] # Make sure we don't try to output past the end of the signal if start >= length: break if end >= length: - click_signal[start:] = click[:length - start] + click_signal[start:] = click[: length - start] break # Normally, just add a click here click_signal[start:end] = click return click_signal -def time_frequency(gram, frequencies, times, fs, function=np.sin, length=None, - n_dec=1): - """Reverse synthesis of a time-frequency representation of a signal +def time_frequency( + gram, frequencies, times, fs, function=np.sin, length=None, n_dec=1, threshold=0.01 +): + r"""Reverse synthesis of a time-frequency representation of a signal Parameters ---------- @@ -73,108 +75,134 @@ def time_frequency(gram, frequencies, times, fs, function=np.sin, length=None, Non-positive magnitudes are interpreted as silence. frequencies : np.ndarray - array of size ``gram.shape[0]`` denoting the frequency of + array of size ``gram.shape[0]`` denoting the frequency (in Hz) of each row of gram + times : np.ndarray, shape= ``(gram.shape[1],)`` or ``(gram.shape[1], 2)`` - Either the start time of each column in the gram, - or the time interval corresponding to each column. + Either the start time (in seconds) of each column in the gram, + or the time interval (in seconds) corresponding to each column. + fs : int desired sampling rate of the output signal + function : function function to use to synthesize notes, should be :math:`2\pi`-periodic + length : int desired number of samples in the output signal, defaults to ``times[-1]*fs`` + n_dec : int the number of decimals used to approximate each sonfied frequency. Defaults to 1 decimal place. Higher precision will be slower. + threshold : float + optimizes synthesis to only occur for frequencies that have a + linear magnitude of at least one element in gram above the given threshold. + Returns ------- output : np.ndarray synthesized version of the piano roll """ - # Default value for length + # Convert times to intervals if necessary + time_converted = False if times.ndim == 1: # Convert to intervals - times = util.boundaries_to_intervals(times) + times = np.hstack((times[:-1, np.newaxis], times[1:, np.newaxis])) + # We'll need this to keep track of whether we should pad an interval on + time_converted = True + # Default value for length if length is None: - length = int(times[-1, 1] * fs) - - times, _ = util.adjust_intervals(times, t_max=length) - - # Truncate times so that the shape matches gram - n_times = gram.shape[1] - times = times[:n_times] - - def _fast_synthesize(frequency): - """A faster way to synthesize a signal. - Generate one cycle, and simulate arbitrary repetitions - using array indexing tricks. - """ - # hack so that we can ensure an integer number of periods and samples - # rounds frequency to 1st decimal, s.t. 10 * frequency will be an int - frequency = np.round(frequency, n_dec) - - # Generate 10*frequency periods at this frequency - # Equivalent to n_samples = int(n_periods * fs / frequency) - # n_periods = 10*frequency is the smallest integer that guarantees - # that n_samples will be an integer, since assuming 10*frequency - # is an integer - n_samples = int(10.0**n_dec * fs) - - short_signal = function(2.0 * np.pi * np.arange(n_samples) * - frequency / fs) - - # Calculate the number of loops we need to fill the duration - n_repeats = int(np.ceil(length/float(short_signal.shape[0]))) - - # Simulate tiling the short buffer by using stride tricks - long_signal = as_strided(short_signal, - shape=(n_repeats, len(short_signal)), - strides=(0, short_signal.itemsize)) - - # Use a flatiter to simulate a long 1D buffer - return long_signal.flat - - def _const_interpolator(value): - """Return a function that returns `value` - no matter the input. - """ - def __interpolator(x): - return value - return __interpolator - - # Threshold the tfgram to remove non-positive values + length = int(np.max(times) * fs) + + last_time_in_secs = float(length) / fs + + if time_converted and times.shape[0] != gram.shape[1]: + times = np.vstack((times, [np.max(times), last_time_in_secs])) + + if times.shape[0] != gram.shape[1]: + raise ValueError( + f"times.shape={times.shape} is incompatible with gram.shape={gram.shape}" + ) + + if frequencies.shape[0] != gram.shape[0]: + raise ValueError( + f"frequencies.shape={frequencies.shape} is incompatible with gram.shape={gram.shape}" + ) + + padding = [0, 0] + stacking = [] + + if times.min() > 0: + # We need to pad a silence column on to gram at the beginning + padding[0] = 1 + stacking.append([0, times.min()]) + + stacking.append(times) + + if times.max() < last_time_in_secs: + # We need to pad a silence column onto gram at the end + padding[1] = 1 + stacking.append([times.max(), last_time_in_secs]) + + gram = np.pad(gram, ((0, 0), padding), mode="constant") + times = np.vstack(stacking) + + # Identify the time intervals that have some overlap with the duration + idx = np.logical_and(times[:, 1] >= 0, times[:, 0] <= last_time_in_secs) + gram = gram[:, idx] + times = np.clip(times[idx], 0, last_time_in_secs) + + n_times = times.shape[0] + + # Threshold the tfgram to remove negative values gram = np.maximum(gram, 0) # Pre-allocate output signal output = np.zeros(length) - time_centers = np.mean(times, axis=1) * float(fs) + if gram.shape[1] == 0: + # There are no time intervals to process, so return + # the empty signal. + return output + + # Discard frequencies below threshold + freq_keep = np.max(gram, axis=1) >= threshold + + gram = gram[freq_keep, :] + frequencies = frequencies[freq_keep] + + # Interpolate the values in gram over the time grid. + if n_times > 1: + interpolator = interp1d( + times[:, 0] * fs, + gram[:, :n_times], + kind="previous", + bounds_error=False, + fill_value=(gram[:, 0], gram[:, -1]), + ) + signal = interpolator(np.arange(length)) + else: + # NOTE: This is a special case where there is only one time interval. + # scipy 1.10 and above handle this case directly with the interp1d above, + # but older scipy's do not. This is a workaround for that. + # + # In the 0.9 release, we can bump the minimum scipy to 1.10 and remove this + signal = np.tile(gram[:, 0], (1, length)) for n, frequency in enumerate(frequencies): # Get a waveform of length samples at this frequency - wave = _fast_synthesize(frequency) - - # Interpolate the values in gram over the time grid - if len(time_centers) > 1: - gram_interpolator = interp1d( - time_centers, gram[n, :], - kind='linear', bounds_error=False, - fill_value=(gram[n, 0], gram[n, -1])) - # If only one time point, create constant interpolator - else: - gram_interpolator = _const_interpolator(gram[n, 0]) - - # Scale each time interval by the piano roll magnitude - for m, (start, end) in enumerate((times * fs).astype(int)): - # Clip the timings to make sure the indices are valid - start, end = max(start, 0), min(end, length) - # add to waveform - output[start:end] += ( - wave[start:end] * gram_interpolator(np.arange(start, end))) + wave = _fast_synthesize(frequency, n_dec, fs, function, length) + + # Use a two-cycle ramp to smooth over transients + period = 2 * int(fs / frequency) + filter = np.ones(period) / period + signal_n = scipy.signal.convolve(signal[n], filter, mode="same") + + # Mix the signal into the output + output[:] += wave[: len(signal_n)] * signal_n # Normalize, but only if there's non-zero values norm = np.abs(output).max() @@ -184,33 +212,60 @@ def __interpolator(x): return output -def pitch_contour(times, frequencies, fs, amplitudes=None, function=np.sin, - length=None, kind='linear'): - '''Sonify a pitch contour. +def _fast_synthesize(frequency, n_dec, fs, function, length): + """Efficiently synthesize a signal. + Generate one cycle, and simulate arbitrary repetitions + using array indexing tricks. + """ + # hack so that we can ensure an integer number of periods and samples + # rounds frequency to 1st decimal, s.t. 10 * frequency will be an int + frequency = np.round(frequency, n_dec) + + # Generate 10*frequency periods at this frequency + # Equivalent to n_samples = int(n_periods * fs / frequency) + # n_periods = 10*frequency is the smallest integer that guarantees + # that n_samples will be an integer, since assuming 10*frequency + # is an integer + n_samples = int(10.0**n_dec * fs) + + short_signal = function(2.0 * np.pi * np.arange(n_samples) * frequency / fs) + + # Calculate the number of loops we need to fill the duration + n_repeats = int(np.ceil(length / float(short_signal.shape[0]))) + + # Simulate tiling the short buffer by using stride tricks + long_signal = as_strided( + short_signal, + shape=(n_repeats, len(short_signal)), + strides=(0, short_signal.itemsize), + ) + + # Use a flatiter to simulate a long 1D buffer + return long_signal.flat + + +def pitch_contour( + times, frequencies, fs, amplitudes=None, function=np.sin, length=None, kind="linear" +): + r"""Sonify a pitch contour. Parameters ---------- times : np.ndarray time indices for each frequency measurement, in seconds - frequencies : np.ndarray frequency measurements, in Hz. - Non-positive measurements will be interpreted as un-voiced samples. - + Non-positive measurements or NaNs will be interpreted as un-voiced samples. fs : int desired sampling rate of the output signal - amplitudes : np.ndarray - amplitude measurments, nonnegative + amplitude measurements, nonnegative defaults to ``np.ones((length,))`` - function : function function to use to synthesize notes, should be :math:`2\pi`-periodic - length : int desired number of samples in the output signal, defaults to ``max(times)*fs`` - kind : str Interpolation mode for the frequency and amplitude values. See: ``scipy.interpolate.interp1d`` for valid settings. @@ -219,8 +274,7 @@ def pitch_contour(times, frequencies, fs, amplitudes=None, function=np.sin, ------- output : np.ndarray synthesized version of the pitch contour - ''' - + """ fs = float(fs) if length is None: @@ -229,21 +283,34 @@ def pitch_contour(times, frequencies, fs, amplitudes=None, function=np.sin, # Squash the negative frequencies. # wave(0) = 0, so clipping here will un-voice the corresponding instants frequencies = np.maximum(frequencies, 0.0) + # Convert nans to zeros to unvoice + frequencies = np.nan_to_num(frequencies, copy=False) # Build a frequency interpolator - f_interp = interp1d(times * fs, 2 * np.pi * frequencies / fs, kind=kind, - fill_value=0.0, bounds_error=False, copy=False) + f_interp = interp1d( + times * fs, + 2 * np.pi * frequencies / fs, + kind=kind, + fill_value=0.0, + bounds_error=False, + copy=False, + ) # Estimate frequency at sample points f_est = f_interp(np.arange(length)) if amplitudes is None: - a_est = np.ones((length, )) + a_est = np.ones((length,)) else: # build an amplitude interpolator a_interp = interp1d( - times * fs, amplitudes, kind=kind, - fill_value=0.0, bounds_error=False, copy=False) + times * fs, + amplitudes, + kind=kind, + fill_value=0.0, + bounds_error=False, + copy=False, + ) a_est = a_interp(np.arange(length)) # Sonify the waveform @@ -259,12 +326,12 @@ def chroma(chromagram, times, fs, **kwargs): Chromagram matrix, where each row represents a semitone [C->Bb] i.e., ``chromagram[3, j]`` is the magnitude of D# from ``times[j]`` to ``times[j + 1]`` - times: np.ndarray, shape=(len(chord_labels),) or (len(chord_labels), 2) + times : np.ndarray, shape=(len(chord_labels),) or (len(chord_labels), 2) Either the start time of each column in the chromagram, or the time interval corresponding to each column. fs : int Sampling rate to synthesize audio data at - kwargs + **kwargs Additional keyword arguments to pass to :func:`mir_eval.sonify.time_frequency` @@ -284,8 +351,8 @@ def chroma(chromagram, times, fs, **kwargs): # and std 6 (one half octave) mean = 72 std = 6 - notes = np.arange(12*n_octaves) + base_note - shepard_weight = np.exp(-(notes - mean)**2./(2.*std**2.)) + notes = np.arange(12 * n_octaves) + base_note + shepard_weight = np.exp(-((notes - mean) ** 2.0) / (2.0 * std**2.0)) # Copy the chromagram matrix vertically n_octaves times gram = np.tile(chromagram.T, n_octaves).T # This fixes issues if the supplied chromagram is int type @@ -293,7 +360,7 @@ def chroma(chromagram, times, fs, **kwargs): # Apply Sheppard weighting gram *= shepard_weight.reshape(-1, 1) # Compute frequencies - frequencies = 440.0*(2.0**((notes - 69)/12.0)) + frequencies = 440.0 * (2.0 ** ((notes - 69) / 12.0)) return time_frequency(gram, frequencies, times, fs, **kwargs) @@ -308,7 +375,7 @@ def chords(chord_labels, intervals, fs, **kwargs): Start and end times of each chord label fs : int Sampling rate to synthesize at - kwargs + **kwargs Additional keyword arguments to pass to :func:`mir_eval.sonify.time_frequency` @@ -322,8 +389,11 @@ def chords(chord_labels, intervals, fs, **kwargs): # Convert from labels to chroma roots, interval_bitmaps, _ = chord.encode_many(chord_labels) - chromagram = np.array([np.roll(interval_bitmap, root) - for (interval_bitmap, root) - in zip(interval_bitmaps, roots)]).T + chromagram = np.array( + [ + np.roll(interval_bitmap, root) + for (interval_bitmap, root) in zip(interval_bitmaps, roots) + ] + ).T return chroma(chromagram, intervals, fs, **kwargs) diff --git a/mir_eval/tempo.py b/mir_eval/tempo.py index 935e81cf..f47efecd 100644 --- a/mir_eval/tempo.py +++ b/mir_eval/tempo.py @@ -1,4 +1,4 @@ -''' +""" The goal of a tempo estimation algorithm is to automatically detect the tempo of a piece of music, measured in beats per minute (BPM). @@ -18,7 +18,7 @@ * :func:`mir_eval.tempo.detection`: Relative error, hits, and weighted precision of tempo estimation. -''' +""" import warnings import numpy as np @@ -27,42 +27,38 @@ def validate_tempi(tempi, reference=True): - """Checks that there are two non-negative tempi. + """Check that there are two non-negative tempi. For a reference value, at least one tempo has to be greater than zero. Parameters ---------- tempi : np.ndarray length-2 array of tempo, in bpm - reference : bool indicates a reference value - """ - if tempi.size != 2: - raise ValueError('tempi must have exactly two values') + raise ValueError("tempi must have exactly two values") if not np.all(np.isfinite(tempi)) or np.any(tempi < 0): - raise ValueError('tempi={} must be non-negative numbers'.format(tempi)) + raise ValueError(f"tempi={tempi} must be non-negative numbers") if reference and np.all(tempi == 0): - raise ValueError('reference tempi={} must have one' - ' value greater than zero'.format(tempi)) + raise ValueError( + "reference tempi={} must have one" " value greater than zero".format(tempi) + ) def validate(reference_tempi, reference_weight, estimated_tempi): - """Checks that the input annotations to a metric look like valid tempo + """Check that the input annotations to a metric look like valid tempo annotations. Parameters ---------- reference_tempi : np.ndarray reference tempo values, in bpm - reference_weight : float perceptual weight of slow vs fast in reference - estimated_tempi : np.ndarray estimated tempo values, in bpm @@ -71,7 +67,7 @@ def validate(reference_tempi, reference_weight, estimated_tempi): validate_tempi(estimated_tempi, reference=False) if reference_weight < 0 or reference_weight > 1: - raise ValueError('Reference weight must lie in range [0, 1]') + raise ValueError("Reference weight must lie in range [0, 1]") def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08): @@ -81,14 +77,11 @@ def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08): ---------- reference_tempi : np.ndarray, shape=(2,) Two non-negative reference tempi - reference_weight : float > 0 The relative strength of ``reference_tempi[0]`` vs ``reference_tempi[1]``. - estimated_tempi : np.ndarray, shape=(2,) Two non-negative estimated tempi. - tol : float in [0, 1]: The maximum allowable deviation from a reference tempo to count as a hit. @@ -100,10 +93,8 @@ def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08): p_score : float in [0, 1] Weighted average of recalls: ``reference_weight * hits[0] + (1 - reference_weight) * hits[1]`` - one_correct : bool True if at least one reference tempo was correctly estimated - both_correct : bool True if both reference tempi were correctly estimated @@ -116,15 +107,14 @@ def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08): If ``tol < 0`` or ``tol > 1``. """ - validate(reference_tempi, reference_weight, estimated_tempi) if tol < 0 or tol > 1: - raise ValueError('invalid tolerance {}: must lie in the range ' - '[0, 1]'.format(tol)) - if tol == 0.: - warnings.warn('A tolerance of 0.0 may not ' - 'lead to the results you expect.') + raise ValueError( + "invalid tolerance {}: must lie in the range " "[0, 1]".format(tol) + ) + if tol == 0.0: + warnings.warn("A tolerance of 0.0 may not " "lead to the results you expect.") hits = [False, False] @@ -137,7 +127,7 @@ def detection(reference_tempi, reference_weight, estimated_tempi, tol=0.08): # Count the hits hits[i] = relative_error <= tol - p_score = reference_weight * hits[0] + (1.0-reference_weight) * hits[1] + p_score = reference_weight * hits[0] + (1.0 - reference_weight) * hits[1] one_correct = bool(np.max(hits)) both_correct = bool(np.min(hits)) @@ -152,15 +142,12 @@ def evaluate(reference_tempi, reference_weight, estimated_tempi, **kwargs): ---------- reference_tempi : np.ndarray, shape=(2,) Two non-negative reference tempi - reference_weight : float > 0 The relative strength of ``reference_tempi[0]`` vs ``reference_tempi[1]``. - estimated_tempi : np.ndarray, shape=(2,) Two non-negative estimated tempi. - - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -173,11 +160,12 @@ def evaluate(reference_tempi, reference_weight, estimated_tempi, **kwargs): # Compute all metrics scores = collections.OrderedDict() - (scores['P-score'], - scores['One-correct'], - scores['Both-correct']) = util.filter_kwargs(detection, reference_tempi, - reference_weight, - estimated_tempi, - **kwargs) + ( + scores["P-score"], + scores["One-correct"], + scores["Both-correct"], + ) = util.filter_kwargs( + detection, reference_tempi, reference_weight, estimated_tempi, **kwargs + ) return scores diff --git a/mir_eval/transcription.py b/mir_eval/transcription.py index db93bd3b..65504279 100644 --- a/mir_eval/transcription.py +++ b/mir_eval/transcription.py @@ -1,4 +1,4 @@ -''' +""" The aim of a transcription algorithm is to produce a symbolic representation of a recorded piece of music in the form of a set of discrete notes. There are different ways to represent notes symbolically. Here we use the piano-roll @@ -102,7 +102,7 @@ account, meaning two notes could be matched even if they have very different pitch values. -''' +""" import numpy as np import collections @@ -115,7 +115,7 @@ def validate(ref_intervals, ref_pitches, est_intervals, est_pitches): - """Checks that the input annotations to a metric look like time intervals + """Check that the input annotations to a metric look like time intervals and a pitch list, and throws helpful errors if not. Parameters @@ -134,23 +134,19 @@ def validate(ref_intervals, ref_pitches, est_intervals, est_pitches): # Make sure intervals and pitches match in length if not ref_intervals.shape[0] == ref_pitches.shape[0]: - raise ValueError('Reference intervals and pitches have different ' - 'lengths.') + raise ValueError("Reference intervals and pitches have different " "lengths.") if not est_intervals.shape[0] == est_pitches.shape[0]: - raise ValueError('Estimated intervals and pitches have different ' - 'lengths.') + raise ValueError("Estimated intervals and pitches have different " "lengths.") # Make sure all pitch values are positive if ref_pitches.size > 0 and np.min(ref_pitches) <= 0: - raise ValueError("Reference contains at least one non-positive pitch " - "value") + raise ValueError("Reference contains at least one non-positive pitch " "value") if est_pitches.size > 0 and np.min(est_pitches) <= 0: - raise ValueError("Estimate contains at least one non-positive pitch " - "value") + raise ValueError("Estimate contains at least one non-positive pitch " "value") def validate_intervals(ref_intervals, est_intervals): - """Checks that the input annotations to a metric look like time intervals, + """Check that the input annotations to a metric look like time intervals, and throws helpful errors if not. Parameters @@ -171,8 +167,13 @@ def validate_intervals(ref_intervals, est_intervals): util.validate_intervals(est_intervals) -def match_note_offsets(ref_intervals, est_intervals, offset_ratio=0.2, - offset_min_tolerance=0.05, strict=False): +def match_note_offsets( + ref_intervals, + est_intervals, + offset_ratio=0.2, + offset_min_tolerance=0.05, + strict=False, +): """Compute a maximum matching between reference and estimated notes, only taking note offsets into account. @@ -229,17 +230,16 @@ def match_note_offsets(ref_intervals, est_intervals, offset_ratio=0.2, cmp_func = np.less_equal # check for offset matches - offset_distances = np.abs(np.subtract.outer(ref_intervals[:, 1], - est_intervals[:, 1])) + offset_distances = np.abs( + np.subtract.outer(ref_intervals[:, 1], est_intervals[:, 1]) + ) # Round distances to a target precision to avoid the situation where # if the distance is exactly 50ms (and strict=False) it erroneously # doesn't match the notes because of precision issues. offset_distances = np.around(offset_distances, decimals=N_DECIMALS) ref_durations = util.intervals_to_durations(ref_intervals) - offset_tolerances = np.maximum(offset_ratio * ref_durations, - offset_min_tolerance) - offset_hit_matrix = ( - cmp_func(offset_distances, offset_tolerances.reshape(-1, 1))) + offset_tolerances = np.maximum(offset_ratio * ref_durations, offset_min_tolerance) + offset_hit_matrix = cmp_func(offset_distances, offset_tolerances.reshape(-1, 1)) # check for hits hits = np.where(offset_hit_matrix) @@ -260,8 +260,7 @@ def match_note_offsets(ref_intervals, est_intervals, offset_ratio=0.2, return matching -def match_note_onsets(ref_intervals, est_intervals, onset_tolerance=0.05, - strict=False): +def match_note_onsets(ref_intervals, est_intervals, onset_tolerance=0.05, strict=False): """Compute a maximum matching between reference and estimated notes, only taking note onsets into account. @@ -306,8 +305,9 @@ def match_note_onsets(ref_intervals, est_intervals, onset_tolerance=0.05, cmp_func = np.less_equal # check for onset matches - onset_distances = np.abs(np.subtract.outer(ref_intervals[:, 0], - est_intervals[:, 0])) + onset_distances = np.abs( + np.subtract.outer(ref_intervals[:, 0], est_intervals[:, 0]) + ) # Round distances to a target precision to avoid the situation where # if the distance is exactly 50ms (and strict=False) it erroneously # doesn't match the notes because of precision issues. @@ -333,9 +333,17 @@ def match_note_onsets(ref_intervals, est_intervals, onset_tolerance=0.05, return matching -def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches, - onset_tolerance=0.05, pitch_tolerance=50.0, offset_ratio=0.2, - offset_min_tolerance=0.05, strict=False): +def match_notes( + ref_intervals, + ref_pitches, + est_intervals, + est_pitches, + onset_tolerance=0.05, + pitch_tolerance=50.0, + offset_ratio=0.2, + offset_min_tolerance=0.05, + strict=False, +): """Compute a maximum matching between reference and estimated notes, subject to onset, pitch and (optionally) offset constraints. @@ -414,8 +422,9 @@ def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches, cmp_func = np.less_equal # check for onset matches - onset_distances = np.abs(np.subtract.outer(ref_intervals[:, 0], - est_intervals[:, 0])) + onset_distances = np.abs( + np.subtract.outer(ref_intervals[:, 0], est_intervals[:, 0]) + ) # Round distances to a target precision to avoid the situation where # if the distance is exactly 50ms (and strict=False) it erroneously # doesn't match the notes because of precision issues. @@ -423,23 +432,25 @@ def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches, onset_hit_matrix = cmp_func(onset_distances, onset_tolerance) # check for pitch matches - pitch_distances = np.abs(1200*np.subtract.outer(np.log2(ref_pitches), - np.log2(est_pitches))) + pitch_distances = np.abs( + 1200 * np.subtract.outer(np.log2(ref_pitches), np.log2(est_pitches)) + ) pitch_hit_matrix = cmp_func(pitch_distances, pitch_tolerance) # check for offset matches if offset_ratio is not None if offset_ratio is not None: - offset_distances = np.abs(np.subtract.outer(ref_intervals[:, 1], - est_intervals[:, 1])) + offset_distances = np.abs( + np.subtract.outer(ref_intervals[:, 1], est_intervals[:, 1]) + ) # Round distances to a target precision to avoid the situation where # if the distance is exactly 50ms (and strict=False) it erroneously # doesn't match the notes because of precision issues. offset_distances = np.around(offset_distances, decimals=N_DECIMALS) ref_durations = util.intervals_to_durations(ref_intervals) - offset_tolerances = np.maximum(offset_ratio * ref_durations, - offset_min_tolerance) - offset_hit_matrix = ( - cmp_func(offset_distances, offset_tolerances.reshape(-1, 1))) + offset_tolerances = np.maximum( + offset_ratio * ref_durations, offset_min_tolerance + ) + offset_hit_matrix = cmp_func(offset_distances, offset_tolerances.reshape(-1, 1)) else: offset_hit_matrix = True @@ -463,11 +474,18 @@ def match_notes(ref_intervals, ref_pitches, est_intervals, est_pitches, return matching -def precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals, - est_pitches, onset_tolerance=0.05, - pitch_tolerance=50.0, offset_ratio=0.2, - offset_min_tolerance=0.05, strict=False, - beta=1.0): +def precision_recall_f1_overlap( + ref_intervals, + ref_pitches, + est_intervals, + est_pitches, + onset_tolerance=0.05, + pitch_tolerance=50.0, + offset_ratio=0.2, + offset_min_tolerance=0.05, + strict=False, + beta=1.0, +): """Compute the Precision, Recall and F-measure of correct vs incorrectly transcribed notes, and the Average Overlap Ratio for correctly transcribed notes (see :func:`average_overlap_ratio`). "Correctness" is determined @@ -548,21 +566,25 @@ def precision_recall_f1_overlap(ref_intervals, ref_pitches, est_intervals, validate(ref_intervals, ref_pitches, est_intervals, est_pitches) # When reference notes are empty, metrics are undefined, return 0's if len(ref_pitches) == 0 or len(est_pitches) == 0: - return 0., 0., 0., 0. - - matching = match_notes(ref_intervals, ref_pitches, est_intervals, - est_pitches, onset_tolerance=onset_tolerance, - pitch_tolerance=pitch_tolerance, - offset_ratio=offset_ratio, - offset_min_tolerance=offset_min_tolerance, - strict=strict) - - precision = float(len(matching))/len(est_pitches) - recall = float(len(matching))/len(ref_pitches) + return 0.0, 0.0, 0.0, 0.0 + + matching = match_notes( + ref_intervals, + ref_pitches, + est_intervals, + est_pitches, + onset_tolerance=onset_tolerance, + pitch_tolerance=pitch_tolerance, + offset_ratio=offset_ratio, + offset_min_tolerance=offset_min_tolerance, + strict=strict, + ) + + precision = float(len(matching)) / len(est_pitches) + recall = float(len(matching)) / len(ref_pitches) f_measure = util.f_measure(precision, recall, beta=beta) - avg_overlap_ratio = average_overlap_ratio(ref_intervals, est_intervals, - matching) + avg_overlap_ratio = average_overlap_ratio(ref_intervals, est_intervals, matching) return precision, recall, f_measure, avg_overlap_ratio @@ -608,9 +630,9 @@ def average_overlap_ratio(ref_intervals, est_intervals, matching): for match in matching: ref_int = ref_intervals[match[0]] est_int = est_intervals[match[1]] - overlap_ratio = ( - (min(ref_int[1], est_int[1]) - max(ref_int[0], est_int[0])) / - (max(ref_int[1], est_int[1]) - min(ref_int[0], est_int[0]))) + overlap_ratio = (min(ref_int[1], est_int[1]) - max(ref_int[0], est_int[0])) / ( + max(ref_int[1], est_int[1]) - min(ref_int[0], est_int[0]) + ) ratios.append(overlap_ratio) if len(ratios) == 0: @@ -619,8 +641,9 @@ def average_overlap_ratio(ref_intervals, est_intervals, matching): return np.mean(ratios) -def onset_precision_recall_f1(ref_intervals, est_intervals, - onset_tolerance=0.05, strict=False, beta=1.0): +def onset_precision_recall_f1( + ref_intervals, est_intervals, onset_tolerance=0.05, strict=False, beta=1.0 +): """Compute the Precision, Recall and F-measure of note onsets: an estimated onset is considered correct if it is within +-50ms of a reference onset. Note that this metric completely ignores note offset and note pitch. This @@ -629,7 +652,6 @@ def onset_precision_recall_f1(ref_intervals, est_intervals, different pitches (i.e. notes that would not match with :func:`match_notes`). - Examples -------- >>> ref_intervals, _ = mir_eval.io.load_valued_intervals( @@ -669,21 +691,26 @@ def onset_precision_recall_f1(ref_intervals, est_intervals, validate_intervals(ref_intervals, est_intervals) # When reference notes are empty, metrics are undefined, return 0's if len(ref_intervals) == 0 or len(est_intervals) == 0: - return 0., 0., 0. + return 0.0, 0.0, 0.0 - matching = match_note_onsets(ref_intervals, est_intervals, - onset_tolerance=onset_tolerance, - strict=strict) + matching = match_note_onsets( + ref_intervals, est_intervals, onset_tolerance=onset_tolerance, strict=strict + ) - onset_precision = float(len(matching))/len(est_intervals) - onset_recall = float(len(matching))/len(ref_intervals) + onset_precision = float(len(matching)) / len(est_intervals) + onset_recall = float(len(matching)) / len(ref_intervals) onset_f_measure = util.f_measure(onset_precision, onset_recall, beta=beta) return onset_precision, onset_recall, onset_f_measure -def offset_precision_recall_f1(ref_intervals, est_intervals, offset_ratio=0.2, - offset_min_tolerance=0.05, strict=False, - beta=1.0): +def offset_precision_recall_f1( + ref_intervals, + est_intervals, + offset_ratio=0.2, + offset_min_tolerance=0.05, + strict=False, + beta=1.0, +): """Compute the Precision, Recall and F-measure of note offsets: an estimated offset is considered correct if it is within +-50ms (or 20% of the ref note duration, which ever is greater) of a reference offset. Note @@ -693,7 +720,6 @@ def offset_precision_recall_f1(ref_intervals, est_intervals, offset_ratio=0.2, different pitches (i.e. notes that would not match with :func:`match_notes`). - Examples -------- >>> ref_intervals, _ = mir_eval.io.load_valued_intervals( @@ -740,17 +766,19 @@ def offset_precision_recall_f1(ref_intervals, est_intervals, offset_ratio=0.2, validate_intervals(ref_intervals, est_intervals) # When reference notes are empty, metrics are undefined, return 0's if len(ref_intervals) == 0 or len(est_intervals) == 0: - return 0., 0., 0. - - matching = match_note_offsets(ref_intervals, est_intervals, - offset_ratio=offset_ratio, - offset_min_tolerance=offset_min_tolerance, - strict=strict) - - offset_precision = float(len(matching))/len(est_intervals) - offset_recall = float(len(matching))/len(ref_intervals) - offset_f_measure = util.f_measure(offset_precision, offset_recall, - beta=beta) + return 0.0, 0.0, 0.0 + + matching = match_note_offsets( + ref_intervals, + est_intervals, + offset_ratio=offset_ratio, + offset_min_tolerance=offset_min_tolerance, + strict=strict, + ) + + offset_precision = float(len(matching)) / len(est_intervals) + offset_recall = float(len(matching)) / len(ref_intervals) + offset_f_measure = util.f_measure(offset_precision, offset_recall, beta=beta) return offset_precision, offset_recall, offset_f_measure @@ -776,7 +804,7 @@ def evaluate(ref_intervals, ref_pitches, est_intervals, est_pitches, **kwargs): Array of estimated notes time intervals (onset and offset times) est_pitches : np.ndarray, shape=(m,) Array of estimated pitch values in Hertz - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -790,40 +818,57 @@ def evaluate(ref_intervals, ref_pitches, est_intervals, est_pitches, **kwargs): scores = collections.OrderedDict() # Precision, recall and f-measure taking note offsets into account - kwargs.setdefault('offset_ratio', 0.2) - orig_offset_ratio = kwargs['offset_ratio'] - if kwargs['offset_ratio'] is not None: - (scores['Precision'], - scores['Recall'], - scores['F-measure'], - scores['Average_Overlap_Ratio']) = util.filter_kwargs( - precision_recall_f1_overlap, ref_intervals, ref_pitches, - est_intervals, est_pitches, **kwargs) + kwargs.setdefault("offset_ratio", 0.2) + orig_offset_ratio = kwargs["offset_ratio"] + if kwargs["offset_ratio"] is not None: + ( + scores["Precision"], + scores["Recall"], + scores["F-measure"], + scores["Average_Overlap_Ratio"], + ) = util.filter_kwargs( + precision_recall_f1_overlap, + ref_intervals, + ref_pitches, + est_intervals, + est_pitches, + **kwargs + ) # Precision, recall and f-measure NOT taking note offsets into account - kwargs['offset_ratio'] = None - (scores['Precision_no_offset'], - scores['Recall_no_offset'], - scores['F-measure_no_offset'], - scores['Average_Overlap_Ratio_no_offset']) = ( - util.filter_kwargs(precision_recall_f1_overlap, - ref_intervals, ref_pitches, - est_intervals, est_pitches, **kwargs)) + kwargs["offset_ratio"] = None + ( + scores["Precision_no_offset"], + scores["Recall_no_offset"], + scores["F-measure_no_offset"], + scores["Average_Overlap_Ratio_no_offset"], + ) = util.filter_kwargs( + precision_recall_f1_overlap, + ref_intervals, + ref_pitches, + est_intervals, + est_pitches, + **kwargs + ) # onset-only metrics - (scores['Onset_Precision'], - scores['Onset_Recall'], - scores['Onset_F-measure']) = ( - util.filter_kwargs(onset_precision_recall_f1, - ref_intervals, est_intervals, **kwargs)) + ( + scores["Onset_Precision"], + scores["Onset_Recall"], + scores["Onset_F-measure"], + ) = util.filter_kwargs( + onset_precision_recall_f1, ref_intervals, est_intervals, **kwargs + ) # offset-only metrics - kwargs['offset_ratio'] = orig_offset_ratio - if kwargs['offset_ratio'] is not None: - (scores['Offset_Precision'], - scores['Offset_Recall'], - scores['Offset_F-measure']) = ( - util.filter_kwargs(offset_precision_recall_f1, - ref_intervals, est_intervals, **kwargs)) + kwargs["offset_ratio"] = orig_offset_ratio + if kwargs["offset_ratio"] is not None: + ( + scores["Offset_Precision"], + scores["Offset_Recall"], + scores["Offset_F-measure"], + ) = util.filter_kwargs( + offset_precision_recall_f1, ref_intervals, est_intervals, **kwargs + ) return scores diff --git a/mir_eval/transcription_velocity.py b/mir_eval/transcription_velocity.py index c7aac282..866ac97e 100644 --- a/mir_eval/transcription_velocity.py +++ b/mir_eval/transcription_velocity.py @@ -59,9 +59,15 @@ from . import util -def validate(ref_intervals, ref_pitches, ref_velocities, est_intervals, - est_pitches, est_velocities): - """Checks that the input annotations have valid time intervals, pitches, +def validate( + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, +): + """Check that the input annotations have valid time intervals, pitches, and velocities, and throws helpful errors if not. Parameters @@ -79,27 +85,39 @@ def validate(ref_intervals, ref_pitches, ref_velocities, est_intervals, est_velocities : np.ndarray, shape=(m,) Array of MIDI velocities (i.e. between 0 and 127) of estimated notes """ - transcription.validate(ref_intervals, ref_pitches, est_intervals, - est_pitches) + transcription.validate(ref_intervals, ref_pitches, est_intervals, est_pitches) # Check that velocities have the same length as intervals/pitches if not ref_velocities.shape[0] == ref_pitches.shape[0]: - raise ValueError('Reference velocities must have the same length as ' - 'pitches and intervals.') + raise ValueError( + "Reference velocities must have the same length as " + "pitches and intervals." + ) if not est_velocities.shape[0] == est_pitches.shape[0]: - raise ValueError('Estimated velocities must have the same length as ' - 'pitches and intervals.') + raise ValueError( + "Estimated velocities must have the same length as " + "pitches and intervals." + ) # Check that the velocities are positive if ref_velocities.size > 0 and np.min(ref_velocities) < 0: - raise ValueError('Reference velocities must be positive.') + raise ValueError("Reference velocities must be positive.") if est_velocities.size > 0 and np.min(est_velocities) < 0: - raise ValueError('Estimated velocities must be positive.') + raise ValueError("Estimated velocities must be positive.") def match_notes( - ref_intervals, ref_pitches, ref_velocities, est_intervals, est_pitches, - est_velocities, onset_tolerance=0.05, pitch_tolerance=50.0, - offset_ratio=0.2, offset_min_tolerance=0.05, strict=False, - velocity_tolerance=0.1): + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + onset_tolerance=0.05, + pitch_tolerance=50.0, + offset_ratio=0.2, + offset_min_tolerance=0.05, + strict=False, + velocity_tolerance=0.1, +): """Match notes, taking note velocity into consideration. This function first calls :func:`mir_eval.transcription.match_notes` to @@ -162,15 +180,22 @@ def match_notes( """ # Compute note matching as usual using standard transcription function matching = transcription.match_notes( - ref_intervals, ref_pitches, est_intervals, est_pitches, - onset_tolerance, pitch_tolerance, offset_ratio, offset_min_tolerance, - strict) + ref_intervals, + ref_pitches, + est_intervals, + est_pitches, + onset_tolerance, + pitch_tolerance, + offset_ratio, + offset_min_tolerance, + strict, + ) # Rescale reference velocities to the range [0, 1] min_velocity, max_velocity = np.min(ref_velocities), np.max(ref_velocities) # Make the smallest possible range 1 to avoid divide by zero velocity_range = max(1, max_velocity - min_velocity) - ref_velocities = (ref_velocities - min_velocity)/float(velocity_range) + ref_velocities = (ref_velocities - min_velocity) / float(velocity_range) # Convert matching list-of-tuples to array for fancy indexing matching = np.array(matching) @@ -183,16 +208,17 @@ def match_notes( # Find slope and intercept of line which produces best least-squares fit # between matched est and ref velocities slope, intercept = np.linalg.lstsq( - np.vstack([est_matched_velocities, - np.ones(len(est_matched_velocities))]).T, - ref_matched_velocities)[0] + np.vstack([est_matched_velocities, np.ones(len(est_matched_velocities))]).T, + ref_matched_velocities, + rcond=None, + )[0] # Re-scale est velocities to match ref - est_matched_velocities = slope*est_matched_velocities + intercept + est_matched_velocities = slope * est_matched_velocities + intercept # Compute the absolute error of (rescaled) estimated velocities vs. # normalized reference velocities. Error will be in [0, 1] velocity_diff = np.abs(est_matched_velocities - ref_matched_velocities) # Check whether each error is within the provided tolerance - velocity_within_tolerance = (velocity_diff < velocity_tolerance) + velocity_within_tolerance = velocity_diff < velocity_tolerance # Only keep matches whose velocity was within the provided tolerance matching = matching[velocity_within_tolerance] # Convert back to list-of-tuple format @@ -202,10 +228,20 @@ def match_notes( def precision_recall_f1_overlap( - ref_intervals, ref_pitches, ref_velocities, est_intervals, est_pitches, - est_velocities, onset_tolerance=0.05, pitch_tolerance=50.0, - offset_ratio=0.2, offset_min_tolerance=0.05, strict=False, - velocity_tolerance=0.1, beta=1.0): + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + onset_tolerance=0.05, + pitch_tolerance=50.0, + offset_ratio=0.2, + offset_min_tolerance=0.05, + strict=False, + velocity_tolerance=0.1, + beta=1.0, +): """Compute the Precision, Recall and F-measure of correct vs incorrectly transcribed notes, and the Average Overlap Ratio for correctly transcribed notes (see :func:`mir_eval.transcription.average_overlap_ratio`). @@ -282,29 +318,53 @@ def precision_recall_f1_overlap( avg_overlap_ratio : float The computed Average Overlap Ratio score """ - validate(ref_intervals, ref_pitches, ref_velocities, est_intervals, - est_pitches, est_velocities) + validate( + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + ) # When reference notes are empty, metrics are undefined, return 0's if len(ref_pitches) == 0 or len(est_pitches) == 0: - return 0., 0., 0., 0. + return 0.0, 0.0, 0.0, 0.0 matching = match_notes( - ref_intervals, ref_pitches, ref_velocities, est_intervals, est_pitches, - est_velocities, onset_tolerance, pitch_tolerance, offset_ratio, - offset_min_tolerance, strict, velocity_tolerance) - - precision = float(len(matching))/len(est_pitches) - recall = float(len(matching))/len(ref_pitches) + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + onset_tolerance, + pitch_tolerance, + offset_ratio, + offset_min_tolerance, + strict, + velocity_tolerance, + ) + + precision = float(len(matching)) / len(est_pitches) + recall = float(len(matching)) / len(ref_pitches) f_measure = util.f_measure(precision, recall, beta=beta) avg_overlap_ratio = transcription.average_overlap_ratio( - ref_intervals, est_intervals, matching) + ref_intervals, est_intervals, matching + ) return precision, recall, f_measure, avg_overlap_ratio -def evaluate(ref_intervals, ref_pitches, ref_velocities, est_intervals, - est_pitches, est_velocities, **kwargs): +def evaluate( + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + **kwargs +): """Compute all metrics for the given reference and estimated annotations. Parameters @@ -321,7 +381,7 @@ def evaluate(ref_intervals, ref_pitches, ref_velocities, est_intervals, Array of estimated pitch values in Hertz est_velocities : np.ndarray, shape=(n,) Array of MIDI velocities (i.e. between 0 and 127) of estimated notes - kwargs + **kwargs Additional keyword arguments which will be passed to the appropriate metric or preprocessing functions. @@ -335,23 +395,40 @@ def evaluate(ref_intervals, ref_pitches, ref_velocities, est_intervals, scores = collections.OrderedDict() # Precision, recall and f-measure taking note offsets into account - kwargs.setdefault('offset_ratio', 0.2) - if kwargs['offset_ratio'] is not None: - (scores['Precision'], - scores['Recall'], - scores['F-measure'], - scores['Average_Overlap_Ratio']) = util.filter_kwargs( - precision_recall_f1_overlap, ref_intervals, ref_pitches, - ref_velocities, est_intervals, est_pitches, est_velocities, - **kwargs) + kwargs.setdefault("offset_ratio", 0.2) + if kwargs["offset_ratio"] is not None: + ( + scores["Precision"], + scores["Recall"], + scores["F-measure"], + scores["Average_Overlap_Ratio"], + ) = util.filter_kwargs( + precision_recall_f1_overlap, + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + **kwargs + ) # Precision, recall and f-measure NOT taking note offsets into account - kwargs['offset_ratio'] = None - (scores['Precision_no_offset'], - scores['Recall_no_offset'], - scores['F-measure_no_offset'], - scores['Average_Overlap_Ratio_no_offset']) = util.filter_kwargs( - precision_recall_f1_overlap, ref_intervals, ref_pitches, - ref_velocities, est_intervals, est_pitches, est_velocities, **kwargs) + kwargs["offset_ratio"] = None + ( + scores["Precision_no_offset"], + scores["Recall_no_offset"], + scores["F-measure_no_offset"], + scores["Average_Overlap_Ratio_no_offset"], + ) = util.filter_kwargs( + precision_recall_f1_overlap, + ref_intervals, + ref_pitches, + ref_velocities, + est_intervals, + est_pitches, + est_velocities, + **kwargs + ) return scores diff --git a/mir_eval/util.py b/mir_eval/util.py index 600677ac..15e812ea 100644 --- a/mir_eval/util.py +++ b/mir_eval/util.py @@ -1,11 +1,12 @@ -''' -This submodule collects useful functionality required across the task -submodules, such as preprocessing, validation, and common computations. -''' +""" +Useful functionality required across the task submodules, +such as preprocessing, validation, and common computations. +""" import os import inspect -import six +import warnings +from decorator import decorator import numpy as np @@ -18,7 +19,6 @@ def index_labels(labels, case_sensitive=False): labels : list of strings, shape=(n,) A list of annotations, e.g., segment or chord labels from an annotation file. - case_sensitive : bool Set to True to enable case-sensitive label indexing (Default value = False) @@ -30,9 +30,7 @@ def index_labels(labels, case_sensitive=False): index_to_label : dict Mapping to convert numerical indices back to labels. ``labels[i] == index_to_label[indices[i]]`` - """ - label_to_index = {} index_to_label = {} @@ -52,7 +50,7 @@ def index_labels(labels, case_sensitive=False): return indices, index_to_label -def generate_labels(items, prefix='__'): +def generate_labels(items, prefix="__"): """Given an array of items (e.g. events, intervals), create a synthetic label for each event of the form '(label prefix)(item number)' @@ -70,11 +68,10 @@ def generate_labels(items, prefix='__'): Synthetically generated labels """ - return ['{}{}'.format(prefix, n) for n in range(len(items))] + return [f"{prefix}{n}" for n in range(len(items))] -def intervals_to_samples(intervals, labels, offset=0, sample_size=0.1, - fill_value=None): +def intervals_to_samples(intervals, labels, offset=0, sample_size=0.1, fill_value=None): """Convert an array of labeled time intervals to annotated samples. Parameters @@ -85,18 +82,14 @@ def intervals_to_samples(intervals, labels, offset=0, sample_size=0.1, :func:`mir_eval.io.load_labeled_intervals()`. The ``i`` th interval spans time ``intervals[i, 0]`` to ``intervals[i, 1]``. - labels : list, shape=(n,) The annotation for each interval - offset : float > 0 Phase offset of the sampled time grid (in seconds) (Default value = 0) - sample_size : float > 0 duration of each sample to be generated (in seconds) (Default value = 0.1) - fill_value : type(labels[0]) Object to use for the label with out-of-range time points. (Default value = None) @@ -105,7 +98,6 @@ def intervals_to_samples(intervals, labels, offset=0, sample_size=0.1, ------- sample_times : list list of sample times - sample_labels : list array of labels for each generated sample @@ -113,15 +105,12 @@ def intervals_to_samples(intervals, labels, offset=0, sample_size=0.1, ----- Intervals will be rounded down to the nearest multiple of ``sample_size``. - """ - # Round intervals to the sample size num_samples = int(np.floor(intervals.max() / sample_size)) sample_indices = np.arange(num_samples, dtype=np.float32) - sample_times = (sample_indices*sample_size + offset).tolist() - sampled_labels = interpolate_intervals( - intervals, labels, sample_times, fill_value) + sample_times = (sample_indices * sample_size + offset).tolist() + sampled_labels = interpolate_intervals(intervals, labels, sample_times, fill_value) return sample_times, sampled_labels @@ -162,33 +151,31 @@ def interpolate_intervals(intervals, labels, time_points, fill_value=None): ValueError If `time_points` is not in non-decreasing order. """ - # Verify that time_points is sorted time_points = np.asarray(time_points) if np.any(time_points[1:] < time_points[:-1]): - raise ValueError('time_points must be in non-decreasing order') + raise ValueError("time_points must be in non-decreasing order") aligned_labels = [fill_value] * len(time_points) - starts = np.searchsorted(time_points, intervals[:, 0], side='left') - ends = np.searchsorted(time_points, intervals[:, 1], side='right') + starts = np.searchsorted(time_points, intervals[:, 0], side="left") + ends = np.searchsorted(time_points, intervals[:, 1], side="right") - for (start, end, lab) in zip(starts, ends, labels): + for start, end, lab in zip(starts, ends, labels): aligned_labels[start:end] = [lab] * (end - start) return aligned_labels def sort_labeled_intervals(intervals, labels=None): - '''Sort intervals, and optionally, their corresponding labels + """Sort intervals, and optionally, their corresponding labels according to start time. Parameters ---------- intervals : np.ndarray, shape=(n, 2) The input intervals - labels : list, optional Labels for each interval @@ -196,8 +183,7 @@ def sort_labeled_intervals(intervals, labels=None): ------- intervals_sorted or (intervals_sorted, labels_sorted) Labels are only returned if provided as input - ''' - + """ idx = np.argsort(intervals[:, 0]) intervals_sorted = intervals[idx] @@ -225,13 +211,11 @@ def f_measure(precision, recall, beta=1.0): ------- f_measure : float The weighted f-measure - """ - if precision == 0 and recall == 0: return 0.0 - return (1 + beta**2)*precision*recall/((beta**2)*precision + recall) + return (1 + beta**2) * precision * recall / ((beta**2) * precision + recall) def intervals_to_boundaries(intervals, q=5): @@ -248,9 +232,7 @@ def intervals_to_boundaries(intervals, q=5): ------- boundaries : np.ndarray Interval boundary times, including the end of the final interval - """ - return np.unique(np.ravel(np.round(intervals, decimals=q))) @@ -268,21 +250,22 @@ def boundaries_to_intervals(boundaries): intervals : np.ndarray, shape=(n_intervals, 2) Start and end time for each interval """ - if not np.allclose(boundaries, np.unique(boundaries)): - raise ValueError('Boundary times are not unique or not ascending.') + raise ValueError("Boundary times are not unique or not ascending.") intervals = np.asarray(list(zip(boundaries[:-1], boundaries[1:]))) return intervals -def adjust_intervals(intervals, - labels=None, - t_min=0.0, - t_max=None, - start_label='__T_MIN', - end_label='__T_MAX'): +def adjust_intervals( + intervals, + labels=None, + t_min=0.0, + t_max=None, + start_label="__T_MIN", + end_label="__T_MAX", +): """Adjust a list of time intervals to span the range ``[t_min, t_max]``. Any intervals lying completely outside the specified range will be removed. @@ -321,9 +304,7 @@ def adjust_intervals(intervals, Intervals spanning ``[t_min, t_max]`` new_labels : list List of labels for ``new_labels`` - """ - # When supplied intervals are empty and t_max and t_min are supplied, # create one interval from t_min to t_max with the label start_label if t_min is not None and t_max is not None and intervals.size == 0: @@ -331,8 +312,7 @@ def adjust_intervals(intervals, # When intervals are empty and either t_min or t_max are not supplied, # we can't append new intervals elif (t_min is None or t_max is None) and intervals.size == 0: - raise ValueError("Supplied intervals are empty, can't append new" - " intervals") + raise ValueError("Supplied intervals are empty, can't append new" " intervals") if t_min is not None: # Find the intervals that end at or after t_min @@ -341,9 +321,9 @@ def adjust_intervals(intervals, if len(first_idx) > 0: # If we have events below t_min, crop them out if labels is not None: - labels = labels[int(first_idx[0]):] + labels = labels[first_idx[0, 0] :] # Clip to the range (t_min, +inf) - intervals = intervals[int(first_idx[0]):] + intervals = intervals[first_idx[0, 0] :] intervals = np.maximum(t_min, intervals) if intervals.min() > t_min: @@ -361,9 +341,9 @@ def adjust_intervals(intervals, # We have boundaries above t_max. # Trim to only boundaries <= t_max if labels is not None: - labels = labels[:int(last_idx[0])] + labels = labels[: last_idx[0, 0]] # Clip to the range (-inf, t_max) - intervals = intervals[:int(last_idx[0])] + intervals = intervals[: last_idx[0, 0]] intervals = np.minimum(t_max, intervals) @@ -376,8 +356,7 @@ def adjust_intervals(intervals, return intervals, labels -def adjust_events(events, labels=None, t_min=0.0, - t_max=None, label_prefix='__'): +def adjust_events(events, labels=None, t_min=0.0, t_max=None, label_prefix="__"): """Adjust the given list of event times to span the range ``[t_min, t_max]``. @@ -416,15 +395,15 @@ def adjust_events(events, labels=None, t_min=0.0, # We have events below t_min # Crop them out if labels is not None: - labels = labels[int(first_idx[0]):] - events = events[int(first_idx[0]):] + labels = labels[first_idx[0, 0] :] + events = events[first_idx[0, 0] :] if events[0] > t_min: # Lowest boundary is higher than t_min: # add a new boundary and label events = np.concatenate(([t_min], events)) if labels is not None: - labels.insert(0, '%sT_MIN' % label_prefix) + labels.insert(0, "%sT_MIN" % label_prefix) if t_max is not None: last_idx = np.argwhere(events > t_max) @@ -433,14 +412,14 @@ def adjust_events(events, labels=None, t_min=0.0, # We have boundaries above t_max. # Trim to only boundaries <= t_max if labels is not None: - labels = labels[:int(last_idx[0])] - events = events[:int(last_idx[0])] + labels = labels[: last_idx[0, 0]] + events = events[: last_idx[0, 0]] if events[-1] < t_max: # Last boundary is below t_max: add a new boundary and label events = np.concatenate((events, [t_max])) if labels is not None: - labels.append('%sT_MAX' % label_prefix) + labels.append("%sT_MAX" % label_prefix) return events, labels @@ -474,21 +453,22 @@ def intersect_files(flist1, flist2): corresponding filepaths from ``flist2`` """ + def fname(abs_path): - """Returns the filename given an absolute path. + """Return the filename given an absolute path. Parameters ---------- - abs_path : - + abs_path Returns ------- + filename """ return os.path.splitext(os.path.split(abs_path)[-1])[0] - fmap = dict([(fname(f), f) for f in flist1]) + fmap = {fname(f): f for f in flist1} pairs = [list(), list()] for f in flist2: if fname(f) in fmap: @@ -522,16 +502,17 @@ def merge_labeled_intervals(x_intervals, x_labels, y_intervals, y_labels): New labels for the sequence ``y`` """ - align_check = [x_intervals[0, 0] == y_intervals[0, 0], - x_intervals[-1, 1] == y_intervals[-1, 1]] + align_check = [ + x_intervals[0, 0] == y_intervals[0, 0], + x_intervals[-1, 1] == y_intervals[-1, 1], + ] if False in align_check: raise ValueError( "Time intervals do not align; did you mean to call " - "'adjust_intervals()' first?") - time_boundaries = np.unique( - np.concatenate([x_intervals, y_intervals], axis=0)) - output_intervals = np.array( - [time_boundaries[:-1], time_boundaries[1:]]).T + "'adjust_intervals()' first?" + ) + time_boundaries = np.unique(np.concatenate([x_intervals, y_intervals], axis=0)) + output_intervals = np.array([time_boundaries[:-1], time_boundaries[1:]]).T x_labels_out, y_labels_out = [], [] x_label_range = np.arange(len(x_labels)) @@ -584,7 +565,7 @@ def _bipartite_match(graph): # layer preds = {} unmatched = [] - pred = dict([(u, unmatched) for u in graph]) + pred = {u: unmatched for u in graph} for v in matching: del pred[matching[v]] layer = list(pred) @@ -711,7 +692,7 @@ def match_events(ref, est, window, distance=None): def _fast_hit_windows(ref, est, window): - '''Fast calculation of windowed hits for time events. + """Fast calculation of windowed hits for time events. Given two lists of event times ``ref`` and ``est``, and a tolerance window, computes a list of pairings @@ -736,15 +717,14 @@ def _fast_hit_windows(ref, est, window): hit_ref : np.ndarray hit_est : np.ndarray indices such that ``|hit_ref[i] - hit_est[i]| <= window`` - ''' - + """ ref = np.asarray(ref) est = np.asarray(est) ref_idx = np.argsort(ref) ref_sorted = ref[ref_idx] - left_idx = np.searchsorted(ref_sorted, est - window, side='left') - right_idx = np.searchsorted(ref_sorted, est + window, side='right') + left_idx = np.searchsorted(ref_sorted, est - window, side="left") + right_idx = np.searchsorted(ref_sorted, est + window, side="right") hit_ref, hit_est = [], [] @@ -756,32 +736,32 @@ def _fast_hit_windows(ref, est, window): def validate_intervals(intervals): - """Checks that an (n, 2) interval ndarray is well-formed, and raises errors + """Check that an (n, 2) interval ndarray is well-formed, and raises errors if not. Parameters ---------- intervals : np.ndarray, shape=(n, 2) Array of interval start/end locations. - """ - # Validate interval shape if intervals.ndim != 2 or intervals.shape[1] != 2: - raise ValueError('Intervals should be n-by-2 numpy ndarray, ' - 'but shape={}'.format(intervals.shape)) + raise ValueError( + "Intervals should be n-by-2 numpy ndarray, " + "but shape={}".format(intervals.shape) + ) # Make sure no times are negative if (intervals < 0).any(): - raise ValueError('Negative interval times found') + raise ValueError("Negative interval times found") # Make sure all intervals have strictly positive duration if (intervals[:, 1] <= intervals[:, 0]).any(): - raise ValueError('All interval durations must be strictly positive') + raise ValueError("All interval durations must be strictly positive") -def validate_events(events, max_time=30000.): - """Checks that a 1-d event location ndarray is well-formed, and raises +def validate_events(events, max_time=30000.0): + """Check that a 1-d event location ndarray is well-formed, and raises errors if not. Parameters @@ -791,26 +771,28 @@ def validate_events(events, max_time=30000.): max_time : float If an event is found above this time, a ValueError will be raised. (Default value = 30000.) - """ # Make sure no event times are huge if (events > max_time).any(): - raise ValueError('An event at time {} was found which is greater than ' - 'the maximum allowable time of max_time = {} (did you' - ' supply event times in ' - 'seconds?)'.format(events.max(), max_time)) + raise ValueError( + "An event at time {} was found which is greater than " + "the maximum allowable time of max_time = {} (did you" + " supply event times in " + "seconds?)".format(events.max(), max_time) + ) # Make sure event locations are 1-d np ndarrays if events.ndim != 1: - raise ValueError('Event times should be 1-d numpy ndarray, ' - 'but shape={}'.format(events.shape)) + raise ValueError( + "Event times should be 1-d numpy ndarray, " + "but shape={}".format(events.shape) + ) # Make sure event times are increasing if (np.diff(events) < 0).any(): - raise ValueError('Events should be in increasing order.') + raise ValueError("Events should be in increasing order.") -def validate_frequencies(frequencies, max_freq, min_freq, - allow_negatives=False): - """Checks that a 1-d frequency ndarray is well-formed, and raises +def validate_frequencies(frequencies, max_freq, min_freq, allow_negatives=False): + """Check that a 1-d frequency ndarray is well-formed, and raises errors if not. Parameters @@ -831,24 +813,30 @@ def validate_frequencies(frequencies, max_freq, min_freq, frequencies = np.abs(frequencies) # Make sure no frequency values are huge if (np.abs(frequencies) > max_freq).any(): - raise ValueError('A frequency of {} was found which is greater than ' - 'the maximum allowable value of max_freq = {} (did ' - 'you supply frequency values in ' - 'Hz?)'.format(frequencies.max(), max_freq)) + raise ValueError( + "A frequency of {} was found which is greater than " + "the maximum allowable value of max_freq = {} (did " + "you supply frequency values in " + "Hz?)".format(frequencies.max(), max_freq) + ) # Make sure no frequency values are tiny if (np.abs(frequencies) < min_freq).any(): - raise ValueError('A frequency of {} was found which is less than the ' - 'minimum allowable value of min_freq = {} (did you ' - 'supply frequency values in ' - 'Hz?)'.format(frequencies.min(), min_freq)) + raise ValueError( + "A frequency of {} was found which is less than the " + "minimum allowable value of min_freq = {} (did you " + "supply frequency values in " + "Hz?)".format(frequencies.min(), min_freq) + ) # Make sure frequency values are 1-d np ndarrays if frequencies.ndim != 1: - raise ValueError('Frequencies should be 1-d numpy ndarray, ' - 'but shape={}'.format(frequencies.shape)) + raise ValueError( + "Frequencies should be 1-d numpy ndarray, " + "but shape={}".format(frequencies.shape) + ) def has_kwargs(function): - r'''Determine whether a function has \*\*kwargs. + r"""Determine whether a function has \*\*kwargs. Parameters ---------- @@ -859,22 +847,18 @@ def has_kwargs(function): ------- True if function accepts arbitrary keyword arguments. False otherwise. - ''' - - if six.PY2: - return inspect.getargspec(function).keywords is not None - else: - sig = inspect.signature(function) + """ + sig = inspect.signature(function) - for param in sig.parameters.values(): - if param.kind == param.VAR_KEYWORD: - return True + for param in list(sig.parameters.values()): + if param.kind == param.VAR_KEYWORD: + return True - return False + return False def filter_kwargs(_function, *args, **kwargs): - """Given a function and args and keyword args to pass to it, call the function + r"""Given a function and args and keyword args to pass to it, call the function but using only the keyword arguments which it accepts. This is equivalent to redefining the function with an additional \*\*kwargs to accept slop keyword args. @@ -886,15 +870,16 @@ def filter_kwargs(_function, *args, **kwargs): ---------- _function : callable Function to call. Can take in any number of args or kwargs - + *args + **kwargs + Arguments and keyword arguments to _function. """ - if has_kwargs(_function): return _function(*args, **kwargs) # Get the list of function arguments - func_code = six.get_function_code(_function) - function_args = func_code.co_varnames[:func_code.co_argcount] + func_code = _function.__code__ + function_args = func_code.co_varnames[: func_code.co_argcount] # Construct a dict of those kwargs which appear in the function filtered_kwargs = {} for kwarg, value in list(kwargs.items()): @@ -905,7 +890,7 @@ def filter_kwargs(_function, *args, **kwargs): def intervals_to_durations(intervals): - """Converts an array of n intervals to their n durations. + """Convert an array of n intervals to their n durations. Parameters ---------- @@ -926,7 +911,7 @@ def intervals_to_durations(intervals): def hz_to_midi(freqs): - '''Convert Hz to MIDI numbers + """Convert Hz to MIDI numbers Parameters ---------- @@ -938,12 +923,12 @@ def hz_to_midi(freqs): midi : number or ndarray MIDI note numbers corresponding to input frequencies. Note that these may be fractional. - ''' + """ return 12.0 * (np.log2(freqs) - np.log2(440.0)) + 69.0 def midi_to_hz(midi): - '''Convert MIDI numbers to Hz + """Convert MIDI numbers to Hz Parameters ---------- @@ -954,5 +939,24 @@ def midi_to_hz(midi): ------- freqs : number or ndarray Frequency/frequencies in Hz corresponding to `midi` - ''' - return 440.0 * (2.0 ** ((midi - 69.0)/12.0)) + """ + return 440.0 * (2.0 ** ((midi - 69.0) / 12.0)) + + +def deprecated(*, version, version_removed): + """Mark a function as deprecated. + + Using the decorated (old) function will result in a warning. + """ + + def __wrapper(func, *args, **kwargs): + """Warn the user, and then proceed.""" + warnings.warn( + f"{func.__module__}.{func.__name__}\n\tDeprecated as of mir_eval version {version}." + f"\n\tIt will be removed in mir_eval version {version_removed}.", + category=FutureWarning, + stacklevel=3, # Would be 2, but the decorator adds a level + ) + return func(*args, **kwargs) + + return decorator(__wrapper) diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..bc6f82bc --- /dev/null +++ b/setup.cfg @@ -0,0 +1,72 @@ +[tool:pytest] +addopts = --cov-report term-missing --cov mir_eval --cov-report=xml --mpl --mpl-baseline-path=baseline_images/test_display + +[coverage:report] +show_missing = True + +[coverage:run] +omit = + mir_eval/separation.py + +[pydocstyle] +# convention = numpy +# Below is equivalent to numpy convention + D400 and D205 +ignore = D107,D203,D205,D212,D213,D400,D402,D413,D415,D416,D417 + +[flake8] +count = True +statistics = True +show_source = True +select = + E9, + F63, + F7, + F82 + +[metadata] +name = mir_eval +version = attr: mir_eval.__version__ +description = Common metrics for common audio/music processing tasks. +author = Colin Raffel +author_email = craffel@gmail.com +url = https://github.com/mir-evaluation/mir_eval +long_description = file: README.rst +long_description_content_type = text/x-rst; charset=UTF-8 +license = MIT +python_requires = ">=3.7" +classifiers = + License :: OSI Approved :: MIT License + Programming Language :: Python + Development Status :: 5 - Production/Stable + Intended Audience :: Developers + Topic :: Multimedia :: Sound/Audio :: Analysis + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.7 + Programming Language :: Python :: 3.8 + Programming Language :: Python :: 3.9 + Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 + Programming Language :: Python :: 3.13 + + +[options] +packages = find: +keywords = audio music mir dsp +install_requires = + numpy >= 1.15.4 + scipy >= 1.4.0 + decorator + +[options.extras_require] +display = + matplotlib >= 3.3.0 +docs = + numpydoc + sphinx_rtd_theme + matplotlib >= 3.3.0 +tests = + matplotlib >= 3.3.0 + pytest + pytest-cov + pytest-mpl diff --git a/setup.py b/setup.py index 4d1e7f55..7f1a1763 100644 --- a/setup.py +++ b/setup.py @@ -1,37 +1,4 @@ from setuptools import setup -with open('README.rst') as file: - long_description = file.read() - -setup( - name='mir_eval', - version='0.6', - description='Common metrics for common audio/music processing tasks.', - author='Colin Raffel', - author_email='craffel@gmail.com', - url='https://github.com/craffel/mir_eval', - packages=['mir_eval'], - long_description=long_description, - classifiers=[ - "License :: OSI Approved :: MIT License", - "Programming Language :: Python", - 'Development Status :: 5 - Production/Stable', - "Intended Audience :: Developers", - "Topic :: Multimedia :: Sound/Audio :: Analysis", - "Programming Language :: Python :: 2.7", - "Programming Language :: Python :: 3", - ], - keywords='audio music mir dsp', - license='MIT', - install_requires=[ - 'numpy >= 1.7.0', - 'scipy >= 1.0.0', - 'future', - 'six' - ], - extras_require={ - 'display': ['matplotlib>=1.5.0', - 'scipy>=1.0.0'], - 'testing': ['matplotlib>=2.1.0,<3'] - } -) +if __name__ == "__main__": + setup() diff --git a/tests/baseline_images/test_display/events.png b/tests/baseline_images/test_display/events.png deleted file mode 100644 index 48673b1fb891c20d81b97669107b118678b9d753..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9671 zcmeI2c~sNay7yzXTC2$Mcv?gR>=B`@0tzxiqQ@2*6{sR0vzAFf2ve9s2(49WWoUzd zjH${XhJc765+G4grZC8q0D%Mr5<&=(gaDby-M`*@*E(z6yWVyGdjG0xnHUT^dq4Y` zzMm(zzjt%d`t-o3Fc?e=`Q0}cVX%)fVX!STpKJ#|u^O7115eu${)N2s3HX!#NysnY z_0HJud=g+V?W@q=Esqd25ir;x81kFbmr}|^)0lgs=xNOg>nY=D=E1+)MQ8qiFZt<* zN2%YD&aX;(XUn||dSe`Z_@%(NxVq{=SXCIJH)B0Im75f`Lp!;>{e@Jz#zrzS!Qx=Pr4o#Mh<;CT-L@(kP4EB9?AlU#0 z%d{ziZG(N8{nx|4+2H&c(vHgn&#wy{*QXM6k6$^pxVRYUvl# zdKc&2-1)-l!&Jgx=<31{*p~FI)PsdX z_L`~zMw=`_yN2t0gGz}J$e4F;wlqYX?YfOhESH6FtlE|^X;@NzOWIR{RX|Zikhc>p z+AN^8%ZHVW!dX#CiNZB{UrCHQy}@0W_B@O;ff`&8_lZ@>ZQpzptJS7>CVuhbR+cSH zZ?dq`n~+h?XH7f2si-Hufph!NN9%eXR1OZ;2zKo8pD%y?sgT+j=E|%h(Guzvk@RJK zu8_HUb|kVPYQOChOp#!5nP{TC+>_y@3~%u!OD$0QZI7$17dfJpF(C=wZtSr^ z=wQ%lSSeu`SJFz ziD_ChQEYSH(L&0e1zWBeYc&MX;5RaVqApM9)mTlV!L_uszNL4HbHJy3H_T)f@(DON zi9|x;&yq_#I8tUGN$J(KeGpC>abzEN>5s)9le8QByyG-rW zP96A4p_S9bppcoEx?4C)HDj&{S>}gba?b|`2d}RM?cegrsgNu`k;5Dd^I(i=4$;;6TpH>9h1#%p0RWq~uDt-=pI zbg2oV=Tbr<8ZM0xuU;()%pq1+*Kd7tKe#opp!3qjqH5@6tYA&#>RKCh`Dvm~K%Jl` zBal+f2)=B{BK_dXrC~=}lf{m*QICkIsB<139_wLW{$jK}v&!+}vv;upN&Qt@Vc&y8 z!akfp%5GkY_nC0={V=ygA?*Q=ESI#_ivnhT|~emF6Xtcw~Kq?y7Y@8 zMyf;eU9EfOdh(_3@0jz(gY){|-alg*`@v7b(ax~YhO1KWb1n$Qh0Z)fg3&~}wBN(q zyU2u)Zh?#!ky&9``bpn`gIIHa@E_P%(|!uPYXCboHWu2Mt!p=vxpkP1)0Cl^K1h6) zLE4u}5hcia^4i1Qh7)BE;p<~6wV`IopTY{ngf5QO+6gN2pRRQ3g}?q~ubp~zq{tyx zCn)^{!Cug}n;RoODH9T+Dbv=hfaC!Fx|k}gZ&OZ--GEgsfu+uu+gt7W{Mcn3ort~L z4#5)dqvNV(-P!nh&9Uj2ckkDZ`A=~~B0qNFF-o;bhHBOja-C!@HkqbTi}^PURabm{ zed9Z{Z00sG;h|Y8etc{e&R(<8g^1sHm&Gsii$UW1O013!A~fmFc?RXJ%haPThSrJz&a-!^$FrYU)E&uI6T*(Jo!7kKmPTbFy|2&Iz`8M_*`Y|Auj z5ANJ`j-Ga(AqpIs7J1Rs`TQk%i7?-^dHkRJ^5j5#u1jxNe@Q`MVd1J^#H98aUp3;^ zZDliENYJQsjrUV&LQ<+m$tC#3v>a&)n=W-hX};xbHoNBUEe)5NcA}Sq6{`Zf z_rDyrh@N~lAkzs*ys!pJQJ<}7BqDOLX|l*i0PeqzZ|qt!Dwo~RhToG*`N__K_)6cN zqq}tGZzB+mz0#BqFLxxYPbVkCNu@?2(YdFw?;bpBZm#A!Z}l2y5rr|WqUP(*hMXB9 zb3w{n%mmDz+5P+Bk>O#yQLz&y$vNl1*j^m*Ui3hjU5iS>2x5Oro?O{!HmEw{m&Z;o z=z7x&0&U^dVS*s#=VKidks|-prsDgqwE|ZUWI(fl-4mhvM?O@n@=3I0VV8bGC_8QL zg+c!~h?ao{{vd(y`(YQ}8XJX+4#FvMNc_uL4!Ku)4}r@Vu$|357~lQH(K^4i{F8;# zV6X5GKWfhb69inH?%s(>jA4N!GtUG;SanhG$r~D0nUtC?Qe0v3 z_|>eo53hDbT)WmCRvOLR8={9!XU1AjR5H+cPHUH&$A7p}Wogz_f5&|cyj5RvCrI_r z(&%QN0|yQi2CY*s`xoroXFN9%PUxR?cO(zHY=KsERDI}Z=wEvb()O#e7}1S+`H{cD z#xs}N*EtiRMFH_|Pwxoj^%bW|dQ7BjjA7qbeE%}#ax1Dp=;k1L=ASeXYB!s0Df!T$ z%^z|x?Ej3#x!X{oZ1=23;=l5!o-@>JxWGXi@;MlB|2}ue-%DdO(Ybc-q9RZzF|!Y| z+!DSvK09r(Wp54;!o~2cewTvNjo| z{(JX-mPlkYWXOn1ja>$IynDftHB$3H7F;>+C@bPyrf+=ktS@y;jR$c_?JKjh zbXa~K%pnV7QgY_3=<7=`t006q)GXx?r-~y&v)x5PyOCyh;tNYthZXmy?Qvz%t_)$j2F z_QXIbYe4zEtL1Ikx_$1!9kbqVbpoELMk)_Gn;@IH8OlqHbg$y0pVMW;AXj|4oDnntRZv&! z2_wSBYZJUtO?i@USl_+hfiKSmIEe0X+A;iW^9*C%p+-7z7G5%48IYkEA@H7b>|r?T zMc)M}HEig?w+*7!;R=mPTwHtUv7C-*GeXPsZ^sL&io!n3ylhdgu|eHaCMuuaj{@gn zdblIxd{AO|YB5d|Uqd02L*Ku?4fQz*3-_~lKl;%7;TYP{9oV_?C91O38-v)_>As+% zf)@MCt=&BvD-EH-5AQnijZ95V%WJS=7jGUH7k3@%_1Yx8M?vSu00o0PpJ9+bx1tjC zgj-yBJ2USV^gQ-zMauizV-=;I`!Zo~3xW%2&Cwgp0f>)PtSr2dwzswG5p-%-_m8#W z#jj?n*c5HHabDY#Lr8mOUFS6tNncn`-&C~8NT`5i3|6t)voQ2fIy!~mD+sDXlqs}3 zXMGcE5!!pV47VvKHWpPTi#ZH+dwqRO%gExQeQwjmS|eor^R?>q6+N9fKR$sFkTh+& zb^Gpa8@jw7lr5)#czu_b^4Gq47Z}XaC*vQ{mgR|qL-qo zgj&T;-kZrNLy?nZWNh6nN)Fos+7#rMPC&I*v3G-;H*`>CrZ0WiDp_vF5%#&m?SLrB z;0pzlY}O`9$-#=3mJ|a!Zx{$y3ewz1))1cR2L07wKD=c6#K96bl)BDG2}l{n(6t86xv7;07wTh=)k z2b|Ovc<}Wm=zmf{MS)y+_kkxDa&1HC=?R^ob{}SLNM#MU!$55Jdbm3k5bLR5o0KPB z1k01XcpdB)uor0}VUWfdDvjp}a3?=hI~s~Hf1FyT>yUK)fS#Uo-ovfu=XS(hr#*Jo zGMTL9{jW#yMnK55EIha)0ln~{{{DWu4==W|CeIV>C2tQtXuKu@&bLdUuqe3TEC<~OZHyBxPzptNPl;h4Q29Y%{M0Dov z;q3TZ=KC=gD;xxOL_`D<4{6ha$csM1g*V#^;rM5 z`G;W$0IC3?4A$zId3nPlNUYxG?F{?q6dvlb zk>T6C9*?O+Z@h^6cSJOB-U1%X^~t7o|IpY`vC#Xzz$1>9~cSJ z{!Q4{-?0MKX1UGH=Xcn)%*6BuouG%c1U?Cg!-i+3J^cDjbjBF}sJ>phWT3K<+p9w% z8`JF_krtst6Fzr4{nuNjM<{WhY5L5X)JMCf3f%(}aQMXbe?P~sR_&PewQjB+)C^?< zSw6~Q)eJ`=D{!fCC7&ZkR&uK0$l#G)9a*G1FZueq>eZXnBQn9r=-q11n-dmnZ^cQT zxp{K>l5bHIKr$?nK(e3Ng?5_O$=JCK_Qy$(2L3qy*TY{n_{$Ie3WL8=!C#s1kKe3HNwcoL7Rl>PotiG=`$LK4dt{|HnwB_ z3&lsD=KG88__ijeH~4uVL&+@iKq;%8k@(+J1fsghQA1|5p*d~GYDdwi>0u$sX_sX~ zed+%T#X~Dn=m(~~sAy$Sm@6fN#;QR(`CFm{CEgl+^9upwxrae#fF8IG;9-F91T51a zg;yMI5jpzEH}AqeFj%EvyZvDx=sLoX&J?=^UZt4bOr4vggdtH-v8Uxj$8A8+}P6_ zoaf{vY~=i7q{iJnARwUHwbV;^fq9#*nm4xt7}XN=yHLkG+m$y6x@&J?Ba^zewpNz5 zSPM0;iZxLwrCk&)@3f&co=*RK0d%Cr+UYY}BVuCQz;g91ypnlZ5XmcHGNL9Fh*3zL zuL16(TFJxJ4LyshCI$zzrNp_x<#XAF2)J6g?w*j4kPBBVKr=6^vgdrxLRtV5LS>_4 zWAi`lGE3QK;*X&w)In-?kL^t6?gJ;k1iv#%$eKwj$hk(mA4W)4t+29wGMaquAJ zBunyQt9A$=hz!8h0W-WP9AZ8OhB)j|eZB{%KjjqQ=ssW@YXF2DyZq!2=0lPP=>Ab= zEpzzu_m1CZKCIiPQSlK??3mUnpr=~i-O>Zb$(!@zD!e$7Lds~`Gx9RfPW{=smhmfZ zbpGUCE&Pg7>41YdSG}AZxbcP$FEs?C0kv1v{S`^>FXss_pn2q%9z-2Oaysc`6T>20LcO^3Mmz%mjEI zYB(f{tEVr7m>N(RPJYp3<LT4aj+unu@f@l9Ce69Ma~HC$k%BM&-Le4q!_Pa(CDYH3Ijy+UMppTd6noULkU9Uv5Db@^BBJ(9LwKI-)+|pvazP(S{ zVYmqe`#wMkqhi`+OvlZQStNdkVfyjpX|y*`_Vfep%`yGu4mRchz{VrGZfw`;9dHo0 zz2v$d(O>sOqtUvKDX}CMA0c33x%P{X%77}J0FXA)X~5p2_ENlB#^Q@mIE=A-MHz_3 zPymdUC11sH?!o-G0-xlLa5xq3IQi%*QSRtqRO@ zK^VuRf{7eLCotuzLCVaJS_*Dy!GB85g{NhZQA9B zihfSnw0UVh~V^Q*z|3aigEduGaDYUv@J8^DFyJ7y??R%2$iDs1 zFp}6nv!`#$8Z48R#zFZQXcJ?_48zS2S$!2wtriY4_6 z@0|#K3JUEUZj+CbfM9k2vyCn&*O!7o*aY5*-o!3XwLl{(Yfj26s1C*>)vk=<@QPv( zouNzPjdtJ;N2aFAZtSstbjxG?ym!|mc!iS};d-E$7S^|#0S_r(_k?IaU`0_@NqKEU zGKJ!~Xu37oIeY)_z-W?w19VRNP$LAs>_Ch(bs1phoW!Z_F>NcIDoGK&4|sB5>wQ)- zS$$+qs{;_!=|!;j$H0_h>BD-bMHuS(ZuelTQ zhyo5Ap%}^oU-!wS`~;ns*pa2(04=7Vt+zKQz>s?^X~AG|@iKp)ytP;P7}Q7-^I=1? z4K*+bh0V~rw|r1YSF`D!6fn+;1mkTqSd|!<&~HW7OWFFpHz@{&x}gm_MM(u;s`$Uo hJN|#aZxeCK@=)fM9nx{|O)MDlTeolO|8?c3{{RiwEja)H diff --git a/tests/baseline_images/test_display/hierarchy_label.png b/tests/baseline_images/test_display/hierarchy_label.png deleted file mode 100644 index c8eb0502ec6e3b1089c1c1cbb1091c7440b414a1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 12968 zcmeHtby!vV*7ZWg02PB02?I$f5$P}hHv$3@f|OFS>F!Vu7A1{@bfRc#8L|=f-Vit4SEu*e#iQQ|50MJ(kIiX6DO6Sal8MFsA>Ytl z6Y-4ihb150!V!4J*@rJ&ORg2aex3BXBdPeoH{q?Zdgo#kTffc@(oyH6XNVC`nwC~Z z<(H86IJ5~lUhCD4S7-G3E)*Om&qMod}5G&2*US#-(fgJNpucD+;#r{ zf&a~tg!17DrgCy}uFUl2-+A@w)uO#n!7wcAu(nsr&h}=IpI=NUC2!Y9CU-{du1t-( zhK4+v{RpD?sVP#hBS-HJo+~{XdFTH5^DWE^#y5j=g%J>8)V|uB$qfw+wbMnO@WG+l zhgy|`cL@$4(LNUI%WS+P&hQm{UVq+2zC$MvQT7*_FolWt21pc=v>d5eT-T@HsPJH6LxXrOUZKUol7(# zS2x@5-8K9gef_nsuS$|^WFEGI#blsN)n#*4X}HGQrip+2aeXkG-(aa;J^8USS8g?i z^Qzpq@w7=~)0j;)=X2*L>!$&P^W(7UN%icd(iH0EPZDJ#v4Y1DBwqneJay{S^2T%) zD=E?TciVeOvK=o^UJMKgN%El)#CB&tEcy2Q>{W+F-3--Si5kj&=cPu!8PC_Rm(ev> z%h7$+neia$$^Jvw`JoTJ?{y?)gf27kYa#V@bytnMIjd`Gri#bk4YREy$kx-MBEHeF zG3-E@gOseSa+HY6RlPb=m!0+T(SptGIk&8|%o3@jDAy7OnXq^he1YN5`EOL5+U^22 zQ}GMKIHR5%J-?Y!o9V=}rbX8^O04B!@px>e6!T5+%CIr|$3$<~@o6DR(r;sx(v&B? zc?ZxmpgDxNtC>uEjp@xd3K?)-k~v1rCkG35Z*8_eC_MaZPohy*rk@+H*?_u#d%~R% zOPxa`w2GXRf!6kf^G$+tim>K^5fSOX{dT~lJNpS)6_aHNQUQCj(H6B6Pod>3KZ7sa zyNW=}!%xpkCSpCU*ZS?I<9tP0ljY;A=gQY4GS?i-{oLsIbCi4TtE#@GpM>viH_9qc zJVfU#N9X!8W^r*5u1ygX7});#lZAgSAtwjN-7XEgNLt5<^T#MTt8A$eWGbDziSOp2 zN1!EK2$ZEw!sOxCUCxn<3#iVQkVvzZK5XuugNj*gBV(+~2o)w>P|GS7YF$dRdzv=IG< z&=|G{1x(b(S-H9IK~9z@O21W@85DHcu-35aw-FeRunG(dOT`2*h&PekvbOi)sAl%|7L|iB~l-SvT{wasl0-+GJ}rS$8XtyWAz$1qiclOQkIsM zfsv6K51m_L#UZN--yBBdD5|Tgnc3K6#Kpy(+cAOgGl8RXYljVrzWL&KP-$jSQMW>B z?=veTEjov+s-Z!ejGSD+={SMA08>&zf|Mja^ z^3q3>h9z%GOM3~?-c87ADbrtK0AxnS<a5)DyXT_t8)AWai}snq7bY5`}!(?&Zn04v0G*)4uy}-@YxGObl04 zR3t4H;pJ8N^5u);QX{WT%^8E*@g+l?4~>BJ#2eG{73E8ejA^iSVOI_PC48t;08}np z;-PSjj*KJ&cw*a={YoH9D|ff&uF>k3Uq`RYc$tAg!dG;ss546|%VBA3vV3izjc)fR z&Js^bK{5U91zo4s?uLrX`e+EYGyN^C>(7V6yIV_+V{Z-eEJvjXpM4;D5o%>=x%?xy zdFl&eRC|b)YYN`9RHjSQDI=IeOEZAvz2-M+C<*2t8Kgr`tlLd@X4vd*uNnR5e((7G z72}2BZ|3*!3N8Eww?=*@oZl*u;qJTidX7qU|Eorwm&> zAGh3q+P#sryTLKjlPd|y`qH>9`ucvGnO+`C{u75E1L{omI9D+nQsg>GhpcQ)}&Xo^mfKRW(+xq-;8snbJTqQYNiLs5t=ovUd3(UV*6ofIF>bm`J)Xd%hLXFyF9 z&Yghz;;Sr;@D*b>@C+B$9^{)vCO4xHknx}}6-|Zi?3fw8d?|P|w1}bg4GL_=>Ks1` zWZ`6DXyO6i%SD+SBMPdpORad|sc@o-;;5X4m1j6-K0uY|SH?T3?opvYiqnu%8T zmB}_qE$0P~;LED9vXKHkKFV4AmHQ}(JQU^Rm`U>b^!h8}z#H{&U4)a0Y*Mt&@28sf;isIJ*q{;sBjpX}Ffz_E7 zez(I#EH`4_+=!7Iu|(_R&-x6v4JBSgF0K0hM)%1bSqWq*8Yx;O~u`-s(w`rxDlV& zAKEKs_09)lo2FN$mfP;CU@)&{m{xqcrrFk%EQ2)W1Wa;k zCF)uRTrwW2QI8K0;1fI+1^c_dFRerEXTV@vVm(Ui<}v|9yQaFbqAabfR$hp1eXzU% z5@N#KmN5HD<$9x8d&UFa@{Osqf;gxow~i6c8wl9`NCxK6;{!$I1P^Id6jx2{@Iv<= z|Mp}*e5O`tY9a_~Labd5ar=%UH_)CKV7|RRpmO#2 zic%LPys~LXK)+%m*xZRJ*cBZeOTK^qZe7Qml>DI3)NyM8$74BiyZUoBCurm;4n|^a z2yeK36F?^<`)L9@yx{nX_ubcsk-Hru$7IP*c~r_D3Of8|+!q^tM5Vl2&|rY9P-gXz zR88Mtv_h^b^dVWSW#@2+ z8xJ)Z_*1=e(%eF%0h-`=u^)T(*|*8?LTPxsGSz|YE3(ktYT#~ex9QR_;5xGh01k(L z4({&k?ViPOO%SV_Bt0YJgWH4v2HX*s-NSj!g1IChx$t>_wlRm7XipWYXz!+!)m z-)NPC6qpTa()voCy=I*K$2s>D6s~GlA=~TCZk-?$Skx=c21STdU}dn+0|@2(-`RgZ z5)>SqYS8>{11Gu@1LG?4(bBVGd2ZuI)^}Q4(wvlvSZrz=;hURPN<&NQBTMTnUQXqj4&|D} zuVlq~uEXtUYB>2ATbj6X@?xK7I8-lrMx|I~^~hK4#noR?z(0r2-*LmgIM7611?3Im z;g%~XMZSyZRo)7)0%N<3rhoAD$9Q z-M;;Xo`%m{Y;tl^EnEA^qem0f5Li2wC6nracy;yl_cS%%iL8&D)^J{^E*bH^ljpK& zi*MNrh)J@l8t=1dKq>s{fk??IDEPl~-7xG2p-&zF0e{Q4jn!Rre8Had^yxS{x2+ai z(!INKB;@>sRsMN)bDC(F0Oh9=vd%iCpp6#l2e@w>z4d@KNB1^!eW=^+W>_rd#A#>5yvl&nl)$!o>e zL_^Q=eCjOwzFytjUflJAY5s38q^umAlbg%x)f^?NwXm>24prMdm4%(XGyCBu(!&Qv z);3LW@1O3wPZ*Se=uB9h>fMS^hHED+{+l_CZ??s=X|Q*l_}tC(Yvl&EsjD zvpnY=>V?QpMIu>Uwt&>>^NUoq|4U3-BAf?ht6*%Lz84ID*<#-26$qG6Dkx9^70;Zj z*la@es(Z3k2|^3o%Tc+NV_LQ#vAtSY2?cM-SQ2jLr9cw#&>fEQ_~SGJ$xzfvR(p+j z>}J)^=osVB!z^$(g5Sz|CcjI{ABc9c*tzJ@`j7{*fITNPAb7pc@?rtWHu~pW88N`1 z+d5RVGK#In)nuVmozOt5;}wG@YLt<(spY-&I->(AcGYT3`qHIKs?ul$6_(9JXyI_P z$YwgN%d-6F=%~2s&YIh<%9%4~9KU%`7)ye5c}c~nf>2z(9}SchyFAfCro8d|I87?Q z)!1Xu7QwA(tc3)yx?7Dk$nWlKvkEr|T2IKKYC6hor`TgJs>Vl%hD3MQo3+X^0p^@Z zA##$t08$wrU{d5UDKn%>iVBYe;hU5L=c#TJq`wCKC&xruOnvm=B}|)gP#wkid$Kgg zMA`gE2sT#N4YgSoB5ksfkrAdnND9^usYMHs^}cH+IeVu-b(}fe>2Q>>jHd7xU4k5B zU*C51`~pC@Ky>g~U2U!0xfABAz(cE8=GUDU*PY2~&hnb3Ls~t!4G*50nwOUuS>(3J zzs?1rkFdufurdER_4(xzM{u~~3rqrV!zuG^g=Sj6viU-jErP$o%^egKMOUmYCl?Kl z=k(T6b73e1*f~d`8cH3(TP7CsL)Re@RiHw2oHZ+p2@Q?ocysXCb=Xkm6NhW(*1F`R zKmcB_-_|G0j%bOKG=lw5E3;qVs;7fE)etAx5?vc~t3>HZN16&N7gx)#IE_K1_DT;F z&AW4DOFqE6j8K0Ja83*90>ygrqQv9Yc#ObO1Gkc#+&K*PYxENZ1%=B=C>LCh5N$C5 zAVJf0du=caxOHtanTq;DR2qsY&a)4lt||q$FWCvbF;uYgH%0BgPxe=Q2H=m#zA+I| z#>mS0>>&b&THftHBSHbq2Z1Jn^rwH~!2Ac0sdr7Pb85wdE2{g``#9#U{ssq8`)eUg zemaY+J68}5Yi6lvW_8`TfFzg5rQqte7wt~v6I%K4K7q~Ud+}!z_W%qNBr81zQ9V9r zhAW}@n*-khcIMrs@n)4u*KK-Cv(S>)l`B^+Gs3#RWH(SjD%Z!(0?e-b=;6K+92kg# z$$e?`^Da%~Vme;*s1g={os`q!e}F*Tv1WCp!PZOGJpB-T&7@bCiww4CncwNS_M=C! zsGITXG$YHUOX9GdZ9sI>LPJA|(w@Tlth~x)g^~y&b7i*w!Lx(M6d<-blJ3&kOtoY7 z?5SUbjNKioh9h{IHbYeeP(6(rLb-a2EaJd@G}w!WcHkoS{ZZF6t8zz8*lAVq+=;I) zYyFHEsw=uL|0qI%@uJ(4chOH$U40x)E9^M&i9{3#12zHw{Kk2Y8!@$`+MG-nq8Fdw zm&cz1kqzYNY)Yb;-`i99MM{yik)r>AvP4FTC6@~rls}lkXEsxrp9(<4W=n@Xks|<(7Dq>qZLLgC=`riHK-u7X#`Z z;sM?RVyeG~1nqn78iM-^B84_0&;xFdAiSMd0cw)hemy-nm1*ZliI_==#Muu$yN*%# z-K0#0%~IUllP$1hz#07m0^)xB_(3!b27rRIbD77jDvt2Z+&$S+cvtfTlB@?M&O|c1 zNyjT?=gJcumInl2Dk_3R-g2$1*5#v(%pJ(5iIK=?rqUL?5&5Anc~ITS)xtO-=8=iR zGWMTKm(;^Sj>eM7@(o{xald~^ui?nR0l@kps{f0fy>Vh~koe%ewwKr#L9^#J*hy`m zH|+55Vf|tm&=LSgm;0=mC&A%U1b2Vs!y&o})W8N`+U5fxMK>yZ&0B$d{tKs09Q7Ml z{M!i4VfRr?7N^iqG%bTBjX|VMaDpOStYy@i;4%w~KRw#_0qHDWJnbwJ<&DOZ{J1rN z(6ARbaD_Y_-0|aUunY!#8pywMs`6++4h+Qu$_fLXoJf zR+5*ij^)snBR|!1l4S)#m$4+uc!1Z(H)H%|v3%vi)_2`Uvh)AOp#E36lV~t2vn)l`{zJ!6GY02Q=BS?;*<+6(Md zl#Y_$Lq8vtOZb)YA_mOf-~c$e{jveEcB$~VyFp8heA z#4tXKSX5Q%H-=4Smd+|jd*w#LlktHHXTtajExvsH`t?=&dF@LK3{yUWvp~eMX~5h) zmm0zD4qcQD{NPSv<@Su3-+L%XYS6fZibg$mfQ^CSBZ8a~L1zIVSiC^uD3Sd|uqX{X zzfbHf6mHqpU8=BfHWroyE5m!7g7LJ~jm6N5Q7@kA{09-!S$w3Of4Uz#XzS96u&z19 zu6YluP>dh4id!9C7dJWidzlzHVg8StBX){7NO^aE_T7>9?`pT-O}GsI?5JJ`l-jm; zm4Cn)WqC94VXyNssKAhae2obb*UnSlIE{FJExE#v)KOf2Dn>k(ZUAZ;8b_-HIi@k|t zOkxJNnlwv2upQ4%?_7}kf*Bzcheb|{`>*ug9tGel6MLDnwwo9;YFWJY5hd@8u zz)*z|bo5c-iON)OI?;i@-QbSQIq*@xLjNooh67ljbu^6g?Ezpx7j z3(C(y{PEPT->ibyObbWC?9~5eOMJs_%W9=Zn$e_~YJ%x1pt+ycd&bqvT$-u^ zaJPd?g}^`HG3u1oa#;xyu$whz-ct*FmHUbe+ZZ&g8hNm7iQZ6J!60YD!1Tk5U-bRk z7SRwiz&cWuScSKj+fbp6c7W_=)BXKV{ViNTA{jK(n)DUP-xb}?gLwcUOzUeh*2_tI z5dj;@oD>lYw?Dz*w(bdTR6dZAZLlT^e?7-2KnVbVBgA38?TnhuY@cdjzil|J!^p{B zbgFKWObjzMSXo&CcE6=g4dop}c>dVt^?mkNLR+(?4U5zYM;jT?nx^RlyMO9$Rb>AY zclwma9rcT!9nw|=cL#SjC*pMhlW^m;M#c4#TS7IXl3L=bs;YHOO$BJ%D&l~k-K-)s zEKP9Ps6i#FMbKAh7KIM7fK|-I%>3oil~M=^Q1|&(bPgUr&@$ClAz~o~|wvLrvq_C(FY{M8L_MMYKE?AZL?ypNV{&Tg#>U?8^7_kB|L$ z@d^9M)!gBmgyoiwHawsDEtLG3OB*YLp$7K8D)>|zt21due1*P=xpc>M{^7bmUi3cs zC4Zmq<;&j}lHbX1$MU%7u4+=GZ5XhrWX*C%L-3snE{ZFGMn@ZQP{h$LV#$x3`aCes zE%D1>QqcM)^)+|DhaC6uNl#BF@_i5J?Ke#V-9b1f1sdqUX1;!nxtR|$L!PhVhGAqN zEt^h=)))L2#uV3P4-%1|VlaN8EFtk6-qm#5Ua{;tt7F|Pyv78zz?+H(m7vg~PPD{t zZ4T~s&VTzfy0#evVGq_vT5@tS16|D#!Cu$HFFv8}Y>D+`EPOTV#fvvlFv-*?t=smg z8*8~qBCOa(GWr{}kqAhp)Q>vUeCFE4CR3ZvZxkm7iGis~(mK%R$hd@(*c>6urqXnr zywvZyIa9JW;DqY+6lmk1V>%VPJ5KnPSpD_IQMB_Qvh!1iO|$GiwB4aw^!_+>v`kB< zvnn;AujLm7cdaT#Eewrq{bF3vqw3eSHfa3rK$Bo9NJX<{akL)wt28XC$;jIza4;35 zW=eM%zRurzG)P6FASKJ{Th){OQ=J*%P@z&ai&DDA&<>ICSIx&N$%yAY+WNVG&iW(*UQrI$sB_j(V#e~jG(FYX zSy{Oa4GqWqBxzrH!BA2&DD@?{NwApLr_!>#=$9swq9h6&pyBR?8|E5xS<`6r5PjuZ zLC2B{ZT^d2j0Y=(uXF-xaqHB0J;tSR1hE5P{tj*D#@r2$q8j>)f=J^Gy4E=F>VAFwT-M|h310fZPoDmGRfhH?B z$&m{7OT3rYQFO-Q(+iR=7&oKGz$lgyA2mBKFWQ!bsf&EgW^ceupg>k7vnA6|eJ>2N zH+V$hOKj4Jh=}SMp8z&MGO}Jdq@1Ca0R8)IoZFV9ZaDadtT1sB<-CaTHppa#_6)S8 zpf@;LLkAO8Jcg}%Vi1K(@P}&^K2E3>7{3kUF-imjX`<$=Sv!={3th#0bhdK?Fg@XT zCr;8g(>#)RDW+gZrzkBsJ6jHVo#-fA@xoYR_%sA|L13zWP*9K|m;h64NxlI0)Mw~G zeW6GQ&xy{tp=o%{v@aP3t2!R_2BfKEY1TD1Dgie^ds#nz7^9;WYL%|#@NDhT*Pjt_ zn4{BtFuaiPxn&K8^o#~8T&Q@kiniHe2br(fO#WfME(*5L7AxJ%46q<`vs_ z)I#=hF!#{P)1`i~a**y!B?#S)awj|34;b_iX$Ag^a^A_dqzBSYmT0?KMcu*!>R1M7 zp~rQB%zNDy*hrY*6X2(VM@)hSI8hR=h?5R`nrH~~$cwpfLp}=pJ=y85EDpQb@OaDd zbnD4hacJhC9ca{(fzhW<>xSea=-gb`o$bv}b6lB9fQ_1jWK@om@MbBOafvz$#(B%7 z5cOCuFR$8AVyhSyFRs_G_%{ebHU#d1jh$IC#d1dHDvVTGcA>cA=Vmw4s|4Kn>oMNG zXlOHow;AR`MOL}lKL=N7P4Gkq+ThYy7#M^*8RQtqtY|ufffjq_a}v5t;O)U@?H-D> zGf?l#jD?{9ctmA;gK=<%wZ83cOviy{S0yID{1tLE3r0+6ZSVm382@W9A$Yna{WkPj zA&)X)V_0Ec4C22RY;3^GrM_;MSw)92M7O??NcJ~*6raODKKfg?SCYUh*#M{Q#MkpP z{iTHijR6-&%|IW3Ic-tgQCYV<)o}-9sOViYL=*lpxUeIYgdt;b7i3r>o}H826WS zlr7Yp1plAp;s&6|4dA;rg6Av)FeL*P;`+|!oHtjE7mZ+NgyU4o7+AjC zMxE(gK;W;zUqOU`06ckPO42(la4~R^Dw;jEH`l^pMc%^S8H_$m3G0KAsX7MLi3^6xI02z~yVR|`GhQgNN#aSq znRlx@P9`M6lQD2`Wc3Yt$ua4Aag9N?UV%m6bZ~Ic2bMJkNehSgL`QW$jeLILx_Kx1^<#wYtLPBwEl3+Z@q>BHn7x{SO4EH&dxZ$VSP1y3SO z?3_05PmJwKhe|kDujrPw#flSKq5s2fNM`PWLDDG0+pmkxRG)@^IeX=nWk6Jv8JER! z8SIsWe@~HxPVvHq2n9NvEw%fAlHW3(YiwTfLmw#LL>Ll8FXy_wq0}u2$>p#^n|8Ea0?GogSRrG2tvi8CgAsBJlTbVbs GpZ*te#!@f< diff --git a/tests/baseline_images/test_display/hierarchy_nolabel.png b/tests/baseline_images/test_display/hierarchy_nolabel.png deleted file mode 100644 index d80b2241e67dd5479980fee116b7be73e85c59fd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11832 zcmeHNWmuGJw;tGHfMTLZsas5>q=rUB6p#>*E*Yh}q(KxBB@{%E8kLmJQ5r-B5fI4% zhEigrbLcqh1^51H?{m(#zw`5Wjn`bvThIGE_qx};?)CU9DO}w{%|MMpq4vmJlTt>Z zD7{fAioTuO;TOKft^xSkW-lqDx)c7n@4Wp8{@!JCP17EQ+ItK6PmwI1WQsx^N6AQC zQgx0V@Ah(d-g<9ix`FlO?rj$<_8s@%{VCH|jrV4uQcA0a@;#MgH3b4Si^SVBYHCAG zYR^_F%l>!eq^9OFpE?l| z8X6W7@_o*D^~(b|)%&uUx%o{!y^P362I7qa=h@li1qB6V<>m946Ry00oA*6}w-9xk zKPxDxSx`^_M=b4AZ{z7(+uBBE8^j5*qHZmj`r^iAV@nPQX3%eNIJHI-)q4^WrlzL2 ze%9+N^W%eL*L>?1G9RsKh(a;-e|Q{0(=e@*

~Tx=gVRUNRsQOQBQb zl%7#A6zS#c;pti0qXuqM1l1D~ET2-?Mu(_20aW%EuFvKx4m zQMT@?>p^aHUHs-}Sq+hN?JSALfR~}QJsT4i9i4Wk_C(qSQ=HhOu0`I|iNJTw}%Y?xJ@~_bE`` z(w05N1wF@XBe*dSSxrAY*meHgx$NgD=fkm(43H zD?g8kNl!^R4`AvH3RTL_&lhqUi)hQ#&pdcWp`|DzF%hqv^5T923jaEru3X~ytIwZh zjm0mcUeTSOup-$Sdzq1J_uH>7+A^bUB#!7hJUfV`=qkFK36T|trKtP-*>0J~I}L@e zAkEg0X8GtBTuWwLOW^4e2c?3EE;>DD(o<7YL#z95C@Y7AgmliEqGV)bW}HO=C*nGc z?GFHldfWdgd!lLR0WH2q>sOsCPIr&joWZsfyUeQIxbeEUSVVOC<3R=y=P4Ij!hALo z$6CHe^qep87}b5fzgH|dUHjd6PEN&2zrBRlm;EfRnmZzod37^F+^f{8Xq>;>u9kgy znNXIYzp+e^m6K}@IVJb|Syid&>8(htC@Jk@l#`WJzI5pxzeT$Nx8|r65_s%-ckU#N zjEuCvQ(XJI3msLiT=7j%O%HFTvibheAKq_|r*r`QEn#3HG!35KMTCXOt;I}gpk(1T zd=Tp~-{Fc0Tr1H>p$rt&)LsD1kUNaAt&HLhKa}p@ycGSina#VYCMP^5Mt73rL&K59 zOoKwXUokUF*}jJ%p3l7PBmPZea$A~a9K4&>^)?~FvdtOrJIZ5Zvbs3EJS0%|w&MPd z(cX=<-kZ9*>DpEWRZe+o)k~D`4Xdaf$J(=d+M@;Sc_Db^{2{(DR>jlO9_vda*S>o@ zkO{VS?`PC1ush}F9aX+W>0Hl8Fll^qg^a(*!^56HfhrFXK$naRhi=mux&l5xR_`K7Ab3j0AO{cKCAnsZ*ye;&~TeuR@sN>V^^OeGLKqwQpxKPz~fB!;7!>S9M2u_I6-n z%a~AFhcE%O%tD8RAGfM4t$8|*cjhV_9Jms9gE);W$mbf$voa^81C(4`3g#!eQ-M%* z2|D(HiSkj2H{zs+AzkA7f!0~CKORTn7Tv|}7#qula1>8T>Uu5>F1NiaO}JJ0@NIm& zl=xDO;)OG?{MU9HtD_iayC*%>A?>&hCxlyEMG^x&b>F$|yXjw*xGi)B!4FTl4wb9W zV+N+3EvjKCo5yk1y%~BXt{uM}&vYJ5e>ayTz9xgg2V^r-fo!}LElPuCq6NB_baZrF z9?NaWvm2D9J95OlCHV$}xM%UWGu&-gzXgR(fn5SH`}2^HH#8jYC6K|aPFM5Y`mpWP zscbF~u$>%dXbfaz{AFcjH;0hrlfVJjRzoIiSN$UlO7w?x@*&HIvZVhL#GwO=lK>$E zJz+0F)GOb&Z{KD!R|o}j&Qk;;u--n4;ww6VV7p z%P8|q&M#*jL{iee%TnFHWW3P(2lRV_qp=@%ke734paWx-XPHq_iayC{t{=S(WLh7? zU)8M99Nu(vbcLfCMJ+{lt(V-zs3<7{SwywW+tRX`_iWDlns&L{!bB^by3T8b7$MV% zu7bk3COJBPsX08OwaMq_=Q^`7eu%O_?fPaxQ9S!acyQV|&$RipIA{G7^2SQ)3|NK1|6d7zh}Z?`GB^3B5oxEZ(%yvNVN z9qYf;P?`ihV+APJ%D~@|*ifo0{W4ErYCcQS?#=8ns5V6xAES>>$+ZnV20p=IaJ=Zh!O7a{> z)z(%?;)Ifgp46EcJICRsEBCkWX?kOeel(KZ}WO<82FCM z4I+2iOay1yp}oec8XCzU zXtwv)hI6WGX+`=!F@3uYKe{ycR`WP_? zYR0EZUk*#iFhJ`3Tw9CiZ^(dWPo26FCmooBWX9vK0h(f1#g2}S&tJZ@36xInsgLB7 zgZkhr3F~ErWMPR9XkAnpqy*Up(S_*c`SH{*F~USdK|bX=hx?^HZ!0a{2vLP>86C}( zu8}P%BQpoW`>T^+w*v@lhOieeQu6Xt83n9b@(FsW&rV)Nv>`4Ip}qh}*vvu>O7H_F zdJK%rzfPeerR^Clm{mDZ1apUQUC$ye? z_#g>^&PT6b>R!;))O3Lr#yXwy_H835cDmi$2NzZ>tFwJ5>HY9(s9)}RpEowH>CFgZ zmdCEW9S3&T*uRdzwQPY(plQcbNb^s3w00qx5GO{uIO0$+8~>>d6V2%j`3iKN=~O5dYrV-7lJV`d2HHTFpa#b5K-g1LQ#-WYRM)%? z+Ttmi1p_MMO#<3;^*xl8tqF2RN@hN@$;t|?QJVf%@&WgH@g%^gNj}I6k*`Pc%3KX* zX@%miL1P_8uf;~)x8VZhfCv-I&ODjn;o&Wq?xs>|Lb8tN+}F#lYh*%elHxI3U{7RY z8Li$|IPtJxmu}MYJLaHPDSPI9vErKq0}i|Yp&mTz4fF7Cw`}-DW`9IZ;hzdab2G6|#LV3ln4Qr%y^y z)UYQ4M6Zc}0^pVo0TIi<4AE(l8qMzQu{`&TFxm6-A}iM$A)ujA9vYr73x zT?=a_oo2lMPVF&n zuCG$$G;Zu^Ccud^PefvZKpsLLGJdc(_?sDW^UgxGWy&4sk2PA1dGAff9>dw@?*b>n1Y8_k6;&vV2(Acafp1xOnRS8TT+({%MY@?dmxPANfS;+hF8weX50-xiz zHeHebE>V_eMu>R`^{&C$og!OrzsORKe(*=;QsB&~sMEdSOS@mO_lpKm(ue;>;`;e! zVOzO>J-gukX1U@vzDh;meK>I|@7;;92_o#JZo8vL@Q!OTd;9(wB_8Xog3BP&{|t2H!qV%H~mAo zaE`*|kyOiSgli{(Tp$3-7OObthN7&^etdl$5)X(N?Y`UFs-h@*1>slA5U_*@8M=si zS}0v8aqR~Wu5c`;_Cf;kv$ie`r7NoG`>vBE3j+%)Naaf059s+Kj0CRe>Qv=O zXMQTU9%Mq<`pp<2$ELw>{R~0-q4rH-z3j|NBK4)=e!=3{gp zD~L%H>3=07{|O_C&#>6c;pQabPlG`MZuC=+_B&9dWn{F0InH~pB=4B(N*I14tflHw?92?*Dx|lTwa*8=0-}>?}FN>K<*IM;A)H@sAq{;5S{bm zM0LGP^<(Z+mn_DnzEv*Q>aQFAJbUG6c@VR(SyQ}BB#cVv?nGLKZqZ2T%7lt?NNx<+ zs^||LVCz6nKn3t89n6wGH|NmpHdz8#!eJ2lc@BCR3YF!0YtOje)ggiGY)D92&{{w} zb!M7`FB_!jnFetD3qH)8N^dt0ti+BwiZ(hop)4(I>P1f+t++!h8LEc+OwA z05;=|`_qH9;cV>e^^(7$Q2R2WuM)^CtX5&51Xh|86WdvDTa(}1wVYP@SD-9-mr9A$ ze%dZw_JEU;HneyIBm1#TPOZm3bq39GKkxp&*`{tecWI_`HEbH`}e<&7MQ7jEK|VOgdR|VJt@Vyw-i+DNMoFz=lU#` z7=)vsiuB3#<`AVSLfM$la!C(x)jm^ z8W4%|7oo_)1U374zXKD@2Au}J+GZj7)dIL^ocQh?Go0|M1FDr3@@Nv^!yM{hTpZw! zmk|_LdxowsQW$T{TW=(`w5aOZR?#i5&Nh%q;v1Twr{!~)_d0rv`V*jQU&gA#kM z$dorNi%2q*#NE(Z`Z>l~Dz0-j3hQa9UL9)yP5YWRQut7 zq>Y~?C?XVy@x4bgW~0s1PMezPff5&|7qi`Go3swIa zF|2hD&sAq;VW;beLWEiy@qKGT&-?_m!jbZPa{_+d4cY_qJ}?^Ilcc>AkRRXUhv~|l z|FxqfzZ_F~TiDAn{;dlI%4ED9DKM3Fbka7hETBzgEiJ83Pf}L}DKf=W&kY=>-93;& z_IyG10toNjjq<(&9vAMU28nz18}Vm3sf2aYSIWsWrLxifHZ@BJFpl=!A_S<^L6{J2 z(+sG%Xu%^A!)8`ETgF_Kag3Four<=lT+ji!^G0V)4*uV=0CYD-KC@-2CMm?U6xbUF zGVsZQI>c@IRkr{Xt*xz*&^Z)aow}T^nJW!#vE^@jy^>9M&Hh$}5}~&^qN`WzoL%TN zo}wtWB=yS<+PvpTrC=MaPtjFipJY`ye40Gbt?gN+U*bv*VIA*<2KZ)CV&h3=Gxwu$ zq$4*e)@A7E;Ly-{hCs8v@_Q-i>G_*}EfVSa?$f>?=I82o8h!}(E(Ng;gM7)4Mjnw3 zhmYvk+(WWQ64p}-ipkD0uQJd1Tt`~71Q4Q{|LyKf zymQo?@J8>3+eYziz?3nm)}x^I>Z%BS(D+!=R!y5azCMFq5c^pRL5=B7eTQ={j zK}QJ2@-}e>J4nheGtcT+^2Z-NTag{O_exdEOXY!wq8w4~q z#$64Iib{=V6=?7q{F{9>4TfIQ#wD+s$%S0zk4}sKc;$ofzL^h!1B--m2!QvGHp4$+ z-~$X0?6lm0f!l*565FY&@wnlwM`bfB7z#&!bVB;(S!V`ZP^AL~9N zqwCOk#f*><^KkdkH}X;Z9YN2Ps;a7%NnY#52}wz*=7fyG_2tohKOk4&MW6MY@8oc2Pbo5@IP`J#Bw9h+#H2J-fSmU^iLyR zJ+y0eG|D_n5On|+6!_|E2tx0ca7!a;0s1D0e5N~e$b|GIhHN#9c=i4Gz|h`FbpSoD zX%j1S*ZD}zXyYUzEwrl<*Wy&T&Kt0F&Vt|$9l~wcMq08oy6c#@JU8`3v0{U2{@B+)gcSt>d<=6eX zZsYkW85tROi-=Z5@*w%Mv5=#+WTVkX^xbVsNQ;oj@EJv7{ z@{rLgD}tXJ0=3eWXh0rm4T(PnJ07x$d94LX;1)wKnqB98R^uqlLTdFfFP$fc*_-fz zjIH1}>^UZ;QyqLPxZ9}fz3uWWb|bAdRo%SmF`X97>uJT))zy};vnRH6`5X(Ei-Ih_rIqT)cJF;VTy1Fqn_4Ry!n8PZ3nUE)r z${0cKIy)OSRCHdC;;%PM`RF!Pu|r-@PcM$ZENJ)9xkPXF`x8z)9-3}etS`a&;8mAm zg9Ko-JU5Ji7Bn|Lu)rXIJEX3bvy7>TDNtJ8$qDyTz*^5@?9;Q8c+%`kFyMRWF?DCK z?uM){p}ysEih>4Wx3dV#?7OnH^ra8jt4UUVc#F=wnrjzf_sf$&@>oan1&yrZjnk3x z7~(D1STO`Wrxm{iTSqY+n1rFC-ZiU|Y#jeFEg>~jzpA!&DhTT2SFeuqzW{UH$A{9g zJ*&bnX9#M|Q!+1Kz9i$ZV1ht%!xYFbQq&D+W@cvbQ7#$U3}E4?L$k3B45B|#4u*CzS@j%os zjQkc#0Q))6%REj|FbZg5n?g9$#}*W9yMGCy2bx!JdlmudDUhE>b80!7VK;-r_m8{e zb{#Z^&M;D`Aqif1;{9cy#8##E;mH^hFfuWtoprfKFHy-iEI&RH@S zQk6#{JUww)6}kuOYb%I7bmhvGUZzbD`UzS$<|^EqfZNFk#nG z0XUW9^_5BU`LPbzo)i??U4j?XHd)BQ^D5U_U z&0X9F4PjSO_C_gmBB8L_2l!G!x&T04E7FtAZ6rLH=X`bp*C0dU@j4MS(_Hwo=Z5EY z=yZ;V_)Gn|9PYJrLa*M&i97vxxNK6jGVc#oC!|Ciw2VUMDnib{$m?x zIH+Ui&rvd`Odf8&3XI&Eb*pmy+gj&{`_p*n2$%p70EO$8*dlN8;Y1%AfO$?(u%O~) zj2|tt*_79sWh02id}v6Otj!ZgdP-JIt%YISGEGpzMV^eLL#lZ3wPtZsxDD7sYL=Gi zK`>jB{FW|OW5jog)}>2{o-?7?=y@lOIw~1B$6yl`bw>Y1 zna9e!DKHvYZG(Xa9)a{HAc1Or zmjd8}f!E|IY(hE6_(VERE7vISA&j3})9@&U7xsmetj^TBLTkPyRXvlZCkZ-dA23Dt zqzY{6Lab9iDPoAC_?qfwVDYrCpQNaXo8^Kx(z4l|d7$sm0 z_1zeEX7SA9Vc05GFp`pb;f#3MM9G3)uG6?yu5km)ESSwNfgwV}!tOvaydA(jG*7e^ z5Otm7vMwRN=qPr{Ju2jojQk&L@xK8|-gWNBEu;F#o7TnY4ZhGGjD)RjpWNpfuS$o! zgbhtgKjMN7jo*Ql8<(h(q%ahwuE%}MYc;z7d|aeUT?ag1ScZY5F$A>311(J6$+^%W!V%@e@~9qTC} zUeLzXISe;d1soIanc;cTFpR4N)l{$OwXBkRt8)A18-gJg2|1x_Zj{i#BSw&CJire;k@id-MI^-N1TdF`$8|eelx~%(0neVQHr|B^Yo^*B4SF|p4 z95n)u`6$1I#(m2<=(j{fSryhejI}o|u6f$mvcHRfmC$nLm%m#$o&_xHwHP}M>&S=+ zfOPTX=*8QQYCe790-W&ZuXGaXI6bGIxK<_eZKZo|f}UGC(Atw}*OElIcnC+dGSAZZ z+D57jk98e6*9_9i9(Zf}f;HRhYd+YEd|fh}9@O*KH&aI?KxY2$dZhnkZo&2cXjl9j h?biRpTh(ps`0$-mq;c|5AiN1m=8A$;hNOYde*xS?@W}uG diff --git a/tests/baseline_images/test_display/labeled_events.png b/tests/baseline_images/test_display/labeled_events.png deleted file mode 100644 index 1851048c782e8c1b6bb6796352b25b98eecb7fa4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9319 zcmeHNcT|&Umk;BJIu=GzLF%ZZpeTq`X;EpSA|ky;2~wmPkY3_9j!KV?(%TFupp;1O zL?sZBCPIQz5+M{JgdS>0_P+D&IlFt#{=fUjc#c8ftq+u-Gig>dilM{rb&=g7nad zm9=yw{F(NKiM5UmdML?nHk4uaylu=^0CoOsspN`AU&Mu?~pza(_YSRx64!$=tcrI1hXtCyuK(%n90!Hfg?b|j40aS0( z=a*M*L^WWGO(wmYDKC5I*)oeesEb)a20 zH{b6yP#MQ0pmm6hT8yKkRGCXhv_*lXzI@WaOjpA5=XylPhLKOYLLCEv*$sT&#yoei z%Mey94n0t4Q}+F9nX3ejyf?Ta^W(>lLCarVv1?$n1KDZ=ebDhY=z)i9Z2~u#P;V8A;?sa~CBwUEHF-yV}3+)2-nwONPRU6H%Kh4D=q& z={NPYo3xfkyTzTx+f(Md4W;${{W;^aA_opM(?e$^DUyNq3S>xD4F%X`^nI!iUaVdniNU`AeTS*N zeSJ)BLEY=uudCOm3Ya3KRef`F^WJq|41c|k$=jGE_=L0U>h9LouEBM&U*6nO3}0W8 zG?=b@o{-S2y)r2LKyY`zeh-!Y`wn5J_GCqWI%OH|m8|T3W7nax&LYN^noxdmWnDU& zjzPl@#qgEzEv`#@h5RZnvsV0gL4)+>cS9V!4ta`N01pn^d(dYTB@guE8pmud5BP{&wNxvxEQ-c+DfrmDtcZ}u=)MY%zG~QHCRx~I z(D)e@BcBvw=UsEGg&-}>q80tSBFJkKS(|E9 za6QIq%`c3ejl3=>{&;UUCV*;Tg?#U{^u2q2Jk7^IU%z=zpBa>?33~J1Jr?b>ppcY= zLc`Jx+Lg#i1)Y&OAKti+Q)l`)93y9TN8vU(eUzGsOmRGyDeY0QTW4ds z%(~poG>pH&AY^S$yDIvQerO-(k3KVTdV61Bex4#tfl73uzP3=IA3P~1haLNvF#kP= z(6UWB?B^|lyYl_Gs^wj1zScF3$rr0kX}+kC*4+7e*e8q)(u!K}oZStfE23IKciYp{ z6nauTn6STu5-g=uWw;H4P6Fj~Fz`AV78Inro;6}7-> z;>eMz5YCif1Kt0CxSxYhkX9e(5yg*nQq`M-2jHxsH-t`fYOo6J{r&wjh;_BK8i5O~ zG86GS>)#)V1*YAx)di&RbBxF)}5KWRf~`v#@gnj(#`@}+3SsFhAp&)IN+foiUViqHGW;%fv$!_f2GoJ z-~P32ym6E5)8`>wP*~W`ne34X1jJDz=MR|oaD~v^yYfeT@w~+>5C8B32M*M15z;yn z*m-V3c`1LuqK92?SLI!^*r`dR_u<=4SB=`1yFIl7)aZ<_HCFL@0N{hRGBRY)g1Ag98pzI>xTFq*^?)u z|I&Tjl77x_5M>nlo~xKhwc#>i5v!qLXJJ8Bo5CNx(!mt|AxY{Z>3!M|S7fod*W zU9KW!Op3uNQjaAAtd{_$Ezwy*q?OaBZOdip5gTk#jeBklL&y32@2@hF< zE~Zl!*d#s=!ENV8W8|i5qI0RE1DoB;?OABFWwE^tYhw`3(YLU$h*S7io*uj~!X{)9 ziVd|dmuafQTV)ESyebYTeYZ^rcZv~BvBASVv5*RO0CjyA6cnj+FlXB#(Y z>(vEbUS$AR3O$_Z>E*~(r?f)H1`Sp3!7tab)K)mHN)rz#bw!~;xxdm&9Li??qa0C> zETTlTt+-RLWw@-=U0j(QZeh9c8IBH|ia2>>W8fm0KYInk!@7mCKICoWu)lWm{D&Wk zv`pq@HBwg^mn(?+uzqx+N8rNoCqF#fXcJh|TpWLFoYXOOA^ULOlLDsr{(yuG2QwTF z=cwgo7*>*)5$J6(|4Aqd zZp58X)+&dHZ|G3dAMJq^oOa`POi#F(L}_*D^Gk6)6(WII{cS zRC*1#Xup>qey&i#Joe4NWqoO~n}p|o@gO#2JJbaz2F-o5t`EY|U}Yw<@KTHo`bxHO znre4r_(nCr>&|sz<aY-AR6BGzcDy5v0^2PTIEXWZbJ}YCStCI22=VIYUCS7u zpm{jjQK}7ee5|;dJpB3~lt`rRo^!wQSR|ettK<&*{Z55gIb!|U#8Wo=ki~~L(q=uK zDy*;9g4RoS`&xSY-c;|2nZ}J7jfTQhq1Ock%ykx#0I$*lm?R7f)Ee?aL#~_=?B&He zcC+y^O7$#oP0Z7yhAo3Ne#{it%&6nWspLbh=}%WoCVVE}ZIh18HBM`h=FL4$K6C40 ztTb9DVm=1rtZmt_xrpZbaldUtN2d8So;xUSFKU)|Eshw@_OKyXDC8K#MW-maw)o&T z27&;*;NFK6@jPxJ<@GZhv!z*ANP9mY+DU5X!Bdg2rmCGl(8=fgTxdDekHF6(l>^=} z@(#6RAeJfrh}yz9iF0Q&LuYDMXW_Je}3bK6Fqkmi}l`87|*w%(f4@KMvx%%60JC7Veq!dyHu`&g9K;SJPfTYcGuYTq3 z83qQqn@4vmsj~nG0a!VN@FsM!!jU7jfwNM;3Z8?2usH5K#Z~K8M}N7y@XMDkLDfhA zX_?TNy*3EHtbTi-=s(rlvC$2X`g4Y6U}sQi?c_j}&!BJH>AmyKb>TAC(nY(p7O(D- zP(Q-~Dmy%Gu1qWrQ<|=tVRlhJ+Afob=%Y z0&gh~4fUgcA>D#>9I1QIlj>9XIjVRfDAWoaV<~m35?zgvY)YsH;;$x|h z{BL^_WX%y?1FfMLueI=@gW&-Z=K|YsFXqQ@i5mV>`=ujRZyEHQ9|{(H^At4IrF-mt z6@>f0Lw%WBSs4O}gHAZh>i(lQTR*1I(CtiGeAB7&t{p?4bGy)+L$eVg8*scOzYnC} zDTB+M%gZ?zH9J?@W;0KpJb5x-*Xym){kRXWEeb|BeHcQTckPL5TJvP}xwl)CG`}u( zCM&siTJaF|JV8Anf5(Hrxk7+WA}e)iL0fAUxprlC127j=cFAOg?L(O7kr}n)!ZykM z9cgNWV^HKFi*}}fr`|GXTerSq zhr!U^~F95|A!&c`+RqmLVd-Ta`1Wb$O7!ry(J8~8;xVyXC z2M+n;*cE*SfE7!#474TDQ4&O|eYy8g$RaLvS_8@o_in2QIo~uV8#c5X#7Xs7qFuJp zv*TMgr_b&e6ZjIT?Ed*zDd5ZMsn-SBCRsY&b%C?1KwM%Vf5$ z8X5rd+J|$fTQ?8W)fi!Z%3l4-yfF;#7~51TUouI-e+qYqlhjm2UW3^Z7gI$Kky{4ws=o9|m)Roj(;iapjNpoV-E=7;tKc;> zkkzq z=afm^&a1dXJsGjqhG|j2ueEhB#9j51g(8Y|0FqvoInsbGE$vg-g7oH_Tn-Kcgp*RJDCqf93vkH>8P`C3z@DcfiG+x z2g6rELqEj``WTkbyOV+ph$T#=ZeJ>dlZ%H6i4HmZ|I)tD-iT$r>^(d-w*Z}#CG z3QV4~t_RSsS3NBB8Sr;7)M>YW=EcS^R$J7ZpuGngt{yG~;!Tr1lg7Iku`(3M3YSci zA1HRHLzIJ<@9wpQ*b9&MNFK@N$qYI|fjbu62Aa|2yj_7Vb*atF7?OhHfg_0NvtEId zpZ?O#n83hsNZ~3-jfy#mJBMo$yHr<83w?I(#S~&EKfkQdkPgfXt zG)VEq2!}ee<&}NCM$l8tw_Ai_PnhNQvN|L|5zo&x2~mKVg<*;M0Uwffas9$a$YX|#&1S6uMq))3j{qqPgO-j3Rcv5m^L3Ajc%Gu!dX{sL8@U zWb!3)wwTb<{MWCe0m8|>ugtLEh7t4k1u-=}y>^<+SS6?FFC_?}iE0LTrr-N)WNK?$ z+caF=54=AOj#DRZyi-b zbx0caK_0>Gdi0}W_nuq?lUw4{=%UAt9m93&i|AK*4^{S)5Ea(2LXUwXnv1IFVj>Rr zOkId!V=pB*mEZ!wh5>*N{n=Mn?1U|5?Iujr(&vJ_9hzIbcrW@4TkTF1H6T&!{pXNNM|;=&U(A7BVN${NQ5U$9w!8b>WG9N&L-Or7S)?a@%gv zGk;ntfi;8(MA&5da|)~MQ5JVprUpDdl@*|Kr#IgrM3dOTy^?sUl{3pAbuJp}%|J7> zowlp=OzOZS`G0oJju*|h99$Kb_yS5m(di;U$GDt*q?pza+gVx=8-~Sb~1rsXWAq+?g5x54y!E2U$ z%#9oS8bUbHr1b*Q$F@y3J9y!d+%)fK(;S0l2wbG4p-wEbDE9vuI`TF*3;}+W(v{IW z-u^&?2;@71gr{MrpCjsCjkMNLNO8Lk>e>}il`KUuJt#Wt+_!f&gc38cCiV+>oZth( z{#1}o?eR0axKt|VTafT})On{LfBEO*pEdZW z4*qF_f1uzWB>X=>!c-US^M@fG-@CU2Ts*|>oP;bx|K|0MzRh5xVXxelH;+!MxntBYiWwV@;ia8gyZy?sI?OiWGX$EdR}AYjsmw1|d9p*U~uO@96k z(@GlF`66oOcgXpy8^eO1Cng$z%cL=#onTsH3ha(dlRyo)YVW}~ZH%f7$nd`!tQ9p5 z$o$Vqs9W`We5SfC_*kQ5g~tjWL$wu}6fKwkIzwk54+K)3Tk88KF3Ufle!Yrfov%o4dmsz-5a$bKF&U4u-moZ2U_P2vg1daFL_h60i_B6;TVYw5`xSDK`YquE+xiJ!tATv)1h-MENHy;N*^L2`WAj+Oy6j2*}lWNA;e!#uiR@um1(qE3TiwEW)fJ)7P`Ur z6;f{Pw6e$PbJ+}@7vezRQB{#hNh5GZ0?F=>t3$f`2j|k(W~KX|K3G^;Q6RS$M#2b) zxn)?m)fgnBT<5Svvci{bkW37y;{lpK8Z=vX!Qw79$2+z(gf58&qjl?UpetZSXj_Wlw)CWU|g%v^Kt6eg3 z5^(8~+3?xWdGJvb(2m*A5jKRTgNTBLByR6s9LQPxldyK}T5Kk&&?ML#V0|fA`&1WUv^U>+}p_ zJ$*g$eSs~N>pUbh1{*XyBOO@;EwBv-rupyLEHavaIcv~Jl#J82xBr4rZR;%vaun}< z|72Zp#{#|_USFA&MqJMxNeyRh+jp`bVMPlcqfj?ja8l<{_sQG;Uw&|Wsxp(G%xnK4&!!W?hzdp~ey(FHtwch%* z;@1d*Y(4&?kmHu$JBDY^3`aeA&xu*#bp@Hda5D)qNh3+RYiJOE{pHl5v(i<`=M8k# z^ltC2P0Yy1EJ`sv_T$x`$CLDZZ|kOQ7;vc#ZAm+?7yieO8}ttxJ#?b`Q_&Bb+HJnC zU@|f5RfJC)jGvikjx9vK{_Z(eFrDj-!E{oFugDbUA0Wa4lI42bVR1Gm5lQ4*bwmRB zO*S4uUT!*q#9!X<4RYkL8m4!7$cfz%M!d&Oo|3N z9)B$nzTGr*dbg}>9RGE3cYe{Z1hZ?3-DT!@QZoLo^a#InJ-0cyqbNLVK4X3%{1}Wi z9XVKUWz&4?>u0t(rYa?SC61ifIB%Z4T8ZaGvz3rSUlO?xVUc2ne(3We6xnR1Q-LlY0x)9h{IB*h%_q7N&&tvMs=YPKevySptSZwv)@S@gJnjT|Z15ie$vV;2h!5c3B*5X9%k zL8fG`X~?9iSIpXCA{i%Yn2?u3-n6wy0uY^QcSdC1-_yF_*>O*c)1hSOpOBDXs73PW zpMS>0j=<)>uiT0pF1_|_VJ_uXcxRyYhKo|CGQQhGCu69L6|yupA#^5Gd2SuM^xk1( z^&#X(Zb~&jZnntAj76khXutHpapm*#*i!DUXYQ?$v`hoPe1}K28>6XGjP=RWhsbU2 zm@s!5=O#PWiQ6|Dyli-{TQuu+l;3P(@PD~3WtfEpuC5IX3^dHQYqIMG1I>OHY($K+YgUH2vGR7f^&8p2ohG`W z(|t!H$E-0iH);TtzOQc-iv;Z8Jb~-t#Jl}!YOxDh%r?C)a|hkA``)>U>F$x&sN4m> z4RRE>)hKqknMo@}pLM5l8XYAa*3RKq{sT={rQMzO{niGOZGVv})L zTkPuR1|D6>t25hV$u`q8#P`O;+r340d8+%fum1q^Yf}~rb!Y%%o&z=4y6f>w_MgoR#{aL-8{^c)J{7=5U(#YX!0J)GXu3G2`TBeR6FcBUae~6_=ASIKR0)zM=j5) zEDwiyxP}dfR1w(phS{bm7DX^if-_AU^Jh{lDyX8h@u9}x_a8pc%j-uJt?chsQdvYz z&y~4PgdqZh_HuuoGW@l(f8Ucma)(?_=LXNe+;8pX=4Q_C^V&$sLXbN*iBze+!jtI< z3A^>KzEZ{42M*L6oP}w^#~<8bO+3m=xv%FJ5G0U_#AtP>j zX>$q&ht+PLe(Lh%;`hQHk9DUcRf{HJ$>2sz^mYL9BslnLbEbEihu!1s zk*w|3)TT~0KO-CeuE>UTw){M;Lx-x2TYYKAs%V}i3b<72NF_C7>cdKZSFBu)Sy;+D zTeEUjQ+21Qrh7Dx7UQpgJo{8WB6D+bx?eAHdDcxAu(>Y7JR&GNc5Q;d@Aqb;!%buu z_?-EM)YR_j zc&J+0TXyAIbS%JLhZ}_R9bUstAq2XDciTX%AA7z-haAR?sW*QFJJY@P_M!2opLciy z2lC4CF`D~qG97s#kDw@2Om8H>!sri=r+U(p6znftxS;!}qCy)OrVcYGja*+TF+%pA z%GfTR*Abu5{ZQf7Jln^`Tr-JN-mq zBqMZosQ;s&KP1bo@E?&b!w4p2pn$mEmv1lTU(lv&oK=l{$sQJhp6>e z_w}Y!nCs6qgGXN_LStA04~swd4m1XP`}aRB^qgMm^&r2q!?j)_Nh8LuYFt{v2`$i% z1ZWn;Mq*Ku}SrQI|vO7FPL919el${HsuSv4%$CTeP zHXw7?-uZrfc9XE!ZO!=j$1Sh<&o5y@pQ6dBF?h08(_!P8;&)4}Y7!6%Ny1Many?_7Vk?4ij8HNGp0f+WAh0ZAEt-kVRLE9w( zEHp-A?96$Vi?hRKl(~QC=(n>ak(C;sTEXD&vf|}nG4|T43|-x;uQG@zPS6?%Y<^_3 zUF>OfHH}z9rgk%R#vCsAacL>(4kp(4Sh8YmFU{JKqSc~#5myY=!y7Lk&o0Eq&Zp+J z$;8gxis=NeF=Q}F{VMt!(1Ajz9@5NUy-|O!YxPwd*Pxes1-r5VJDS%;JFhz^>ieAD zqNxA)CB;1;AVByWCwz__3xJzq-BSOep2Px$#KF1%vyYV??}ECVk3HBEiihdf}km;R0d;aN9czKm@)n@ zVvY81j4TRG_U5q|7%dYl)g3tR?<|cx?Y51!;-G8Sh{)*QBpw#t=w}s&HJw?LR zFP0?L@1BG)H0*)26C4k>=FsF{)vf72eYLryRUe5jfnV;HMKRf>G3ZK}YvbYE7=FJ` zPl;1BkK|Fte~KU_75&l|ZES2%%N4meE_+LQD={mu>EPKi7uh(_FYUo7o~#UwVdbH} zlW$6Gq;J+~IZfLo1t#vQQheUxr#l$Bl#KcIb-^+jDCZu7y4=d8g;52=;WM?pp1>1? zKfaQ00~+?ArAZ?C6!@!mkL<|XR7@@JY*RsF3*P5cJNk1OoU+z=A#aaI-1-t>qA;C) ze1Fc))ctlTojGQVy9XUg8hnv>dau9K94NKK)rB#JaCO0JcBm1RTE=4Gf>;03D;$%f zceZJXDWD4pqxZe(O86H0WDZPw|7?%@5wB~Y2!T+j(GhbW|LATE9$OiU6OA>jO_qv3 z&j3O>9FjqP_syKqjBnOVJJFqQMe1vb2#)Xvg~nk5hgtwp%J0t&_cXY-5qWbNeegn^ ze7<`)S9*>?(l;9Mzuva$gYCAx;kR1 zG~yHe`Y`gyHb)WAYCW}g8KgBaA1tPQ!7+pG5(L4jWL`lGYs)Xx(?677C~e*Hcg;}$ z(uNjLLjNsbhB9w$8Y@$r&O-cu0a?)o+M+!jLu4B^#{Mhq zBF61Gv;!|;bOcJe6~Bb)49o|+@%P24>nq}EtGxoAH9!Bi^%eCf`{F9Tx?r=BNwxex z^Qu!FzMQdOnuN2jp9Z7U#G*xdeT!5gWr+Vm^LLRO&*>H#{4ywbeC97L4gBDoSXBL*KL&rn@Ur#p~SC1{S>s zRGbCBz)^z+gR198;(0cgip`!FjzE_ z<|wM9bfNX(kQ`9|K@lxgGJxLu?#i%#=^u>sAFR^~Y?>%C=G=A2j}h?got@=T_l4$( zyO!;l5t+sWPY2xHTXmJysC2MU9eeSsy5{I*6D3)D?(#2Zig!X};~ileY@*b&50QC> zHoDPBxYtnVU;uBjw32@9QI6bLPMCX{du!9cNxvI9EptC4shAm#zPi5ytV9BwA}v8y zodue@BqQv#L2p}onq~)ewt=0ZQQknviJpJCiyZu+>>OuGG|LrfixzNVJyr01;h+8? z&uU#BA)VB2d`K(}aII{aKS6T`jRRVdBIM|#kLmXY^SZ4uG;|HHA$)5ulfY2taXZNuAajPxoMBB`4=M7rk>r&Y~uDo&(x^XGlm0?dQMkC}}8}bZLzYk(H&P z;H)YjQCCQ;+e(3tStXftG#Sed9k{mS4z{`}AKIUW+9lN}cCryM;lS0d++oL)WhF|1 z4G&PiPz8B%{(<>yQo;e%sOhSC4mc(`i^#g7Gw zC!I+o?WS?Nk$r;xhVgt7kt*z3N5j*7=$*RXg2Z zUE1(;(uEouV`-@z^4@+ITgH6u2{y>YZ&C@PDJjvzzD zR80Bs5x);*iE!JN;Nvvu&0$aYHaV=_`1vqVnXN;b2Ell=}wrxM{A+~nkByVto+nn;7FcD|-hkoDc`^%fKa(PZKPQ4M^tD`7a< zXv9UdUi>ImF<_u35%m$t0PdDRI}SRv-LA2#91b%tsQn;eviQ+4YV5ivfEl_Z(aS-D zW$1a{LY~62Yo5-WHzB zx_Z+o2vQ&`aAm7qR&;dv5WCYs9r0IVzE$h$H+~NcUVH7`K4}?MWk~4!Bm5zq;Z0X} z(ZNSUw-(v&v~mSq$LsSV(;!ZlX8^Wf@p^G0VM4-qf3-J2LJ3wl_dHxMQ(pkrryD-^ z(Falzn#n3tyv{_Oa5Zi}6;hXc00mwz^hs1sI>%KRAga5-noO)lmBN>h@6Z+%Q++5) zd1!Rat_V1EPgo4+bf9f0jR};;Mw2^GsP!@5&(%Q;@8svU5G&M`NlwG#Of|=bGs&Pks=u8 z%ISG`9=dnv+!^7h5%u708H_t%n;g8l(+0q$@RIR6Tceh>;ax~0tDd?ZpnGHd4Jxyp z+(Xy{RbJNJ01>RBtiY1p_}MW}kTf*E%GR?j`~g3}VYKIXKv!wF9#aCVEgy=91~6!U z$mPlJUl7=fY*B}5-c{_e6yvHm892sKzhsW)6r1V9Knkq!_aep^xbe_o)C;g4N**Tl6`l? zZybjMIEbcoHP_x18Bdi}6HS6g9ixuhsHsh^FOfWi@ZvZhts6AJlebn0akO`Ea6rZX z%E&Usxy*w~Y1)k6;IZ27Oauik2Yq}{G~FdCTWNQuoM~D5aDg*rgdcH5Q&ZZn)~e^w zK)7LrMIxnXI-e!blDW>*H7B54Xafy;mY^|Qv^qAm9?a{hjIBLD6btOXs~?H<>xR2R zWwY_}Q1EhJ!7?+Bib5kA#-XokXuTAEEqX~#;rbi#ctO?Wm%rgwV?M8r<>|m}gWYi6 zqHy>jDu>b6Wvqv(wiOooAbEv4nT<1oVFa{tsWb3vo|; Wc9sdlV)cp***7D%#5&M(n{l_=`fDa#Nl+>p^doD;8L1_y zX`Izs-x|+lY?`n}`DJuoK;@h@m$1C!)AQ}y_%~*L+#zkU0r zv@-0W#;}T_qZ)}+`|IlhjoVs#1{KP8{9jgg6f1hA(l|3dG4V=YU!M=pYKG6Pt!g|x zJkd)&T>(T5L@Qpq7XIc9%g&uUPv-6rSYdB(zk2Q31leFAAD$?+{lqVb2@gNp>-xeTkF%mgc~ZaM;|;cP_Fbapiqd?&yc(YDOV?NA%z{6`As z&BdoXtTUIA+wZPi`{LQNF#B(gQo`RHBHA)dRDR~xrESFx8|vP?bt@n!D89b_n%?c( zzwC?VL+DFty?6^%6Z$r@OB|EV?e!5$rjGo?3vzOEo17g*HU)YRZ#=fk>#x85+Och0 zxY%g^^>Yt5QP#4%+z56bYB}C%T_Zk0qXiTd6^&%~ZqhR_kjb?iu>bM-xgSr*5i28- zCfW3LuI=CN=TT~L)<3J~+PjwYv*f$n=99{NSh*x6B^@0c zj);n?=h+RXRk@EAOgE54C36y)T5IEagMnrbzmecZ_V ztN8B9Abta%w~qac?S1`TZ(Sy&WkV&j)^1{{eXm<$Gc%@JT2^K}`Kzw2$h}B8MLUVO zIjojlWmT}Sd|h2#Tb_N&A^Eq7580e9;qv?<|N3i1=bcLKxskll4D(LW;r1+f_k~_| zB?X0j180lh@ag^)Egzc9TR3wzC4b^ceSQ5Tu0w#@Le|$@v4#}(Kw4ftFeWCZqM|}k zQ}fv2!-u)G^Hu#aoM^^Mii(D^7a()jB6W@%aE*!p~-TaoS{IX58t=b14pK)cj>RxhC$c zj_?;RcC=4#d$^fB`HTNy89)8%H*u>s?lNqMmVNT{X=AWOVchn^a)vc;#V3CM_UCx; z;6W>SD~kMZs7t-FR3MMmqJ3ogN2F*oc;_{_54MJyDU7vG3ns-MH)6 zvxJ0%i@t2!(rsB5p@nl)E|ftlxAoG3W4rw_C)crKjol>=1&!)C7MG@rvN8h$0=|@& zmnU1;*kp807P`)7xG%f4_uUxy*e=3TXcncBW$qWLUEtK(&+PdwENpLIfB$8_6a4&? zEDIftnFQ0WLf5&OUf<#BnSj8+!G+o1oSd9Jy5DH0F-F23jE{}I583BTqo`2w>@B!; z3N`%?$zD5i=5BJ)l2hJkI|Wo$f|d^cfWv5@J}S4{lXwM-n^M?ye@B4_LPxH z%r0?`HW))mP)WcZoS2$QriiXsxssM1x_Cv$eKE7lmt7}jczUo|KGc0Fr}JQYE|roH z_>x;A`g)96qZ-)E%`_r(Csp&Nm*Nu(i`)fjVddUV@cUCbw zLsWRh*RPH5&OMy`<~*vDvp(|Bt#Tiwt5=_m|z8%&{17cFav*$qA85T74P zcN)%YCYn}_$}eO z1u~s-ct3|_G6;25##JZUE}FGv)ZyJJn^;8+XU4jzt?4m-etxEbp${3DqbDbA3YM4V zsa-`y=K13nxmDkvZq~GxP0FHk_M{kRisnA*-!CSn(X8#9G&RzhAmKWvndGCi!Aby}GyR62--*SerxVPNb zWNva`!^VyN!NI}A;{Fa@)^uMS)WIfhjLf`$|NfD~houx06l8OObUdPe)CQj}T$(fK zv}uqfwl*4NW4eKSKtRC2_Ye0j%gQQY6x+@ZX9=r`S65ft)tm>`G4y6}T4Oo!)5A)`d|0p)mzs3_6*13WKi(euU{{A*^@{l zDJxadDaVP(Z|=*kvae4>Rq#LH$(>pCJ3Cv1f$_vAb8Q8M=cuksV+yBP%lk9ILT2@` z3K1D@i%v;8McOGvOS#&PeLIu1@|wt3FoEB?FF6ujaf&q2@l=F2M>U(^z`&vGwlcJFX-y(s&ksAX&fhWFx)TOKK>e4BiqoiawjNil{ z6`Gq$OG~qv_#utqM$B0>=nOG20tfHfRIqUddsy zepH)*Ra7xuzv|lPw}PFltVtKVb`6Y|vuCZ&0PbQHcbbeRQwm+3qoSgWs)7Wm-wN`u z=O%wjF6ZpCTZ!&8*b_my!IQsXWE-wT@%HUV?9Ryd@85Gx{Zz6f)t;ctaAxN&%y({~ z&MwyAw~QKxH9sYMH|`(H^9c4rr!+!Yqq!B3)Fa>;M z1rTX?czA`izW1R*KK-POMXsqVbz&SF*R)`RAKAC>3`X=^_e1s{qoXNx5mF}IA0Nz> z^18h{Am#I`PKuop;VZ!@&B@EFqOKmK|Mk^7T(7j>wr$&pc1z$3r;(hWHq^$Wi&I%e zN_Xz0p*^QyAjhI6F_Rmv%Zux^PyMxe;|n|h>?n0NH#er^H}7k^j9&osF~)0hv7fMS zXR?Uym~K#`j5=OILpGwsD~Ve7KgA8ac=bxPZ}k>i0-Iou;YQoVSMOGu{VKF?#rI`R z7mh2k+qQ2Hc=qgS)FQxtU|ybcRMY1+3zx;Yw&~R*O~Ku}cduT(IxZ|MtRkYggJ+g~ zG1_v;{qT0lQIq)d=6z5hs|k%XOuf$#*BL89{6~G z(*N~!1ZrdO^Ya4);G}-4I{sPOZ4yGzjp#c zYgiOeGP?Gi=1;XP6mTdDn5TPY(>S${4@je0UjieoSg~RU8=JXCbybxtV1-0E za67O|#?8_VKuku(9(0#wWEui>+~OFRg+71zBHgUxUa*T>JyP6x>eud~nFkz3cpPL~ z%&hA_YeJiUeDTk;8Mr2W?cpZYA3uKF&KgFGG6o8+sIFGV^&XTiaGq}JuL{nz8&)qE z`?%4TDOiSRv_LbID&a28VS7@Q4@q_HF0ZJz%97;dWF@w<3<5_Z&i3qCWt^0suj#2v zT!x%nKw;YT0R|=Ss@VKRE}AfX>8DRe4jwE`?yRYiN1s#AwtV;TO>C!PH%rW%W_O!;M?m;MZ71nqVT}z#zkO_dPx1aSP1C=89EzV`p(bJ z*0}TN=;&~3>C~Bl_nb5i;$3{WUah-*ZC<1WqEv{p!<>w zF~FZZc~V~yz!mVyaQXD`nR{ynKCESLESUb5Fg4t6GSZRb7undTDEo@X{ltm zPB}s8!i5VJRaF5gDHZ|t^t~&9af@GoUvqrfwW2O}W^S%+>8Zj_s8KgAd+Ht$5mCj= zx@u$;7g2d;C3bz^*RGv^1dRTnp)0B$1D5!(|4G-<4##{@&v!_TsKjVCRIJ)hDl0Ah z!uOO@Qpc61xy^G6QOCo(=SJguw#-Vm> z7ut5~qT;k>bV}hIF3%9NGIt<4n5ZU(J$$D@1e?%KDWKmNC)DPmrA)cEr;Ierfe4r| zdKVTK`+9pLK=1VQ_1ltyWr`uw(Fe(r^55uHzWh=eMSWb+nr>i%7KAn>4JRc%qsL!H$g6Tq(?7#NtC zoXn8zvTHX}PuKT$adq9v$fy>z%)UG$)#)^tbo@5)Q>S+9 z+NC_WgdR!mVkn6vCxwT96k3GNp)9#H69H6A$+0%T^QVn==>WemToosgPFL2MuDEe) z9%Pk9O-P}V+s*ws2;N4F>3azT(08g@I`k~lq#0v?8sEH~jV&20MHX|cE%nyJLswqy z6&B7~6AP_0OwVYFZtUx8essuODlIZPn)>PSc4|Y6Z%9X zt%@l zAJ3X~<|<(vQR^Ze67ULDx$3kugtN<6t~6qtCNC|{>yU|I{>016tYLO`)-MtO1L#S} zDWq6WEri483@LV}M0(CD&BNp4z6R>D>^$n>2}hQJaTcf_1RXvS6tV>!tkMl z(9y47ub{2{jlz0b+J31~Yg^;^tQ@1K;|cl8w9bp? zp5BseFRY*A<wD+$+vA$I|D z?s6Umap$?%!LPB|cS74@lRZ7v?OnAJ?=!+ zHq$X$+#Q%C%UU%qV!DpO_E^^Bcut%9Ju1Hsx9iuSv)tvhc!_k4$NI~A_Ml<}j{S?) z`F};pE>ooA1L1R@t<+)Bg5K(JPSA_wurOmKzA#oot&j2o)F3pwJ3lU&b!4ZW$wC>h zEp~5qH>|Zq9cBpd`0(rnMu~@8N$~9ztPTfpZr0j|gJafHl-UW#rF*3TPZCEhFQfl9GbqwsY&&eNb?Tjfn25 zo_h1a*>mS8$y#DiTV$aCllwSzRtV9b!=P&br-VA2JT_*Tq>+_eH_eC^^qhr>Bp)Ig z1z==Y@#5&j*jVg|u0Qe^AmqMto-I{q@z;Q-m6O!}ya+&zBc#d0?&8AOf@`sCcYaUOMvc!weQY}T7LIHClj>%epddnGHwY`S^6O1*ZQjQ5mMs^w*5(t~}xysm3%zR}WfamnxL?KRQl5ff`SpX-r5 z64;Z*k7hH>;qqQ@v{YMKUD*VF{sbicljf~4uBX5PKyU@Iulm^(Ol zQ3CVn#8R_8GsAHoL9_YGgGJeiC;a@})=vA1%fd}}C6 z%!;8_rUSQ{b{L;bR-wJ_yu7E0ntU%@{eRi5pcJV&Iy;NyvOL^NYC2>(I?}5 z9umS`!uR6!>w#Y|9Z5{G>FYeoec78!HcKY*XytysnRcXn!F z#6x&{uAHb!@C|HrIdp;Rj*dBIIWSw!LDYwla1B}~E;I(Puib@*4tt8<(vWn@mihv! zL>L;~=aQ0$3Ku|M>1(J|PkP3i!YQ$_m%umFGmQg?7olevHO4Ba7r7M>{RJ!y62ioj zrerM>jp)e8^AJk&u`y!c5vjDcwxWhjzG@MZ*^b3&{mg&sE#T<7&C2TP@lfh*huf6= z1khLtmgWZks0;%GaU-31T!#)_WOto>K2k8PEnSbfZ&pd@O}HdN^W3mu!;gW1)?#M> z!P-+N1xPSkrTujG2u{gdxsq7lKQdyz(cN#yo;|U6DtUW%@7ZH$|5*6Ifpe&LLa~<% zl`z{FfSs#^CY54LfFAV?54Bfp+PpbXcMqDv>5xEMTieWFA^M0mve&*8sQ#b{{#jXB z%E_8>1UU_{|MuLbyPgj#rzRpmnwy<9f^DOocKb1u(uA*)*8NxRnQ?J($sCq{3t!I< zL@4p|=Sz9%_wZr*-B5OPpm0OZHvtKrga4BZSzrJj4q;>9%Ry-ZWP-vZ8;gM(DnokA zFs$3}x=3qQ&oU2z(m3ncg4X7-tF)VLtj#2a5)Z5z3s+qs zCElY}lRWP$ItXqik^asC+&_Zp;>ZH*rDDc>Dq>&|H_uoJDd+TQkGPe413dB0b zJGM|LnHA?^SwF_cwl%R(SKw7fyFyzxmC5eKD*pRi{4!lSK0p_6F@FAZB{TG#upE;N z(w^PB50~(nNZE?UdnZ9P4s z+ORWstgWvrD!u?`(d-rN?dh@k8Dei-{pREbdiocDLvNi&P7?b6;`+e@*xt;9n7Fo#zWuIG`AI$_otAY^&=+}h%YXtfNJCuP^ z62=DK%M~VsVAJGegS-g@wz(hv@;`cd!XIxxEc8nPT~a{Gn~BuW(9kz9a1{DGAy~nY zAUxtXZ`|638sn}DI!zkd4mQbRk4=n^*G=?Qn*E018}H#k%A*l20b1#kM~`aJzW@i{ zg@^AKr!`*Ruz7PcutsCFEHmI#QrVogL-%UJXeIP!!nQT*D%677Wzlj5kHAo$i6&zO z;s3I<^mC37QQOPBMGI->dBZ}(-wN8B489lNmE76e6aMR$Ar$iBCNg{u&cR`D8L)k` z2B=PU!VV&0VwKR|;XeyDHA4hxAN}y*gI^>jf-Qy<`Hx4@BVbIL92>i;s`|=|5YYpD zeSKf@XvM>XT28BE^i=3|($dmOm`q6q6AW(}%*n3e;wAuEz2JZ~iJ|0~zrR0-wt-1| zjx{{Lw6qiO^pBi4k#@8ac$+T(Ri}!54rNBh4-Gg8Few4T4OTCMriQ-xv805Oq#;ar zS%mwL$JiTKv=DXCn<*;pdW?{M9G*pVT`Zj?8Zmja39E7R#EEMrO^J2S_ML+)!!3RJ z@?|s>!Z!u_i^jUNc1~Ks_6Ft?)PGI9{ibS?azcccVH~XA|73>kb$yrE4=S5^y{1gs z;(neA=G9oNwQR{l(tz}RPd;{5kkj_NOldC|OTW&e3FbfK#hfF|H$IEJZ*)|C0y4H7Yu1c|qf+&dlg{ zM(^}*RDnsd`V*P*A9rqDZOGO#YVFq_T!f$E)~T1rkip0F+0)m& zKrjV0cinyMGk%YM1zcMMm&#|l?qxDNxjbcJYcXq*^!l&Aw=CAMt@rYrk#mv}%V50F zyf$-^t&4GjM6co0ag)<7`qqD>oPmE^L<#=f*FVt=c(^u->+SwVK!d+dU^$tzB32i} zsbloiogbf`RxveAoEq)wQVq`Moc`vVQn)Z7BNd1xto_AAqrZWl-nny!K=9to!WE^Z z7YcO*2tp&1h-Da| ze1d?s^*{9V*EuC)x&auSAy320=Hq=|xuSsj=02fKJQW20mOjX(Ug+Pkm|50mKg#Q>Jj$SN8du4`*+6EFiY5$?k= zp+aJ|R7Gz#adpCx1@VE6oZ&Kir+ooZC?|lvc>zotLK26tNrYm6PGbQe<;7_)u&V`4 znvNllOqO$TK3yV?DRs{k%MJm*goDkE2w2aw8@WPk8I>``(AOUBT_0gBa zS`{WFm;@|QGC?&j`PmJ%yc_Q=H-R00Szf;RTY)nTvm+rgGLp@iV3}$En?`shE6;V_ zj*z&W<_2O2{}GHIIjqPtSwxv$6QCaa4j9BL>89ytGdF2O#AQ$sOn6074fIA>Daq!Y zxkgwtQP;T)!v76kuGCq&7ID$L9uI3!OA0*MhfN%L1?4#RJ3zBm<&Il3M!3bDs36G;~m+fjUi({K*DkvF(IiStMLk(QH+e%7?SM zlBL*P6(Qg%g-W<);9kUkf8|%nl*7Y(8DCdTijImxfeoh-lUNUgUm8h^0@I?Qp&`z} zBvVim6-{%@n;M5(Y_FCed?){qZKF#W~s^IZg%-#?Z!l zjtZhTc?^Fi8USm=qALmfa;QVx7J#EZ>(`oVX*HwSuWc0r+6k4|9 zKjCCU9ie52XoB*#XITUyiF5Jkp1LE0n;}>ta)!3Yq_>Dya7*JAP~79$+-7vmzU8ZR zxlU`{OX3aikMOAELPU=W3R1{d`ecehf57kBvaKSJ1|Uox!eT+3#5*^IXkK?q zR(=P?CO{VA6^mWVi=ohR^=~Nhtb+(TqMB zgLe7HuvxkJR2aGuxy(#*ak3#u*y63k?2qj^+1Z9YpB{UsdJ&)X1fK<@hhm#{_ls<4 zQ6-2G%rC?qYYuR|}#Aw+$Au#vbNph?dC`_BS>J5l3eh#jn!t{(?JjTokece@+> zIGalDtXRY>oF={;&cvpIUs>2~zm4D|R=)y!2s#Fvcn}sr`GcF9nnd`6N>S0F0QYc6 z!}m%|)BG=7GAp^2d z0)-PeABhM=)neNEG_|ybqyM4%hnEfO@(1`eDwbz<~aT?FmR;ZQ?_T8PFC4TI(CK8%OK+p4^-{Trwpl9DpX0qy?Pb< zr$G<_!-+jiq`)-yS9zOQbLVJ{&a9-b$!JD(2Sr%A=>EYw?;ly@z1~;Xv_${?u;jd1 zQuBQ6v-fKf&{p^_w@wCx7daOwyKb!6Y4YiS*X*Xf#^nU_jBD<3>Hc}G+N`a~sQ|XH ziUt$lYkxQC-GP@L(rY9Af*f|zy8lmz z2Y3a`ire!)Zbo_$sSt7T5A_sxCP(}k2`8dAdH7BCc<~ySKG`{kuPjRwN5vs71C4nFPv%eCI+c~F`;;Nn+i}~0xjy6RWmN%B#@jtB#P-I*AxHs{gqA)qo|8K&8&NUS; zQKWIAn$&c+^yc%@n{o9cR4S*6O0!c+b48q?W>iIaD8(|FoOGuvsO@VkxmPuC++3FG zd_~JtxMYu(v~B2476XO|p3oAN->*B>|5wiM|Gg@`60xQjU4OBU*FEm{9rwA0(8-6A z3-6R&AJQ|1BV!d=-JXqv;9F6fUqrW`if_Uy*KQMmnF*oF5Md4snp^jhT~HdCo9^r; z02weO0ooG2GM~78seno(PZ9q@0)r40h*a61p!eo=>uQPEf%DXmQxZGY@SxS)PC8*O&1WHaW#lzPfz%+vI3?1SbBA)Ju>QfJtP~4YHB|CkL=2vt;c@I@Ac^ zm&lk9*)@=nCcy zU@FAzcW~D)uup1p3L#i0buIJ@6H$CqSJ;|F3=J&HkO;ufbU)0*w|T?SF)@|kZfLmb z?{D0dnEAeP^{Q1Zw1Jk?2p^V{*SvIzshktf;DPiJ5pf=B1Hz@?Kt{U_m@Di_C|yR) zFN1?Mde=D9sCx6AB_-#;1j%~1#lzAN`O;}$s;is8H&dY96A}z=dAiHJ2ni!IS0afF zE*{JVrixtTZTpj=xaZfeU$+&y<{_(KTO&TL7>6SbWZ2>Hscr;osE#7DLRI=tq)`oB z>1!$f9Y{dRF!K*e!=Jq#U`x*`nnBiUclak;`;(Xhn(C_zga)KWmH=E#GlNYVmYF8D z(gDt&6lI?@VKSmk(n@#hD2lm*fKbRPg^qeaK@i~pZhz}#!HswaDv?Jf;(SzK-p(f& ztA@I7h-fDW=aG{q)$rv72Lhoo+YenaoV)qvjeyOstAutZSBbFp6@LA%wIA-Sr8Q|- z7$JcgB;lIN5r<_Z_^;haM;s)sBQPJZYa6j=WAa7|+sQ!E`-w9zc#m1FD|(}$xhDF2 zBlLMK_eI(tu@CC(FF>J9M}I-fY=(8@ToodwA}5#WtndY0#%84BBvFw_ZxJ6kOxH-{ zxGE~@aTEtO-C=ikcOsjCf+b>krh^W|DHQm+3~Z;e6{h}(#+Gp|jEcEq9Yr#jT?kE7 zCv!_kXu~BDi)8DpAVKmr2bI5K059Q8XDJ{YT|j1GW7raDU&<) z=nXU-CyTE4FLiRZU%iKUgQ59Ns>hVt2tD*v*A~Oe$|XzBrI3_?Cq1HZJ(!VV$1-^0TavV^Z=Bb# z{+)Jy1WiEtps!qmE+7Dzi`W&&+`XrL zK{tC4rhsY9C2g;R=+U@ zu_(g%E4cvT3|*3hNdIVX5DbX(Q8>dzoFzJWaxtS4ngj(~g%FUDpZ{E5E`^kW{S68p zwRZ|-P4VCHE>3>_>zLE^#0+gW&y$t%VM(PiqFoK!? zU;8A|6odyxWCPLfmh}Sp=_8I-;7%-GDLb!uf1-pBanwjeKw8_dQ3giFI4Ej{pP%hR==CKBqF1mVq_LPV9vkpXc5K=5y*M6A z!=#fy?}5`M@SXF92pSAJEM1Su6Z{9A_rFtGQyi`QPlmLeiz`!d2_kyDSwcrvv=D#% zut<|$zJ~uC+x6Y;ReTp!t~t_k`0z{J=)g{bQ~bxWn?yG5pfutCB5xB>v0`%Id4pZU zVWG@~|8WoZ9e*ZL&C=atFk6~%SO2zZRsOW?-@41HKO!tsF?tN4no0$BZ&_6=8tbx# zh=Jwy2btJR{!+mB7I~l6zYiR6-@bjKIK26X1NxjhzH4faZz$h?{Fll4RHAZ%+rqdM zXldODgRcI^gWpBps~)3hQ)%Pl-MoVb@kU19W!+;#$Jr$R`Cq{%hgXZAyieZZMfpD_ z&M5-op9HG;R(B!eHd-vxiQA7fHM4_wjZS0#ghwc6^{J=622ifLd{*P)Zu^_ zNy3l;;HTLVbPPcpA(@?@Cni6^t`S-NUi^ev#n8|&V4RQs9MT5Q8*_cVxHZTGvL)I$ zk~Ek>Cc<4f8Vh8pWNG;xYD5B#NKuiAv2z8Yj6$7Owzq2(78VkxnXt8k?g%qok?mE0 zZ~vO#ARbZ}B1r^#6GUdl6^HTc24hDYP(H>~tsG+&Q6U1s4H-ao*J4j+VfKq(6#m3V zUocrOLzYPDYq1nqaYH;2;w&*dan_GE5EB}2Kl+V0Kmwq_s-+&$p6{3*edTpH>~HG5 z%@Rc5)VzSE4BXU4n_nDtPl18iUW5EBtD;pep%!+!nYWXxghq)|36jgR@`SJnVWtT? zHYIN)xAHDUC>e~W#k&-TMfXbQ&X44oxvCFZDX6N(Ld^X;v=a((5|N+)KfiFOLSKM@ zU^q44Ih(oSG+PRBB#hl<>@E^sguD1BP!_Lccva9FN1&b)VIe{jor5k+!x_SJ#DNv) zrrJ@XfTjla`KJ(0lT~oLms_8@kLap)^RqwztyRL6sr$Z|v%sO4L~gQzj~;4!Te6Oc zh!9Kl6H#VJ)3zWF}?6PzoRYAYe_iO{wr-6}<`} z(5f^Js`;{SQL5jAw#`C%stDO*h*0f_xXJ@wWuzE-%LjGDBCZ~j0 z?!vkvPC!3>N)Nu!*wl}Pijp+O?QTl^dc69rIF~ehyL1yBkRIY-!Oopggq9W?`*~}h zU@md00?ILw{3F&0TLDKXPn=LxS2wnQbN=GRcr@o|5jhm-WLb6n)n_EqT6yB{1z@%y zw7nDbB~9ipdwZ@Cr>lL~CC%-R&44C&GMm3=-AJ4Z1X&#l+kg z+aN%9501HIn#fHb#Vb<;Ertg>M}ADDZ24ZtQ*Y@RMr;4jtL_ z{BxV?<@ft8@t@CxR@rTf7pPw?^WHP6!U^z{6J2EnA9H#20HmywzI1#^dk zWm6>*NVf-@l1_+-9C?1}yp5>VN8SwRZ;#LKXHv?yc`M<{S!bw-BQY$@*Q0osOqYHZ z)SkTMrp>YKIl4vmEKDqa#LhED13^Tr@;ZC&Q5W9G#o)wJS()O1SJ6g~+qXH2*87hv;zJKW%H2t-@91C= z9tD+5#T6UZ4cOf2`LsUA@ptqu+FTTmpdLhk-NDK6@qYLzN_b0X&0!>t+;g;Zr#&Bi z#wnH`Fu43eLRf{9FIz-1S@?rK_4(<&?{`EQmEGX`3}yS^mUWNn3}1|P70C!Nd23vJAi>DSw#&Et0K!kQxo^Qfa&55V=f zrmDI}j7M>ZVe9iiUY&~C+FJRV1N*6yWjAk;7A}Z}$e9WS+t}EAD(mm>kBo>o$Fk|6 zX8<Nj#Z=N|B z^7Ua6{vLrx<-N}#VJzCP7IpCgm95V@*H8rvcH7kSn4M;96Z3C~d!O{vzaS;&o}0*7 zkorr&g|5$i5J!7z2W$*YOiI8Dt&lC=L%-(y{;f$eGC2U8Kp^ZAp_{ucMW$ZAd9#YN zY6YEUksE74LBYBQXALbZq#PYj8k?91CcBJlWn0qWrSGn;OVKGR!+AZVEFRU>DUga0 zT+tN@?xs1_wYP74SVH%(7%|irDx6g{sH@ zmwz)ZupQh*zh-4yd;4)dJ`Xf<{7n}(jg2{{Sg|8oGK{u6IXR6{DO~V8Kla@*%9LKd zoVIwJpZ^}_X?S8Hy<3sw$f=3pc45+K6_wrQU4;xqUA{N?9${pK#m259CYM8{n^u}D zvL|=FcJqekVmbk^~bq^MrL)Cz(d%4 zE7M!)L*yRfZx`srzv_V*9OrL;kT)^mBHg%sdsSv;=8+>uJhW5Fg9J@12kH+#_4PH; zaM&OqAW(e!#F>}|UBs~gVovYf@~A2_DCi?*TPlu~f=b*3URw9Cgh~00Q%$Jk!$&s^ z4Ud?g>YNWxPq+Ti+XDb9x6mX`k>3QoV>GY z`}XZjrd36OrtbI8zAHNAYh!1(f>57sWVJsJ4fTbkoTyX8CT!mEXyb+D-Cw3kfBfJe zTXYfB9p(AqX6ko<>Giv}6VCw9<$|2trtmt$zn{T81L>O696i_jh2ntG05+vs+nKpomfP#n;1XOYop~y*T zKtOVmj3Su=kt870eU{z(8+(s)_PyWtYu_`*adc}7RMlE@&G&tuH2eM)`SY7LvTY=h zNSnwP&M1;dtGq}gy1w;m@fFdAuCMrGwcTm*we|Srw%*_#{=UKbf|?zPv{{$e|il@osn39S$qBsW$8WQxj{pTbEtfeMO>u6m*rWe*Ke5QY2+qLXWLE-Oz&r~kp0NlxKZZJzLWl4;k{g`2R8q{ zYSp@DL;ZO zwf!=r)g+P->#E;LB(KUXBsvo5k~e8RiFEzQ>XZ0>(C+{HlK=LKS=I_fi6$i{H_!Ka z3UYCAv3h?Vx5c`1d@>hX{?Q*W+LfE3srk{F{~_rnYY}b5_3quf;`xjuQa$N>0H;!j zfL6Gc_th;V!P@}=0o)@g&Yq7{mK>R;=j?OyYsa2{_q}&61rIVqZ81_%w6AkUVwPXZ zPC0vd#A z`E>bFJns8j|M_vdt=&Spt}YfPH!v`e7$0x#$dhx4s&$CFn#MUAe~dR+BSPZ}!+GqM zXN~O{M!{iWNr8dC8HL)>FlfL7kH|M7|Xrpwa&n>PnYr1Y2kL4Q$RzBSCMC}Au$ zU!Ft4KHjO@xSOb>%p+4%Qx`8^R#8)XJ~z>w)Vn<4bc9i+PI6u|;;FRTk2l;Str}K^ z$#Id@RaIXnCVKh?KA-4XwU!hWoUY2Zal-}$Rn=#f`Bbq@yTmR{7A>W^uQa?m?=TeM zRMp$7^T!{5n+Zd5H5?t`JE;r$|%MW#*BOkI&0juV&vf2!0wG z`1SzZs#jZAS2tA5BJ_c;FSXj+$H%NT zTs-uc%}X~ow`Pi&$}-i;)9zE8Rq?X2eucl%$&+3M1qIu8?Ml+hcWfUjUY_e5?JX|G z`!=U&9Nl-|KoY(^wYKF~JIU0eEl+w<^edPsW^M9~xkx#8pM2=hp+>)>J@Hsv`LVXNwj%eXnK zp9<<&7UzY$cp>2F>1okZpmwy|KIrH2@;>tJ-Mi-;YF|id;L_Tt*}Vefyzf=KRNXPB z;-!efUi!@^u21eI&v)4clm~F7t>e6TGH^R9YeIALweFk!eivnBGdu-s{=By}^n~+U zZWAnvdX9Cpm{m{m@UU^=T!)#2$IqO(@%C3