From 73c6fc701d0545b48e67172dab2d524d680cbff5 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 24 Sep 2024 18:07:45 +0200 Subject: [PATCH 01/73] Modernize the build system using `pyproject.toml` Also, remove vendored `versioneer.py` --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 5476c70b..e045deb7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,6 @@ include CHANGELOG.md include README.md +exclude README_pkg.md include LICENSE.txt include CONTRIBUTORS.txt # include only files in docs, not subdirs like _build From 9a18e8922c37fa0f3877edbf08359741cea14b6f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 2 Jul 2024 17:39:17 +0200 Subject: [PATCH 02/73] Temporarily rename project to sourcespec2 Also add "2" suffix to all console scripts. --- .gitignore | 2 +- pyproject.toml | 26 +++++++++---------- {sourcespec => sourcespec2}/__init__.py | 0 {sourcespec => sourcespec2}/_version.py | 0 .../adjustText/LICENSE | 0 .../adjustText/__init__.py | 0 {sourcespec => sourcespec2}/cached_tiler.py | 0 .../clipping_detection.py | 0 {sourcespec => sourcespec2}/config.py | 0 .../config_files/configspec.conf | 0 .../config_files/ssp_event.yaml | 0 {sourcespec => sourcespec2}/configobj/LICENSE | 0 .../configobj/__init__.py | 0 .../configobj/_version.py | 0 .../configobj/validate.py | 0 .../html_report_template/index.html | 0 .../html_report_template/misfit.html | 0 .../misfit_table_column.html | 0 .../quakeml_file_link.html | 0 .../html_report_template/spectra_plot.html | 0 .../station_table_row.html | 0 .../html_report_template/style.css | 0 .../supplementary_file_links.html | 0 .../html_report_template/traces_plot.html | 0 {sourcespec => sourcespec2}/kdtree.py | 0 {sourcespec => sourcespec2}/map_tiles.py | 0 .../plot_sourcepars.py | 0 {sourcespec => sourcespec2}/savefig.py | 0 {sourcespec => sourcespec2}/source_model.py | 0 .../source_residuals.py | 0 {sourcespec => sourcespec2}/source_spec.py | 0 {sourcespec => sourcespec2}/spectrum.py | 0 .../ssp_build_spectra.py | 0 {sourcespec => sourcespec2}/ssp_correction.py | 0 {sourcespec => sourcespec2}/ssp_data_types.py | 0 .../ssp_db_definitions.py | 0 {sourcespec => sourcespec2}/ssp_event.py | 0 .../ssp_geom_spreading.py | 2 +- .../ssp_grid_sampling.py | 0 .../ssp_html_report.py | 0 {sourcespec => sourcespec2}/ssp_inversion.py | 0 .../ssp_local_magnitude.py | 0 {sourcespec => sourcespec2}/ssp_output.py | 0 .../ssp_parse_arguments.py | 0 {sourcespec => sourcespec2}/ssp_pick.py | 0 .../ssp_plot_params_stats.py | 0 .../ssp_plot_spectra.py | 0 .../ssp_plot_stacked_spectra.py | 0 .../ssp_plot_stations.py | 0 .../ssp_plot_traces.py | 0 .../ssp_process_traces.py | 0 {sourcespec => sourcespec2}/ssp_qml_output.py | 0 .../ssp_radiated_energy.py | 0 .../ssp_radiation_pattern.py | 0 .../ssp_read_event_metadata.py | 0 .../ssp_read_sac_header.py | 0 .../ssp_read_station_metadata.py | 0 .../ssp_read_traces.py | 0 {sourcespec => sourcespec2}/ssp_residuals.py | 0 {sourcespec => sourcespec2}/ssp_setup.py | 0 .../ssp_spectral_model.py | 0 .../ssp_sqlite_output.py | 0 .../ssp_summary_statistics.py | 0 {sourcespec => sourcespec2}/ssp_update_db.py | 0 {sourcespec => sourcespec2}/ssp_util.py | 0 .../ssp_wave_arrival.py | 0 .../ssp_wave_picking.py | 0 67 files changed, 15 insertions(+), 15 deletions(-) rename {sourcespec => sourcespec2}/__init__.py (100%) rename {sourcespec => sourcespec2}/_version.py (100%) rename {sourcespec => sourcespec2}/adjustText/LICENSE (100%) rename {sourcespec => sourcespec2}/adjustText/__init__.py (100%) rename {sourcespec => sourcespec2}/cached_tiler.py (100%) rename {sourcespec => sourcespec2}/clipping_detection.py (100%) rename {sourcespec => sourcespec2}/config.py (100%) rename {sourcespec => sourcespec2}/config_files/configspec.conf (100%) rename {sourcespec => sourcespec2}/config_files/ssp_event.yaml (100%) rename {sourcespec => sourcespec2}/configobj/LICENSE (100%) rename {sourcespec => sourcespec2}/configobj/__init__.py (100%) rename {sourcespec => sourcespec2}/configobj/_version.py (100%) rename {sourcespec => sourcespec2}/configobj/validate.py (100%) rename {sourcespec => sourcespec2}/html_report_template/index.html (100%) rename {sourcespec => sourcespec2}/html_report_template/misfit.html (100%) rename {sourcespec => sourcespec2}/html_report_template/misfit_table_column.html (100%) rename {sourcespec => sourcespec2}/html_report_template/quakeml_file_link.html (100%) rename {sourcespec => sourcespec2}/html_report_template/spectra_plot.html (100%) rename {sourcespec => sourcespec2}/html_report_template/station_table_row.html (100%) rename {sourcespec => sourcespec2}/html_report_template/style.css (100%) rename {sourcespec => sourcespec2}/html_report_template/supplementary_file_links.html (100%) rename {sourcespec => sourcespec2}/html_report_template/traces_plot.html (100%) rename {sourcespec => sourcespec2}/kdtree.py (100%) rename {sourcespec => sourcespec2}/map_tiles.py (100%) rename {sourcespec => sourcespec2}/plot_sourcepars.py (100%) rename {sourcespec => sourcespec2}/savefig.py (100%) rename {sourcespec => sourcespec2}/source_model.py (100%) rename {sourcespec => sourcespec2}/source_residuals.py (100%) rename {sourcespec => sourcespec2}/source_spec.py (100%) rename {sourcespec => sourcespec2}/spectrum.py (100%) rename {sourcespec => sourcespec2}/ssp_build_spectra.py (100%) rename {sourcespec => sourcespec2}/ssp_correction.py (100%) rename {sourcespec => sourcespec2}/ssp_data_types.py (100%) rename {sourcespec => sourcespec2}/ssp_db_definitions.py (100%) rename {sourcespec => sourcespec2}/ssp_event.py (100%) rename {sourcespec => sourcespec2}/ssp_geom_spreading.py (99%) rename {sourcespec => sourcespec2}/ssp_grid_sampling.py (100%) rename {sourcespec => sourcespec2}/ssp_html_report.py (100%) rename {sourcespec => sourcespec2}/ssp_inversion.py (100%) rename {sourcespec => sourcespec2}/ssp_local_magnitude.py (100%) rename {sourcespec => sourcespec2}/ssp_output.py (100%) rename {sourcespec => sourcespec2}/ssp_parse_arguments.py (100%) rename {sourcespec => sourcespec2}/ssp_pick.py (100%) rename {sourcespec => sourcespec2}/ssp_plot_params_stats.py (100%) rename {sourcespec => sourcespec2}/ssp_plot_spectra.py (100%) rename {sourcespec => sourcespec2}/ssp_plot_stacked_spectra.py (100%) rename {sourcespec => sourcespec2}/ssp_plot_stations.py (100%) rename {sourcespec => sourcespec2}/ssp_plot_traces.py (100%) rename {sourcespec => sourcespec2}/ssp_process_traces.py (100%) rename {sourcespec => sourcespec2}/ssp_qml_output.py (100%) rename {sourcespec => sourcespec2}/ssp_radiated_energy.py (100%) rename {sourcespec => sourcespec2}/ssp_radiation_pattern.py (100%) rename {sourcespec => sourcespec2}/ssp_read_event_metadata.py (100%) rename {sourcespec => sourcespec2}/ssp_read_sac_header.py (100%) rename {sourcespec => sourcespec2}/ssp_read_station_metadata.py (100%) rename {sourcespec => sourcespec2}/ssp_read_traces.py (100%) rename {sourcespec => sourcespec2}/ssp_residuals.py (100%) rename {sourcespec => sourcespec2}/ssp_setup.py (100%) rename {sourcespec => sourcespec2}/ssp_spectral_model.py (100%) rename {sourcespec => sourcespec2}/ssp_sqlite_output.py (100%) rename {sourcespec => sourcespec2}/ssp_summary_statistics.py (100%) rename {sourcespec => sourcespec2}/ssp_update_db.py (100%) rename {sourcespec => sourcespec2}/ssp_util.py (100%) rename {sourcespec => sourcespec2}/ssp_wave_arrival.py (100%) rename {sourcespec => sourcespec2}/ssp_wave_picking.py (100%) diff --git a/.gitignore b/.gitignore index 7d4c4061..2c3b3878 100644 --- a/.gitignore +++ b/.gitignore @@ -22,7 +22,7 @@ docs/generated .vscode #Build products -sourcespec.egg-info/ +*.egg-info/ dist/ build/ RELEASE-VERSION diff --git a/pyproject.toml b/pyproject.toml index 88734ed3..7fe7a952 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["setuptools>=64", "versioneer[toml]"] build-backend = "setuptools.build_meta" [project] -name = "sourcespec" +name = "sourcespec2" dynamic = ["version", "readme"] authors = [ { name = "Claudio Satriano", email = "satriano@ipgp.fr" }, @@ -48,11 +48,11 @@ Source = "https://github.com/SeismicSource/sourcespec" Documentation = "https://sourcespec.readthedocs.io" [project.scripts] -source_spec = "sourcespec.source_spec:main" -source_model = "sourcespec.source_model:main" -source_residuals = "sourcespec.source_residuals:main" -clipping_detection = "sourcespec.clipping_detection:main" -plot_sourcepars = "sourcespec.plot_sourcepars:main" +source_spec2 = "sourcespec2.source_spec:main" +source_model2 = "sourcespec2.source_model:main" +source_residuals2 = "sourcespec2.source_residuals:main" +clipping_detection2 = "sourcespec2.clipping_detection:main" +plot_sourcepars2 = "sourcespec2.plot_sourcepars:main" [tool.setuptools] include-package-data = true @@ -62,23 +62,23 @@ platforms = [ ] [tool.setuptools.packages.find] -include = ["sourcespec", "sourcespec.*"] +include = ["sourcespec2", "sourcespec2.*"] [tool.setuptools.package-data] "*" = ["LICENSE"] -"sourcespec.config_files" = ["*.yaml", "*.conf"] -"sourcespec.html_report_template" = ["*.html", "*.css"] +"sourcespec2.config_files" = ["*.yaml", "*.conf"] +"sourcespec2.html_report_template" = ["*.html", "*.css"] [tool.setuptools.dynamic] -version = {attr = "sourcespec.__version__"} +version = {attr = "sourcespec2.__version__"} [tool.versioneer] VCS = "git" style = "pep440" -versionfile_source = "sourcespec/_version.py" -versionfile_build = "sourcespec/_version.py" +versionfile_source = "sourcespec2/_version.py" +versionfile_build = "sourcespec2/_version.py" tag_prefix = "v" -parentdir_prefix = "sourcespec-" +parentdir_prefix = "sourcespec2-" [tool.pylama] skip = "build/*,versioneer.py,*/_version.py,*/configobj/*,*/adjustText/*" diff --git a/sourcespec/__init__.py b/sourcespec2/__init__.py similarity index 100% rename from sourcespec/__init__.py rename to sourcespec2/__init__.py diff --git a/sourcespec/_version.py b/sourcespec2/_version.py similarity index 100% rename from sourcespec/_version.py rename to sourcespec2/_version.py diff --git a/sourcespec/adjustText/LICENSE b/sourcespec2/adjustText/LICENSE similarity index 100% rename from sourcespec/adjustText/LICENSE rename to sourcespec2/adjustText/LICENSE diff --git a/sourcespec/adjustText/__init__.py b/sourcespec2/adjustText/__init__.py similarity index 100% rename from sourcespec/adjustText/__init__.py rename to sourcespec2/adjustText/__init__.py diff --git a/sourcespec/cached_tiler.py b/sourcespec2/cached_tiler.py similarity index 100% rename from sourcespec/cached_tiler.py rename to sourcespec2/cached_tiler.py diff --git a/sourcespec/clipping_detection.py b/sourcespec2/clipping_detection.py similarity index 100% rename from sourcespec/clipping_detection.py rename to sourcespec2/clipping_detection.py diff --git a/sourcespec/config.py b/sourcespec2/config.py similarity index 100% rename from sourcespec/config.py rename to sourcespec2/config.py diff --git a/sourcespec/config_files/configspec.conf b/sourcespec2/config_files/configspec.conf similarity index 100% rename from sourcespec/config_files/configspec.conf rename to sourcespec2/config_files/configspec.conf diff --git a/sourcespec/config_files/ssp_event.yaml b/sourcespec2/config_files/ssp_event.yaml similarity index 100% rename from sourcespec/config_files/ssp_event.yaml rename to sourcespec2/config_files/ssp_event.yaml diff --git a/sourcespec/configobj/LICENSE b/sourcespec2/configobj/LICENSE similarity index 100% rename from sourcespec/configobj/LICENSE rename to sourcespec2/configobj/LICENSE diff --git a/sourcespec/configobj/__init__.py b/sourcespec2/configobj/__init__.py similarity index 100% rename from sourcespec/configobj/__init__.py rename to sourcespec2/configobj/__init__.py diff --git a/sourcespec/configobj/_version.py b/sourcespec2/configobj/_version.py similarity index 100% rename from sourcespec/configobj/_version.py rename to sourcespec2/configobj/_version.py diff --git a/sourcespec/configobj/validate.py b/sourcespec2/configobj/validate.py similarity index 100% rename from sourcespec/configobj/validate.py rename to sourcespec2/configobj/validate.py diff --git a/sourcespec/html_report_template/index.html b/sourcespec2/html_report_template/index.html similarity index 100% rename from sourcespec/html_report_template/index.html rename to sourcespec2/html_report_template/index.html diff --git a/sourcespec/html_report_template/misfit.html b/sourcespec2/html_report_template/misfit.html similarity index 100% rename from sourcespec/html_report_template/misfit.html rename to sourcespec2/html_report_template/misfit.html diff --git a/sourcespec/html_report_template/misfit_table_column.html b/sourcespec2/html_report_template/misfit_table_column.html similarity index 100% rename from sourcespec/html_report_template/misfit_table_column.html rename to sourcespec2/html_report_template/misfit_table_column.html diff --git a/sourcespec/html_report_template/quakeml_file_link.html b/sourcespec2/html_report_template/quakeml_file_link.html similarity index 100% rename from sourcespec/html_report_template/quakeml_file_link.html rename to sourcespec2/html_report_template/quakeml_file_link.html diff --git a/sourcespec/html_report_template/spectra_plot.html b/sourcespec2/html_report_template/spectra_plot.html similarity index 100% rename from sourcespec/html_report_template/spectra_plot.html rename to sourcespec2/html_report_template/spectra_plot.html diff --git a/sourcespec/html_report_template/station_table_row.html b/sourcespec2/html_report_template/station_table_row.html similarity index 100% rename from sourcespec/html_report_template/station_table_row.html rename to sourcespec2/html_report_template/station_table_row.html diff --git a/sourcespec/html_report_template/style.css b/sourcespec2/html_report_template/style.css similarity index 100% rename from sourcespec/html_report_template/style.css rename to sourcespec2/html_report_template/style.css diff --git a/sourcespec/html_report_template/supplementary_file_links.html b/sourcespec2/html_report_template/supplementary_file_links.html similarity index 100% rename from sourcespec/html_report_template/supplementary_file_links.html rename to sourcespec2/html_report_template/supplementary_file_links.html diff --git a/sourcespec/html_report_template/traces_plot.html b/sourcespec2/html_report_template/traces_plot.html similarity index 100% rename from sourcespec/html_report_template/traces_plot.html rename to sourcespec2/html_report_template/traces_plot.html diff --git a/sourcespec/kdtree.py b/sourcespec2/kdtree.py similarity index 100% rename from sourcespec/kdtree.py rename to sourcespec2/kdtree.py diff --git a/sourcespec/map_tiles.py b/sourcespec2/map_tiles.py similarity index 100% rename from sourcespec/map_tiles.py rename to sourcespec2/map_tiles.py diff --git a/sourcespec/plot_sourcepars.py b/sourcespec2/plot_sourcepars.py similarity index 100% rename from sourcespec/plot_sourcepars.py rename to sourcespec2/plot_sourcepars.py diff --git a/sourcespec/savefig.py b/sourcespec2/savefig.py similarity index 100% rename from sourcespec/savefig.py rename to sourcespec2/savefig.py diff --git a/sourcespec/source_model.py b/sourcespec2/source_model.py similarity index 100% rename from sourcespec/source_model.py rename to sourcespec2/source_model.py diff --git a/sourcespec/source_residuals.py b/sourcespec2/source_residuals.py similarity index 100% rename from sourcespec/source_residuals.py rename to sourcespec2/source_residuals.py diff --git a/sourcespec/source_spec.py b/sourcespec2/source_spec.py similarity index 100% rename from sourcespec/source_spec.py rename to sourcespec2/source_spec.py diff --git a/sourcespec/spectrum.py b/sourcespec2/spectrum.py similarity index 100% rename from sourcespec/spectrum.py rename to sourcespec2/spectrum.py diff --git a/sourcespec/ssp_build_spectra.py b/sourcespec2/ssp_build_spectra.py similarity index 100% rename from sourcespec/ssp_build_spectra.py rename to sourcespec2/ssp_build_spectra.py diff --git a/sourcespec/ssp_correction.py b/sourcespec2/ssp_correction.py similarity index 100% rename from sourcespec/ssp_correction.py rename to sourcespec2/ssp_correction.py diff --git a/sourcespec/ssp_data_types.py b/sourcespec2/ssp_data_types.py similarity index 100% rename from sourcespec/ssp_data_types.py rename to sourcespec2/ssp_data_types.py diff --git a/sourcespec/ssp_db_definitions.py b/sourcespec2/ssp_db_definitions.py similarity index 100% rename from sourcespec/ssp_db_definitions.py rename to sourcespec2/ssp_db_definitions.py diff --git a/sourcespec/ssp_event.py b/sourcespec2/ssp_event.py similarity index 100% rename from sourcespec/ssp_event.py rename to sourcespec2/ssp_event.py diff --git a/sourcespec/ssp_geom_spreading.py b/sourcespec2/ssp_geom_spreading.py similarity index 99% rename from sourcespec/ssp_geom_spreading.py rename to sourcespec2/ssp_geom_spreading.py index 936daa5d..642d2178 100644 --- a/sourcespec/ssp_geom_spreading.py +++ b/sourcespec2/ssp_geom_spreading.py @@ -12,7 +12,7 @@ import logging import numpy as np from obspy.taup import TauPyModel -from sourcespec.ssp_util import MediumProperties +from .ssp_util import MediumProperties model = TauPyModel(model='iasp91') v_model = model.model.s_mod.v_mod logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec/ssp_grid_sampling.py b/sourcespec2/ssp_grid_sampling.py similarity index 100% rename from sourcespec/ssp_grid_sampling.py rename to sourcespec2/ssp_grid_sampling.py diff --git a/sourcespec/ssp_html_report.py b/sourcespec2/ssp_html_report.py similarity index 100% rename from sourcespec/ssp_html_report.py rename to sourcespec2/ssp_html_report.py diff --git a/sourcespec/ssp_inversion.py b/sourcespec2/ssp_inversion.py similarity index 100% rename from sourcespec/ssp_inversion.py rename to sourcespec2/ssp_inversion.py diff --git a/sourcespec/ssp_local_magnitude.py b/sourcespec2/ssp_local_magnitude.py similarity index 100% rename from sourcespec/ssp_local_magnitude.py rename to sourcespec2/ssp_local_magnitude.py diff --git a/sourcespec/ssp_output.py b/sourcespec2/ssp_output.py similarity index 100% rename from sourcespec/ssp_output.py rename to sourcespec2/ssp_output.py diff --git a/sourcespec/ssp_parse_arguments.py b/sourcespec2/ssp_parse_arguments.py similarity index 100% rename from sourcespec/ssp_parse_arguments.py rename to sourcespec2/ssp_parse_arguments.py diff --git a/sourcespec/ssp_pick.py b/sourcespec2/ssp_pick.py similarity index 100% rename from sourcespec/ssp_pick.py rename to sourcespec2/ssp_pick.py diff --git a/sourcespec/ssp_plot_params_stats.py b/sourcespec2/ssp_plot_params_stats.py similarity index 100% rename from sourcespec/ssp_plot_params_stats.py rename to sourcespec2/ssp_plot_params_stats.py diff --git a/sourcespec/ssp_plot_spectra.py b/sourcespec2/ssp_plot_spectra.py similarity index 100% rename from sourcespec/ssp_plot_spectra.py rename to sourcespec2/ssp_plot_spectra.py diff --git a/sourcespec/ssp_plot_stacked_spectra.py b/sourcespec2/ssp_plot_stacked_spectra.py similarity index 100% rename from sourcespec/ssp_plot_stacked_spectra.py rename to sourcespec2/ssp_plot_stacked_spectra.py diff --git a/sourcespec/ssp_plot_stations.py b/sourcespec2/ssp_plot_stations.py similarity index 100% rename from sourcespec/ssp_plot_stations.py rename to sourcespec2/ssp_plot_stations.py diff --git a/sourcespec/ssp_plot_traces.py b/sourcespec2/ssp_plot_traces.py similarity index 100% rename from sourcespec/ssp_plot_traces.py rename to sourcespec2/ssp_plot_traces.py diff --git a/sourcespec/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py similarity index 100% rename from sourcespec/ssp_process_traces.py rename to sourcespec2/ssp_process_traces.py diff --git a/sourcespec/ssp_qml_output.py b/sourcespec2/ssp_qml_output.py similarity index 100% rename from sourcespec/ssp_qml_output.py rename to sourcespec2/ssp_qml_output.py diff --git a/sourcespec/ssp_radiated_energy.py b/sourcespec2/ssp_radiated_energy.py similarity index 100% rename from sourcespec/ssp_radiated_energy.py rename to sourcespec2/ssp_radiated_energy.py diff --git a/sourcespec/ssp_radiation_pattern.py b/sourcespec2/ssp_radiation_pattern.py similarity index 100% rename from sourcespec/ssp_radiation_pattern.py rename to sourcespec2/ssp_radiation_pattern.py diff --git a/sourcespec/ssp_read_event_metadata.py b/sourcespec2/ssp_read_event_metadata.py similarity index 100% rename from sourcespec/ssp_read_event_metadata.py rename to sourcespec2/ssp_read_event_metadata.py diff --git a/sourcespec/ssp_read_sac_header.py b/sourcespec2/ssp_read_sac_header.py similarity index 100% rename from sourcespec/ssp_read_sac_header.py rename to sourcespec2/ssp_read_sac_header.py diff --git a/sourcespec/ssp_read_station_metadata.py b/sourcespec2/ssp_read_station_metadata.py similarity index 100% rename from sourcespec/ssp_read_station_metadata.py rename to sourcespec2/ssp_read_station_metadata.py diff --git a/sourcespec/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py similarity index 100% rename from sourcespec/ssp_read_traces.py rename to sourcespec2/ssp_read_traces.py diff --git a/sourcespec/ssp_residuals.py b/sourcespec2/ssp_residuals.py similarity index 100% rename from sourcespec/ssp_residuals.py rename to sourcespec2/ssp_residuals.py diff --git a/sourcespec/ssp_setup.py b/sourcespec2/ssp_setup.py similarity index 100% rename from sourcespec/ssp_setup.py rename to sourcespec2/ssp_setup.py diff --git a/sourcespec/ssp_spectral_model.py b/sourcespec2/ssp_spectral_model.py similarity index 100% rename from sourcespec/ssp_spectral_model.py rename to sourcespec2/ssp_spectral_model.py diff --git a/sourcespec/ssp_sqlite_output.py b/sourcespec2/ssp_sqlite_output.py similarity index 100% rename from sourcespec/ssp_sqlite_output.py rename to sourcespec2/ssp_sqlite_output.py diff --git a/sourcespec/ssp_summary_statistics.py b/sourcespec2/ssp_summary_statistics.py similarity index 100% rename from sourcespec/ssp_summary_statistics.py rename to sourcespec2/ssp_summary_statistics.py diff --git a/sourcespec/ssp_update_db.py b/sourcespec2/ssp_update_db.py similarity index 100% rename from sourcespec/ssp_update_db.py rename to sourcespec2/ssp_update_db.py diff --git a/sourcespec/ssp_util.py b/sourcespec2/ssp_util.py similarity index 100% rename from sourcespec/ssp_util.py rename to sourcespec2/ssp_util.py diff --git a/sourcespec/ssp_wave_arrival.py b/sourcespec2/ssp_wave_arrival.py similarity index 100% rename from sourcespec/ssp_wave_arrival.py rename to sourcespec2/ssp_wave_arrival.py diff --git a/sourcespec/ssp_wave_picking.py b/sourcespec2/ssp_wave_picking.py similarity index 100% rename from sourcespec/ssp_wave_picking.py rename to sourcespec2/ssp_wave_picking.py From d5d78cc5d4a49b167c7c9cdaaea17e48190db291 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 3 Jul 2024 12:17:24 +0200 Subject: [PATCH 03/73] Replace absolute imports with relative imports This allows for actually using "sourcespec2" modules instead of "sourcespec" modules. --- sourcespec2/source_model.py | 22 +++++++-------- sourcespec2/source_residuals.py | 8 +++--- sourcespec2/source_spec.py | 34 ++++++++++++------------ sourcespec2/ssp_build_spectra.py | 14 +++++----- sourcespec2/ssp_correction.py | 6 ++--- sourcespec2/ssp_grid_sampling.py | 4 +-- sourcespec2/ssp_html_report.py | 4 +-- sourcespec2/ssp_inversion.py | 10 +++---- sourcespec2/ssp_local_magnitude.py | 6 ++--- sourcespec2/ssp_output.py | 6 ++--- sourcespec2/ssp_parse_arguments.py | 2 +- sourcespec2/ssp_plot_params_stats.py | 4 +-- sourcespec2/ssp_plot_spectra.py | 6 ++--- sourcespec2/ssp_plot_stacked_spectra.py | 8 +++--- sourcespec2/ssp_plot_stations.py | 10 +++---- sourcespec2/ssp_plot_traces.py | 4 +-- sourcespec2/ssp_process_traces.py | 10 +++---- sourcespec2/ssp_qml_output.py | 2 +- sourcespec2/ssp_radiated_energy.py | 2 +- sourcespec2/ssp_read_event_metadata.py | 6 ++--- sourcespec2/ssp_read_sac_header.py | 6 ++--- sourcespec2/ssp_read_station_metadata.py | 2 +- sourcespec2/ssp_read_traces.py | 10 +++---- sourcespec2/ssp_residuals.py | 8 +++--- sourcespec2/ssp_setup.py | 10 +++---- sourcespec2/ssp_sqlite_output.py | 6 ++--- sourcespec2/ssp_summary_statistics.py | 4 +-- sourcespec2/ssp_update_db.py | 2 +- 28 files changed, 108 insertions(+), 108 deletions(-) diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 5b4c5d52..c61efaf9 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -23,9 +23,9 @@ def make_synth(config, spec_st, trace_spec=None): import math import numpy as np from copy import deepcopy - from sourcespec.spectrum import Spectrum - from sourcespec.ssp_spectral_model import spectral_model, objective_func - from sourcespec.ssp_util import mag_to_moment, moment_to_mag + from .spectrum import Spectrum + from .ssp_spectral_model import spectral_model, objective_func + from .ssp_util import mag_to_moment, moment_to_mag fdelta = 0.01 fmin = config.options.fmin fmax = config.options.fmax + fdelta @@ -82,9 +82,9 @@ def main(): """ # pylint: disable=import-outside-toplevel # Lazy-import modules for speed - from sourcespec.ssp_parse_arguments import parse_args + from .ssp_parse_arguments import parse_args options = parse_args(progname='source_model') - from sourcespec.ssp_setup import configure, ssp_exit + from .ssp_setup import configure, ssp_exit plot_show = bool(options.plot) conf_overrides = { 'plot_show': plot_show, @@ -93,10 +93,10 @@ def main(): } config = configure( options, progname='source_model', config_overrides=conf_overrides) - from sourcespec.spectrum import SpectrumStream - from sourcespec.ssp_read_traces import read_traces - from sourcespec.ssp_process_traces import process_traces - from sourcespec.ssp_build_spectra import build_spectra + from .spectrum import SpectrumStream + from .ssp_read_traces import read_traces + from .ssp_process_traces import process_traces + from .ssp_build_spectra import build_spectra # We don't use weighting in source_model config.weighting = 'no_weight' @@ -121,8 +121,8 @@ def main(): spec_st = SpectrumStream() make_synth(config, spec_st) - from sourcespec.ssp_plot_spectra import plot_spectra - from sourcespec.ssp_plot_traces import plot_traces + from .ssp_plot_spectra import plot_spectra + from .ssp_plot_traces import plot_traces plot_traces(config, proc_st, ncols=2, block=False) plot_spectra(config, spec_st, ncols=1, stack_plots=True) diff --git a/sourcespec2/source_residuals.py b/sourcespec2/source_residuals.py index e3e41028..d47bc915 100644 --- a/sourcespec2/source_residuals.py +++ b/sourcespec2/source_residuals.py @@ -18,10 +18,10 @@ from argparse import ArgumentParser import matplotlib import matplotlib.pyplot as plt -from sourcespec._version import get_versions -from sourcespec.spectrum import read_spectra -from sourcespec.ssp_util import moment_to_mag, mag_to_moment -from sourcespec.spectrum import SpectrumStream +from ._version import get_versions +from .spectrum import read_spectra +from .ssp_util import moment_to_mag, mag_to_moment +from .spectrum import SpectrumStream matplotlib.use('Agg') # NOQA diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 46096465..55fc1851 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -21,17 +21,17 @@ def main(): """Main routine for source_spec.""" # pylint: disable=import-outside-toplevel # Lazy-import modules for speed - from sourcespec.ssp_parse_arguments import parse_args + from .ssp_parse_arguments import parse_args options = parse_args(progname='source_spec') # Setup stage - from sourcespec.ssp_setup import ( + from .ssp_setup import ( configure, move_outdir, remove_old_outdir, setup_logging, save_config, ssp_exit) config = configure(options, progname='source_spec') setup_logging(config) - from sourcespec.ssp_read_traces import read_traces + from .ssp_read_traces import read_traces st = read_traces(config) # Now that we have an evid, we can rename the outdir and the log file @@ -43,59 +43,59 @@ def main(): save_config(config) # Deconvolve, filter, cut traces: - from sourcespec.ssp_process_traces import process_traces + from .ssp_process_traces import process_traces proc_st = process_traces(config, st) # Build spectra (amplitude in magnitude units) - from sourcespec.ssp_build_spectra import build_spectra + from .ssp_build_spectra import build_spectra spec_st, specnoise_st, weight_st = build_spectra(config, proc_st) - from sourcespec.ssp_plot_traces import plot_traces + from .ssp_plot_traces import plot_traces plot_traces(config, st, suffix='raw') plot_traces(config, proc_st) # Spectral inversion - from sourcespec.ssp_inversion import spectral_inversion + from .ssp_inversion import spectral_inversion sspec_output = spectral_inversion(config, spec_st, weight_st) # Radiated energy and apparent stress - from sourcespec.ssp_radiated_energy import ( + from .ssp_radiated_energy import ( radiated_energy_and_apparent_stress) radiated_energy_and_apparent_stress( config, spec_st, specnoise_st, sspec_output) # Local magnitude if config.compute_local_magnitude: - from sourcespec.ssp_local_magnitude import local_magnitude + from .ssp_local_magnitude import local_magnitude local_magnitude(config, st, proc_st, sspec_output) # Compute summary statistics from station spectral parameters - from sourcespec.ssp_summary_statistics import compute_summary_statistics + from .ssp_summary_statistics import compute_summary_statistics compute_summary_statistics(config, sspec_output) # Save output - from sourcespec.ssp_output import write_output, save_spectra + from .ssp_output import write_output, save_spectra write_output(config, sspec_output) save_spectra(config, spec_st) # Save residuals - from sourcespec.ssp_residuals import spectral_residuals + from .ssp_residuals import spectral_residuals spectral_residuals(config, spec_st, sspec_output) # Plotting - from sourcespec.ssp_plot_spectra import plot_spectra + from .ssp_plot_spectra import plot_spectra plot_spectra(config, spec_st, specnoise_st, plot_type='regular') plot_spectra(config, weight_st, plot_type='weight') - from sourcespec.ssp_plot_stacked_spectra import plot_stacked_spectra + from .ssp_plot_stacked_spectra import plot_stacked_spectra plot_stacked_spectra(config, spec_st, weight_st, sspec_output) - from sourcespec.ssp_plot_params_stats import box_plots + from .ssp_plot_params_stats import box_plots box_plots(config, sspec_output) if config.plot_station_map: - from sourcespec.ssp_plot_stations import plot_stations + from .ssp_plot_stations import plot_stations plot_stations(config, sspec_output) if config.html_report: - from sourcespec.ssp_html_report import html_report + from .ssp_html_report import html_report html_report(config, sspec_output) ssp_exit() diff --git a/sourcespec2/ssp_build_spectra.py b/sourcespec2/ssp_build_spectra.py index 0d659400..9084f845 100644 --- a/sourcespec2/ssp_build_spectra.py +++ b/sourcespec2/ssp_build_spectra.py @@ -25,16 +25,16 @@ from scipy.integrate import cumulative_trapezoid as cumtrapz from scipy.interpolate import interp1d from obspy.core import Stream -from sourcespec.spectrum import Spectrum, SpectrumStream -from sourcespec.ssp_setup import ssp_exit -from sourcespec.ssp_util import ( +from .spectrum import Spectrum, SpectrumStream +from .ssp_setup import ssp_exit +from .ssp_util import ( smooth, cosine_taper, moment_to_mag, MediumProperties) -from sourcespec.ssp_geom_spreading import ( +from .ssp_geom_spreading import ( geom_spread_r_power_n, geom_spread_r_power_n_segmented, geom_spread_boatwright, geom_spread_teleseismic) -from sourcespec.ssp_process_traces import filter_trace -from sourcespec.ssp_correction import station_correction -from sourcespec.ssp_radiation_pattern import get_radiation_pattern_coefficient +from .ssp_process_traces import filter_trace +from .ssp_correction import station_correction +from .ssp_radiation_pattern import get_radiation_pattern_coefficient logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_correction.py b/sourcespec2/ssp_correction.py index 1f229cf6..1bb9a3ab 100644 --- a/sourcespec2/ssp_correction.py +++ b/sourcespec2/ssp_correction.py @@ -14,9 +14,9 @@ """ import logging from scipy.interpolate import interp1d -from sourcespec.spectrum import read_spectra -from sourcespec.ssp_util import moment_to_mag, mag_to_moment -from sourcespec.ssp_setup import ssp_exit +from .spectrum import read_spectra +from .ssp_util import moment_to_mag, mag_to_moment +from .ssp_setup import ssp_exit logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_grid_sampling.py b/sourcespec2/ssp_grid_sampling.py index dc038da3..301e4677 100644 --- a/sourcespec2/ssp_grid_sampling.py +++ b/sourcespec2/ssp_grid_sampling.py @@ -21,8 +21,8 @@ # pylint: disable=no-name-in-module from scipy.signal._peak_finding_utils import PeakPropertyWarning import matplotlib.pyplot as plt -from sourcespec.kdtree import KDTree -from sourcespec.savefig import savefig +from .kdtree import KDTree +from .savefig import savefig logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_html_report.py b/sourcespec2/ssp_html_report.py index ce4e5808..72e6cd30 100644 --- a/sourcespec2/ssp_html_report.py +++ b/sourcespec2/ssp_html_report.py @@ -16,8 +16,8 @@ import contextlib from urllib.parse import urlparse import numpy as np -from sourcespec._version import get_versions -from sourcespec.ssp_data_types import SpectralParameter +from ._version import get_versions +from .ssp_data_types import SpectralParameter logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) VALID_FIGURE_FORMATS = ('.png', '.svg') diff --git a/sourcespec2/ssp_inversion.py b/sourcespec2/ssp_inversion.py index 3e0de068..37706b5f 100644 --- a/sourcespec2/ssp_inversion.py +++ b/sourcespec2/ssp_inversion.py @@ -20,16 +20,16 @@ from scipy.optimize import curve_fit, minimize, basinhopping from scipy.signal import argrelmax from obspy.geodetics import gps2dist_azimuth -from sourcespec.spectrum import SpectrumStream -from sourcespec.ssp_spectral_model import ( +from .spectrum import SpectrumStream +from .ssp_spectral_model import ( spectral_model, objective_func, callback) -from sourcespec.ssp_util import ( +from .ssp_util import ( mag_to_moment, source_radius, static_stress_drop, quality_factor, select_trace, smooth) -from sourcespec.ssp_data_types import ( +from .ssp_data_types import ( InitialValues, Bounds, SpectralParameter, StationParameters, SourceSpecOutput) -from sourcespec.ssp_grid_sampling import GridSampling +from .ssp_grid_sampling import GridSampling logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_local_magnitude.py b/sourcespec2/ssp_local_magnitude.py index 9b6ebe7e..0fef8164 100644 --- a/sourcespec2/ssp_local_magnitude.py +++ b/sourcespec2/ssp_local_magnitude.py @@ -23,9 +23,9 @@ from obspy.signal.invsim import WOODANDERSON from obspy.signal.util import smooth from obspy.signal.trigger import trigger_onset -from sourcespec.ssp_data_types import SpectralParameter -from sourcespec.ssp_util import cosine_taper -from sourcespec.ssp_util import remove_instr_response +from .ssp_data_types import SpectralParameter +from .ssp_util import cosine_taper +from .ssp_util import remove_instr_response logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_output.py b/sourcespec2/ssp_output.py index fb60f761..7a56a437 100644 --- a/sourcespec2/ssp_output.py +++ b/sourcespec2/ssp_output.py @@ -22,9 +22,9 @@ from datetime import datetime from tzlocal import get_localzone import numpy as np -from sourcespec.ssp_qml_output import write_qml -from sourcespec.ssp_sqlite_output import write_sqlite -from sourcespec._version import get_versions +from .ssp_qml_output import write_qml +from .ssp_sqlite_output import write_sqlite +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_parse_arguments.py b/sourcespec2/ssp_parse_arguments.py index 836b43a1..9190a6a4 100644 --- a/sourcespec2/ssp_parse_arguments.py +++ b/sourcespec2/ssp_parse_arguments.py @@ -12,7 +12,7 @@ import sys from argparse import ArgumentParser, RawTextHelpFormatter from itertools import zip_longest -from sourcespec._version import get_versions +from ._version import get_versions def _parse_values(value_str): diff --git a/sourcespec2/ssp_plot_params_stats.py b/sourcespec2/ssp_plot_params_stats.py index 29d48a6d..6b819241 100644 --- a/sourcespec2/ssp_plot_params_stats.py +++ b/sourcespec2/ssp_plot_params_stats.py @@ -17,8 +17,8 @@ import matplotlib.pyplot as plt import matplotlib.patheffects as mpe -from sourcespec._version import get_versions -from sourcespec.savefig import savefig +from ._version import get_versions +from .savefig import savefig logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) # Reduce logging level for Matplotlib to avoid DEBUG messages mpl_logger = logging.getLogger('matplotlib') diff --git a/sourcespec2/ssp_plot_spectra.py b/sourcespec2/ssp_plot_spectra.py index 0a65ffe9..3ad10d21 100644 --- a/sourcespec2/ssp_plot_spectra.py +++ b/sourcespec2/ssp_plot_spectra.py @@ -25,9 +25,9 @@ import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages import matplotlib.patheffects as PathEffects -from sourcespec.ssp_util import spec_minmax, moment_to_mag, mag_to_moment -from sourcespec.savefig import savefig -from sourcespec._version import get_versions +from .ssp_util import spec_minmax, moment_to_mag, mag_to_moment +from .savefig import savefig +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) # Reduce logging level for Matplotlib to avoid DEBUG messages mpl_logger = logging.getLogger('matplotlib') diff --git a/sourcespec2/ssp_plot_stacked_spectra.py b/sourcespec2/ssp_plot_stacked_spectra.py index 53da2bed..73156ca8 100644 --- a/sourcespec2/ssp_plot_stacked_spectra.py +++ b/sourcespec2/ssp_plot_stacked_spectra.py @@ -17,10 +17,10 @@ import matplotlib.pyplot as plt import matplotlib.patheffects as PathEffects from matplotlib.collections import LineCollection -from sourcespec.ssp_util import moment_to_mag, mag_to_moment -from sourcespec.ssp_spectral_model import spectral_model -from sourcespec.savefig import savefig -from sourcespec._version import get_versions +from .ssp_util import moment_to_mag, mag_to_moment +from .ssp_spectral_model import spectral_model +from .savefig import savefig +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) # Reduce logging level for Matplotlib to avoid DEBUG messages mpl_logger = logging.getLogger('matplotlib') diff --git a/sourcespec2/ssp_plot_stations.py b/sourcespec2/ssp_plot_stations.py index 386508fe..34641ffe 100644 --- a/sourcespec2/ssp_plot_stations.py +++ b/sourcespec2/ssp_plot_stations.py @@ -25,17 +25,17 @@ from matplotlib import colors import matplotlib.patheffects as PathEffects from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable -from sourcespec.adjustText import adjust_text -from sourcespec.cached_tiler import CachedTiler -from sourcespec.map_tiles import ( +from .adjustText import adjust_text +from .cached_tiler import CachedTiler +from .map_tiles import ( EsriHillshade, EsriHillshadeDark, EsriOcean, EsriImagery, StamenTerrain, ) -from sourcespec.savefig import savefig -from sourcespec._version import get_versions +from .savefig import savefig +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) # Reduce logging level for Matplotlib to avoid DEBUG messages mpl_logger = logging.getLogger('matplotlib') diff --git a/sourcespec2/ssp_plot_traces.py b/sourcespec2/ssp_plot_traces.py index a7a21126..06ff3cd0 100644 --- a/sourcespec2/ssp_plot_traces.py +++ b/sourcespec2/ssp_plot_traces.py @@ -21,8 +21,8 @@ from matplotlib import patches import matplotlib.patheffects as PathEffects from matplotlib.ticker import ScalarFormatter as sf -from sourcespec.savefig import savefig -from sourcespec._version import get_versions +from .savefig import savefig +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) # Reduce logging level for Matplotlib to avoid DEBUG messages mpl_logger = logging.getLogger('matplotlib') diff --git a/sourcespec2/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py index fe8f593b..73f924f9 100644 --- a/sourcespec2/ssp_process_traces.py +++ b/sourcespec2/ssp_process_traces.py @@ -20,12 +20,12 @@ from scipy.signal import savgol_filter from obspy.core import Stream from obspy.core.util import AttribDict -from sourcespec.ssp_setup import ssp_exit -from sourcespec.ssp_util import ( +from .ssp_setup import ssp_exit +from .ssp_util import ( remove_instr_response, station_to_event_position) -from sourcespec.ssp_wave_arrival import add_arrival_to_trace -from sourcespec.ssp_wave_picking import refine_trace_picks -from sourcespec.clipping_detection import ( +from .ssp_wave_arrival import add_arrival_to_trace +from .ssp_wave_picking import refine_trace_picks +from .clipping_detection import ( check_min_amplitude, compute_clipping_score, clipping_peaks) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_qml_output.py b/sourcespec2/ssp_qml_output.py index e2443185..bb22ea72 100644 --- a/sourcespec2/ssp_qml_output.py +++ b/sourcespec2/ssp_qml_output.py @@ -19,7 +19,7 @@ MomentTensor, QuantityError, ResourceIdentifier, StationMagnitude, StationMagnitudeContribution, WaveformStreamID) -from sourcespec._version import get_versions +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_radiated_energy.py b/sourcespec2/ssp_radiated_energy.py index 561b4a1d..337f1a81 100644 --- a/sourcespec2/ssp_radiated_energy.py +++ b/sourcespec2/ssp_radiated_energy.py @@ -18,7 +18,7 @@ import contextlib import logging import numpy as np -from sourcespec.ssp_data_types import SpectralParameter +from .ssp_data_types import SpectralParameter logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_event_metadata.py b/sourcespec2/ssp_read_event_metadata.py index dfa1f0f2..44024ec0 100644 --- a/sourcespec2/ssp_read_event_metadata.py +++ b/sourcespec2/ssp_read_event_metadata.py @@ -20,9 +20,9 @@ import yaml from obspy import UTCDateTime from obspy import read_events -from sourcespec.ssp_setup import ssp_exit, TRACEID_MAP -from sourcespec.ssp_event import SSPEvent -from sourcespec.ssp_pick import SSPPick +from .ssp_setup import ssp_exit, TRACEID_MAP +from .ssp_event import SSPEvent +from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_sac_header.py b/sourcespec2/ssp_read_sac_header.py index e065e157..70a21c7d 100644 --- a/sourcespec2/ssp_read_sac_header.py +++ b/sourcespec2/ssp_read_sac_header.py @@ -13,9 +13,9 @@ import logging import contextlib from obspy.core.util import AttribDict -from sourcespec.ssp_setup import ssp_exit -from sourcespec.ssp_event import SSPEvent -from sourcespec.ssp_pick import SSPPick +from .ssp_setup import ssp_exit +from .ssp_event import SSPEvent +from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_station_metadata.py b/sourcespec2/ssp_read_station_metadata.py index 250876fd..b63e772e 100644 --- a/sourcespec2/ssp_read_station_metadata.py +++ b/sourcespec2/ssp_read_station_metadata.py @@ -15,7 +15,7 @@ import logging from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response -from sourcespec.ssp_setup import INSTR_CODES_VEL, INSTR_CODES_ACC +from .ssp_setup import INSTR_CODES_VEL, INSTR_CODES_ACC logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index c2b85c32..feb52b3c 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -26,14 +26,14 @@ from obspy import read from obspy.core import Stream from obspy.core.util import AttribDict -from sourcespec.ssp_setup import ( +from .ssp_setup import ( ssp_exit, INSTR_CODES_VEL, INSTR_CODES_ACC, TRACEID_MAP) -from sourcespec.ssp_util import MediumProperties -from sourcespec.ssp_read_station_metadata import ( +from .ssp_util import MediumProperties +from .ssp_read_station_metadata import ( read_station_metadata, PAZ) -from sourcespec.ssp_read_event_metadata import ( +from .ssp_read_event_metadata import ( parse_qml, parse_hypo_file, parse_hypo71_picks) -from sourcespec.ssp_read_sac_header import ( +from .ssp_read_sac_header import ( compute_sensitivity_from_SAC, get_instrument_from_SAC, get_station_coordinates_from_SAC, get_event_from_SAC, get_picks_from_SAC) diff --git a/sourcespec2/ssp_residuals.py b/sourcespec2/ssp_residuals.py index 2eeac653..7a105d0d 100644 --- a/sourcespec2/ssp_residuals.py +++ b/sourcespec2/ssp_residuals.py @@ -14,10 +14,10 @@ """ import os import logging -from sourcespec._version import get_versions -from sourcespec.spectrum import SpectrumStream -from sourcespec.ssp_spectral_model import spectral_model -from sourcespec.ssp_util import mag_to_moment +from ._version import get_versions +from .spectrum import SpectrumStream +from .ssp_spectral_model import spectral_model +from .ssp_util import mag_to_moment logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index 3f055ad4..254eaae7 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -28,11 +28,11 @@ from copy import copy from datetime import datetime from collections import defaultdict -from sourcespec import __version__, __banner__ -from sourcespec.configobj import ConfigObj -from sourcespec.configobj.validate import Validator -from sourcespec.config import Config -from sourcespec.ssp_update_db import update_db_file +from . import __version__, __banner__ +from .configobj import ConfigObj +from .configobj.validate import Validator +from .config import Config +from .ssp_update_db import update_db_file # define ipshell(), if possible # note: ANSI colors do not work on Windows standard terminal diff --git a/sourcespec2/ssp_sqlite_output.py b/sourcespec2/ssp_sqlite_output.py index 8ea6886e..d659f669 100644 --- a/sourcespec2/ssp_sqlite_output.py +++ b/sourcespec2/ssp_sqlite_output.py @@ -12,11 +12,11 @@ import os.path import logging import sqlite3 -from sourcespec.ssp_setup import ssp_exit -from sourcespec.ssp_db_definitions import ( +from .ssp_setup import ssp_exit +from .ssp_db_definitions import ( DB_VERSION, STATIONS_TABLE, STATIONS_PRIMARY_KEYS, EVENTS_TABLE, EVENTS_PRIMARY_KEYS) -from sourcespec._version import get_versions +from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_summary_statistics.py b/sourcespec2/ssp_summary_statistics.py index a8ae3c97..c708f696 100644 --- a/sourcespec2/ssp_summary_statistics.py +++ b/sourcespec2/ssp_summary_statistics.py @@ -13,8 +13,8 @@ import numpy as np from scipy.stats import norm from scipy.integrate import quad -from sourcespec.ssp_setup import ssp_exit -from sourcespec.ssp_data_types import ( +from .ssp_setup import ssp_exit +from .ssp_data_types import ( SummarySpectralParameter, SummaryStatistics) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_update_db.py b/sourcespec2/ssp_update_db.py index 98b56a5a..e01e3cc0 100644 --- a/sourcespec2/ssp_update_db.py +++ b/sourcespec2/ssp_update_db.py @@ -14,7 +14,7 @@ import sys import shutil import sqlite3 -from sourcespec.ssp_db_definitions import ( +from .ssp_db_definitions import ( DB_VERSION, STATIONS_TABLE, STATIONS_PRIMARY_KEYS, EVENTS_TABLE, EVENTS_PRIMARY_KEYS) From cb6647f42b32e5666fa266d92c44ebf769b4f9fe Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 3 Jul 2024 12:25:23 +0200 Subject: [PATCH 04/73] Move all config-related files to a "config" directory --- pyproject.toml | 2 +- sourcespec2/config/__init__.py | 12 ++++++++++++ sourcespec2/{ => config}/config.py | 0 sourcespec2/{ => config}/configobj/LICENSE | 0 sourcespec2/{ => config}/configobj/__init__.py | 0 sourcespec2/{ => config}/configobj/_version.py | 0 sourcespec2/{ => config}/configobj/validate.py | 0 sourcespec2/{config_files => config}/configspec.conf | 0 sourcespec2/{config_files => config}/ssp_event.yaml | 0 sourcespec2/ssp_setup.py | 8 ++++---- 10 files changed, 17 insertions(+), 5 deletions(-) create mode 100644 sourcespec2/config/__init__.py rename sourcespec2/{ => config}/config.py (100%) rename sourcespec2/{ => config}/configobj/LICENSE (100%) rename sourcespec2/{ => config}/configobj/__init__.py (100%) rename sourcespec2/{ => config}/configobj/_version.py (100%) rename sourcespec2/{ => config}/configobj/validate.py (100%) rename sourcespec2/{config_files => config}/configspec.conf (100%) rename sourcespec2/{config_files => config}/ssp_event.yaml (100%) diff --git a/pyproject.toml b/pyproject.toml index 7fe7a952..0428d2ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ include = ["sourcespec2", "sourcespec2.*"] [tool.setuptools.package-data] "*" = ["LICENSE"] -"sourcespec2.config_files" = ["*.yaml", "*.conf"] +"sourcespec2.config" = ["*.yaml", "*.conf"] "sourcespec2.html_report_template" = ["*.html", "*.css"] [tool.setuptools.dynamic] diff --git a/sourcespec2/config/__init__.py b/sourcespec2/config/__init__.py new file mode 100644 index 00000000..99d4a06a --- /dev/null +++ b/sourcespec2/config/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Configuration classes and functions for SourceSpec + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +from .config import Config # noqa diff --git a/sourcespec2/config.py b/sourcespec2/config/config.py similarity index 100% rename from sourcespec2/config.py rename to sourcespec2/config/config.py diff --git a/sourcespec2/configobj/LICENSE b/sourcespec2/config/configobj/LICENSE similarity index 100% rename from sourcespec2/configobj/LICENSE rename to sourcespec2/config/configobj/LICENSE diff --git a/sourcespec2/configobj/__init__.py b/sourcespec2/config/configobj/__init__.py similarity index 100% rename from sourcespec2/configobj/__init__.py rename to sourcespec2/config/configobj/__init__.py diff --git a/sourcespec2/configobj/_version.py b/sourcespec2/config/configobj/_version.py similarity index 100% rename from sourcespec2/configobj/_version.py rename to sourcespec2/config/configobj/_version.py diff --git a/sourcespec2/configobj/validate.py b/sourcespec2/config/configobj/validate.py similarity index 100% rename from sourcespec2/configobj/validate.py rename to sourcespec2/config/configobj/validate.py diff --git a/sourcespec2/config_files/configspec.conf b/sourcespec2/config/configspec.conf similarity index 100% rename from sourcespec2/config_files/configspec.conf rename to sourcespec2/config/configspec.conf diff --git a/sourcespec2/config_files/ssp_event.yaml b/sourcespec2/config/ssp_event.yaml similarity index 100% rename from sourcespec2/config_files/ssp_event.yaml rename to sourcespec2/config/ssp_event.yaml diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index 254eaae7..f9342076 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -29,8 +29,8 @@ from datetime import datetime from collections import defaultdict from . import __version__, __banner__ -from .configobj import ConfigObj -from .configobj.validate import Validator +from .config.configobj import ConfigObj +from .config.configobj.validate import Validator from .config import Config from .ssp_update_db import update_db_file @@ -266,7 +266,7 @@ def _read_config(config_file, configspec=None): def _parse_configspec(): configspec_file = os.path.join( - os.path.dirname(__file__), 'config_files', 'configspec.conf') + os.path.dirname(__file__), 'config', 'configspec.conf') return _read_config(configspec_file) @@ -578,7 +578,7 @@ def _init_traceid_map(traceid_map_file): def _write_sample_ssp_event_file(): ssp_event_file = 'ssp_event.yaml' src_path = os.path.join( - os.path.dirname(__file__), 'config_files', 'ssp_event.yaml') + os.path.dirname(__file__), 'config', 'ssp_event.yaml') dest_path = os.path.join('.', ssp_event_file) write_file = True if os.path.exists(dest_path): From 1463ab28314b754795b1fc24be31630301a41751 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 3 Jul 2024 16:56:10 +0200 Subject: [PATCH 05/73] Move library version check to its own module --- sourcespec2/config/__init__.py | 1 + sourcespec2/config/library_versions.py | 240 +++++++++++++++++++++++++ sourcespec2/ssp_setup.py | 217 ++-------------------- 3 files changed, 255 insertions(+), 203 deletions(-) create mode 100644 sourcespec2/config/library_versions.py diff --git a/sourcespec2/config/__init__.py b/sourcespec2/config/__init__.py index 99d4a06a..5ca4e1a7 100644 --- a/sourcespec2/config/__init__.py +++ b/sourcespec2/config/__init__.py @@ -10,3 +10,4 @@ (http://www.cecill.info/licences.en.html) """ from .config import Config # noqa +from .library_versions import library_versions # noqa diff --git a/sourcespec2/config/library_versions.py b/sourcespec2/config/library_versions.py new file mode 100644 index 00000000..6531ecd8 --- /dev/null +++ b/sourcespec2/config/library_versions.py @@ -0,0 +1,240 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Retrieve and check library versions for SourceSpec. + +:copyright: + 2012 Claudio Satriano + + 2013-2014 Claudio Satriano , + Emanuela Matrullo , + Agnes Chounet + + 2015-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import sys +import os +import contextlib +import warnings + + +# ---- Helper functions for cartopy feature download ---- +def _cartopy_download_gshhs(): + """ + Download GSHHS data for cartopy. + """ + # pylint: disable=import-outside-toplevel + from cartopy.io.shapereader import GSHHSShpDownloader as Downloader + from cartopy import config as cartopy_config + from pathlib import Path + gshhs_downloader = Downloader.from_config(('shapefiles', 'gshhs')) + format_dict = {'config': cartopy_config, 'scale': 'f', 'level': 1} + target_path = gshhs_downloader.target_path(format_dict) + if not os.path.exists(target_path): + sys.stdout.write( + 'Downloading GSHHS data for cartopy.\n' + 'This is needed only the first time you use cartopy and may take ' + 'a while...\n') + sys.stdout.flush() + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + try: + path = gshhs_downloader.path(format_dict) + except Exception: + sys.stderr.write( + '\nUnable to download data. ' + 'Check your internet connection.\n') + sys.exit(1) + sys.stdout.write(f'Done! Data cached to {Path(path).parents[1]}\n\n') + + +def _cartopy_download_borders(): + """ + Download borders data for cartopy. + + Inspired from + https://github.com/SciTools/cartopy/blob/main/tools/cartopy_feature_download.py + """ + # pylint: disable=import-outside-toplevel + from cartopy.io import Downloader + from cartopy import config as cartopy_config + from pathlib import Path + category = 'cultural' + name = 'admin_0_boundary_lines_land' + scales = ('10m', '50m') + for scale in scales: + downloader = Downloader.from_config(( + 'shapefiles', 'natural_earth', category, name, scale)) + format_dict = { + 'config': cartopy_config, 'category': category, 'name': name, + 'resolution': scale} + target_path = downloader.target_path(format_dict) + if not os.path.exists(target_path): + sys.stdout.write( + 'Downloading border data for cartopy...\n') + sys.stdout.flush() + with warnings.catch_warnings(): + warnings.simplefilter('ignore') + try: + path = downloader.path(format_dict) + except Exception: + sys.stderr.write( + '\nUnable to download data. ' + 'Check your internet connection.\n') + sys.exit(1) + sys.stdout.write( + f'Done! Data cached to {Path(path).parents[0]}\n\n') + + +class _LibraryVersions(): + """ + Check library versions and download data for cartopy. + + This is a private class, only the instance `library_versions` is + accessible from the outside. + """ + def __init__(self): + self.MAX_NUMPY_VERSION = (2, 0, 0) + self.MAX_MATPLOTLIB_VERSION = (3, 10, 0) + self.MIN_OBSPY_VERSION = (1, 2, 0) + self.MIN_CARTOPY_VERSION = (0, 21, 0) + self.MIN_NLLGRID_VERSION = (1, 4, 2) + self.OBSPY_VERSION = None + self.OBSPY_VERSION_STR = None + self.NUMPY_VERSION_STR = None + self.SCIPY_VERSION_STR = None + self.MATPLOTLIB_VERSION_STR = None + self.CARTOPY_VERSION_STR = None + self.PYTHON_VERSION_STR = None + # check base library versions and obspy version + # the other checks are called on demand + self.check_base_library_versions() + self.check_obspy_version() + + def check_base_library_versions(self): + """ + Check base library versions. + """ + # pylint: disable=import-outside-toplevel + self.PYTHON_VERSION_STR = '.'.join(map(str, sys.version_info[:3])) + import numpy + self.NUMPY_VERSION_STR = numpy.__version__ + numpy_version = tuple(map(int, self.NUMPY_VERSION_STR.split('.')[:3])) + if numpy_version >= self.MAX_NUMPY_VERSION: + max_numpy_version_str = '.'.join(map(str, self.MAX_NUMPY_VERSION)) + raise ImportError( + f'ERROR: Numpy >= {max_numpy_version_str} ' + 'is not yet supported. Please use a less recent version. ' + f'You have version: {self.NUMPY_VERSION_STR}' + ) + import scipy + self.SCIPY_VERSION_STR = scipy.__version__ + import matplotlib + self.MATPLOTLIB_VERSION_STR = matplotlib.__version__ + matplotlib_version = self.MATPLOTLIB_VERSION_STR.split('.')[:3] + matplotlib_version = tuple(map(int, matplotlib_version)) + if matplotlib_version >= self.MAX_MATPLOTLIB_VERSION: + max_matplotlib_version_str =\ + '.'.join(map(str, self.MAX_MATPLOTLIB_VERSION)) + raise ImportError( + f'ERROR: Matplotlib >= {max_matplotlib_version_str} ' + 'is not yet supported. Please use a less recent version' + f' You have version: {self.MATPLOTLIB_VERSION_STR}' + ) + + def check_obspy_version(self): + """ + Check ObsPy version. + """ + # pylint: disable=import-outside-toplevel + import obspy + self.OBSPY_VERSION_STR = obspy.__version__ + obspy_version = self.OBSPY_VERSION_STR.split('.')[:3] + # special case for "rc" versions: + obspy_version[2] = obspy_version[2].split('rc')[0] + obspy_version = tuple(map(int, obspy_version)) + with contextlib.suppress(IndexError): + # add half version number for development versions + # check if there is a fourth field in version string: + obspy_version = obspy_version[:2] + (obspy_version[2] + 0.5,) + if obspy_version < self.MIN_OBSPY_VERSION: + min_obspy_version_str = '.'.join(map(str, self.MIN_OBSPY_VERSION)) + raise ImportError( + f'ERROR: ObsPy >= {min_obspy_version_str} is required. ' + f'You have version: {self.OBSPY_VERSION_STR}' + ) + + def check_cartopy_version(self): + """ + Check cartopy version and download data if needed. + """ + try: + cartopy_ver = None + import cartopy # NOQA pylint: disable=import-outside-toplevel + self.CARTOPY_VERSION_STR = cartopy.__version__ + cartopy_ver = tuple(map(int, cartopy.__version__.split('.')[:3])) + if cartopy_ver < self.MIN_CARTOPY_VERSION: + raise ImportError + _cartopy_download_gshhs() + _cartopy_download_borders() + except ImportError as e: + cartopy_min_ver_str = '.'.join(map(str, self.MIN_CARTOPY_VERSION)) + msg = ( + f'\nPlease install cartopy >= {cartopy_min_ver_str} ' + 'to plot maps.\nHow to install: ' + 'https://scitools.org.uk/cartopy/docs/latest/installing.html\n' + '\nAlternatively, set "plot_station_map" to "False" ' + 'in config file.\n' + ) + if cartopy_ver is not None: + msg += ( + f'Installed cartopy version: {self.CARTOPY_VERSION_STR}.\n' + ) + raise ImportError(msg) from e + + def check_pyproj_version(self): + """ + Check pyproj version. + """ + # pylint: disable=import-outside-toplevel + try: + import pyproj # noqa pylint: disable=unused-import + except ImportError as e: + msg = '\nPlease install pyproj to plot maps.\n' + raise ImportError(msg) from e + + def check_nllgrid_version(self): + """ + Check nllgrid version. + """ + # pylint: disable=import-outside-toplevel + try: + nllgrid_ver = None + import nllgrid # NOQA + nllgrid_ver_str = nllgrid.__version__.split('+')[0] + nllgrid_ver = tuple(map(int, nllgrid_ver_str.split('.'))) + # nllgrid versions are sometimes X.Y, other times X.Y.Z + while len(nllgrid_ver) < 3: + nllgrid_ver = (*nllgrid_ver, 0) + if nllgrid_ver < self.MIN_NLLGRID_VERSION: + raise ImportError + except ImportError as e: + nllgrid_min_ver_str = '.'.join(map(str, self.MIN_NLLGRID_VERSION)) + msg = ( + f'\nPlease install nllgrid >= {nllgrid_min_ver_str} to use ' + 'NonLinLoc grids.\n' + 'How to install: https://github.com/claudiodsf/nllgrid\n' + ) + if nllgrid_ver is not None: + msg += f'Installed nllgrid version: {nllgrid.__version__}\n' + raise ImportError(msg) from e + + +# class instance exposed to the outside +try: + library_versions = _LibraryVersions() +except ImportError as _error: + sys.exit(_error) diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index f9342076..ec116218 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -24,11 +24,11 @@ import uuid import json import contextlib -import warnings from copy import copy from datetime import datetime from collections import defaultdict from . import __version__, __banner__ +from .config import library_versions from .config.configobj import ConfigObj from .config.configobj.validate import Validator from .config import Config @@ -46,7 +46,6 @@ IPSHELL = None # global variables -OS = os.name OLDLOGFILE = None LOGGER = None SSP_EXIT_CALLED = False @@ -55,190 +54,6 @@ # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ INSTR_CODES_VEL = ['H', 'L'] INSTR_CODES_ACC = ['N', ] -OBSPY_VERSION = None -OBSPY_VERSION_STR = None -NUMPY_VERSION_STR = None -SCIPY_VERSION_STR = None -MATPLOTLIB_VERSION_STR = None -CARTOPY_VERSION_STR = None -PYTHON_VERSION_STR = None - - -def _check_obspy_version(): - global OBSPY_VERSION, OBSPY_VERSION_STR # pylint: disable=global-statement - # check ObsPy version - # pylint: disable=import-outside-toplevel - import obspy - MIN_OBSPY_VERSION = (1, 2, 0) - OBSPY_VERSION_STR = obspy.__version__ - OBSPY_VERSION = OBSPY_VERSION_STR.split('.')[:3] - # special case for "rc" versions: - OBSPY_VERSION[2] = OBSPY_VERSION[2].split('rc')[0] - OBSPY_VERSION = tuple(map(int, OBSPY_VERSION)) - with contextlib.suppress(IndexError): - # add half version number for development versions - # check if there is a fourth field in version string: - OBSPY_VERSION = OBSPY_VERSION[:2] + (OBSPY_VERSION[2] + 0.5,) - if OBSPY_VERSION < MIN_OBSPY_VERSION: - MIN_OBSPY_VERSION_STR = '.'.join(map(str, MIN_OBSPY_VERSION)) - sys.stderr.write( - f'ERROR: ObsPy >= {MIN_OBSPY_VERSION_STR} is required. ' - f'You have version: {OBSPY_VERSION_STR}\n') - sys.exit(1) - - -def _cartopy_download_gshhs(): - """ - Download GSHHS data for cartopy. - """ - # pylint: disable=import-outside-toplevel - from cartopy.io.shapereader import GSHHSShpDownloader as Downloader - from cartopy import config as cartopy_config - from pathlib import Path - gshhs_downloader = Downloader.from_config(('shapefiles', 'gshhs')) - format_dict = {'config': cartopy_config, 'scale': 'f', 'level': 1} - target_path = gshhs_downloader.target_path(format_dict) - if not os.path.exists(target_path): - sys.stdout.write( - 'Downloading GSHHS data for cartopy.\n' - 'This is needed only the first time you use cartopy and may take ' - 'a while...\n') - sys.stdout.flush() - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - try: - path = gshhs_downloader.path(format_dict) - except Exception: - sys.stderr.write( - '\nUnable to download data. ' - 'Check your internet connection.\n') - sys.exit(1) - sys.stdout.write(f'Done! Data cached to {Path(path).parents[1]}\n\n') - - -def _cartopy_download_borders(): - """ - Download borders data for cartopy. - - Inspired from - https://github.com/SciTools/cartopy/blob/main/tools/cartopy_feature_download.py - """ - # pylint: disable=import-outside-toplevel - from cartopy.io import Downloader - from cartopy import config as cartopy_config - from pathlib import Path - category = 'cultural' - name = 'admin_0_boundary_lines_land' - scales = ('10m', '50m') - for scale in scales: - downloader = Downloader.from_config(( - 'shapefiles', 'natural_earth', category, name, scale)) - format_dict = { - 'config': cartopy_config, 'category': category, 'name': name, - 'resolution': scale} - target_path = downloader.target_path(format_dict) - if not os.path.exists(target_path): - sys.stdout.write( - 'Downloading border data for cartopy...\n') - sys.stdout.flush() - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - try: - path = downloader.path(format_dict) - except Exception: - sys.stderr.write( - '\nUnable to download data. ' - 'Check your internet connection.\n') - sys.exit(1) - sys.stdout.write( - f'Done! Data cached to {Path(path).parents[0]}\n\n') - - -def _check_cartopy_version(): - cartopy_min_ver = (0, 21, 0) - try: - cartopy_ver = None - import cartopy # NOQA pylint: disable=import-outside-toplevel - global CARTOPY_VERSION_STR # pylint: disable=global-statement - CARTOPY_VERSION_STR = cartopy.__version__ - cartopy_ver = tuple(map(int, cartopy.__version__.split('.')[:3])) - if cartopy_ver < cartopy_min_ver: - raise ImportError - _cartopy_download_gshhs() - _cartopy_download_borders() - except ImportError as e: - cartopy_min_ver_str = '.'.join(map(str, cartopy_min_ver)) - msg = ( - f'\nPlease install cartopy >= {cartopy_min_ver_str} to plot maps.' - '\nHow to install: ' - 'https://scitools.org.uk/cartopy/docs/latest/installing.html\n\n' - 'Alternatively, set "plot_station_map" to "False" ' - 'in config file.\n' - ) - if cartopy_ver is not None: - msg += f'Installed cartopy version: {CARTOPY_VERSION_STR}.\n' - raise ImportError(msg) from e - - -def _check_pyproj_version(): - # pylint: disable=import-outside-toplevel - try: - import pyproj # noqa pylint: disable=unused-import - except ImportError as e: - msg = '\nPlease install pyproj to plot maps.\n' - raise ImportError(msg) from e - - -def _check_nllgrid_version(): - # pylint: disable=import-outside-toplevel - nllgrid_min_ver = (1, 4, 2) - try: - nllgrid_ver = None - import nllgrid # NOQA - nllgrid_ver_str = nllgrid.__version__.split('+')[0] - nllgrid_ver = tuple(map(int, nllgrid_ver_str.split('.'))) - # nllgrid versions are sometimes X.Y, other times X.Y.Z - while len(nllgrid_ver) < 3: - nllgrid_ver = (*nllgrid_ver, 0) - if nllgrid_ver < nllgrid_min_ver: - raise ImportError - except ImportError as e: - nllgrid_min_ver_str = '.'.join(map(str, nllgrid_min_ver)) - msg = ( - f'\nPlease install nllgrid >= {nllgrid_min_ver_str} to use ' - 'NonLinLoc grids.\n' - 'How to install: https://github.com/claudiodsf/nllgrid\n' - ) - if nllgrid_ver is not None: - msg += f'Installed nllgrid version: {nllgrid.__version__}\n' - raise ImportError(msg) from e - - -def _check_library_versions(): - # pylint: disable=import-outside-toplevel - # pylint: disable=global-statement - global PYTHON_VERSION_STR - global NUMPY_VERSION_STR - global SCIPY_VERSION_STR - global MATPLOTLIB_VERSION_STR - PYTHON_VERSION_STR = '.'.join(map(str, sys.version_info[:3])) - import numpy - NUMPY_VERSION_STR = numpy.__version__ - import scipy - SCIPY_VERSION_STR = scipy.__version__ - import matplotlib - MATPLOTLIB_VERSION_STR = matplotlib.__version__ - MATPLOTLIB_VERSION = MATPLOTLIB_VERSION_STR.split('.')[:3] - MATPLOTLIB_VERSION = tuple(map(int, MATPLOTLIB_VERSION)) - MAX_MATPLOTLIB_VERSION = (3, 10, 0) - if MATPLOTLIB_VERSION >= MAX_MATPLOTLIB_VERSION: - MAX_MATPLOTLIB_VERSION_STR = '.'.join(map(str, MAX_MATPLOTLIB_VERSION)) - sys.stderr.write( - f'ERROR: Matplotlib >= {MAX_MATPLOTLIB_VERSION_STR} ' - 'is not yet supported. Please use a less recent version' - f' You have version: {MATPLOTLIB_VERSION_STR}\n' - ) - sys.exit(1) def _read_config(config_file, configspec=None): @@ -653,9 +468,6 @@ def configure(options, progname, config_overrides=None): extend those defined in the config file :return: A ``Config`` object with both command line and config options. """ - _check_obspy_version() - _check_library_versions() - configspec = _parse_configspec() if options.sampleconf: _write_sample_config(configspec, progname) @@ -803,20 +615,18 @@ def configure(options, progname, config_overrides=None): if config.plot_station_map: try: - _check_cartopy_version() - _check_pyproj_version() + library_versions.check_cartopy_version() + library_versions.check_pyproj_version() except ImportError as err: for msg in config.warnings: print(msg) - sys.stderr.write(str(err)) - sys.exit(1) + sys.exit(err) if config.NLL_time_dir is not None or config.NLL_model_dir is not None: try: - _check_nllgrid_version() + library_versions.check_nllgrid_version() except ImportError as err: - sys.stderr.write(str(err)) - sys.exit(1) + sys.exit(err) _init_instrument_codes(config) _init_traceid_map(config.traceid_mapping_file) @@ -986,13 +796,14 @@ def _log_debug_information(): uname = platform.uname() uname_str = f'{uname[0]} {uname[2]} {uname[4]}' LOGGER.debug(f'Platform: {uname_str}') - LOGGER.debug(f'Python version: {PYTHON_VERSION_STR}') - LOGGER.debug(f'ObsPy version: {OBSPY_VERSION_STR}') - LOGGER.debug(f'NumPy version: {NUMPY_VERSION_STR}') - LOGGER.debug(f'SciPy version: {SCIPY_VERSION_STR}') - LOGGER.debug(f'Matplotlib version: {MATPLOTLIB_VERSION_STR}') - if CARTOPY_VERSION_STR is not None: - LOGGER.debug(f'Cartopy version: {CARTOPY_VERSION_STR}') + lv = library_versions + LOGGER.debug(f'Python version: {lv.PYTHON_VERSION_STR}') + LOGGER.debug(f'ObsPy version: {lv.OBSPY_VERSION_STR}') + LOGGER.debug(f'NumPy version: {lv.NUMPY_VERSION_STR}') + LOGGER.debug(f'SciPy version: {lv.SCIPY_VERSION_STR}') + LOGGER.debug(f'Matplotlib version: {lv.MATPLOTLIB_VERSION_STR}') + if lv.CARTOPY_VERSION_STR is not None: + LOGGER.debug(f'Cartopy version: {lv.CARTOPY_VERSION_STR}') LOGGER.debug('Running arguments:') LOGGER.debug(' '.join(sys.argv)) From 67fef9245f166c64989d1803bcd8bd64a3d01add Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 3 Jul 2024 23:09:08 +0200 Subject: [PATCH 06/73] Move `configure()` to config.py --- sourcespec2/config/__init__.py | 2 +- sourcespec2/config/config.py | 606 +++++++++++++++++++++++ sourcespec2/source_model.py | 3 +- sourcespec2/source_spec.py | 5 +- sourcespec2/ssp_read_event_metadata.py | 12 +- sourcespec2/ssp_read_station_metadata.py | 6 +- sourcespec2/ssp_read_traces.py | 19 +- sourcespec2/ssp_setup.py | 595 ---------------------- 8 files changed, 631 insertions(+), 617 deletions(-) diff --git a/sourcespec2/config/__init__.py b/sourcespec2/config/__init__.py index 5ca4e1a7..2bcb1809 100644 --- a/sourcespec2/config/__init__.py +++ b/sourcespec2/config/__init__.py @@ -9,5 +9,5 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -from .config import Config # noqa +from .config import configure # noqa from .library_versions import library_versions # noqa diff --git a/sourcespec2/config/config.py b/sourcespec2/config/config.py index 121c06a3..3f15eb41 100644 --- a/sourcespec2/config/config.py +++ b/sourcespec2/config/config.py @@ -9,6 +9,23 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ +import os +import sys +import shutil +import uuid +import json +import contextlib +from copy import copy +from datetime import datetime +from collections import defaultdict +from .library_versions import library_versions +from .configobj import ConfigObj +from .configobj.validate import Validator +from ..ssp_update_db import update_db_file + +# TODO: remove these when the global config object will be implemented +INSTR_CODES_VEL = [] +INSTR_CODES_ACC = [] class Config(dict): @@ -27,3 +44,592 @@ def __getattr__(self, key): raise AttributeError(err) from err __setattr__ = __setitem__ + + +def _read_config(config_file, configspec=None): + kwargs = { + 'configspec': configspec, + 'file_error': True, + 'default_encoding': 'utf8' + } + if configspec is None: + kwargs.update({ + 'interpolation': False, + 'list_values': False, + '_inspec': True + }) + try: + config_obj = ConfigObj(config_file, **kwargs) + except IOError as err: + sys.stderr.write(f'{err}\n') + sys.exit(1) + except Exception as err: + sys.stderr.write(f'Unable to read "{config_file}": {err}\n') + sys.exit(1) + return config_obj + + +def _parse_configspec(): + configspec_file = os.path.join( + os.path.dirname(__file__), 'configspec.conf') + return _read_config(configspec_file) + + +def _write_sample_config(configspec, progname): + c = ConfigObj(configspec=configspec, default_encoding='utf8') + val = Validator() + c.validate(val) + c.defaults = [] + c.initial_comment = configspec.initial_comment + c.comments = configspec.comments + c.final_comment = configspec.final_comment + configfile = f'{progname}.conf' + write_file = True + if os.path.exists(configfile): + ans = input( + f'{configfile} already exists. Do you want to overwrite it? [y/N] ' + ) + write_file = ans in ['y', 'Y'] + if write_file: + with open(configfile, 'wb') as fp: + c.write(fp) + print(f'Sample config file written to: {configfile}') + note = """ +Note that the default config parameters are suited for a M<5 earthquake +recorded within ~100 km. Adjust `win_length`, `noise_pre_time`, and the +frequency bands (`bp_freqmin_*`, `bp_freqmax_*`, `freq1_*`, `freq2_*`) +according to your setup.""" + print(note) + + +def _update_config_file(config_file, configspec): + config_obj = _read_config(config_file, configspec) + val = Validator() + config_obj.validate(val) + mod_time = datetime.fromtimestamp(os.path.getmtime(config_file)) + mod_time_str = mod_time.strftime('%Y%m%d_%H%M%S') + config_file_old = f'{config_file}.{mod_time_str}' + ans = input( + f'Ok to update {config_file}? [y/N]\n' + f'(Old file will be saved as {config_file_old}) ' + ) + if ans not in ['y', 'Y']: + sys.exit(0) + config_new = ConfigObj(configspec=configspec, default_encoding='utf8') + config_new = _read_config(None, configspec) + config_new.validate(val) + config_new.defaults = [] + config_new.comments = configspec.comments + config_new.initial_comment = config_obj.initial_comment + config_new.final_comment = configspec.final_comment + for k, v in config_obj.items(): + if k not in config_new: + continue + # Fix for force_list(default=None) + if v == ['None', ]: + v = None + config_new[k] = v + migrate_options = { + 's_win_length': 'win_length', + 'traceids': 'traceid_mapping_file', + 'ignore_stations': 'ignore_traceids', + 'use_stations': 'use_traceids', + 'dataless': 'station_metadata', + 'clip_nmax': 'clip_max_percent', + 'PLOT_SHOW': 'plot_show', + 'PLOT_SAVE': 'plot_save', + 'PLOT_SAVE_FORMAT': 'plot_save_format', + 'vp': 'vp_source', + 'vs': 'vs_source', + 'rho': 'rho_source', + 'pre_p_time': 'noise_pre_time', + 'pre_s_time': 'signal_pre_time', + 'rps_from_focal_mechanism': 'rp_from_focal_mechanism', + 'paz': 'station_metadata', + 'pi_bsd_min_max': 'pi_ssd_min_max', + 'max_epi_dist': 'epi_dist_ranges' + } + for old_opt, new_opt in migrate_options.items(): + if old_opt in config_obj and config_obj[old_opt] != 'None': + # max_epi_dist needs to be converted to a list + if old_opt == 'max_epi_dist': + config_new[new_opt] = [0, config_obj[old_opt]] + else: + config_new[new_opt] = config_obj[old_opt] + shutil.copyfile(config_file, config_file_old) + with open(config_file, 'wb') as fp: + config_new.write(fp) + print(f'{config_file}: updated') + + +def _write_config(config_obj, progname, outdir): + if progname != 'source_spec': + return + configfile = f'{progname}.conf' + configfile = os.path.join(outdir, configfile) + if not os.path.exists(outdir): + os.makedirs(outdir) + with open(configfile, 'wb') as fp: + # create a copy of config_obj and remove the basemap API key + _tmp_config_obj = copy(config_obj) + _tmp_config_obj['plot_map_api_key'] = None + _tmp_config_obj.write(fp) + + +def _check_deprecated_config_options(config_obj): + deprecation_msgs = [] + if 's_win_length' in config_obj or 'noise_win_length' in config_obj: + deprecation_msgs.append( + '> "s_win_length" and "noise_win_length" config parameters ' + 'are no more\n' + ' supported. Both are replaced by "win_length".\n' + ) + if 'traceids' in config_obj: + deprecation_msgs.append( + '> "traceids" config parameter has been renamed to ' + '"traceid_mapping_file".\n' + ) + if 'ignore_stations' in config_obj or 'use_stations' in config_obj: + deprecation_msgs.append( + '> "ignore_stations" and "use_stations" config parameters ' + 'have been renamed to\n' + ' "ignore_traceids" and "use_traceids", respectively.\n' + ) + if 'dataless' in config_obj: + deprecation_msgs.append( + '> "dataless" config parameter has been renamed to ' + '"station_metadata".\n' + ) + if 'clip_nmax' in config_obj: + deprecation_msgs.append( + '> "clip_nmax" config parameter has been renamed to ' + '"clip_max_percent".\n' + ' Note that the new default is 5% (current value in your config ' + f'file: {config_obj["clip_nmax"]}%)\n' + ) + if 'trace_format' in config_obj: + deprecation_msgs.append( + '> "trace_format" config parameter is no more supported.\n' + ' Use "sensitivity" to manually specify how sensor sensitivity ' + 'should be computed.\n' + ) + if 'PLOT_SHOW' in config_obj: + deprecation_msgs.append( + '> "PLOT_SHOW" config parameter has been renamed to "plot_show".\n' + ) + if 'PLOT_SAVE' in config_obj: + deprecation_msgs.append( + '> "PLOT_SAVE" config parameter has been renamed to "plot_save".\n' + ) + if 'PLOT_SAVE_FORMAT' in config_obj: + deprecation_msgs.append( + '> "PLOT_SAVE_FORMAT" config parameter has been renamed to ' + '"plot_save_format".\n' + ) + if 'vp' in config_obj: + deprecation_msgs.append( + '> "vp" config parameter has been renamed to "vp_source".\n' + ) + if 'vs' in config_obj: + deprecation_msgs.append( + '> "vs" config parameter has been renamed to "vs_source".\n' + ) + if 'rho' in config_obj: + deprecation_msgs.append( + '> "rho" config parameter has been renamed to "rho_source".\n' + ) + if 'pre_p_time' in config_obj: + deprecation_msgs.append( + '> "pre_p_time" config parameter has been renamed to ' + '"noise_pre_time".\n' + ) + if 'pre_s_time' in config_obj: + deprecation_msgs.append( + '> "pre_s_time" config parameter has been renamed to ' + '"signal_pre_time".\n' + ) + if 'rps_from_focal_mechanism' in config_obj: + deprecation_msgs.append( + '> "rps_from_focal_mechanism" config parameter has been renamed ' + 'to "rp_from_focal_mechanism".\n' + ) + if 'paz' in config_obj: + deprecation_msgs.append( + '> "paz" config parameter has been removed and merged with ' + '"station_metadata".\n' + ) + if 'max_epi_dist' in config_obj: + deprecation_msgs.append( + '> "max_epi_dist" config parameter has been removed and replaced ' + 'by "epi_dist_ranges".\n' + ) + if 'max_freq_Er' in config_obj: + deprecation_msgs.append( + '> "max_freq_Er" config parameter has been removed and replaced ' + 'by "Er_freq_min_max".\n' + ) + if deprecation_msgs: + sys.stderr.write( + 'Error: your config file contains deprecated parameters:\n\n') + for msg in deprecation_msgs: + sys.stderr.write(msg) + if deprecation_msgs: + sys.stderr.write( + '\nPlease upgrade your config file manually or ' + 'via the "-U" option.\n' + ) + sys.exit(1) + + +def _init_plotting(plot_show): + # pylint: disable=import-outside-toplevel + import matplotlib.pyplot as plt + if not plot_show: + plt.switch_backend('Agg') + + +def _check_mandatory_config_params(config_obj): + mandatory_params = [ + 'p_arrival_tolerance', + 's_arrival_tolerance', + 'noise_pre_time', + 'signal_pre_time', + 'win_length', + 'taper_halfwidth', + 'spectral_smooth_width_decades', + 'bp_freqmin_acc', + 'bp_freqmax_acc', + 'bp_freqmin_shortp', + 'bp_freqmax_shortp', + 'bp_freqmin_broadb', + 'bp_freqmax_broadb', + 'freq1_acc', + 'freq2_acc', + 'freq1_shortp', + 'freq2_shortp', + 'freq1_broadb', + 'freq2_broadb', + 'rmsmin', + 'sn_min', + 'spectral_sn_min', + 'rpp', + 'rps', + 'geom_spread_n_exponent', + 'geom_spread_cutoff_distance', + 'f_weight', + 'weight', + 't_star_0', + 't_star_0_variability', + 'a', 'b', 'c', + 'ml_bp_freqmin', + 'ml_bp_freqmax', + 'n_sigma', + 'lower_percentage', + 'mid_percentage', + 'upper_percentage', + 'nIQR', + 'plot_spectra_maxrows', + 'plot_traces_maxrows', + 'plot_station_text_size' + ] + messages = [] + for par in mandatory_params: + if config_obj[par] is None: + msg = f'"{par}" is mandatory and cannot be None' + messages.append(msg) + if messages: + msg = '\n'.join(messages) + sys.exit(msg) + + +def _init_instrument_codes(config): + """ + Initialize instrument codes from config file. + """ + # SEED standard instrument codes: + # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ + config.INSTR_CODES_VEL = ['H', 'L'] + config.INSTR_CODES_ACC = ['N', ] + # User-defined instrument codes: + instr_code_acc_user = config.instrument_code_acceleration + instr_code_vel_user = config.instrument_code_velocity + # Remove user-defined instrument codes if they conflict + # with another instrument + with contextlib.suppress(ValueError): + config.INSTR_CODES_VEL.remove(instr_code_acc_user) + with contextlib.suppress(ValueError): + config.INSTR_CODES_ACC.remove(instr_code_vel_user) + # Add user-defined instrument codes + if instr_code_vel_user is not None: + config.INSTR_CODES_VEL.append(instr_code_vel_user) + if instr_code_acc_user is not None: + config.INSTR_CODES_ACC.append(instr_code_acc_user) + # TODO: remove these when the global config object will be implemented + global INSTR_CODES_VEL + global INSTR_CODES_ACC + INSTR_CODES_VEL = config.INSTR_CODES_VEL + INSTR_CODES_ACC = config.INSTR_CODES_ACC + + +def _init_traceid_map(config): + """ + Initialize trace ID map from file. + """ + config.TRACEID_MAP = None + if config.traceid_mapping_file is None: + return + try: + with open(config.traceid_mapping_file, 'r', encoding='utf-8') as fp: + config.TRACEID_MAP = json.loads(fp.read()) + except Exception: + sys.exit( + f'traceid mapping file "{config.traceid_map_file}" not found ' + 'or not in json format.\n') + + +def _write_sample_ssp_event_file(): + ssp_event_file = 'ssp_event.yaml' + src_path = os.path.join( + os.path.dirname(__file__), 'ssp_event.yaml') + dest_path = os.path.join('.', ssp_event_file) + write_file = True + if os.path.exists(dest_path): + ans = input( + f'{ssp_event_file} already exists. ' + 'Do you want to overwrite it? [y/N] ' + ) + write_file = ans in ['y', 'Y'] + if write_file: + shutil.copyfile(src_path, dest_path) + print(f'Sample SourceSpec Event File written to: {ssp_event_file}') + + +def _fix_and_expand_path(path): + """ + Fix any path issues and expand it. + + :param str path: Path specification + :return: The fixed and expanded path + :rtype: str + """ + fixed_path = os.path.normpath(path).split(os.sep) + fixed_path = os.path.join(*fixed_path) + if path.startswith(os.sep): + fixed_path = os.path.join(os.sep, fixed_path) + elif path.startswith('~'): + fixed_path = os.path.expanduser(fixed_path) + return fixed_path + + +def _float_list(input_list, max_length=None, accepted_values=None): + """ + Convert an input list to a list of floats. + + :param list input_list: Input list or None + :return: A list of floats or None + :rtype: list + """ + if input_list is None: + return None + if accepted_values is None: + accepted_values = [] + + def _parse_float(val): + val = None if val == 'None' else val + return val if val in accepted_values else float(val) + + try: + return [_parse_float(val) for val in input_list[:max_length]] + except ValueError as e: + raise ValueError('Cannot parse all values in list') from e + + +def _none_lenght(input_list): + """ + Return the length of input list, or 1 if input list is None + + :param list input_list: Input list or None + :return: List length or 1 + :rtype: int + """ + return 1 if input_list is None else len(input_list) + + +def configure(options, progname, config_overrides=None): + """ + Parse command line arguments and read config file. + + :param object options: An object containing command line options + :param str progname: The name of the program + :param dict config_overrides: A dictionary with parameters that override or + extend those defined in the config file + :return: A ``Config`` object with both command line and config options. + """ + configspec = _parse_configspec() + if options.sampleconf: + _write_sample_config(configspec, progname) + sys.exit(0) + if options.updateconf: + _update_config_file(options.updateconf, configspec) + sys.exit(0) + if options.updatedb: + update_db_file(options.updatedb) + sys.exit(0) + if options.samplesspevent: + _write_sample_ssp_event_file() + sys.exit(0) + + if options.config_file: + options.config_file = _fix_and_expand_path(options.config_file) + config_obj = _read_config(options.config_file, configspec) + + # Apply overrides + if config_overrides is not None: + try: + for key, value in config_overrides.items(): + config_obj[key] = value + except AttributeError as e: + raise ValueError('"config_override" must be a dict-like.') from e + + # Set to None all the 'None' strings + for key, value in config_obj.dict().items(): + if value == 'None': + config_obj[key] = None + + val = Validator() + test = config_obj.validate(val) + # test is: + # - True if everything is ok + # - False if no config value is provided + # - A dict if invalid values are present, with the invalid values as False + if isinstance(test, dict): + for entry in [e for e in test if not test[e]]: + sys.stderr.write( + f'Invalid value for "{entry}": "{config_obj[entry]}"\n') + sys.exit(1) + if not test: + sys.stderr.write('No configuration value present!\n') + sys.exit(1) + + _check_deprecated_config_options(config_obj) + _check_mandatory_config_params(config_obj) + + # Fix and expand paths in options + options.outdir = _fix_and_expand_path(options.outdir) + if options.trace_path: + # trace_path is a list + options.trace_path = [ + _fix_and_expand_path(path) for path in options.trace_path] + if options.qml_file: + options.qml_file = _fix_and_expand_path(options.qml_file) + if options.hypo_file: + options.hypo_file = _fix_and_expand_path(options.hypo_file) + if options.pick_file: + options.pick_file = _fix_and_expand_path(options.pick_file) + + # Create a 'no_evid_' subdir into outdir. + # The random hex string will make it sure that this name is unique + # It will be then renamed once an evid is available + hexstr = uuid.uuid4().hex + options.outdir = os.path.join(options.outdir, f'no_evid_{hexstr}') + _write_config(config_obj, progname, options.outdir) + + # Create a Config object + config = Config(config_obj.dict().copy()) + + # Add options to config: + config.options = options + + # Override station_metadata config option with command line option + if options.station_metadata is not None: + config.station_metadata = options.station_metadata + + # Additional config values + config.vertical_channel_codes = ['Z'] + config.horizontal_channel_codes_1 = ['N', 'R'] + config.horizontal_channel_codes_2 = ['E', 'T'] + msc = config.mis_oriented_channels + if msc is not None: + config.vertical_channel_codes.append(msc[0]) + config.horizontal_channel_codes_1.append(msc[1]) + config.horizontal_channel_codes_2.append(msc[2]) + + # Fix and expand paths in config + if config.database_file: + config.database_file = _fix_and_expand_path(config.database_file) + if config.traceid_mapping_file: + config.traceid_mapping_file = _fix_and_expand_path( + config.traceid_mapping_file) + if config.station_metadata: + config.station_metadata = _fix_and_expand_path(config.station_metadata) + if config.residuals_filepath: + config.residuals_filepath = _fix_and_expand_path( + config.residuals_filepath) + + # Parse force_list options into lists of float + try: + for param in [ + 'vp_source', 'vs_source', 'rho_source', 'layer_top_depths']: + config[param] = _float_list(config[param]) + except ValueError as msg: + sys.exit(f'Error parsing parameter "{param}": {msg}') + n_vp_source = _none_lenght(config.vp_source) + n_vs_source = _none_lenght(config.vs_source) + n_rho_source = _none_lenght(config.rho_source) + n_layer_top_depths = _none_lenght(config.layer_top_depths) + try: + assert n_vp_source == n_vs_source == n_rho_source == n_layer_top_depths + except AssertionError: + sys.exit( + 'Error: "vp_source", "vs_source", "rho_source", and ' + '"layer_top_depths" must have the same length.' + ) + + # Check the Er_freq_range parameter + if config.Er_freq_range is None: + config.Er_freq_range = [None, None] + try: + config.Er_freq_range = _float_list( + config.Er_freq_range, max_length=2, + accepted_values=[None, 'noise']) + except ValueError as msg: + sys.exit(f'Error parsing parameter "Er_freq_range": {msg}') + + # A list of warnings to be issued when logger is set up + config.warnings = [] + + if config.html_report: + if not config.plot_save: + msg = ( + 'The "html_report" option is selected but "plot_save" ' + 'is "False". HTML report will have no plots.') + config.warnings.append(msg) + if config.plot_save_format not in ['png', 'svg']: + msg = ( + 'The "html_report" option is selected but "plot_save_format" ' + 'is not "png" or "svg". HTML report will have no plots.') + config.warnings.append(msg) + + if config.plot_station_map: + try: + library_versions.check_cartopy_version() + library_versions.check_pyproj_version() + except ImportError as err: + for msg in config.warnings: + print(msg) + sys.exit(err) + + if config.NLL_time_dir is not None or config.NLL_model_dir is not None: + try: + library_versions.check_nllgrid_version() + except ImportError as err: + sys.exit(err) + + _init_instrument_codes(config) + _init_traceid_map(config) + _init_plotting(config.plot_show) + # Create a dict to store figure paths + config.figures = defaultdict(list) + # store the absolute path of the current working directory + config.workdir = os.getcwd() + return config diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index c61efaf9..1478fbe8 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -84,7 +84,8 @@ def main(): # Lazy-import modules for speed from .ssp_parse_arguments import parse_args options = parse_args(progname='source_model') - from .ssp_setup import configure, ssp_exit + from .config import configure + from .ssp_setup import ssp_exit plot_show = bool(options.plot) conf_overrides = { 'plot_show': plot_show, diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 55fc1851..91603c35 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -25,10 +25,11 @@ def main(): options = parse_args(progname='source_spec') # Setup stage + from .config import configure + config = configure(options, progname='source_spec') from .ssp_setup import ( - configure, move_outdir, remove_old_outdir, setup_logging, + move_outdir, remove_old_outdir, setup_logging, save_config, ssp_exit) - config = configure(options, progname='source_spec') setup_logging(config) from .ssp_read_traces import read_traces diff --git a/sourcespec2/ssp_read_event_metadata.py b/sourcespec2/ssp_read_event_metadata.py index 44024ec0..07bbc20c 100644 --- a/sourcespec2/ssp_read_event_metadata.py +++ b/sourcespec2/ssp_read_event_metadata.py @@ -20,7 +20,7 @@ import yaml from obspy import UTCDateTime from obspy import read_events -from .ssp_setup import ssp_exit, TRACEID_MAP +from .ssp_setup import ssp_exit from .ssp_event import SSPEvent from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -505,7 +505,7 @@ def _is_hypo71_picks(pick_file): raise TypeError(f'{pick_file}: Not a hypo71 phase file') -def _correct_station_name(station): +def _correct_station_name(station, config): """ Correct station name, based on a traceid map. @@ -513,16 +513,16 @@ def _correct_station_name(station): :return: corrected station name """ - if TRACEID_MAP is None: + if config.TRACEID_MAP is None: return station # get all the keys containing station name in it - keys = [key for key in TRACEID_MAP if station == key.split('.')[1]] + keys = [key for key in config.TRACEID_MAP if station == key.split('.')[1]] # then take just the first one try: key = keys[0] except IndexError: return station - traceid = TRACEID_MAP[key] + traceid = config.TRACEID_MAP[key] return traceid.split('.')[1] @@ -560,7 +560,7 @@ def parse_hypo71_picks(config): continue pick = SSPPick() pick.station = line[:4].strip() - pick.station = _correct_station_name(pick.station) + pick.station = _correct_station_name(pick.station, config) pick.flag = line[4:5] pick.phase = line[5:6] pick.polarity = line[6:7] diff --git a/sourcespec2/ssp_read_station_metadata.py b/sourcespec2/ssp_read_station_metadata.py index b63e772e..811b4851 100644 --- a/sourcespec2/ssp_read_station_metadata.py +++ b/sourcespec2/ssp_read_station_metadata.py @@ -15,7 +15,7 @@ import logging from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response -from .ssp_setup import INSTR_CODES_VEL, INSTR_CODES_ACC +from .config.config import INSTR_CODES_VEL, INSTR_CODES_ACC logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -32,7 +32,7 @@ class PAZ(): input_units = None linenum = None - def __init__(self, file=None): + def __init__(self, file=None, INST_CODES_VEL=None, INST_CODES_ACC=None): """ Init PAZ object. @@ -41,6 +41,8 @@ def __init__(self, file=None): """ if file is not None: self._read(file) + self.INSTR_CODES_VEL = INST_CODES_VEL + self.INSTR_CODES_ACC = INST_CODES_ACC def __str__(self): return ( diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index feb52b3c..9e461560 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -26,8 +26,7 @@ from obspy import read from obspy.core import Stream from obspy.core.util import AttribDict -from .ssp_setup import ( - ssp_exit, INSTR_CODES_VEL, INSTR_CODES_ACC, TRACEID_MAP) +from .ssp_setup import ssp_exit from .ssp_util import MediumProperties from .ssp_read_station_metadata import ( read_station_metadata, PAZ) @@ -41,11 +40,11 @@ # TRACE MANIPULATION ---------------------------------------------------------- -def _correct_traceid(trace): - if TRACEID_MAP is None: +def _correct_traceid(trace, config): + if config.TRACEID_MAP is None: return with contextlib.suppress(KeyError): - traceid = TRACEID_MAP[trace.get_id()] + traceid = config.TRACEID_MAP[trace.get_id()] net, sta, loc, chan = traceid.split('.') trace.stats.network = net trace.stats.station = sta @@ -53,7 +52,7 @@ def _correct_traceid(trace): trace.stats.channel = chan -def _add_instrtype(trace): +def _add_instrtype(trace, config): """Add instrtype to trace.""" instrtype = None band_code = None @@ -64,14 +63,14 @@ def _add_instrtype(trace): if len(chan) > 2: band_code = chan[0] instr_code = chan[1] - if instr_code in INSTR_CODES_VEL: + if instr_code in config.INSTR_CODES_VEL: # SEED standard band codes from higher to lower sampling rate # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ if band_code in ['G', 'D', 'E', 'S']: instrtype = 'shortp' if band_code in ['F', 'C', 'H', 'B']: instrtype = 'broadb' - if instr_code in INSTR_CODES_ACC: + if instr_code in config.INSTR_CODES_ACC: instrtype = 'acc' if instrtype is None: # Let's see if there is an instrument name in SAC header (ISNet format) @@ -343,9 +342,9 @@ def _read_trace_files(config, inventory, ssp_event, picks): if (config.options.station is not None and trace.stats.station != config.options.station): continue - _correct_traceid(trace) + _correct_traceid(trace, config) try: - _add_instrtype(trace) + _add_instrtype(trace, config) _add_inventory(trace, inventory, config) _check_instrtype(trace) _add_coords(trace) diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index ec116218..92f86219 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -21,18 +21,10 @@ import shutil import logging import signal -import uuid -import json import contextlib -from copy import copy from datetime import datetime -from collections import defaultdict from . import __version__, __banner__ from .config import library_versions -from .config.configobj import ConfigObj -from .config.configobj.validate import Validator -from .config import Config -from .ssp_update_db import update_db_file # define ipshell(), if possible # note: ANSI colors do not work on Windows standard terminal @@ -49,593 +41,6 @@ OLDLOGFILE = None LOGGER = None SSP_EXIT_CALLED = False -TRACEID_MAP = None -# SEED standard instrument codes: -# https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ -INSTR_CODES_VEL = ['H', 'L'] -INSTR_CODES_ACC = ['N', ] - - -def _read_config(config_file, configspec=None): - kwargs = { - 'configspec': configspec, - 'file_error': True, - 'default_encoding': 'utf8' - } - if configspec is None: - kwargs.update({ - 'interpolation': False, - 'list_values': False, - '_inspec': True - }) - try: - config_obj = ConfigObj(config_file, **kwargs) - except IOError as err: - sys.stderr.write(f'{err}\n') - sys.exit(1) - except Exception as err: - sys.stderr.write(f'Unable to read "{config_file}": {err}\n') - sys.exit(1) - return config_obj - - -def _parse_configspec(): - configspec_file = os.path.join( - os.path.dirname(__file__), 'config', 'configspec.conf') - return _read_config(configspec_file) - - -def _write_sample_config(configspec, progname): - c = ConfigObj(configspec=configspec, default_encoding='utf8') - val = Validator() - c.validate(val) - c.defaults = [] - c.initial_comment = configspec.initial_comment - c.comments = configspec.comments - c.final_comment = configspec.final_comment - configfile = f'{progname}.conf' - write_file = True - if os.path.exists(configfile): - ans = input( - f'{configfile} already exists. Do you want to overwrite it? [y/N] ' - ) - write_file = ans in ['y', 'Y'] - if write_file: - with open(configfile, 'wb') as fp: - c.write(fp) - print(f'Sample config file written to: {configfile}') - note = """ -Note that the default config parameters are suited for a M<5 earthquake -recorded within ~100 km. Adjust `win_length`, `noise_pre_time`, and the -frequency bands (`bp_freqmin_*`, `bp_freqmax_*`, `freq1_*`, `freq2_*`) -according to your setup.""" - print(note) - - -def _update_config_file(config_file, configspec): - config_obj = _read_config(config_file, configspec) - val = Validator() - config_obj.validate(val) - mod_time = datetime.fromtimestamp(os.path.getmtime(config_file)) - mod_time_str = mod_time.strftime('%Y%m%d_%H%M%S') - config_file_old = f'{config_file}.{mod_time_str}' - ans = input( - f'Ok to update {config_file}? [y/N]\n' - f'(Old file will be saved as {config_file_old}) ' - ) - if ans not in ['y', 'Y']: - sys.exit(0) - config_new = ConfigObj(configspec=configspec, default_encoding='utf8') - config_new = _read_config(None, configspec) - config_new.validate(val) - config_new.defaults = [] - config_new.comments = configspec.comments - config_new.initial_comment = config_obj.initial_comment - config_new.final_comment = configspec.final_comment - for k, v in config_obj.items(): - if k not in config_new: - continue - # Fix for force_list(default=None) - if v == ['None', ]: - v = None - config_new[k] = v - migrate_options = { - 's_win_length': 'win_length', - 'traceids': 'traceid_mapping_file', - 'ignore_stations': 'ignore_traceids', - 'use_stations': 'use_traceids', - 'dataless': 'station_metadata', - 'clip_nmax': 'clip_max_percent', - 'PLOT_SHOW': 'plot_show', - 'PLOT_SAVE': 'plot_save', - 'PLOT_SAVE_FORMAT': 'plot_save_format', - 'vp': 'vp_source', - 'vs': 'vs_source', - 'rho': 'rho_source', - 'pre_p_time': 'noise_pre_time', - 'pre_s_time': 'signal_pre_time', - 'rps_from_focal_mechanism': 'rp_from_focal_mechanism', - 'paz': 'station_metadata', - 'pi_bsd_min_max': 'pi_ssd_min_max', - 'max_epi_dist': 'epi_dist_ranges' - } - for old_opt, new_opt in migrate_options.items(): - if old_opt in config_obj and config_obj[old_opt] != 'None': - # max_epi_dist needs to be converted to a list - if old_opt == 'max_epi_dist': - config_new[new_opt] = [0, config_obj[old_opt]] - else: - config_new[new_opt] = config_obj[old_opt] - shutil.copyfile(config_file, config_file_old) - with open(config_file, 'wb') as fp: - config_new.write(fp) - print(f'{config_file}: updated') - - -def _write_config(config_obj, progname, outdir): - if progname != 'source_spec': - return - configfile = f'{progname}.conf' - configfile = os.path.join(outdir, configfile) - if not os.path.exists(outdir): - os.makedirs(outdir) - with open(configfile, 'wb') as fp: - # create a copy of config_obj and remove the basemap API key - _tmp_config_obj = copy(config_obj) - _tmp_config_obj['plot_map_api_key'] = None - _tmp_config_obj.write(fp) - - -def _check_deprecated_config_options(config_obj): - deprecation_msgs = [] - if 's_win_length' in config_obj or 'noise_win_length' in config_obj: - deprecation_msgs.append( - '> "s_win_length" and "noise_win_length" config parameters ' - 'are no more\n' - ' supported. Both are replaced by "win_length".\n' - ) - if 'traceids' in config_obj: - deprecation_msgs.append( - '> "traceids" config parameter has been renamed to ' - '"traceid_mapping_file".\n' - ) - if 'ignore_stations' in config_obj or 'use_stations' in config_obj: - deprecation_msgs.append( - '> "ignore_stations" and "use_stations" config parameters ' - 'have been renamed to\n' - ' "ignore_traceids" and "use_traceids", respectively.\n' - ) - if 'dataless' in config_obj: - deprecation_msgs.append( - '> "dataless" config parameter has been renamed to ' - '"station_metadata".\n' - ) - if 'clip_nmax' in config_obj: - deprecation_msgs.append( - '> "clip_nmax" config parameter has been renamed to ' - '"clip_max_percent".\n' - ' Note that the new default is 5% (current value in your config ' - f'file: {config_obj["clip_nmax"]}%)\n' - ) - if 'trace_format' in config_obj: - deprecation_msgs.append( - '> "trace_format" config parameter is no more supported.\n' - ' Use "sensitivity" to manually specify how sensor sensitivity ' - 'should be computed.\n' - ) - if 'PLOT_SHOW' in config_obj: - deprecation_msgs.append( - '> "PLOT_SHOW" config parameter has been renamed to "plot_show".\n' - ) - if 'PLOT_SAVE' in config_obj: - deprecation_msgs.append( - '> "PLOT_SAVE" config parameter has been renamed to "plot_save".\n' - ) - if 'PLOT_SAVE_FORMAT' in config_obj: - deprecation_msgs.append( - '> "PLOT_SAVE_FORMAT" config parameter has been renamed to ' - '"plot_save_format".\n' - ) - if 'vp' in config_obj: - deprecation_msgs.append( - '> "vp" config parameter has been renamed to "vp_source".\n' - ) - if 'vs' in config_obj: - deprecation_msgs.append( - '> "vs" config parameter has been renamed to "vs_source".\n' - ) - if 'rho' in config_obj: - deprecation_msgs.append( - '> "rho" config parameter has been renamed to "rho_source".\n' - ) - if 'pre_p_time' in config_obj: - deprecation_msgs.append( - '> "pre_p_time" config parameter has been renamed to ' - '"noise_pre_time".\n' - ) - if 'pre_s_time' in config_obj: - deprecation_msgs.append( - '> "pre_s_time" config parameter has been renamed to ' - '"signal_pre_time".\n' - ) - if 'rps_from_focal_mechanism' in config_obj: - deprecation_msgs.append( - '> "rps_from_focal_mechanism" config parameter has been renamed ' - 'to "rp_from_focal_mechanism".\n' - ) - if 'paz' in config_obj: - deprecation_msgs.append( - '> "paz" config parameter has been removed and merged with ' - '"station_metadata".\n' - ) - if 'max_epi_dist' in config_obj: - deprecation_msgs.append( - '> "max_epi_dist" config parameter has been removed and replaced ' - 'by "epi_dist_ranges".\n' - ) - if 'max_freq_Er' in config_obj: - deprecation_msgs.append( - '> "max_freq_Er" config parameter has been removed and replaced ' - 'by "Er_freq_min_max".\n' - ) - if deprecation_msgs: - sys.stderr.write( - 'Error: your config file contains deprecated parameters:\n\n') - for msg in deprecation_msgs: - sys.stderr.write(msg) - if deprecation_msgs: - sys.stderr.write( - '\nPlease upgrade your config file manually or ' - 'via the "-U" option.\n' - ) - sys.exit(1) - - -def _init_plotting(plot_show): - # pylint: disable=import-outside-toplevel - import matplotlib.pyplot as plt - if not plot_show: - plt.switch_backend('Agg') - - -def _check_mandatory_config_params(config_obj): - mandatory_params = [ - 'p_arrival_tolerance', - 's_arrival_tolerance', - 'noise_pre_time', - 'signal_pre_time', - 'win_length', - 'taper_halfwidth', - 'spectral_smooth_width_decades', - 'bp_freqmin_acc', - 'bp_freqmax_acc', - 'bp_freqmin_shortp', - 'bp_freqmax_shortp', - 'bp_freqmin_broadb', - 'bp_freqmax_broadb', - 'freq1_acc', - 'freq2_acc', - 'freq1_shortp', - 'freq2_shortp', - 'freq1_broadb', - 'freq2_broadb', - 'rmsmin', - 'sn_min', - 'spectral_sn_min', - 'rpp', - 'rps', - 'geom_spread_n_exponent', - 'geom_spread_cutoff_distance', - 'f_weight', - 'weight', - 't_star_0', - 't_star_0_variability', - 'a', 'b', 'c', - 'ml_bp_freqmin', - 'ml_bp_freqmax', - 'n_sigma', - 'lower_percentage', - 'mid_percentage', - 'upper_percentage', - 'nIQR', - 'plot_spectra_maxrows', - 'plot_traces_maxrows', - 'plot_station_text_size' - ] - messages = [] - for par in mandatory_params: - if config_obj[par] is None: - msg = f'"{par}" is mandatory and cannot be None' - messages.append(msg) - if messages: - msg = '\n'.join(messages) - sys.stderr.write(msg + '\n') - ssp_exit(1) - - -def _init_instrument_codes(config): - """ - Initialize instrument codes from config file. - """ - # User-defined instrument codes: - instr_code_acc_user = config.instrument_code_acceleration - instr_code_vel_user = config.instrument_code_velocity - # Remove user-defined instrument codes if they conflict - # with another instrument - with contextlib.suppress(ValueError): - INSTR_CODES_VEL.remove(instr_code_acc_user) - with contextlib.suppress(ValueError): - INSTR_CODES_ACC.remove(instr_code_vel_user) - # Add user-defined instrument codes - if instr_code_vel_user is not None: - INSTR_CODES_VEL.append(instr_code_vel_user) - if instr_code_acc_user is not None: - INSTR_CODES_ACC.append(instr_code_acc_user) - - -def _init_traceid_map(traceid_map_file): - """ - Initialize trace ID map from file. - """ - global TRACEID_MAP # pylint: disable=global-statement - if traceid_map_file is None: - return - try: - with open(traceid_map_file, 'r', encoding='utf-8') as fp: - TRACEID_MAP = json.loads(fp.read()) - except Exception: - sys.stderr.write( - f'traceid mapping file "{traceid_map_file}" not found ' - 'or not in json format.\n') - ssp_exit(1) - - -def _write_sample_ssp_event_file(): - ssp_event_file = 'ssp_event.yaml' - src_path = os.path.join( - os.path.dirname(__file__), 'config', 'ssp_event.yaml') - dest_path = os.path.join('.', ssp_event_file) - write_file = True - if os.path.exists(dest_path): - ans = input( - f'{ssp_event_file} already exists. ' - 'Do you want to overwrite it? [y/N] ' - ) - write_file = ans in ['y', 'Y'] - if write_file: - shutil.copyfile(src_path, dest_path) - print(f'Sample SourceSpec Event File written to: {ssp_event_file}') - - -def _fix_and_expand_path(path): - """ - Fix any path issues and expand it. - - :param str path: Path specification - :return: The fixed and expanded path - :rtype: str - """ - fixed_path = os.path.normpath(path).split(os.sep) - fixed_path = os.path.join(*fixed_path) - if path.startswith(os.sep): - fixed_path = os.path.join(os.sep, fixed_path) - elif path.startswith('~'): - fixed_path = os.path.expanduser(fixed_path) - return fixed_path - - -def _float_list(input_list, max_length=None, accepted_values=None): - """ - Convert an input list to a list of floats. - - :param list input_list: Input list or None - :return: A list of floats or None - :rtype: list - """ - if input_list is None: - return None - if accepted_values is None: - accepted_values = [] - - def _parse_float(val): - val = None if val == 'None' else val - return val if val in accepted_values else float(val) - - try: - return [_parse_float(val) for val in input_list[:max_length]] - except ValueError as e: - raise ValueError('Cannot parse all values in list') from e - - -def _none_lenght(input_list): - """ - Return the length of input list, or 1 if input list is None - - :param list input_list: Input list or None - :return: List length or 1 - :rtype: int - """ - return 1 if input_list is None else len(input_list) - - -def configure(options, progname, config_overrides=None): - """ - Parse command line arguments and read config file. - - :param object options: An object containing command line options - :param str progname: The name of the program - :param dict config_overrides: A dictionary with parameters that override or - extend those defined in the config file - :return: A ``Config`` object with both command line and config options. - """ - configspec = _parse_configspec() - if options.sampleconf: - _write_sample_config(configspec, progname) - sys.exit(0) - if options.updateconf: - _update_config_file(options.updateconf, configspec) - sys.exit(0) - if options.updatedb: - update_db_file(options.updatedb) - sys.exit(0) - if options.samplesspevent: - _write_sample_ssp_event_file() - sys.exit(0) - - if options.config_file: - options.config_file = _fix_and_expand_path(options.config_file) - config_obj = _read_config(options.config_file, configspec) - - # Apply overrides - if config_overrides is not None: - try: - for key, value in config_overrides.items(): - config_obj[key] = value - except AttributeError as e: - raise ValueError('"config_override" must be a dict-like.') from e - - # Set to None all the 'None' strings - for key, value in config_obj.dict().items(): - if value == 'None': - config_obj[key] = None - - val = Validator() - test = config_obj.validate(val) - # test is: - # - True if everything is ok - # - False if no config value is provided - # - A dict if invalid values are present, with the invalid values as False - if isinstance(test, dict): - for entry in [e for e in test if not test[e]]: - sys.stderr.write( - f'Invalid value for "{entry}": "{config_obj[entry]}"\n') - sys.exit(1) - if not test: - sys.stderr.write('No configuration value present!\n') - sys.exit(1) - - _check_deprecated_config_options(config_obj) - _check_mandatory_config_params(config_obj) - - # Fix and expand paths in options - options.outdir = _fix_and_expand_path(options.outdir) - if options.trace_path: - # trace_path is a list - options.trace_path = [ - _fix_and_expand_path(path) for path in options.trace_path] - if options.qml_file: - options.qml_file = _fix_and_expand_path(options.qml_file) - if options.hypo_file: - options.hypo_file = _fix_and_expand_path(options.hypo_file) - if options.pick_file: - options.pick_file = _fix_and_expand_path(options.pick_file) - - # Create a 'no_evid_' subdir into outdir. - # The random hex string will make it sure that this name is unique - # It will be then renamed once an evid is available - hexstr = uuid.uuid4().hex - options.outdir = os.path.join(options.outdir, f'no_evid_{hexstr}') - _write_config(config_obj, progname, options.outdir) - - # Create a Config object - config = Config(config_obj.dict().copy()) - - # Add options to config: - config.options = options - - # Override station_metadata config option with command line option - if options.station_metadata is not None: - config.station_metadata = options.station_metadata - - # Additional config values - config.vertical_channel_codes = ['Z'] - config.horizontal_channel_codes_1 = ['N', 'R'] - config.horizontal_channel_codes_2 = ['E', 'T'] - msc = config.mis_oriented_channels - if msc is not None: - config.vertical_channel_codes.append(msc[0]) - config.horizontal_channel_codes_1.append(msc[1]) - config.horizontal_channel_codes_2.append(msc[2]) - - # Fix and expand paths in config - if config.database_file: - config.database_file = _fix_and_expand_path(config.database_file) - if config.traceid_mapping_file: - config.traceid_mapping_file = _fix_and_expand_path( - config.traceid_mapping_file) - if config.station_metadata: - config.station_metadata = _fix_and_expand_path(config.station_metadata) - if config.residuals_filepath: - config.residuals_filepath = _fix_and_expand_path( - config.residuals_filepath) - - # Parse force_list options into lists of float - try: - for param in [ - 'vp_source', 'vs_source', 'rho_source', 'layer_top_depths']: - config[param] = _float_list(config[param]) - except ValueError as msg: - sys.exit(f'Error parsing parameter "{param}": {msg}') - n_vp_source = _none_lenght(config.vp_source) - n_vs_source = _none_lenght(config.vs_source) - n_rho_source = _none_lenght(config.rho_source) - n_layer_top_depths = _none_lenght(config.layer_top_depths) - try: - assert n_vp_source == n_vs_source == n_rho_source == n_layer_top_depths - except AssertionError: - sys.exit( - 'Error: "vp_source", "vs_source", "rho_source", and ' - '"layer_top_depths" must have the same length.' - ) - - # Check the Er_freq_range parameter - if config.Er_freq_range is None: - config.Er_freq_range = [None, None] - try: - config.Er_freq_range = _float_list( - config.Er_freq_range, max_length=2, - accepted_values=[None, 'noise']) - except ValueError as msg: - sys.exit(f'Error parsing parameter "Er_freq_range": {msg}') - - # A list of warnings to be issued when logger is set up - config.warnings = [] - - if config.html_report: - if not config.plot_save: - msg = ( - 'The "html_report" option is selected but "plot_save" ' - 'is "False". HTML report will have no plots.') - config.warnings.append(msg) - if config.plot_save_format not in ['png', 'svg']: - msg = ( - 'The "html_report" option is selected but "plot_save_format" ' - 'is not "png" or "svg". HTML report will have no plots.') - config.warnings.append(msg) - - if config.plot_station_map: - try: - library_versions.check_cartopy_version() - library_versions.check_pyproj_version() - except ImportError as err: - for msg in config.warnings: - print(msg) - sys.exit(err) - - if config.NLL_time_dir is not None or config.NLL_model_dir is not None: - try: - library_versions.check_nllgrid_version() - except ImportError as err: - sys.exit(err) - - _init_instrument_codes(config) - _init_traceid_map(config.traceid_mapping_file) - _init_plotting(config.plot_show) - # Create a dict to store figure paths - config.figures = defaultdict(list) - # store the absolute path of the current working directory - config.workdir = os.getcwd() - return config def save_config(config): From 81ee0779a09cd03306659df7ef138c509b3ea5b8 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Thu, 4 Jul 2024 11:05:28 +0200 Subject: [PATCH 07/73] Make it possible to call configure() without any arguments --- sourcespec2/config/config.py | 61 ++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/sourcespec2/config/config.py b/sourcespec2/config/config.py index 3f15eb41..f590cdf6 100644 --- a/sourcespec2/config/config.py +++ b/sourcespec2/config/config.py @@ -11,6 +11,7 @@ """ import os import sys +import types import shutil import uuid import json @@ -46,7 +47,7 @@ def __getattr__(self, key): __setattr__ = __setitem__ -def _read_config(config_file, configspec=None): +def _read_config_file(config_file, configspec=None): kwargs = { 'configspec': configspec, 'file_error': True, @@ -72,17 +73,22 @@ def _read_config(config_file, configspec=None): def _parse_configspec(): configspec_file = os.path.join( os.path.dirname(__file__), 'configspec.conf') - return _read_config(configspec_file) + return _read_config_file(configspec_file) -def _write_sample_config(configspec, progname): - c = ConfigObj(configspec=configspec, default_encoding='utf8') +def _get_default_config_obj(configspec): + config_obj = ConfigObj(configspec=configspec, default_encoding='utf8') val = Validator() - c.validate(val) - c.defaults = [] - c.initial_comment = configspec.initial_comment - c.comments = configspec.comments - c.final_comment = configspec.final_comment + config_obj.validate(val) + config_obj.defaults = [] + config_obj.initial_comment = configspec.initial_comment + config_obj.comments = configspec.comments + config_obj.final_comment = configspec.final_comment + return config_obj + + +def _write_sample_config(configspec, progname): + config_obj = _get_default_config_obj(configspec) configfile = f'{progname}.conf' write_file = True if os.path.exists(configfile): @@ -92,7 +98,7 @@ def _write_sample_config(configspec, progname): write_file = ans in ['y', 'Y'] if write_file: with open(configfile, 'wb') as fp: - c.write(fp) + config_obj.write(fp) print(f'Sample config file written to: {configfile}') note = """ Note that the default config parameters are suited for a M<5 earthquake @@ -103,7 +109,7 @@ def _write_sample_config(configspec, progname): def _update_config_file(config_file, configspec): - config_obj = _read_config(config_file, configspec) + config_obj = _read_config_file(config_file, configspec) val = Validator() config_obj.validate(val) mod_time = datetime.fromtimestamp(os.path.getmtime(config_file)) @@ -116,7 +122,7 @@ def _update_config_file(config_file, configspec): if ans not in ['y', 'Y']: sys.exit(0) config_new = ConfigObj(configspec=configspec, default_encoding='utf8') - config_new = _read_config(None, configspec) + config_new = _read_config_file(None, configspec) config_new.validate(val) config_new.defaults = [] config_new.comments = configspec.comments @@ -455,7 +461,7 @@ def _none_lenght(input_list): return 1 if input_list is None else len(input_list) -def configure(options, progname, config_overrides=None): +def configure(options=None, progname='source_spec', config_overrides=None): """ Parse command line arguments and read config file. @@ -465,23 +471,28 @@ def configure(options, progname, config_overrides=None): extend those defined in the config file :return: A ``Config`` object with both command line and config options. """ + if options is None: + # create an empty object to support the following getattr() calls + options = types.SimpleNamespace() configspec = _parse_configspec() - if options.sampleconf: + if getattr(options, 'sampleconf', None): _write_sample_config(configspec, progname) sys.exit(0) - if options.updateconf: + if getattr(options, 'updateconf', None): _update_config_file(options.updateconf, configspec) sys.exit(0) - if options.updatedb: + if getattr(options, 'updatedb', None): update_db_file(options.updatedb) sys.exit(0) - if options.samplesspevent: + if getattr(options, 'samplesspevent', None): _write_sample_ssp_event_file() sys.exit(0) - if options.config_file: + # initialize config object to the default values + config_obj = _get_default_config_obj(configspec) + if getattr(options, 'config_file', None): options.config_file = _fix_and_expand_path(options.config_file) - config_obj = _read_config(options.config_file, configspec) + config_obj = _read_config_file(options.config_file, configspec) # Apply overrides if config_overrides is not None: @@ -514,17 +525,19 @@ def configure(options, progname, config_overrides=None): _check_deprecated_config_options(config_obj) _check_mandatory_config_params(config_obj) + # TODO: we should allow outdir to be None and not producing any output + options.outdir = getattr(options, 'outdir', 'sspec_out') # Fix and expand paths in options options.outdir = _fix_and_expand_path(options.outdir) - if options.trace_path: + if getattr(options, 'trace_path', None): # trace_path is a list options.trace_path = [ _fix_and_expand_path(path) for path in options.trace_path] - if options.qml_file: + if getattr(options, 'qml_file', None): options.qml_file = _fix_and_expand_path(options.qml_file) - if options.hypo_file: + if getattr(options, 'hypo_file', None): options.hypo_file = _fix_and_expand_path(options.hypo_file) - if options.pick_file: + if getattr(options, 'pick_file', None): options.pick_file = _fix_and_expand_path(options.pick_file) # Create a 'no_evid_' subdir into outdir. @@ -541,7 +554,7 @@ def configure(options, progname, config_overrides=None): config.options = options # Override station_metadata config option with command line option - if options.station_metadata is not None: + if getattr(options, 'station_metadata', None): config.station_metadata = options.station_metadata # Additional config values From 8da0db409b2cd349b2e40719c72a16f724fbff3b Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Thu, 4 Jul 2024 11:38:25 +0200 Subject: [PATCH 08/73] Export a global config object TODO: not yet used in the rest of the code --- sourcespec2/config/__init__.py | 2 +- sourcespec2/config/config.py | 53 ++++++++++++++++++++-------------- 2 files changed, 33 insertions(+), 22 deletions(-) diff --git a/sourcespec2/config/__init__.py b/sourcespec2/config/__init__.py index 2bcb1809..a5cf61ce 100644 --- a/sourcespec2/config/__init__.py +++ b/sourcespec2/config/__init__.py @@ -9,5 +9,5 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -from .config import configure # noqa +from .config import configure, config # noqa from .library_versions import library_versions # noqa diff --git a/sourcespec2/config/config.py b/sourcespec2/config/config.py index f590cdf6..40d7cc1d 100644 --- a/sourcespec2/config/config.py +++ b/sourcespec2/config/config.py @@ -31,6 +31,27 @@ class Config(dict): """Config class for sourcespec.""" + def __init__(self): + # Initialize config object to the default values + configspec = _parse_configspec() + config_obj = _get_default_config_obj(configspec) + self.update(config_obj.dict()) + # Additional config values + self.vertical_channel_codes = ['Z'] + self.horizontal_channel_codes_1 = ['N', 'R'] + self.horizontal_channel_codes_2 = ['E', 'T'] + # Empty options object, for compatibility with the command line version + self.options = types.SimpleNamespace() + # A list of warnings to be issued when logger is set up + self.warnings = [] + # Create a dict to store figure paths + self.figures = defaultdict(list) + # store the absolute path of the current working directory + self.workdir = os.getcwd() + # SEED standard instrument codes: + # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ + self.INSTR_CODES_VEL = ['H', 'L'] + self.INSTR_CODES_ACC = ['N', ] def __setitem__(self, key, value): """Make Config keys accessible as attributes.""" @@ -348,14 +369,10 @@ def _check_mandatory_config_params(config_obj): sys.exit(msg) -def _init_instrument_codes(config): +def _update_instrument_codes(config): """ - Initialize instrument codes from config file. + Update instrument codes from config file. """ - # SEED standard instrument codes: - # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ - config.INSTR_CODES_VEL = ['H', 'L'] - config.INSTR_CODES_ACC = ['N', ] # User-defined instrument codes: instr_code_acc_user = config.instrument_code_acceleration instr_code_vel_user = config.instrument_code_velocity @@ -461,6 +478,12 @@ def _none_lenght(input_list): return 1 if input_list is None else len(input_list) +# Global config object, initialized with default values +# API users should use this object to access configuration parameters +# and update them as needed +config = Config() + + def configure(options=None, progname='source_spec', config_overrides=None): """ Parse command line arguments and read config file. @@ -547,9 +570,8 @@ def configure(options=None, progname='source_spec', config_overrides=None): options.outdir = os.path.join(options.outdir, f'no_evid_{hexstr}') _write_config(config_obj, progname, options.outdir) - # Create a Config object - config = Config(config_obj.dict().copy()) - + # Update config object with the contents of the config file + config.update(config_obj.dict()) # Add options to config: config.options = options @@ -557,10 +579,6 @@ def configure(options=None, progname='source_spec', config_overrides=None): if getattr(options, 'station_metadata', None): config.station_metadata = options.station_metadata - # Additional config values - config.vertical_channel_codes = ['Z'] - config.horizontal_channel_codes_1 = ['N', 'R'] - config.horizontal_channel_codes_2 = ['E', 'T'] msc = config.mis_oriented_channels if msc is not None: config.vertical_channel_codes.append(msc[0]) @@ -608,9 +626,6 @@ def configure(options=None, progname='source_spec', config_overrides=None): except ValueError as msg: sys.exit(f'Error parsing parameter "Er_freq_range": {msg}') - # A list of warnings to be issued when logger is set up - config.warnings = [] - if config.html_report: if not config.plot_save: msg = ( @@ -638,11 +653,7 @@ def configure(options=None, progname='source_spec', config_overrides=None): except ImportError as err: sys.exit(err) - _init_instrument_codes(config) + _update_instrument_codes(config) _init_traceid_map(config) _init_plotting(config.plot_show) - # Create a dict to store figure paths - config.figures = defaultdict(list) - # store the absolute path of the current working directory - config.workdir = os.getcwd() return config From f213df8039dd0e7e10a673e92b9ce898bcd139f5 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 8 Jul 2024 08:39:01 +0200 Subject: [PATCH 09/73] Move validation and deprecation checks to the Config class --- sourcespec2/config/config.py | 509 ++++++++++----------- sourcespec2/config/mandatory_deprecated.py | 112 +++++ 2 files changed, 342 insertions(+), 279 deletions(-) create mode 100644 sourcespec2/config/mandatory_deprecated.py diff --git a/sourcespec2/config/config.py b/sourcespec2/config/config.py index 40d7cc1d..bbf1e3c1 100644 --- a/sourcespec2/config/config.py +++ b/sourcespec2/config/config.py @@ -20,6 +20,9 @@ from datetime import datetime from collections import defaultdict from .library_versions import library_versions +from .mandatory_deprecated import ( + mandatory_config_params, deprecated_config_params +) from .configobj import ConfigObj from .configobj.validate import Validator from ..ssp_update_db import update_db_file @@ -32,26 +35,27 @@ class Config(dict): """Config class for sourcespec.""" def __init__(self): - # Initialize config object to the default values - configspec = _parse_configspec() - config_obj = _get_default_config_obj(configspec) - self.update(config_obj.dict()) - # Additional config values - self.vertical_channel_codes = ['Z'] - self.horizontal_channel_codes_1 = ['N', 'R'] - self.horizontal_channel_codes_2 = ['E', 'T'] + # Additional config values. Tey must be defined using the dict syntax. + self['running_from_command_line'] = False + self['vertical_channel_codes'] = ['Z'] + self['horizontal_channel_codes_1'] = ['N', 'R'] + self['horizontal_channel_codes_2'] = ['E', 'T'] # Empty options object, for compatibility with the command line version - self.options = types.SimpleNamespace() + self['options'] = types.SimpleNamespace() # A list of warnings to be issued when logger is set up - self.warnings = [] + self['warnings'] = [] # Create a dict to store figure paths - self.figures = defaultdict(list) + self['figures'] = defaultdict(list) # store the absolute path of the current working directory - self.workdir = os.getcwd() + self['workdir'] = os.getcwd() # SEED standard instrument codes: # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ - self.INSTR_CODES_VEL = ['H', 'L'] - self.INSTR_CODES_ACC = ['N', ] + self['INSTR_CODES_VEL'] = ['H', 'L'] + self['INSTR_CODES_ACC'] = ['N', ] + # Initialize config object to the default values + configspec = _parse_configspec() + config_obj = _get_default_config_obj(configspec) + self.update(config_obj.dict()) def __setitem__(self, key, value): """Make Config keys accessible as attributes.""" @@ -67,6 +71,197 @@ def __getattr__(self, key): __setattr__ = __setitem__ + def update(self, other): + """ + Update the configuration with the values from another dictionary. + + :param dict other: The dictionary with the new values + + :raises ValueError: If an error occurs while parsing the options + """ + for key, value in other.items(): + self[key] = value + # Set to None all the 'None' strings + for key, value in self.items(): + if value == 'None': + self[key] = None + # Make sure that self['figures'] is still a defaultdict + self['figures'] = defaultdict(list, self['figures']) + self._update_channel_codes() + self._update_instrument_codes() + + def _update_channel_codes(self): + """ + Update channel codes with mis-oriented channels. + """ + msc = self.mis_oriented_channels + if msc is None: + return + self['vertical_channel_codes'].append(msc[0]) + self['horizontal_channel_codes_1'].append(msc[1]) + self['horizontal_channel_codes_2'].append(msc[2]) + self['vertical_channel_codes'] =\ + list(set(self['vertical_channel_codes'])) + self['horizontal_channel_codes_1'] =\ + list(set(self['horizontal_channel_codes_1'])) + self['horizontal_channel_codes_2'] =\ + list(set(self['horizontal_channel_codes_2'])) + + def _update_instrument_codes(self): + """ + Update instrument codes from user-defined values. + """ + # User-defined instrument codes: + instr_code_acc_user = self['instrument_code_acceleration'] + instr_code_vel_user = self['instrument_code_velocity'] + # Remove user-defined instrument codes if they conflict + # with another instrument + with contextlib.suppress(ValueError): + self['INSTR_CODES_VEL'].remove(instr_code_acc_user) + with contextlib.suppress(ValueError): + self['INSTR_CODES_ACC'].remove(instr_code_vel_user) + # Add user-defined instrument codes + if instr_code_vel_user is not None: + self['INSTR_CODES_VEL'].append(instr_code_vel_user) + if instr_code_acc_user is not None: + self['INSTR_CODES_ACC'].append(instr_code_acc_user) + self['INSTR_CODES_VEL'] = list(set(self['INSTR_CODES_VEL'])) + self['INSTR_CODES_ACC'] = list(set(self['INSTR_CODES_ACC'])) + + def validate(self): + """ + Validate the configuration. + + :raises ValueError: If an error occurs while validating the options + """ + config_obj = ConfigObj(self, configspec=_parse_configspec()) + val = Validator() + test = config_obj.validate(val) + # The variable "test" is: + # - True if everything is ok + # - False if no config value is provided + # - A dict if invalid values are present, + # with the invalid values as False + msg = '' + if isinstance(test, dict): + for entry in [e for e in test if not test[e]]: + msg += f'\nInvalid value for "{entry}": "{config_obj[entry]}"' + raise ValueError(msg) + if not test: + raise ValueError('No configuration value present!') + # Reupdate the config object with the validated values, which include + # type conversion from string to numeric values + self.update(config_obj.dict()) + # Check deprecated and mandatory parameters + self._check_deprecated_config_params() + self._check_mandatory_config_params() + self._check_force_list() + self._check_list_lengths() + self._check_Er_freq_range() + self._check_html_report() + + def _check_deprecated_config_params(self): + deprecation_msgs = [] + for param, msgs in deprecated_config_params.items(): + if param in self: + # add two spaces before each line of the message + msg = ''.join(f' {line}\n' for line in msgs) + # replace first character with '>' + msg = f'>{msg[1:]}' + deprecation_msgs.append(msg) + if not deprecation_msgs: + return + msg = '' + if self['running_from_command_line']: + msg += ( + 'Error: your config file contains deprecated parameters:\n\n' + ) + msg += ''.join(deprecation_msgs) + if self['running_from_command_line']: + msg += ( + '\nPlease upgrade your config file manually or ' + 'via the "-U" option.\n' + ) + raise ValueError(msg) + + def _check_mandatory_config_params(self): + messages = [] + for par in mandatory_config_params: + if self[par] is None: + msg = f'"{par}" is mandatory and cannot be None' + messages.append(msg) + if messages: + msg = '\n'.join(messages) + raise ValueError(msg) + + def _check_force_list(self): + """ + Check the force_list options and convert them to lists of floats. + + :raises ValueError: If an error occurs while parsing the options + """ + try: + for param in [ + 'vp_source', 'vs_source', 'rho_source', 'layer_top_depths' + ]: + self[param] = _float_list(self[param]) + except ValueError as msg: + raise ValueError( + f'Error parsing parameter "{param}": {msg}' + ) from msg + + def _check_list_lengths(self): + """ + Check that the lists describing the source model have the same length. + """ + n_vp_source = _none_lenght(self.vp_source) + n_vs_source = _none_lenght(self.vs_source) + n_rho_source = _none_lenght(self.rho_source) + n_layer_top_depths = _none_lenght(self.layer_top_depths) + try: + assert n_vp_source == n_vs_source == n_rho_source \ + == n_layer_top_depths + except AssertionError as err: + raise ValueError( + 'Error: "vp_source", "vs_source", "rho_source", and ' + '"layer_top_depths" must have the same length.' + ) from err + + def _check_Er_freq_range(self): + """ + Check the Er_freq_range option. + + :raises ValueError: If an error occurs while parsing the options + """ + if self['Er_freq_range'] is None: + self['Er_freq_range'] = [None, None] + try: + self['Er_freq_range'] = _float_list( + self['Er_freq_range'], max_length=2, + accepted_values=[None, 'noise'] + ) + except ValueError as msg: + raise ValueError( + f'Error parsing parameter "Er_freq_range": {msg}' + ) from msg + + def _check_html_report(self): + """ + Check the html_report option. + """ + if self['html_report']: + if not self['plot_save']: + self['warnings'].append( + 'The "html_report" option is selected but "plot_save" ' + 'is "False". HTML report will have no plots.' + ) + if self['plot_save_format'] not in ['png', 'svg']: + self['warnings'].append( + 'The "html_report" option is selected but ' + '"plot_save_format" is not "png" or "svg". ' + 'HTML report will have no plots.' + ) + def _read_config_file(config_file, configspec=None): kwargs = { @@ -203,111 +398,6 @@ def _write_config(config_obj, progname, outdir): _tmp_config_obj.write(fp) -def _check_deprecated_config_options(config_obj): - deprecation_msgs = [] - if 's_win_length' in config_obj or 'noise_win_length' in config_obj: - deprecation_msgs.append( - '> "s_win_length" and "noise_win_length" config parameters ' - 'are no more\n' - ' supported. Both are replaced by "win_length".\n' - ) - if 'traceids' in config_obj: - deprecation_msgs.append( - '> "traceids" config parameter has been renamed to ' - '"traceid_mapping_file".\n' - ) - if 'ignore_stations' in config_obj or 'use_stations' in config_obj: - deprecation_msgs.append( - '> "ignore_stations" and "use_stations" config parameters ' - 'have been renamed to\n' - ' "ignore_traceids" and "use_traceids", respectively.\n' - ) - if 'dataless' in config_obj: - deprecation_msgs.append( - '> "dataless" config parameter has been renamed to ' - '"station_metadata".\n' - ) - if 'clip_nmax' in config_obj: - deprecation_msgs.append( - '> "clip_nmax" config parameter has been renamed to ' - '"clip_max_percent".\n' - ' Note that the new default is 5% (current value in your config ' - f'file: {config_obj["clip_nmax"]}%)\n' - ) - if 'trace_format' in config_obj: - deprecation_msgs.append( - '> "trace_format" config parameter is no more supported.\n' - ' Use "sensitivity" to manually specify how sensor sensitivity ' - 'should be computed.\n' - ) - if 'PLOT_SHOW' in config_obj: - deprecation_msgs.append( - '> "PLOT_SHOW" config parameter has been renamed to "plot_show".\n' - ) - if 'PLOT_SAVE' in config_obj: - deprecation_msgs.append( - '> "PLOT_SAVE" config parameter has been renamed to "plot_save".\n' - ) - if 'PLOT_SAVE_FORMAT' in config_obj: - deprecation_msgs.append( - '> "PLOT_SAVE_FORMAT" config parameter has been renamed to ' - '"plot_save_format".\n' - ) - if 'vp' in config_obj: - deprecation_msgs.append( - '> "vp" config parameter has been renamed to "vp_source".\n' - ) - if 'vs' in config_obj: - deprecation_msgs.append( - '> "vs" config parameter has been renamed to "vs_source".\n' - ) - if 'rho' in config_obj: - deprecation_msgs.append( - '> "rho" config parameter has been renamed to "rho_source".\n' - ) - if 'pre_p_time' in config_obj: - deprecation_msgs.append( - '> "pre_p_time" config parameter has been renamed to ' - '"noise_pre_time".\n' - ) - if 'pre_s_time' in config_obj: - deprecation_msgs.append( - '> "pre_s_time" config parameter has been renamed to ' - '"signal_pre_time".\n' - ) - if 'rps_from_focal_mechanism' in config_obj: - deprecation_msgs.append( - '> "rps_from_focal_mechanism" config parameter has been renamed ' - 'to "rp_from_focal_mechanism".\n' - ) - if 'paz' in config_obj: - deprecation_msgs.append( - '> "paz" config parameter has been removed and merged with ' - '"station_metadata".\n' - ) - if 'max_epi_dist' in config_obj: - deprecation_msgs.append( - '> "max_epi_dist" config parameter has been removed and replaced ' - 'by "epi_dist_ranges".\n' - ) - if 'max_freq_Er' in config_obj: - deprecation_msgs.append( - '> "max_freq_Er" config parameter has been removed and replaced ' - 'by "Er_freq_min_max".\n' - ) - if deprecation_msgs: - sys.stderr.write( - 'Error: your config file contains deprecated parameters:\n\n') - for msg in deprecation_msgs: - sys.stderr.write(msg) - if deprecation_msgs: - sys.stderr.write( - '\nPlease upgrade your config file manually or ' - 'via the "-U" option.\n' - ) - sys.exit(1) - - def _init_plotting(plot_show): # pylint: disable=import-outside-toplevel import matplotlib.pyplot as plt @@ -315,85 +405,6 @@ def _init_plotting(plot_show): plt.switch_backend('Agg') -def _check_mandatory_config_params(config_obj): - mandatory_params = [ - 'p_arrival_tolerance', - 's_arrival_tolerance', - 'noise_pre_time', - 'signal_pre_time', - 'win_length', - 'taper_halfwidth', - 'spectral_smooth_width_decades', - 'bp_freqmin_acc', - 'bp_freqmax_acc', - 'bp_freqmin_shortp', - 'bp_freqmax_shortp', - 'bp_freqmin_broadb', - 'bp_freqmax_broadb', - 'freq1_acc', - 'freq2_acc', - 'freq1_shortp', - 'freq2_shortp', - 'freq1_broadb', - 'freq2_broadb', - 'rmsmin', - 'sn_min', - 'spectral_sn_min', - 'rpp', - 'rps', - 'geom_spread_n_exponent', - 'geom_spread_cutoff_distance', - 'f_weight', - 'weight', - 't_star_0', - 't_star_0_variability', - 'a', 'b', 'c', - 'ml_bp_freqmin', - 'ml_bp_freqmax', - 'n_sigma', - 'lower_percentage', - 'mid_percentage', - 'upper_percentage', - 'nIQR', - 'plot_spectra_maxrows', - 'plot_traces_maxrows', - 'plot_station_text_size' - ] - messages = [] - for par in mandatory_params: - if config_obj[par] is None: - msg = f'"{par}" is mandatory and cannot be None' - messages.append(msg) - if messages: - msg = '\n'.join(messages) - sys.exit(msg) - - -def _update_instrument_codes(config): - """ - Update instrument codes from config file. - """ - # User-defined instrument codes: - instr_code_acc_user = config.instrument_code_acceleration - instr_code_vel_user = config.instrument_code_velocity - # Remove user-defined instrument codes if they conflict - # with another instrument - with contextlib.suppress(ValueError): - config.INSTR_CODES_VEL.remove(instr_code_acc_user) - with contextlib.suppress(ValueError): - config.INSTR_CODES_ACC.remove(instr_code_vel_user) - # Add user-defined instrument codes - if instr_code_vel_user is not None: - config.INSTR_CODES_VEL.append(instr_code_vel_user) - if instr_code_acc_user is not None: - config.INSTR_CODES_ACC.append(instr_code_acc_user) - # TODO: remove these when the global config object will be implemented - global INSTR_CODES_VEL - global INSTR_CODES_ACC - INSTR_CODES_VEL = config.INSTR_CODES_VEL - INSTR_CODES_ACC = config.INSTR_CODES_ACC - - def _init_traceid_map(config): """ Initialize trace ID map from file. @@ -511,42 +522,18 @@ def configure(options=None, progname='source_spec', config_overrides=None): _write_sample_ssp_event_file() sys.exit(0) - # initialize config object to the default values - config_obj = _get_default_config_obj(configspec) if getattr(options, 'config_file', None): options.config_file = _fix_and_expand_path(options.config_file) config_obj = _read_config_file(options.config_file, configspec) - - # Apply overrides - if config_overrides is not None: - try: - for key, value in config_overrides.items(): - config_obj[key] = value - except AttributeError as e: - raise ValueError('"config_override" must be a dict-like.') from e - - # Set to None all the 'None' strings - for key, value in config_obj.dict().items(): - if value == 'None': - config_obj[key] = None - - val = Validator() - test = config_obj.validate(val) - # test is: - # - True if everything is ok - # - False if no config value is provided - # - A dict if invalid values are present, with the invalid values as False - if isinstance(test, dict): - for entry in [e for e in test if not test[e]]: - sys.stderr.write( - f'Invalid value for "{entry}": "{config_obj[entry]}"\n') - sys.exit(1) - if not test: - sys.stderr.write('No configuration value present!\n') - sys.exit(1) - - _check_deprecated_config_options(config_obj) - _check_mandatory_config_params(config_obj) + # Apply overrides + if config_overrides is not None: + try: + for key, value in config_overrides.items(): + config_obj[key] = value + except AttributeError as e: + raise ValueError( + '"config_override" must be a dict-like.' + ) from e # TODO: we should allow outdir to be None and not producing any output options.outdir = getattr(options, 'outdir', 'sspec_out') @@ -572,6 +559,12 @@ def configure(options=None, progname='source_spec', config_overrides=None): # Update config object with the contents of the config file config.update(config_obj.dict()) + config.running_from_command_line = True + try: + config.validate() + except ValueError as msg: + sys.exit(msg) + # Add options to config: config.options = options @@ -579,12 +572,6 @@ def configure(options=None, progname='source_spec', config_overrides=None): if getattr(options, 'station_metadata', None): config.station_metadata = options.station_metadata - msc = config.mis_oriented_channels - if msc is not None: - config.vertical_channel_codes.append(msc[0]) - config.horizontal_channel_codes_1.append(msc[1]) - config.horizontal_channel_codes_2.append(msc[2]) - # Fix and expand paths in config if config.database_file: config.database_file = _fix_and_expand_path(config.database_file) @@ -597,47 +584,6 @@ def configure(options=None, progname='source_spec', config_overrides=None): config.residuals_filepath = _fix_and_expand_path( config.residuals_filepath) - # Parse force_list options into lists of float - try: - for param in [ - 'vp_source', 'vs_source', 'rho_source', 'layer_top_depths']: - config[param] = _float_list(config[param]) - except ValueError as msg: - sys.exit(f'Error parsing parameter "{param}": {msg}') - n_vp_source = _none_lenght(config.vp_source) - n_vs_source = _none_lenght(config.vs_source) - n_rho_source = _none_lenght(config.rho_source) - n_layer_top_depths = _none_lenght(config.layer_top_depths) - try: - assert n_vp_source == n_vs_source == n_rho_source == n_layer_top_depths - except AssertionError: - sys.exit( - 'Error: "vp_source", "vs_source", "rho_source", and ' - '"layer_top_depths" must have the same length.' - ) - - # Check the Er_freq_range parameter - if config.Er_freq_range is None: - config.Er_freq_range = [None, None] - try: - config.Er_freq_range = _float_list( - config.Er_freq_range, max_length=2, - accepted_values=[None, 'noise']) - except ValueError as msg: - sys.exit(f'Error parsing parameter "Er_freq_range": {msg}') - - if config.html_report: - if not config.plot_save: - msg = ( - 'The "html_report" option is selected but "plot_save" ' - 'is "False". HTML report will have no plots.') - config.warnings.append(msg) - if config.plot_save_format not in ['png', 'svg']: - msg = ( - 'The "html_report" option is selected but "plot_save_format" ' - 'is not "png" or "svg". HTML report will have no plots.') - config.warnings.append(msg) - if config.plot_station_map: try: library_versions.check_cartopy_version() @@ -653,7 +599,12 @@ def configure(options=None, progname='source_spec', config_overrides=None): except ImportError as err: sys.exit(err) - _update_instrument_codes(config) + # TODO: remove these when the global config object will be implemented + global INSTR_CODES_VEL + global INSTR_CODES_ACC + INSTR_CODES_VEL = config.INSTR_CODES_VEL + INSTR_CODES_ACC = config.INSTR_CODES_ACC + _init_traceid_map(config) _init_plotting(config.plot_show) return config diff --git a/sourcespec2/config/mandatory_deprecated.py b/sourcespec2/config/mandatory_deprecated.py new file mode 100644 index 00000000..eb665041 --- /dev/null +++ b/sourcespec2/config/mandatory_deprecated.py @@ -0,0 +1,112 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Mandatory and deprecated config parameters for sourcespec. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" + +# Mandatory parameters, which cannot be None +mandatory_config_params = [ + 'p_arrival_tolerance', + 's_arrival_tolerance', + 'noise_pre_time', + 'signal_pre_time', + 'win_length', + 'taper_halfwidth', + 'spectral_smooth_width_decades', + 'bp_freqmin_acc', + 'bp_freqmax_acc', + 'bp_freqmin_shortp', + 'bp_freqmax_shortp', + 'bp_freqmin_broadb', + 'bp_freqmax_broadb', + 'freq1_acc', + 'freq2_acc', + 'freq1_shortp', + 'freq2_shortp', + 'freq1_broadb', + 'freq2_broadb', + 'rmsmin', + 'sn_min', + 'spectral_sn_min', + 'rpp', + 'rps', + 'geom_spread_n_exponent', + 'geom_spread_cutoff_distance', + 'f_weight', + 'weight', + 't_star_0', + 't_star_0_variability', + 'a', 'b', 'c', + 'ml_bp_freqmin', + 'ml_bp_freqmax', + 'n_sigma', + 'lower_percentage', + 'mid_percentage', + 'upper_percentage', + 'nIQR', + 'plot_spectra_maxrows', + 'plot_traces_maxrows', + 'plot_station_text_size' +] + + +# Deprecated parameters (keys) and messages to be shown to the user (values) +# The messages are lists of strings, each string is a line of the message. +deprecated_config_params = { + 's_win_length': + ['"s_win_length" has been removed and replaced by "win_length".'], + 'noise_win_length': + ['"noise_win_length" has been removed and replaced by "win_length".'], + 'traceids': + ['"traceids" has been renamed to "traceid_mapping_file".'], + 'ignore_stations': + ['"ignore_stations" has been replaced by "ignore_traceids".'], + 'use_stations': + ['"use_stations" has been replaced by "use_traceids".'], + 'dataless': + ['"dataless" has been renamed to "station_metadata".'], + 'clip_nmax': + [ + '"clip_nmax" has been renamed to "clip_max_percent".', + 'Note that the new default is 5%.' + ], + 'trace_format': + [ + '"trace_format" is no more supported.', + 'Use "sensitivity" to manually specify how sensor sensitivity ' + 'should be computed.' + ], + 'PLOT_SHOW': + ['"PLOT_SHOW" has been renamed to "plot_show".'], + 'PLOT_SAVE': + ['"PLOT_SAVE" has been renamed to "plot_save".'], + 'PLOT_SAVE_FORMAT': + ['"PLOT_SAVE_FORMAT" has been renamed to "plot_save_format".'], + 'vp': + ['"vp" has been renamed to "vp_source".'], + 'vs': + ['"vs" has been renamed to "vs_source".'], + 'rho': + ['"rho" has been renamed to "rho_source".'], + 'pre_p_time': + ['"pre_p_time" has been renamed to "noise_pre_time".'], + 'pre_s_time': + ['"pre_s_time" has been renamed to "signal_pre_time".'], + 'rps_from_focal_mechanism': + [ + '"rps_from_focal_mechanism" has been renamed to ' + '"rp_from_focal_mechanism".' + ], + 'paz': + ['"paz" has been removed and merged with "station_metadata".'], + 'max_epi_dist': + ['"max_epi_dist" has been removed and replaced by "epi_dist_ranges".'], + 'max_freq_Er': + ['"max_freq_Er" has been removed and replaced by "Er_freq_min_max".'], +} From 32bb403404a60a29c00358241156fb6691d0a5d5 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 15:02:33 +0200 Subject: [PATCH 10/73] Rename `configure()` to `configure_cli()` Also, split the code into multiple submodules. --- sourcespec2/config/__init__.py | 3 +- sourcespec2/config/config.py | 443 ++++------------------- sourcespec2/config/configobj_helpers.py | 81 +++++ sourcespec2/config/configure_cli.py | 337 +++++++++++++++++ sourcespec2/source_model.py | 4 +- sourcespec2/source_spec.py | 4 +- sourcespec2/ssp_read_station_metadata.py | 2 +- 7 files changed, 505 insertions(+), 369 deletions(-) create mode 100644 sourcespec2/config/configobj_helpers.py create mode 100644 sourcespec2/config/configure_cli.py diff --git a/sourcespec2/config/__init__.py b/sourcespec2/config/__init__.py index a5cf61ce..8023baab 100644 --- a/sourcespec2/config/__init__.py +++ b/sourcespec2/config/__init__.py @@ -9,5 +9,6 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -from .config import configure, config # noqa +from .config import config # noqa +from .configure_cli import configure_cli # noqa from .library_versions import library_versions # noqa diff --git a/sourcespec2/config/config.py b/sourcespec2/config/config.py index bbf1e3c1..739bfb2a 100644 --- a/sourcespec2/config/config.py +++ b/sourcespec2/config/config.py @@ -10,30 +10,75 @@ (http://www.cecill.info/licences.en.html) """ import os -import sys import types -import shutil -import uuid -import json import contextlib -from copy import copy -from datetime import datetime from collections import defaultdict -from .library_versions import library_versions +from .configobj_helpers import parse_configspec, get_default_config_obj from .mandatory_deprecated import ( mandatory_config_params, deprecated_config_params ) from .configobj import ConfigObj from .configobj.validate import Validator -from ..ssp_update_db import update_db_file -# TODO: remove these when the global config object will be implemented -INSTR_CODES_VEL = [] -INSTR_CODES_ACC = [] +# ---- Helper functions ---- +def _float_list(input_list, max_length=None, accepted_values=None): + """ + Convert an input list to a list of floats. + + :param input_list: Input list or None + :type input_list: list or None + :param max_length: Maximum length of the list + :type max_length: int or None + :param accepted_values: List of accepted values + :type accepted_values: list or None -class Config(dict): - """Config class for sourcespec.""" + :return: A list of floats or None + :rtype: list + """ + if input_list is None: + return None + if accepted_values is None: + accepted_values = [] + + def _parse_float(val): + val = None if val == 'None' else val + return val if val in accepted_values else float(val) + + try: + return [_parse_float(val) for val in input_list[:max_length]] + except ValueError as e: + raise ValueError('Cannot parse all values in list') from e + + +def _none_lenght(input_list): + """ + Return the length of input list, or 1 if input list is None + + :param input_list: Input list or None + :type input_list: list or None + + :return: List length or 1 + :rtype: int + """ + return 1 if input_list is None else len(input_list) +# ---- End Helper functions ---- + + +class _Config(dict): + """ + Config class for sourcespec. + + This class stores the configuration parameters for sourcespec. + Parameters can be accessed as dictionary keys (e.g. config['param']) + or as attributes (e.g. config.param). + The class is initialized with default values. + + .. note:: + + This class is private and should not be used directly. + Import the global config object instead. + """ def __init__(self): # Additional config values. Tey must be defined using the dict syntax. self['running_from_command_line'] = False @@ -53,8 +98,8 @@ def __init__(self): self['INSTR_CODES_VEL'] = ['H', 'L'] self['INSTR_CODES_ACC'] = ['N', ] # Initialize config object to the default values - configspec = _parse_configspec() - config_obj = _get_default_config_obj(configspec) + configspec = parse_configspec() + config_obj = get_default_config_obj(configspec) self.update(config_obj.dict()) def __setitem__(self, key, value): @@ -75,7 +120,8 @@ def update(self, other): """ Update the configuration with the values from another dictionary. - :param dict other: The dictionary with the new values + :param other: The dictionary with the new values + :type other: dict :raises ValueError: If an error occurs while parsing the options """ @@ -134,7 +180,7 @@ def validate(self): :raises ValueError: If an error occurs while validating the options """ - config_obj = ConfigObj(self, configspec=_parse_configspec()) + config_obj = ConfigObj(self, configspec=parse_configspec()) val = Validator() test = config_obj.validate(val) # The variable "test" is: @@ -161,6 +207,11 @@ def validate(self): self._check_html_report() def _check_deprecated_config_params(self): + """ + Check the deprecated configuration parameters. + + :raises ValueError: If an error occurs while parsing the options + """ deprecation_msgs = [] for param, msgs in deprecated_config_params.items(): if param in self: @@ -185,6 +236,11 @@ def _check_deprecated_config_params(self): raise ValueError(msg) def _check_mandatory_config_params(self): + """ + Check the mandatory configuration parameters. + + :raises ValueError: If an error occurs while parsing the options + """ messages = [] for par in mandatory_config_params: if self[par] is None: @@ -213,11 +269,13 @@ def _check_force_list(self): def _check_list_lengths(self): """ Check that the lists describing the source model have the same length. + + :raises ValueError: If an error occurs while parsing the options """ - n_vp_source = _none_lenght(self.vp_source) - n_vs_source = _none_lenght(self.vs_source) - n_rho_source = _none_lenght(self.rho_source) - n_layer_top_depths = _none_lenght(self.layer_top_depths) + n_vp_source = _none_lenght(self['vp_source']) + n_vs_source = _none_lenght(self['vs_source']) + n_rho_source = _none_lenght(self['rho_source']) + n_layer_top_depths = _none_lenght(self['layer_top_depths']) try: assert n_vp_source == n_vs_source == n_rho_source \ == n_layer_top_depths @@ -263,348 +321,7 @@ def _check_html_report(self): ) -def _read_config_file(config_file, configspec=None): - kwargs = { - 'configspec': configspec, - 'file_error': True, - 'default_encoding': 'utf8' - } - if configspec is None: - kwargs.update({ - 'interpolation': False, - 'list_values': False, - '_inspec': True - }) - try: - config_obj = ConfigObj(config_file, **kwargs) - except IOError as err: - sys.stderr.write(f'{err}\n') - sys.exit(1) - except Exception as err: - sys.stderr.write(f'Unable to read "{config_file}": {err}\n') - sys.exit(1) - return config_obj - - -def _parse_configspec(): - configspec_file = os.path.join( - os.path.dirname(__file__), 'configspec.conf') - return _read_config_file(configspec_file) - - -def _get_default_config_obj(configspec): - config_obj = ConfigObj(configspec=configspec, default_encoding='utf8') - val = Validator() - config_obj.validate(val) - config_obj.defaults = [] - config_obj.initial_comment = configspec.initial_comment - config_obj.comments = configspec.comments - config_obj.final_comment = configspec.final_comment - return config_obj - - -def _write_sample_config(configspec, progname): - config_obj = _get_default_config_obj(configspec) - configfile = f'{progname}.conf' - write_file = True - if os.path.exists(configfile): - ans = input( - f'{configfile} already exists. Do you want to overwrite it? [y/N] ' - ) - write_file = ans in ['y', 'Y'] - if write_file: - with open(configfile, 'wb') as fp: - config_obj.write(fp) - print(f'Sample config file written to: {configfile}') - note = """ -Note that the default config parameters are suited for a M<5 earthquake -recorded within ~100 km. Adjust `win_length`, `noise_pre_time`, and the -frequency bands (`bp_freqmin_*`, `bp_freqmax_*`, `freq1_*`, `freq2_*`) -according to your setup.""" - print(note) - - -def _update_config_file(config_file, configspec): - config_obj = _read_config_file(config_file, configspec) - val = Validator() - config_obj.validate(val) - mod_time = datetime.fromtimestamp(os.path.getmtime(config_file)) - mod_time_str = mod_time.strftime('%Y%m%d_%H%M%S') - config_file_old = f'{config_file}.{mod_time_str}' - ans = input( - f'Ok to update {config_file}? [y/N]\n' - f'(Old file will be saved as {config_file_old}) ' - ) - if ans not in ['y', 'Y']: - sys.exit(0) - config_new = ConfigObj(configspec=configspec, default_encoding='utf8') - config_new = _read_config_file(None, configspec) - config_new.validate(val) - config_new.defaults = [] - config_new.comments = configspec.comments - config_new.initial_comment = config_obj.initial_comment - config_new.final_comment = configspec.final_comment - for k, v in config_obj.items(): - if k not in config_new: - continue - # Fix for force_list(default=None) - if v == ['None', ]: - v = None - config_new[k] = v - migrate_options = { - 's_win_length': 'win_length', - 'traceids': 'traceid_mapping_file', - 'ignore_stations': 'ignore_traceids', - 'use_stations': 'use_traceids', - 'dataless': 'station_metadata', - 'clip_nmax': 'clip_max_percent', - 'PLOT_SHOW': 'plot_show', - 'PLOT_SAVE': 'plot_save', - 'PLOT_SAVE_FORMAT': 'plot_save_format', - 'vp': 'vp_source', - 'vs': 'vs_source', - 'rho': 'rho_source', - 'pre_p_time': 'noise_pre_time', - 'pre_s_time': 'signal_pre_time', - 'rps_from_focal_mechanism': 'rp_from_focal_mechanism', - 'paz': 'station_metadata', - 'pi_bsd_min_max': 'pi_ssd_min_max', - 'max_epi_dist': 'epi_dist_ranges' - } - for old_opt, new_opt in migrate_options.items(): - if old_opt in config_obj and config_obj[old_opt] != 'None': - # max_epi_dist needs to be converted to a list - if old_opt == 'max_epi_dist': - config_new[new_opt] = [0, config_obj[old_opt]] - else: - config_new[new_opt] = config_obj[old_opt] - shutil.copyfile(config_file, config_file_old) - with open(config_file, 'wb') as fp: - config_new.write(fp) - print(f'{config_file}: updated') - - -def _write_config(config_obj, progname, outdir): - if progname != 'source_spec': - return - configfile = f'{progname}.conf' - configfile = os.path.join(outdir, configfile) - if not os.path.exists(outdir): - os.makedirs(outdir) - with open(configfile, 'wb') as fp: - # create a copy of config_obj and remove the basemap API key - _tmp_config_obj = copy(config_obj) - _tmp_config_obj['plot_map_api_key'] = None - _tmp_config_obj.write(fp) - - -def _init_plotting(plot_show): - # pylint: disable=import-outside-toplevel - import matplotlib.pyplot as plt - if not plot_show: - plt.switch_backend('Agg') - - -def _init_traceid_map(config): - """ - Initialize trace ID map from file. - """ - config.TRACEID_MAP = None - if config.traceid_mapping_file is None: - return - try: - with open(config.traceid_mapping_file, 'r', encoding='utf-8') as fp: - config.TRACEID_MAP = json.loads(fp.read()) - except Exception: - sys.exit( - f'traceid mapping file "{config.traceid_map_file}" not found ' - 'or not in json format.\n') - - -def _write_sample_ssp_event_file(): - ssp_event_file = 'ssp_event.yaml' - src_path = os.path.join( - os.path.dirname(__file__), 'ssp_event.yaml') - dest_path = os.path.join('.', ssp_event_file) - write_file = True - if os.path.exists(dest_path): - ans = input( - f'{ssp_event_file} already exists. ' - 'Do you want to overwrite it? [y/N] ' - ) - write_file = ans in ['y', 'Y'] - if write_file: - shutil.copyfile(src_path, dest_path) - print(f'Sample SourceSpec Event File written to: {ssp_event_file}') - - -def _fix_and_expand_path(path): - """ - Fix any path issues and expand it. - - :param str path: Path specification - :return: The fixed and expanded path - :rtype: str - """ - fixed_path = os.path.normpath(path).split(os.sep) - fixed_path = os.path.join(*fixed_path) - if path.startswith(os.sep): - fixed_path = os.path.join(os.sep, fixed_path) - elif path.startswith('~'): - fixed_path = os.path.expanduser(fixed_path) - return fixed_path - - -def _float_list(input_list, max_length=None, accepted_values=None): - """ - Convert an input list to a list of floats. - - :param list input_list: Input list or None - :return: A list of floats or None - :rtype: list - """ - if input_list is None: - return None - if accepted_values is None: - accepted_values = [] - - def _parse_float(val): - val = None if val == 'None' else val - return val if val in accepted_values else float(val) - - try: - return [_parse_float(val) for val in input_list[:max_length]] - except ValueError as e: - raise ValueError('Cannot parse all values in list') from e - - -def _none_lenght(input_list): - """ - Return the length of input list, or 1 if input list is None - - :param list input_list: Input list or None - :return: List length or 1 - :rtype: int - """ - return 1 if input_list is None else len(input_list) - - # Global config object, initialized with default values # API users should use this object to access configuration parameters # and update them as needed -config = Config() - - -def configure(options=None, progname='source_spec', config_overrides=None): - """ - Parse command line arguments and read config file. - - :param object options: An object containing command line options - :param str progname: The name of the program - :param dict config_overrides: A dictionary with parameters that override or - extend those defined in the config file - :return: A ``Config`` object with both command line and config options. - """ - if options is None: - # create an empty object to support the following getattr() calls - options = types.SimpleNamespace() - configspec = _parse_configspec() - if getattr(options, 'sampleconf', None): - _write_sample_config(configspec, progname) - sys.exit(0) - if getattr(options, 'updateconf', None): - _update_config_file(options.updateconf, configspec) - sys.exit(0) - if getattr(options, 'updatedb', None): - update_db_file(options.updatedb) - sys.exit(0) - if getattr(options, 'samplesspevent', None): - _write_sample_ssp_event_file() - sys.exit(0) - - if getattr(options, 'config_file', None): - options.config_file = _fix_and_expand_path(options.config_file) - config_obj = _read_config_file(options.config_file, configspec) - # Apply overrides - if config_overrides is not None: - try: - for key, value in config_overrides.items(): - config_obj[key] = value - except AttributeError as e: - raise ValueError( - '"config_override" must be a dict-like.' - ) from e - - # TODO: we should allow outdir to be None and not producing any output - options.outdir = getattr(options, 'outdir', 'sspec_out') - # Fix and expand paths in options - options.outdir = _fix_and_expand_path(options.outdir) - if getattr(options, 'trace_path', None): - # trace_path is a list - options.trace_path = [ - _fix_and_expand_path(path) for path in options.trace_path] - if getattr(options, 'qml_file', None): - options.qml_file = _fix_and_expand_path(options.qml_file) - if getattr(options, 'hypo_file', None): - options.hypo_file = _fix_and_expand_path(options.hypo_file) - if getattr(options, 'pick_file', None): - options.pick_file = _fix_and_expand_path(options.pick_file) - - # Create a 'no_evid_' subdir into outdir. - # The random hex string will make it sure that this name is unique - # It will be then renamed once an evid is available - hexstr = uuid.uuid4().hex - options.outdir = os.path.join(options.outdir, f'no_evid_{hexstr}') - _write_config(config_obj, progname, options.outdir) - - # Update config object with the contents of the config file - config.update(config_obj.dict()) - config.running_from_command_line = True - try: - config.validate() - except ValueError as msg: - sys.exit(msg) - - # Add options to config: - config.options = options - - # Override station_metadata config option with command line option - if getattr(options, 'station_metadata', None): - config.station_metadata = options.station_metadata - - # Fix and expand paths in config - if config.database_file: - config.database_file = _fix_and_expand_path(config.database_file) - if config.traceid_mapping_file: - config.traceid_mapping_file = _fix_and_expand_path( - config.traceid_mapping_file) - if config.station_metadata: - config.station_metadata = _fix_and_expand_path(config.station_metadata) - if config.residuals_filepath: - config.residuals_filepath = _fix_and_expand_path( - config.residuals_filepath) - - if config.plot_station_map: - try: - library_versions.check_cartopy_version() - library_versions.check_pyproj_version() - except ImportError as err: - for msg in config.warnings: - print(msg) - sys.exit(err) - - if config.NLL_time_dir is not None or config.NLL_model_dir is not None: - try: - library_versions.check_nllgrid_version() - except ImportError as err: - sys.exit(err) - - # TODO: remove these when the global config object will be implemented - global INSTR_CODES_VEL - global INSTR_CODES_ACC - INSTR_CODES_VEL = config.INSTR_CODES_VEL - INSTR_CODES_ACC = config.INSTR_CODES_ACC - - _init_traceid_map(config) - _init_plotting(config.plot_show) - return config +config = _Config() diff --git a/sourcespec2/config/configobj_helpers.py b/sourcespec2/config/configobj_helpers.py new file mode 100644 index 00000000..c8d8cba4 --- /dev/null +++ b/sourcespec2/config/configobj_helpers.py @@ -0,0 +1,81 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Helper functions for using ConfigObj in SourceSpec. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import sys +from .configobj import ConfigObj +from .configobj.validate import Validator + + +def read_config_file(config_file, configspec=None): + """ + Read a configuration file and return a ConfigObj object. + + :param config_file: path to the configuration file + :type config_file: str + :param configspec: path to the configuration specification file + :type configspec: str + + :return: ConfigObj object + :rtype: ConfigObj + """ + kwargs = { + 'configspec': configspec, + 'file_error': True, + 'default_encoding': 'utf8' + } + if configspec is None: + kwargs.update({ + 'interpolation': False, + 'list_values': False, + '_inspec': True + }) + try: + config_obj = ConfigObj(config_file, **kwargs) + except IOError as err: + sys.stderr.write(f'{err}\n') + sys.exit(1) + except Exception as err: + sys.stderr.write(f'Unable to read "{config_file}": {err}\n') + sys.exit(1) + return config_obj + + +def parse_configspec(): + """ + Parse the configuration specification file and return a ConfigObj object. + + :return: ConfigObj object + :rtype: ConfigObj + """ + configspec_file = os.path.join( + os.path.dirname(__file__), 'configspec.conf') + return read_config_file(configspec_file) + + +def get_default_config_obj(configspec): + """ + Return a ConfigObj object with default values. + + :param configspec: ConfigObj object with the configuration specification + :type configspec: ConfigObj + + :return: ConfigObj object + :rtype: ConfigObj + """ + config_obj = ConfigObj(configspec=configspec, default_encoding='utf8') + val = Validator() + config_obj.validate(val) + config_obj.defaults = [] + config_obj.initial_comment = configspec.initial_comment + config_obj.comments = configspec.comments + config_obj.final_comment = configspec.final_comment + return config_obj diff --git a/sourcespec2/config/configure_cli.py b/sourcespec2/config/configure_cli.py new file mode 100644 index 00000000..950ce4d6 --- /dev/null +++ b/sourcespec2/config/configure_cli.py @@ -0,0 +1,337 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Configure SourceSpec from command line arguments and config file. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import sys +import types +import shutil +import uuid +import json +from copy import copy +from datetime import datetime +from .config import config +from .configobj_helpers import ( + read_config_file, parse_configspec, get_default_config_obj +) +from .library_versions import library_versions +from .configobj import ConfigObj +from .configobj.validate import Validator +from ..ssp_update_db import update_db_file + +# TODO: remove these when the global config object will be implemented +INSTR_CODES_VEL = [] +INSTR_CODES_ACC = [] + + +def _write_sample_config(configspec, progname): + """ + Write a sample configuration file. + + :param configspec: The configuration specification + :type configspec: ConfigObj + :param progname: The name of the program + :type progname: str + """ + config_obj = get_default_config_obj(configspec) + configfile = f'{progname}.conf' + write_file = True + if os.path.exists(configfile): + ans = input( + f'{configfile} already exists. Do you want to overwrite it? [y/N] ' + ) + write_file = ans in ['y', 'Y'] + if write_file: + with open(configfile, 'wb') as fp: + config_obj.write(fp) + print(f'Sample config file written to: {configfile}') + note = """ +Note that the default config parameters are suited for a M<5 earthquake +recorded within ~100 km. Adjust `win_length`, `noise_pre_time`, and the +frequency bands (`bp_freqmin_*`, `bp_freqmax_*`, `freq1_*`, `freq2_*`) +according to your setup.""" + print(note) + + +def _update_config_file(config_file, configspec): + """ + Update a configuration file to the latest version. + + :param config_file: The path to the configuration file + :type config_file: str + :param configspec: The configuration specification + :type configspec: ConfigObj + """ + config_obj = read_config_file(config_file, configspec) + val = Validator() + config_obj.validate(val) + mod_time = datetime.fromtimestamp(os.path.getmtime(config_file)) + mod_time_str = mod_time.strftime('%Y%m%d_%H%M%S') + config_file_old = f'{config_file}.{mod_time_str}' + ans = input( + f'Ok to update {config_file}? [y/N]\n' + f'(Old file will be saved as {config_file_old}) ' + ) + if ans not in ['y', 'Y']: + sys.exit(0) + config_new = ConfigObj(configspec=configspec, default_encoding='utf8') + config_new = read_config_file(None, configspec) + config_new.validate(val) + config_new.defaults = [] + config_new.comments = configspec.comments + config_new.initial_comment = config_obj.initial_comment + config_new.final_comment = configspec.final_comment + for k, v in config_obj.items(): + if k not in config_new: + continue + # Fix for force_list(default=None) + if v == ['None', ]: + v = None + config_new[k] = v + migrate_options = { + 's_win_length': 'win_length', + 'traceids': 'traceid_mapping_file', + 'ignore_stations': 'ignore_traceids', + 'use_stations': 'use_traceids', + 'dataless': 'station_metadata', + 'clip_nmax': 'clip_max_percent', + 'PLOT_SHOW': 'plot_show', + 'PLOT_SAVE': 'plot_save', + 'PLOT_SAVE_FORMAT': 'plot_save_format', + 'vp': 'vp_source', + 'vs': 'vs_source', + 'rho': 'rho_source', + 'pre_p_time': 'noise_pre_time', + 'pre_s_time': 'signal_pre_time', + 'rps_from_focal_mechanism': 'rp_from_focal_mechanism', + 'paz': 'station_metadata', + 'pi_bsd_min_max': 'pi_ssd_min_max', + 'max_epi_dist': 'epi_dist_ranges' + } + for old_opt, new_opt in migrate_options.items(): + if old_opt in config_obj and config_obj[old_opt] != 'None': + # max_epi_dist needs to be converted to a list + if old_opt == 'max_epi_dist': + config_new[new_opt] = [0, config_obj[old_opt]] + else: + config_new[new_opt] = config_obj[old_opt] + shutil.copyfile(config_file, config_file_old) + with open(config_file, 'wb') as fp: + config_new.write(fp) + print(f'{config_file}: updated') + + +def _write_config(config_obj, progname, outdir): + """ + Write the configuration file to the output directory. + + :param config_obj: The configuration object + :type config_obj: ConfigObj + :param progname: The name of the program + :type progname: str + :param outdir: The output directory + :type outdir: str + """ + if progname != 'source_spec': + return + configfile = f'{progname}.conf' + configfile = os.path.join(outdir, configfile) + if not os.path.exists(outdir): + os.makedirs(outdir) + with open(configfile, 'wb') as fp: + # create a copy of config_obj and remove the basemap API key + _tmp_config_obj = copy(config_obj) + _tmp_config_obj['plot_map_api_key'] = None + _tmp_config_obj.write(fp) + + +def _init_plotting(): + """ + Initialize plotting backend. + """ + # pylint: disable=import-outside-toplevel + import matplotlib.pyplot as plt + if not config.plot_show: + plt.switch_backend('Agg') + + +def _init_traceid_map(): + """ + Initialize trace ID map from file. + """ + config.TRACEID_MAP = None + if config.traceid_mapping_file is None: + return + try: + with open(config.traceid_mapping_file, 'r', encoding='utf-8') as fp: + config.TRACEID_MAP = json.loads(fp.read()) + except Exception: + sys.exit( + f'traceid mapping file "{config.traceid_map_file}" not found ' + 'or not in json format.\n') + + +def _write_sample_ssp_event_file(): + """ + Write a sample SourceSpec Event file. + """ + ssp_event_file = 'ssp_event.yaml' + src_path = os.path.join( + os.path.dirname(__file__), 'ssp_event.yaml') + dest_path = os.path.join('.', ssp_event_file) + write_file = True + if os.path.exists(dest_path): + ans = input( + f'{ssp_event_file} already exists. ' + 'Do you want to overwrite it? [y/N] ' + ) + write_file = ans in ['y', 'Y'] + if write_file: + shutil.copyfile(src_path, dest_path) + print(f'Sample SourceSpec Event File written to: {ssp_event_file}') + + +def _fix_and_expand_path(path): + """ + Fix any path issues and expand it. + + :param path: Path specification + :type path: str + :return: The fixed and expanded path + :rtype: str + """ + fixed_path = os.path.normpath(path).split(os.sep) + fixed_path = os.path.join(*fixed_path) + if path.startswith(os.sep): + fixed_path = os.path.join(os.sep, fixed_path) + elif path.startswith('~'): + fixed_path = os.path.expanduser(fixed_path) + return fixed_path + + +def configure_cli(options=None, progname='source_spec', config_overrides=None): + """ + Configure SourceSpec from command line arguments and config file. + + :param options: An object containing command line options + :type options: A generic object + :param progname: The name of the program + :type progname: str + :param config_overrides: A dictionary with parameters that override or + extend those defined in the config file + :type config_overrides: dict + + :return: A ``Config`` object with both command line and config options. + :rtype: Config + """ + if options is None: + # create an empty object to support the following getattr() calls + options = types.SimpleNamespace() + configspec = parse_configspec() + if getattr(options, 'sampleconf', None): + _write_sample_config(configspec, progname) + sys.exit(0) + if getattr(options, 'updateconf', None): + _update_config_file(options.updateconf, configspec) + sys.exit(0) + if getattr(options, 'updatedb', None): + update_db_file(options.updatedb) + sys.exit(0) + if getattr(options, 'samplesspevent', None): + _write_sample_ssp_event_file() + sys.exit(0) + + if getattr(options, 'config_file', None): + options.config_file = _fix_and_expand_path(options.config_file) + config_obj = read_config_file(options.config_file, configspec) + # Apply overrides + if config_overrides is not None: + try: + for key, value in config_overrides.items(): + config_obj[key] = value + except AttributeError as e: + raise ValueError( + '"config_override" must be a dict-like.' + ) from e + + # TODO: we should allow outdir to be None and not producing any output + options.outdir = getattr(options, 'outdir', 'sspec_out') + # Fix and expand paths in options + options.outdir = _fix_and_expand_path(options.outdir) + if getattr(options, 'trace_path', None): + # trace_path is a list + options.trace_path = [ + _fix_and_expand_path(path) for path in options.trace_path] + if getattr(options, 'qml_file', None): + options.qml_file = _fix_and_expand_path(options.qml_file) + if getattr(options, 'hypo_file', None): + options.hypo_file = _fix_and_expand_path(options.hypo_file) + if getattr(options, 'pick_file', None): + options.pick_file = _fix_and_expand_path(options.pick_file) + + if getattr(options, 'config_file', None): + # Create a 'no_evid_' subdir into outdir. + # The random hex string will make it sure that this name is unique + # It will be then renamed once an evid is available + hexstr = uuid.uuid4().hex + options.outdir = os.path.join(options.outdir, f'no_evid_{hexstr}') + _write_config(config_obj, progname, options.outdir) + # Update config object with the contents of the config file + config.update(config_obj.dict()) + config.running_from_command_line = True + + try: + config.validate() + except ValueError as msg: + sys.exit(msg) + + # Add options to config: + config.options = options + + # Override station_metadata config option with command line option + if getattr(options, 'station_metadata', None): + config.station_metadata = options.station_metadata + + # Fix and expand paths in config + if config.database_file: + config.database_file = _fix_and_expand_path(config.database_file) + if config.traceid_mapping_file: + config.traceid_mapping_file = _fix_and_expand_path( + config.traceid_mapping_file) + if config.station_metadata: + config.station_metadata = _fix_and_expand_path(config.station_metadata) + if config.residuals_filepath: + config.residuals_filepath = _fix_and_expand_path( + config.residuals_filepath) + + if config.plot_station_map: + try: + library_versions.check_cartopy_version() + library_versions.check_pyproj_version() + except ImportError as err: + for msg in config.warnings: + print(msg) + sys.exit(err) + + if config.NLL_time_dir is not None or config.NLL_model_dir is not None: + try: + library_versions.check_nllgrid_version() + except ImportError as err: + sys.exit(err) + + # TODO: remove these when the global config object will be implemented + global INSTR_CODES_VEL + global INSTR_CODES_ACC + INSTR_CODES_VEL = config.INSTR_CODES_VEL + INSTR_CODES_ACC = config.INSTR_CODES_ACC + + _init_traceid_map() + _init_plotting() + return config diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 1478fbe8..752e9dd2 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -84,7 +84,7 @@ def main(): # Lazy-import modules for speed from .ssp_parse_arguments import parse_args options = parse_args(progname='source_model') - from .config import configure + from .config import configure_cli from .ssp_setup import ssp_exit plot_show = bool(options.plot) conf_overrides = { @@ -92,7 +92,7 @@ def main(): 'plot_save': False, 'html_report': False } - config = configure( + config = configure_cli( options, progname='source_model', config_overrides=conf_overrides) from .spectrum import SpectrumStream from .ssp_read_traces import read_traces diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 91603c35..8e604032 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -25,8 +25,8 @@ def main(): options = parse_args(progname='source_spec') # Setup stage - from .config import configure - config = configure(options, progname='source_spec') + from .config import configure_cli + config = configure_cli(options, progname='source_spec') from .ssp_setup import ( move_outdir, remove_old_outdir, setup_logging, save_config, ssp_exit) diff --git a/sourcespec2/ssp_read_station_metadata.py b/sourcespec2/ssp_read_station_metadata.py index 811b4851..bc9c5a78 100644 --- a/sourcespec2/ssp_read_station_metadata.py +++ b/sourcespec2/ssp_read_station_metadata.py @@ -15,7 +15,7 @@ import logging from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response -from .config.config import INSTR_CODES_VEL, INSTR_CODES_ACC +from .config.configure_cli import INSTR_CODES_VEL, INSTR_CODES_ACC logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) From 4a336745eee27cbbcf4da4c872c59edb9a6e9d7b Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 15:34:58 +0200 Subject: [PATCH 11/73] Use the global `config` object in ssp_setup.py and ssp_read_traces.py --- sourcespec2/config/configure_cli.py | 7 +- sourcespec2/source_model.py | 6 +- sourcespec2/source_spec.py | 16 ++-- sourcespec2/ssp_read_traces.py | 116 +++++++++++++++++++++++----- sourcespec2/ssp_setup.py | 16 ++-- 5 files changed, 123 insertions(+), 38 deletions(-) diff --git a/sourcespec2/config/configure_cli.py b/sourcespec2/config/configure_cli.py index 950ce4d6..861671eb 100644 --- a/sourcespec2/config/configure_cli.py +++ b/sourcespec2/config/configure_cli.py @@ -220,6 +220,9 @@ def configure_cli(options=None, progname='source_spec', config_overrides=None): """ Configure SourceSpec from command line arguments and config file. + The global config object is updated with the configuration parameters + read from the configuration file and the command line options. + :param options: An object containing command line options :type options: A generic object :param progname: The name of the program @@ -227,9 +230,6 @@ def configure_cli(options=None, progname='source_spec', config_overrides=None): :param config_overrides: A dictionary with parameters that override or extend those defined in the config file :type config_overrides: dict - - :return: A ``Config`` object with both command line and config options. - :rtype: Config """ if options is None: # create an empty object to support the following getattr() calls @@ -334,4 +334,3 @@ def configure_cli(options=None, progname='source_spec', config_overrides=None): _init_traceid_map() _init_plotting() - return config diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 752e9dd2..9f529e5e 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -84,7 +84,7 @@ def main(): # Lazy-import modules for speed from .ssp_parse_arguments import parse_args options = parse_args(progname='source_model') - from .config import configure_cli + from .config import config, configure_cli from .ssp_setup import ssp_exit plot_show = bool(options.plot) conf_overrides = { @@ -92,7 +92,7 @@ def main(): 'plot_save': False, 'html_report': False } - config = configure_cli( + configure_cli( options, progname='source_model', config_overrides=conf_overrides) from .spectrum import SpectrumStream from .ssp_read_traces import read_traces @@ -102,7 +102,7 @@ def main(): # We don't use weighting in source_model config.weighting = 'no_weight' if len(config.options.trace_path) > 0: - st = read_traces(config) + st = read_traces() # Deconvolve, filter, cut traces: proc_st = process_traces(config, st) # Build spectra (amplitude in magnitude units) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 8e604032..12c18fb1 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -25,23 +25,23 @@ def main(): options = parse_args(progname='source_spec') # Setup stage - from .config import configure_cli - config = configure_cli(options, progname='source_spec') + from .config import config, configure_cli + configure_cli(options, progname='source_spec') from .ssp_setup import ( move_outdir, remove_old_outdir, setup_logging, save_config, ssp_exit) - setup_logging(config) + setup_logging() from .ssp_read_traces import read_traces - st = read_traces(config) + st = read_traces() # Now that we have an evid, we can rename the outdir and the log file - move_outdir(config) - setup_logging(config, config.event.event_id) - remove_old_outdir(config) + move_outdir() + setup_logging(config.event.event_id) + remove_old_outdir() # Save config to out dir - save_config(config) + save_config() # Deconvolve, filter, cut traces: from .ssp_process_traces import process_traces diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index 9e461560..7c79c0e1 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -26,6 +26,7 @@ from obspy import read from obspy.core import Stream from obspy.core.util import AttribDict +from .config import config from .ssp_setup import ssp_exit from .ssp_util import MediumProperties from .ssp_read_station_metadata import ( @@ -40,7 +41,13 @@ # TRACE MANIPULATION ---------------------------------------------------------- -def _correct_traceid(trace, config): +def _correct_traceid(trace): + """ + Correct traceid from config.TRACEID_MAP, if available. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + """ if config.TRACEID_MAP is None: return with contextlib.suppress(KeyError): @@ -52,8 +59,13 @@ def _correct_traceid(trace, config): trace.stats.channel = chan -def _add_instrtype(trace, config): - """Add instrtype to trace.""" +def _add_instrtype(trace): + """ + Add instrtype to trace. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + """ instrtype = None band_code = None instr_code = None @@ -82,8 +94,15 @@ def _add_instrtype(trace, config): trace.stats.info = f'{trace.id} {trace.stats.instrtype}' -def _add_inventory(trace, inventory, config): - """Add inventory to trace.""" +def _add_inventory(trace, inventory): + """ + Add inventory to trace. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param inventory: ObsPy Inventory object + :type inventory: :class:`obspy.core.inventory.Inventory` + """ net, sta, loc, chan = trace.id.split('.') inv = ( inventory.select( @@ -125,7 +144,12 @@ def _add_inventory(trace, inventory, config): def _check_instrtype(trace): - """Check if instrument type is consistent with units in inventory.""" + """ + Check if instrument type is consistent with units in inventory. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + """ inv = trace.stats.inventory if not inv: raise RuntimeError( @@ -164,7 +188,12 @@ def _check_instrtype(trace): def _add_coords(trace): - """Add coordinates to trace.""" + """ + Add coordinates to trace. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + """ # If we already know that traceid is skipped, raise a silent exception if trace.id in _add_coords.skipped: raise RuntimeError() @@ -203,7 +232,14 @@ def _add_coords(trace): def _add_event(trace, ssp_event=None): - """Add ssp_event object to trace.""" + """ + Add ssp_event object to trace. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param ssp_event: SSPEvent object (default: None) + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + """ if ssp_event is None: # Try to get hypocenter information from the SAC header try: @@ -214,7 +250,14 @@ def _add_event(trace, ssp_event=None): def _add_picks(trace, picks=None): - """Add picks to trace.""" + """ + Add picks to trace. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param picks: list of picks (default: None) + :type picks: list of :class:`sourcespec.ssp_event.Pick` + """ if picks is None: picks = [] trace_picks = [] @@ -232,7 +275,12 @@ def _add_picks(trace, picks=None): def _complete_picks(st): - """Add component-specific picks to all components.""" + """ + Add component-specific picks to all components in a stream. + + :param st: ObsPy Stream object + :type st: :class:`obspy.core.stream.Stream` + """ for station in {tr.stats.station for tr in st}: st_sel = st.select(station=station) # 'code' is band+instrument code @@ -254,7 +302,13 @@ def _complete_picks(st): # FILE PARSING ---------------------------------------------------------------- -def _hypo_vel(hypo, config): +def _hypo_vel(hypo): + """ + Compute velocity at hypocenter. + + :param hypo: Hypocenter object + :type hypo: :class:`sourcespec.ssp_event.Hypocenter` + """ medium_properties = MediumProperties( hypo.longitude, hypo.latitude, hypo.depth.value_in_km, config) hypo.vp = medium_properties.get(mproperty='vp', where='source') @@ -269,6 +323,16 @@ def _hypo_vel(hypo, config): def _build_filelist(path, filelist, tmpdir): + """ + Build a list of files to read. + + :param path: Path to a file or directory + :type path: str + :param filelist: List of files to read + :type filelist: list + :param tmpdir: Temporary directory + :type tmpdir: str + """ if os.path.isdir(path): listing = os.listdir(path) for filename in listing: @@ -299,10 +363,20 @@ def _build_filelist(path, filelist, tmpdir): filelist.append(path) -def _read_trace_files(config, inventory, ssp_event, picks): +def _read_trace_files(inventory, ssp_event, picks): """ Read trace files from a given path. Complete trace metadata and return a stream object. + + :param inventory: ObsPy Inventory object + :type inventory: :class:`obspy.core.inventory.Inventory` + :param ssp_event: SSPEvent object + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + :param picks: list of picks + :type picks: list of :class:`sourcespec.ssp_event.Pick` + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` """ # phase 1: build a file list # ph 1.1: create a temporary dir and run '_build_filelist()' @@ -342,10 +416,10 @@ def _read_trace_files(config, inventory, ssp_event, picks): if (config.options.station is not None and trace.stats.station != config.options.station): continue - _correct_traceid(trace, config) + _correct_traceid(trace) try: - _add_instrtype(trace, config) - _add_inventory(trace, inventory, config) + _add_instrtype(trace) + _add_inventory(trace, inventory) _check_instrtype(trace) _add_coords(trace) _add_event(trace, ssp_event) @@ -361,13 +435,19 @@ def _read_trace_files(config, inventory, ssp_event, picks): def _log_event_info(ssp_event): + """ + Log event information. + + :param ssp_event: SSPEvent object + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + """ for line in str(ssp_event).splitlines(): logger.info(line) logger.info('---------------------------------------------------') # Public interface: -def read_traces(config): +def read_traces(): """Read traces, store waveforms and metadata.""" # read station metadata into an ObsPy ``Inventory`` object inventory = read_station_metadata(config.station_metadata) @@ -390,7 +470,7 @@ def read_traces(config): # finally, read trace files logger.info('Reading traces...') - st = _read_trace_files(config, inventory, ssp_event, picks) + st = _read_trace_files(inventory, ssp_event, picks) logger.info('Reading traces: done') logger.info('---------------------------------------------------') if len(st) == 0: @@ -414,7 +494,7 @@ def read_traces(config): ssp_exit(1) # add velocity info to hypocenter try: - _hypo_vel(ssp_event.hypocenter, config) + _hypo_vel(ssp_event.hypocenter) except Exception as e: logger.error( f'Unable to compute velocity at hypocenter: {e}\n') diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index 92f86219..420347bb 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -24,7 +24,7 @@ import contextlib from datetime import datetime from . import __version__, __banner__ -from .config import library_versions +from .config import config, library_versions # define ipshell(), if possible # note: ANSI colors do not work on Windows standard terminal @@ -43,7 +43,7 @@ SSP_EXIT_CALLED = False -def save_config(config): +def save_config(): """Save config file to output dir.""" # Actually, it renames the file already existing. src = os.path.join(config.options.outdir, 'source_spec.conf') @@ -55,7 +55,7 @@ def save_config(config): os.rename(src, dst) -def move_outdir(config): +def move_outdir(): """Move outdir to a new dir named from evid (and optional run_id).""" try: evid = config.event.event_id @@ -85,7 +85,7 @@ def move_outdir(config): config.options.outdir = dst -def remove_old_outdir(config): +def remove_old_outdir(): """Try to remove the old outdir.""" try: oldoutdir = config.options.oldoutdir @@ -121,7 +121,7 @@ def new(*args): return new -def setup_logging(config, basename=None, progname='source_spec'): +def setup_logging(basename=None, progname='source_spec'): """ Set up the logging infrastructure. @@ -129,6 +129,12 @@ def setup_logging(config, basename=None, progname='source_spec'): and a second time with a basename (typically the eventid). When called the second time, the previous logfile is renamed using the given basename. + + :param basename: The basename for the logfile (default: None). + :type basename: str + :param progname: The program name to be used in the log file (default: + 'source_spec') + :type progname: str """ # pylint: disable=global-statement global OLDLOGFILE From 100f72e201780002d0e477e821c19bd5c3761336 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 10:03:31 +0200 Subject: [PATCH 12/73] Use the global `config` object in ssp_read_event_metadata.py and ssp_read_sac_header.py --- sourcespec2/ssp_read_event_metadata.py | 141 +++++++++++++++++++++++-- sourcespec2/ssp_read_sac_header.py | 53 ++++++++-- sourcespec2/ssp_read_traces.py | 6 +- 3 files changed, 180 insertions(+), 20 deletions(-) diff --git a/sourcespec2/ssp_read_event_metadata.py b/sourcespec2/ssp_read_event_metadata.py index 07bbc20c..4037d8a2 100644 --- a/sourcespec2/ssp_read_event_metadata.py +++ b/sourcespec2/ssp_read_event_metadata.py @@ -20,14 +20,20 @@ import yaml from obspy import UTCDateTime from obspy import read_events +from .config import config from .ssp_setup import ssp_exit from .ssp_event import SSPEvent from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def parse_qml(config): - """Parse event metadata and picks from a QuakeML file.""" +def parse_qml(): + """ + Parse event metadata and picks from a QuakeML file. + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + """ ssp_event = None picks = [] qml_file = config.options.qml_file @@ -76,6 +82,17 @@ def parse_qml(config): def _get_event_from_qml(qml_file, event_id=None): + """ + Get an event from a QuakeML file. + + :param qml_file: QuakeML file + :type qml_file: str + :param event_id: event id + :type event_id: str + + :return: QuakeML event + :rtype: obspy.core.event.Event + """ cat = read_events(qml_file) if event_id is not None: _qml_events = [ev for ev in cat if event_id in str(ev.resource_id)] @@ -98,7 +115,10 @@ def _get_evid_from_resource_id(resource_id): Get evid from resource_id. :param resource_id: resource_id string + :type resource_id: str + :returns: evid string + :rtype: str """ evid = resource_id if '/' in evid: @@ -115,6 +135,19 @@ def _get_evid_from_resource_id(resource_id): def _parse_qml_event( qml_event, parse_event_name_from_description=False, event_description_regex=None): + """ + Parse event metadata from a QuakeML event. + + :param qml_event: QuakeML event + :type qml_event: obspy.core.event.Event + :param parse_event_name_from_description: parse event name from description + :type parse_event_name_from_description: bool + :param event_description_regex: regex to extract event name + :type event_description_regex: str + + :return: SSPEvent object + :rtype: SSPEvent + """ ssp_event = SSPEvent() ssp_event.event_id = _get_evid_from_resource_id( str(qml_event.resource_id.id)) @@ -146,18 +179,42 @@ def _parse_qml_event( def _parse_magnitude_from_qml_event(qml_event, ssp_event): + """ + Parse magnitude from a QuakeML event. + + :param qml_event: QuakeML event + :type qml_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ mag = qml_event.preferred_magnitude() or qml_event.magnitudes[0] ssp_event.magnitude.value = mag.mag ssp_event.magnitude.mag_type = mag.magnitude_type def _parse_scalar_moment_from_qml_event(qml_event, ssp_event): + """ + Parse scalar moment from a QuakeML event. + + :param qml_event: QuakeML event + :type qml_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ fm = qml_event.preferred_focal_mechanism() or qml_event.focal_mechanisms[0] ssp_event.scalar_moment.value = fm.moment_tensor.scalar_moment ssp_event.scalar_moment.units = 'N-m' def _parse_moment_tensor_from_qml_event(qml_event, ssp_event): + """ + Parse moment tensor from a QuakeML event. + + :param qml_event: QuakeML event + :type qml_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ fm = qml_event.preferred_focal_mechanism() or qml_event.focal_mechanisms[0] mt = fm.moment_tensor.tensor ssp_event.moment_tensor.m_rr = mt.m_rr @@ -170,6 +227,14 @@ def _parse_moment_tensor_from_qml_event(qml_event, ssp_event): def _parse_focal_mechanism_from_qml_event(qml_event, ssp_event): + """ + Parse focal mechanism from a QuakeML event. + + :param qml_event: QuakeML event + :type qml_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ fm = qml_event.focal_mechanisms[0] nodal_plane = fm.nodal_planes.nodal_plane_1 ssp_event.focal_mechanism.strike = nodal_plane.strike @@ -178,6 +243,17 @@ def _parse_focal_mechanism_from_qml_event(qml_event, ssp_event): def _parse_picks_from_qml_event(ev, origin): + """ + Parse picks from a QuakeML event. + + :param ev: QuakeML event + :type ev: obspy.core.event.Event + :param origin: QuakeML origin object + :type origin: obspy.core.event.Origin + + :return: list of SSPPick objects + :rtype: list + """ picks = [] for pck in ev.picks: pick = SSPPick() @@ -219,9 +295,12 @@ def _parse_hypo71_hypocenter(hypo_file, _): Parse a hypo71 hypocenter file. :param hypo_file: path to hypo71 hypocenter file + :type hypo_file: str :param _: unused (for consistency with other parsers) + :type _: None :return: a tuple of (SSPEvent, picks) + :rtype: tuple """ with open(hypo_file, encoding='ascii') as fp: line = fp.readline() @@ -257,6 +336,15 @@ def _parse_hypo71_hypocenter(hypo_file, _): def _parse_hypo2000_hypo_line(line): + """ + Parse a line from a hypo2000 hypocenter file. + + :param line: line from hypo2000 hypocenter file + :type line: str + + :return: SSPEvent object + :rtype: SSPEvent + """ word = line.split() ssp_event = SSPEvent() hypo = ssp_event.hypocenter @@ -308,6 +396,19 @@ def _parse_hypo2000_hypo_line(line): def _parse_hypo2000_station_line(line, oldpick, origin_time): + """ + Parse a line from a hypo2000 station file. + + :param line: line from hypo2000 station file + :type line: str + :param oldpick: previous pick + :type oldpick: SSPPick + :param origin_time: origin time + :type origin_time: UTCDateTime + + :return: SSPPick object + :rtype: SSPPick + """ if oldpick is not None: oldstation = oldpick.station oldnetwork = oldpick.network @@ -341,9 +442,12 @@ def _parse_hypo2000_file(hypo_file, _): Parse a hypo2000 hypocenter file. :param hypo_file: path to hypo2000 hypocenter file + :type hypo_file: str :param _: unused (for consistency with other parsers) + :type _: None :return: a tuple of (SSPEvent, picks) + :rtype: tuple """ ssp_event = None picks = [] @@ -391,9 +495,12 @@ def _parse_source_spec_event_file(event_file, event_id=None): Parse a SourceSpec Event File, which is a YAML file. :param event_file: path to SourceSpec event file + :type event_file: str :param evid: event id + :type evid: str :return: SSPEvent object + :rtype: SSPEvent """ try: with open(event_file, encoding='utf-8') as fp: @@ -452,10 +559,13 @@ def parse_hypo_file(hypo_file, event_id=None): """ Parse a SourceSpec Event File, hypo71 or hypo2000 hypocenter file. - :param hypo_file: - Path to the hypocenter file. - :returns: - A tuple of (SSPEvent, picks, format). + :param hypo_file: Path to the hypocenter file. + :type hypo_file: str + :param event_id: Event ID. + :type event_id: str + + :return: A tuple of (SSPEvent, picks, format). + :rtype: tuple """ if not os.path.exists(hypo_file): logger.error(f'{hypo_file}: No such file or directory') @@ -488,6 +598,14 @@ def parse_hypo_file(hypo_file, event_id=None): def _is_hypo71_picks(pick_file): + """ + Check if a file is a hypo71 phase file. + + :param pick_file: path to hypo71 phase file + :type pick_file: str + + :raises TypeError: if the file is not a hypo71 phase file + """ with open(pick_file, encoding='ascii') as fp: for line in fp: # remove newline @@ -505,13 +623,15 @@ def _is_hypo71_picks(pick_file): raise TypeError(f'{pick_file}: Not a hypo71 phase file') -def _correct_station_name(station, config): +def _correct_station_name(station): """ Correct station name, based on a traceid map. :param station: station name + :type station: str :return: corrected station name + :rtype: str """ if config.TRACEID_MAP is None: return station @@ -526,13 +646,12 @@ def _correct_station_name(station, config): return traceid.split('.')[1] -def parse_hypo71_picks(config): +def parse_hypo71_picks(): """ Parse hypo71 picks file - :param config: Config object - :return: list of SSPPick objects + :rtype: list """ picks = [] pick_file = config.options.pick_file @@ -560,7 +679,7 @@ def parse_hypo71_picks(config): continue pick = SSPPick() pick.station = line[:4].strip() - pick.station = _correct_station_name(pick.station, config) + pick.station = _correct_station_name(pick.station) pick.flag = line[4:5] pick.phase = line[5:6] pick.polarity = line[6:7] diff --git a/sourcespec2/ssp_read_sac_header.py b/sourcespec2/ssp_read_sac_header.py index 70a21c7d..41848737 100644 --- a/sourcespec2/ssp_read_sac_header.py +++ b/sourcespec2/ssp_read_sac_header.py @@ -13,14 +13,23 @@ import logging import contextlib from obspy.core.util import AttribDict +from .config import config from .ssp_setup import ssp_exit from .ssp_event import SSPEvent from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def compute_sensitivity_from_SAC(trace, config): - """Compute sensitivity from SAC header fields.""" +def compute_sensitivity_from_SAC(trace): + """ + Compute sensitivity from SAC header fields. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + + :return: Sensitivity + :rtype: float + """ # Securize the string before calling eval() # see https://stackoverflow.com/a/25437733/2021880 inp = re.sub(r'\.(?![0-9])', '', config.sensitivity) @@ -53,7 +62,15 @@ def compute_sensitivity_from_SAC(trace, config): def get_instrument_from_SAC(trace): - """Get instrument information from SAC header.""" + """ + Get instrument information from SAC header. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core + + :return: Instrument type, band code, instrument code + :rtype: tuple + """ try: codes = instruments[trace.stats.sac.kinst] instrtype = codes['instrtype'] @@ -74,7 +91,15 @@ def get_instrument_from_SAC(trace): def get_station_coordinates_from_SAC(trace): - """Get station coordinates from SAC header.""" + """ + Get station coordinates from SAC header. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core + + :return: Station coordinates + :rtype: :class:`AttribDict` or None + """ with contextlib.suppress(Exception): stla = trace.stats.sac.stla stlo = trace.stats.sac.stlo @@ -87,7 +112,15 @@ def get_station_coordinates_from_SAC(trace): def get_event_from_SAC(trace): - """Get event information from SAC header.""" + """ + Get event information from SAC header. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core + + :return: Event information + :rtype: :class:`ssp_event.SSPEvent` + """ try: sac_hdr = trace.stats.sac except AttributeError as e: @@ -133,7 +166,15 @@ def get_event_from_SAC(trace): def get_picks_from_SAC(trace): - """Get picks from SAC header.""" + """ + Get picks from SAC header. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core + + :return: List of picks + :rtype: list of :class:`ssp_pick.SSPPick` + """ try: sac_hdr = trace.stats.sac except AttributeError as e: diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index 7c79c0e1..68d9e302 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -125,7 +125,7 @@ def _add_inventory(trace, inventory): coords = inv.get_coordinates(trace.id, trace.stats.starttime) paz = PAZ() paz.seedID = trace.id - paz.sensitivity = compute_sensitivity_from_SAC(trace, config) + paz.sensitivity = compute_sensitivity_from_SAC(trace) paz.poles = [] paz.zeros = [] if inv: @@ -461,10 +461,10 @@ def read_traces(): config.hypo_file_format = file_format # parse pick file if config.options.pick_file is not None: - picks = parse_hypo71_picks(config) + picks = parse_hypo71_picks() # parse QML file if config.options.qml_file is not None: - ssp_event, picks = parse_qml(config) + ssp_event, picks = parse_qml() if ssp_event is not None: _log_event_info(ssp_event) From 6f3fa528664e676a8a603c22ed5447cf169946d1 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 15:53:02 +0200 Subject: [PATCH 13/73] Use the global `config` object in ssp_process_traces.py. --- sourcespec2/source_model.py | 2 +- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_process_traces.py | 190 ++++++++++++++++++++++++------ 3 files changed, 155 insertions(+), 39 deletions(-) diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 9f529e5e..6c5dc6c4 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -104,7 +104,7 @@ def main(): if len(config.options.trace_path) > 0: st = read_traces() # Deconvolve, filter, cut traces: - proc_st = process_traces(config, st) + proc_st = process_traces(st) # Build spectra (amplitude in magnitude units) spec_st, _specnoise_st, _weight_st = build_spectra(config, proc_st) if len(spec_st) == 0: diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 12c18fb1..c6aa5852 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -45,7 +45,7 @@ def main(): # Deconvolve, filter, cut traces: from .ssp_process_traces import process_traces - proc_st = process_traces(config, st) + proc_st = process_traces(st) # Build spectra (amplitude in magnitude units) from .ssp_build_spectra import build_spectra diff --git a/sourcespec2/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py index 73f924f9..86eb8643 100644 --- a/sourcespec2/ssp_process_traces.py +++ b/sourcespec2/ssp_process_traces.py @@ -20,6 +20,7 @@ from scipy.signal import savgol_filter from obspy.core import Stream from obspy.core.util import AttribDict +from .config import config from .ssp_setup import ssp_exit from .ssp_util import ( remove_instr_response, station_to_event_position) @@ -45,8 +46,18 @@ def _skip_stream_and_raise(st, reason, short_reason=None): raise RuntimeError(f'{st[0].id}: {reason}: skipping trace') -def _get_bandpass_frequencies(config, trace): - """Get frequencies for bandpass filter.""" +def _get_bandpass_frequencies(trace): + """ + Get frequencies for bandpass filter. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :return: Tuple with minimum and maximum frequencies. + :rtype: tuple + + :raises: ValueError if instrument type is unknown. + """ # see if there is a station-specific filter station = trace.stats.station try: @@ -66,9 +77,16 @@ def _get_bandpass_frequencies(config, trace): return bp_freqmin, bp_freqmax -def filter_trace(config, trace): - """Filter trace.""" - bp_freqmin, bp_freqmax = _get_bandpass_frequencies(config, trace) +def filter_trace(trace): + """ + Filter trace in place. + + Save filter info to trace stats. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + """ + bp_freqmin, bp_freqmax = _get_bandpass_frequencies(trace) nyquist = 1. / (2. * trace.stats.delta) if bp_freqmax >= nyquist: bp_freqmax = nyquist * 0.999 @@ -85,7 +103,15 @@ def filter_trace(config, trace): trace.stats.filter = AttribDict(filter_options) -def _check_signal_level(config, trace): +def _check_signal_level(trace): + """ + Check if the trace has significant signal level. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core + + :raises: RuntimeError if trace RMS is smaller than config.rmsmin. + """ rms2 = np.power(trace.data, 2).sum() rms = np.sqrt(rms2) rms_min = config.rmsmin @@ -97,7 +123,13 @@ def _check_signal_level(config, trace): ) -def _check_clipping(config, trace): +def _check_clipping(trace): + """ + Check if the trace is clipped. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + """ trace.stats.clipped = False if config.clipping_detection_algorithm == 'none': return @@ -147,7 +179,15 @@ def _check_clipping(config, trace): 'skipping trace') -def _check_sn_ratio(config, trace): +def _check_sn_ratio(trace): + """ + Check if the trace has significant signal to noise ratio. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if no noise window is available. + """ trace_noise = _get_detrended_trace_copy(trace) t1 = trace_noise.stats.arrivals['N1'][1] t2 = trace_noise.stats.arrivals['N2'][1] @@ -189,6 +229,15 @@ def _check_sn_ratio(config, trace): def _get_detrended_trace_copy(trace): + """ + Get a copy of the trace with the mean and linear trend removed. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :return: Trace with mean and linear trend removed. + :rtype: :class:`obspy.core.trace.Trace` + """ # noise time window for s/n ratio tr_copy = trace.copy() # remove the mean... @@ -198,10 +247,13 @@ def _get_detrended_trace_copy(trace): return tr_copy -def _remove_baseline(config, trace): +def _remove_baseline(trace): """ Get the signal baseline using a Savitzky-Golay filter and subtract it from the trace. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` """ if not config.remove_baseline: return @@ -213,7 +265,18 @@ def _remove_baseline(config, trace): trace.data -= baseline -def _process_trace(config, trace): +def _process_trace(trace): + """ + Process a trace. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :return: Processed trace. + :rtype: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if unable to remove instrument response. + """ # copy trace for manipulation trace_process = trace.copy() comp = trace_process.stats.channel @@ -228,11 +291,11 @@ def _process_trace(config, trace): trace_process.stats.ignore = True trace_process.stats.ignore_reason = 'vertical' # check if the trace has (significant) signal - _check_signal_level(config, trace_process) + _check_signal_level(trace_process) # check if trace is clipped - _check_clipping(config, trace_process) + _check_clipping(trace_process) # Remove instrument response - bp_freqmin, bp_freqmax = _get_bandpass_frequencies(config, trace) + bp_freqmin, bp_freqmax = _get_bandpass_frequencies(trace) if config.correct_instrumental_response: try: pre_filt = ( @@ -245,10 +308,10 @@ def _process_trace(config, trace): reason=f'unable to remove instrument response: {e}', short_reason='no instr response' ) - _remove_baseline(config, trace_process) - filter_trace(config, trace_process) + _remove_baseline(trace_process) + filter_trace(trace_process) # Check if the trace has significant signal to noise ratio - _check_sn_ratio(config, trace_process) + _check_sn_ratio(trace_process) trace_process.stats.processed = True return trace_process @@ -258,6 +321,11 @@ def _add_station_to_event_position(trace): Add to ``trace.stats`` station-to-event distance (hypocentral and epicentral), great-circle distance, azimuth and backazimuth. Raise RuntimeError if unable to compute distances. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if unable to compute hypocentral distance. """ try: station_to_event_position(trace) @@ -269,10 +337,14 @@ def _add_station_to_event_position(trace): ) -def _check_epicentral_distance(config, trace): +def _check_epicentral_distance(trace): """ - Reject traces with hypocentral distance outside the range specified - in the configuration file. + Check if the epicentral distance is within the selected range. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if epicentral distance is outside the range. """ if config.epi_dist_ranges is None: return @@ -299,8 +371,15 @@ def _check_epicentral_distance(config, trace): ) -def _add_arrivals(config, trace): - """Add to trace P and S arrival times, travel times and angles.""" +def _add_arrivals(trace): + """ + Add to trace P and S arrival times, travel times and angles. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if unable to get arrival times. + """ for phase in 'P', 'S': try: add_arrival_to_trace(trace, phase, config) @@ -318,8 +397,15 @@ def _add_arrivals(config, trace): refine_trace_picks(trace, freqmin, debug) -def _define_signal_and_noise_windows(config, trace): - """Define signal and noise windows for spectral analysis.""" +def _define_signal_and_noise_windows(trace): + """ + Define signal and noise windows for spectral analysis. + + :param trace: ObsPy Trace object. + :type trace: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if P or S window is incomplete. + """ p_arrival_time = trace.stats.arrivals['P'][1] if config.wave_type[0] == 'P' and p_arrival_time < trace.stats.starttime: _skip_trace_and_raise(trace, 'P-window incomplete') @@ -381,12 +467,18 @@ def _define_signal_and_noise_windows(config, trace): trace.stats.arrivals['N2'] = ('N2', t2) -def _check_signal_window(config, st): +def _check_signal_window(st): """ Check if the signal window has sufficient amount of signal (i.e., not too many gaps). This is done on the stream, before merging. + + :param st: ObsPy Stream object. + :type st: :class:`obspy.core.stream.Stream` + + :raises: RuntimeError if the cut interval has no signal. + :raises: RuntimeError if too many gaps in the signal window. """ traceid = st[0].id st_cut = st.copy() @@ -424,9 +516,19 @@ def _check_signal_window(config, st): 'of overlaps.') -def _merge_stream(config, st): +def _merge_stream(st): """ Check for gaps and overlaps; remove mean; merge stream. + + :param st: ObsPy Stream object. + :type st: :class:`obspy.core.stream.Stream` + + :return: An ObsPy Trace object. + :rtype: :class:`obspy.core.trace.Trace` + + :raises: RuntimeError if gap duration is larger than config.gap_max. + :raises: RuntimeError if overlap duration is larger than + config.overlap_max. """ traceid = st[0].id # First, compute gap/overlap statistics for the whole trace. @@ -472,9 +574,15 @@ def _merge_stream(config, st): return st[0] -def _skip_ignored(config, st): - """Skip traces ignored from config.""" - # all traces in the stream have the same id +def _skip_ignored(st): + """ + Skip traces ignored from config. + + :param st: ObsPy Stream object. + :type st: :class:`obspy.core.stream.Stream` + + :raises: RuntimeError if traces are ignored from config file. + """ traceid = st[0].id network, station, location, channel = traceid.split('.') # build a list of all possible ids, from station only @@ -513,8 +621,16 @@ def _skip_ignored(config, st): ) -def process_traces(config, st): - """Remove mean, deconvolve and ignore unwanted components.""" +def process_traces(st): + """ + Remove mean, deconvolve and ignore unwanted components. + + :param st: Stream object with traces to process. + :type st: :class:`obspy.core.stream.Stream` + + :return: Stream object with processed traces. + :rtype: :class:`obspy.core.stream.Stream` + """ logger.info('Processing traces...') out_st = Stream() for traceid in sorted({tr.id for tr in st}): @@ -524,16 +640,16 @@ def process_traces(config, st): # Add event-related metadata to trace.stats for _trace in st_sel: _add_station_to_event_position(_trace) - _check_epicentral_distance(config, _trace) - _add_arrivals(config, _trace) - _define_signal_and_noise_windows(config, _trace) + _check_epicentral_distance(_trace) + _add_arrivals( _trace) + _define_signal_and_noise_windows(_trace) # We skip traces ignored from config file here, so that we have # the metadata needed for the raw plot - _skip_ignored(config, st_sel) - _check_signal_window(config, st_sel) - trace = _merge_stream(config, st_sel) + _skip_ignored(st_sel) + _check_signal_window(st_sel) + trace = _merge_stream(st_sel) trace.stats.ignore = False - trace_process = _process_trace(config, trace) + trace_process = _process_trace(trace) out_st.append(trace_process) except (ValueError, RuntimeError) as msg: logger.warning(msg) From 0c2de6725fa1a01afc778d81fdf83328c068a183 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 16:02:02 +0200 Subject: [PATCH 14/73] Use the global `config` object in ssp_wave_arrival.py --- sourcespec2/ssp_process_traces.py | 2 +- sourcespec2/ssp_wave_arrival.py | 140 ++++++++++++++++++++++++++++-- 2 files changed, 132 insertions(+), 10 deletions(-) diff --git a/sourcespec2/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py index 86eb8643..f1f9d472 100644 --- a/sourcespec2/ssp_process_traces.py +++ b/sourcespec2/ssp_process_traces.py @@ -382,7 +382,7 @@ def _add_arrivals(trace): """ for phase in 'P', 'S': try: - add_arrival_to_trace(trace, phase, config) + add_arrival_to_trace(trace, phase) except Exception as e: for line in str(e).splitlines(): logger.warning(line) diff --git a/sourcespec2/ssp_wave_arrival.py b/sourcespec2/ssp_wave_arrival.py index e4324950..f559305d 100644 --- a/sourcespec2/ssp_wave_arrival.py +++ b/sourcespec2/ssp_wave_arrival.py @@ -16,11 +16,29 @@ import warnings from math import asin, degrees from obspy.taup import TauPyModel +from .config import config model = TauPyModel(model='iasp91') logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) def _get_nll_grd(phase, station, grd_type, NLL_time_dir): + """ + Get a NLL grid for a given phase, station and type. + + :param phase: Phase for which to get the grid + :type phase: str + :param station: Station name + :type station: str + :param grd_type: Type of grid (time or angle) + :type grd_type: str + :param NLL_time_dir: Path to NLL time grids + :type NLL_time_dir: str + + :return: NLL grid + :rtype: :class:`nllgrid.NLLGrid` + + :raises RuntimeError: If the grid is not found + """ # Lazy-import here, since nllgrid is not an installation requirement # pylint: disable=import-outside-toplevel from nllgrid import NLLGrid @@ -42,7 +60,23 @@ def _get_nll_grd(phase, station, grd_type, NLL_time_dir): def _wave_arrival_nll(trace, phase, NLL_time_dir, focmec): - """Travel time and takeoff angle using a NLL grid.""" + """ + Travel time and takeoff angle using a NLL grid. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param phase: Phase for which to get arrival + :type phase: str + :param NLL_time_dir: Path to NLL time grids + :type NLL_time_dir: str + :param focmec: Whether to compute takeoff angle from focal mechanism + :type focmec: bool + + :return: Travel time and takeoff angle + :rtype: tuple of float + + :raises RuntimeError: If the grid is not found + """ if NLL_time_dir is None: raise RuntimeError station = trace.stats.station @@ -75,7 +109,19 @@ def _wave_arrival_nll(trace, phase, NLL_time_dir, focmec): def _wave_arrival_vel(trace, vel): - """Travel time and takeoff angle using a constant velocity (in km/s).""" + """ + Travel time and takeoff angle using a constant velocity (in km/s). + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param vel: Velocity in km/s + :type vel: float + + :return: Travel time and takeoff angle + :rtype: tuple of float + + :raises RuntimeError: If velocity is not set + """ if vel is None: raise RuntimeError travel_time = trace.stats.hypo_dist / vel @@ -86,7 +132,17 @@ def _wave_arrival_vel(trace, vel): def _wave_arrival_taup(trace, phase): - """Travel time and takeoff angle using taup.""" + """ + Travel time and takeoff angle using taup. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param phase: Phase for which to get arrival + :type phase: str + + :return: Travel time and takeoff angle + :rtype: tuple of float + """ phase_list = [phase.lower(), phase] hypo_depth = trace.stats.event.hypocenter.depth.value_in_km kwargs = { @@ -114,8 +170,18 @@ def _wave_arrival_taup(trace, phase): return travel_time, takeoff_angle -def _wave_arrival(trace, phase, config): - """Get travel time and takeoff angle.""" +def _wave_arrival(trace, phase): + """ + Get travel time and takeoff angle. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param phase: Phase for which to get arrival + :type phase: str + + :return: Travel time, takeoff angle and method used + :rtype: float, float, str + """ NLL_time_dir = config.NLL_time_dir focmec = config.rp_from_focal_mechanism vel = {'P': config.vp_tt, 'S': config.vs_tt} @@ -137,7 +203,21 @@ def _wave_arrival(trace, phase, config): def _validate_pick(pick, theo_pick_time, tolerance, trace_id): - """Check if a pick is valid, i.e., close enough to theoretical one.""" + """ + Check if a pick is valid, i.e., close enough to theoretical one. + + :param pick: Pick to validate + :type pick: :class:`obspy.core.event.Pick` + :param theo_pick_time: Theoretical pick time + :type theo_pick_time: float + :param tolerance: Tolerance in seconds + :type tolerance: float + :param trace_id: Trace ID + :type trace_id: str + + :return: True if pick is valid, False otherwise + :rtype: bool + """ if theo_pick_time is None: return True delta_t = pick.time - theo_pick_time @@ -150,6 +230,17 @@ def _validate_pick(pick, theo_pick_time, tolerance, trace_id): def _get_theo_pick_time(trace, travel_time): + """ + Get theoretical pick time. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param travel_time: Travel time + :type travel_time: float + + :return: Theoretical pick time + :rtype: float + """ if trace.stats.event.hypocenter.origin_time is None: msg = ( f'{trace.id}: hypocenter origin time not set: ' @@ -163,6 +254,17 @@ def _get_theo_pick_time(trace, travel_time): def _travel_time_from_pick(trace, pick_time): + """ + Compute travel time from pick time. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param pick_time: Pick time + :type pick_time: float + + :return: Travel time + :rtype: float + """ try: travel_time = pick_time - trace.stats.event.hypocenter.origin_time except TypeError: @@ -171,7 +273,21 @@ def _travel_time_from_pick(trace, pick_time): def _find_picks(trace, phase, theo_pick_time, tolerance): - """Search for valid picks in trace stats. Return pick time if found.""" + """ + Search for valid picks in trace stats. Return pick time if found. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + :param phase: Phase for which to search pick + :type phase: str + :param theo_pick_time: Theoretical pick time + :type theo_pick_time: float + :param tolerance: Tolerance in seconds + :type tolerance: float + + :return: Pick time if found, None otherwise + :rtype: float or None + """ for pick in (p for p in trace.stats.picks if p.phase.upper() == phase): if _validate_pick(pick, theo_pick_time, tolerance, trace.id): trace.stats.arrivals[phase] = (phase, pick.time) @@ -179,13 +295,19 @@ def _find_picks(trace, phase, theo_pick_time, tolerance): return None -def add_arrival_to_trace(trace, phase, config): +def add_arrival_to_trace(trace, phase): """ Add arrival time, travel time and takeoff angle to trace for the given phase. Uses the theoretical arrival time if no pick is available or if the pick is too different from the theoretical arrival. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + + :param phase: Phase for which to add arrival + :type phase: str """ tolerance = ( config.p_arrival_tolerance @@ -201,7 +323,7 @@ def add_arrival_to_trace(trace, phase, config): trst.takeoff_angles[phase] = add_arrival_to_trace.angle_cache[key] return # If no cache is available, compute travel_time and takeoff_angle - travel_time, takeoff_angle, method = _wave_arrival(trace, phase, config) + travel_time, takeoff_angle, method = _wave_arrival(trace, phase) theo_pick_time = _get_theo_pick_time(trace, travel_time) pick_time = _find_picks(trace, phase, theo_pick_time, tolerance) if pick_time is not None: From fd8408e48e2aa3cd6df00c0df6530dfae3975d91 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 16:58:08 +0200 Subject: [PATCH 15/73] Use the global `config` object in ssp_build_spectra.py, ssp_correction.py, and ssp_radiation_pattern.py --- sourcespec2/source_model.py | 2 +- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_build_spectra.py | 379 +++++++++++++++++++++++---- sourcespec2/ssp_correction.py | 9 +- sourcespec2/ssp_radiation_pattern.py | 113 +++++++- 5 files changed, 442 insertions(+), 63 deletions(-) diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 6c5dc6c4..19eb766f 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -106,7 +106,7 @@ def main(): # Deconvolve, filter, cut traces: proc_st = process_traces(st) # Build spectra (amplitude in magnitude units) - spec_st, _specnoise_st, _weight_st = build_spectra(config, proc_st) + spec_st, _specnoise_st, _weight_st = build_spectra(proc_st) if len(spec_st) == 0: ssp_exit() # We keep just horizontal component: diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index c6aa5852..9fb54f71 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -49,7 +49,7 @@ def main(): # Build spectra (amplitude in magnitude units) from .ssp_build_spectra import build_spectra - spec_st, specnoise_st, weight_st = build_spectra(config, proc_st) + spec_st, specnoise_st, weight_st = build_spectra(proc_st) from .ssp_plot_traces import plot_traces plot_traces(config, st, suffix='raw') diff --git a/sourcespec2/ssp_build_spectra.py b/sourcespec2/ssp_build_spectra.py index 9084f845..2fb783be 100644 --- a/sourcespec2/ssp_build_spectra.py +++ b/sourcespec2/ssp_build_spectra.py @@ -25,6 +25,7 @@ from scipy.integrate import cumulative_trapezoid as cumtrapz from scipy.interpolate import interp1d from obspy.core import Stream +from .config import config from .spectrum import Spectrum, SpectrumStream from .ssp_setup import ssp_exit from .ssp_util import ( @@ -47,7 +48,18 @@ def __init__(self, message, reason): self.reason = reason -def _get_nint(config, trace): +def _get_nint(trace): + """ + Return the number of integrations to be performed on the trace. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + + :return: Number of integrations + :rtype: int + + :raises ValueError: If the instrument type is unknown + """ if config.trace_units == 'auto': instrtype = trace.stats.instrtype else: @@ -63,23 +75,48 @@ def _get_nint(config, trace): return nint -def _time_integrate(config, trace): - nint = _get_nint(config, trace) +def _time_integrate(trace): + """ + Integrate the trace in time domain. + + Trace is filtered after each integration. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace + """ + nint = _get_nint(trace) trace.detrend(type='constant') trace.detrend(type='linear') for _ in range(nint): trace.data = cumtrapz(trace.data) * trace.stats.delta trace.stats.npts -= 1 - filter_trace(config, trace) + filter_trace(trace) -def _frequency_integrate(config, spec): - nint = _get_nint(config, spec) +def _frequency_integrate(spec): + """ + Integrate the spectrum in frequency domain. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + """ + nint = _get_nint(spec) for _ in range(nint): spec.data /= (2 * math.pi * spec.freq) -def _cut_spectrum(config, spec): +def _cut_spectrum(spec): + """ + Cut the spectrum to the frequency range specified in the configuration. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + + :return: Cut spectrum + :rtype: :class:`~sourcespec.spectrum.Spectrum` + + :raises RuntimeError: If the instrument type is unknown + """ # see if there is a station-specific frequency range station = spec.stats.station try: @@ -103,6 +140,19 @@ def _compute_h(spec_st, code, vertical_channel_codes=None, wave_type='S'): Compute the component 'H' from geometric mean of the stream components. (which can also be all three components) + + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param code: Band+instrument code + :type code: str + :param vertical_channel_codes: List of vertical channel codes + (default: ['Z'] if None) + :type vertical_channel_codes: list + :param wave_type: Wave type ('S' or 'P', default: 'S') + :type wave_type: str + + :return: Spectrum object with the 'H' component + :rtype: :class:`~sourcespec.spectrum.Spectrum` """ if vertical_channel_codes is None: vertical_channel_codes = ['Z'] @@ -136,7 +186,15 @@ def _compute_h(spec_st, code, vertical_channel_codes=None, wave_type='S'): return spec_h -def _check_data_len(config, trace): +def _check_data_len(trace): + """ + Check if data length is sufficient. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + + :raises RuntimeError: If data length is insufficient + """ traceId = trace.get_id() trace_cut = trace.copy() if config.wave_type[0] == 'S': @@ -158,15 +216,24 @@ def _check_data_len(config, trace): 'skipping trace') -def _cut_signal_noise(config, trace): +def _cut_signal_noise(trace): + """ + Cut signal and noise windows from trace. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + + :return: Signal and noise traces + :rtype: tuple of :class:`obspy.core.trace.Trace + """ trace_signal = trace.copy() trace_noise = trace.copy() # Integrate in time domain, if required. # (otherwise frequency-domain integration is performed later) if config.time_domain_int: - _time_integrate(config, trace_signal) - _time_integrate(config, trace_noise) + _time_integrate(trace_signal) + _time_integrate(trace_noise) # trim... if config.wave_type[0] == 'S': @@ -218,8 +285,21 @@ def _cut_signal_noise(config, trace): def _recompute_time_window(trace, wave_type, npts, keep='start'): - """Recompute start or end time of signal or noise window, - based on new number of points""" + """ + Recompute start or end time of signal or noise window, + based on new number of points. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + :param wave_type: Wave type ('P', 'S', 'N') + :type wave_type: str + :param npts: Number of points + :type npts: int + :param keep: Keep 'start' or 'end' of window (default: 'start') + :type keep: str + + :raises ValueError: If keep is not 'start' or 'end' + """ length = npts * trace.stats.delta if keep == 'end': label, _ = trace.stats.arrivals[f'{wave_type}1'] @@ -233,7 +313,17 @@ def _recompute_time_window(trace, wave_type, npts, keep='start'): raise ValueError('keep must be "start" or "end"') -def _check_noise_level(trace_signal, trace_noise, config): +def _check_noise_level(trace_signal, trace_noise): + """ + Check noise level. + + :param trace_signal: Signal trace + :type trace_signal: :class:`obspy.core.trace.Trace` + :param trace_noise: Noise trace + :type trace_noise: :class:`obspy.core.trace.Trace` + + :raises RuntimeError: If noise level is too low + """ traceId = trace_signal.get_id() trace_signal_rms = ((trace_signal.data**2).sum())**0.5 # Scale trace_noise_rms to length of signal window, @@ -253,9 +343,17 @@ def _check_noise_level(trace_signal, trace_noise, config): 'station will be skipped') -def _geometrical_spreading_coefficient(config, spec): +def _geometrical_spreading_coefficient(spec): """ Return the geometrical spreading coefficient for the given spectrum. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + + :return: Geometrical spreading coefficient + :rtype: float + + :raises ValueError: If the geometrical spreading model is unknown """ hypo_dist_in_km = spec.stats.hypo_dist epi_dist_in_km = spec.stats.epi_dist @@ -298,11 +396,14 @@ def _geometrical_spreading_coefficient(config, spec): PROPERTY_LOG_MESSAGES = [] -def _displacement_to_moment(stats, config): +def _displacement_to_moment(stats): """ Return the coefficient for converting displacement to seismic moment. From Aki&Richards,1980 + + :param stats: Stats object + :type stats: :class:`~sourcespec.spectrum.AttributeDict` """ phase = config.wave_type[0] lon = stats.coords.longitude @@ -349,7 +450,15 @@ def _displacement_to_moment(stats, config): def _smooth_spectrum(spec, smooth_width_decades=0.2): - """Smooth spectrum in a log10-freq space.""" + """ + Smooth spectrum in a log10-freq space. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param smooth_width_decades: Width of the smoothing window in decades + (default: 0.2) + :type smooth_width_decades: float + """ # 1. Generate log10-spaced frequencies freq = spec.freq _log_freq = np.log10(freq) @@ -383,7 +492,18 @@ def _smooth_spectrum(spec, smooth_width_decades=0.2): spec.data_logspaced = data_logspaced -def _build_spectrum(config, trace): +def _build_spectrum(trace): + """ + Build a spectrum from a trace. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + + :return: Spectrum object + :rtype: :class:`~sourcespec.spectrum.Spectrum` + + :raises RuntimeError: If an error occurs while building the spectrum + """ spec = Spectrum(obspy_trace=trace) spec.stats.instrtype = trace.stats.instrtype spec.stats.coords = trace.stats.coords @@ -398,12 +518,12 @@ def _build_spectrum(config, trace): # Integrate in frequency domain, if no time-domain # integration has been performed if not config.time_domain_int: - _frequency_integrate(config, spec) + _frequency_integrate(spec) # cut the spectrum - spec = _cut_spectrum(config, spec) + spec = _cut_spectrum(spec) # correct geometrical spreading try: - geom_spread = _geometrical_spreading_coefficient(config, spec) + geom_spread = _geometrical_spreading_coefficient(spec) except Exception as e: raise RuntimeError( f'{spec.id}: Error computing geometrical spreading: ' @@ -412,9 +532,9 @@ def _build_spectrum(config, trace): spec.data *= geom_spread # store the radiation pattern coefficient in the spectrum stats spec.stats.radiation_pattern =\ - get_radiation_pattern_coefficient(spec.stats, config) + get_radiation_pattern_coefficient(spec.stats) # convert to seismic moment - coeff = _displacement_to_moment(spec.stats, config) + coeff = _displacement_to_moment(spec.stats) spec.data *= coeff # store coeff to correct back data in displacement units # for radiated_energy() @@ -425,6 +545,15 @@ def _build_spectrum(config, trace): def _build_uniform_weight(spec): + """ + Build a uniform spectral weight. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + + :return: Uniform weight + :rtype: :class:`~sourcespec.spectrum.Spectrum` + """ weight = spec.copy() weight.snratio = None weight.data = np.ones_like(weight.data) @@ -432,7 +561,16 @@ def _build_uniform_weight(spec): return weight -def _build_weight_from_frequency(config, spec): +def _build_weight_from_frequency(spec): + """ + Build spectral weights from frequency. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + + :return: Spectral weights + :rtype: :class:`~sourcespec.spectrum.Spectrum` + """ weight = spec.copy() freq = weight.freq weight.data = np.ones_like(weight.data) @@ -448,6 +586,17 @@ def _build_weight_from_frequency(config, spec): def _build_weight_from_inv_frequency(spec, power=0.25): """ Build spectral weights from inverse frequency (raised to a power < 1) + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param power: Power to which the inverse frequency is raised + (default: 0.25) + :type power: float + + :return: Spectral weights + :rtype: :class:`~sourcespec.spectrum.Spectrum` + + :raises ValueError: If power is >= 1 """ if power >= 1: raise ValueError('pow must be < 1') @@ -478,6 +627,19 @@ def _build_weight_from_inv_frequency(spec, power=0.25): def _build_weight_from_ratio(spec, specnoise, smooth_width_decades): + """ + Build spectral weights from the ratio of signal to noise. + + :param spec: signal spectrum + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param specnoise: noise spectrum + :type specnoise: :class:`~sourcespec.spectrum.Spectrum` + :param smooth_width_decades: Width of the smoothing window in decades + :type smooth_width_decades: float + + :return: Spectral weights + :rtype: :class:`~sourcespec.spectrum.Spectrum` + """ weight = spec.copy() weight.data /= specnoise.data # save signal-to-noise ratio before log10, smoothing, and normalization @@ -499,7 +661,19 @@ def _build_weight_from_ratio(spec, specnoise, smooth_width_decades): return weight -def _build_weight_from_noise(config, spec, specnoise): +def _build_weight_from_noise(spec, specnoise): + """ + Build spectral weights from signal to noise ratio, if available. + Otherwise, build uniform weights. + + :param spec: Signal spectrum + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param specnoise: Noise spectrum + :type specnoise: :class:`~sourcespec.spectrum.Spectrum` + + :return: Spectral weights + :rtype: :class:`~sourcespec.spectrum.Spectrum` + """ if specnoise is None or np.all(specnoise.data == 0): spec_id = spec.get_id()[:-1] logger.warning( @@ -518,9 +692,19 @@ def _build_weight_from_noise(config, spec, specnoise): return weight -def _build_weight_spectral_stream(config, spec_st, specnoise_st): - """Build a stream of weights from a stream of spectra and a stream of - noise spectra.""" +def _build_weight_spectral_stream(spec_st, specnoise_st): + """ + Build a stream of weights from a stream of spectra and a stream of + noise spectra. + + :param spec_st: Stream of signal spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param specnoise_st: Stream of noise spectra + :type specnoise_st: :class:`~sourcespec.spectrum.SpectrumStream` + + :return: Stream of weights + :rtype: :class:`~sourcespec.spectrum.SpectrumStream` + """ weight_st = SpectrumStream() spec_ids = {sp.id[:-1] for sp in spec_st if not sp.stats.ignore} for specid in spec_ids: @@ -530,9 +714,9 @@ def _build_weight_spectral_stream(config, spec_st, specnoise_st): except Exception: continue if config.weighting == 'noise': - weight = _build_weight_from_noise(config, spec_h, specnoise_h) + weight = _build_weight_from_noise(spec_h, specnoise_h) elif config.weighting == 'frequency': - weight = _build_weight_from_frequency(config, spec_h) + weight = _build_weight_from_frequency(spec_h) elif config.weighting == 'inv_frequency': weight = _build_weight_from_inv_frequency(spec_h) elif config.weighting == 'no_weight': @@ -542,7 +726,17 @@ def _build_weight_spectral_stream(config, spec_st, specnoise_st): def _select_spectra(spec_st, specid): - """Select spectra from stream, based on specid.""" + """ + Select spectra from stream, based on specid. + + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param specid: Spectrum ID + :type specid: str + + :return: Stream of selected spectra + :rtype: :class:`~sourcespec.spectrum.SpectrumStream` + """ network, station, location, channel = specid.split('.') channel = channel + '?' * (3 - len(channel)) spec_st_sel = spec_st.select( @@ -558,6 +752,17 @@ def _build_H(spec_st, specnoise_st=None, vertical_channel_codes=None, Add to spec_st and specnoise_st the "H" component. H component is obtained from the modulus of all the available components. + + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param specnoise_st: Stream of noise spectra + :type specnoise_st: :class:`~sourcespec.spectrum.SpectrumStream` + + :param vertical_channel_codes: List of vertical channel codes + (default: ['Z'] if None) + :type vertical_channel_codes: list + :param wave_type: Wave type ('S' or 'P', default: 'S') + :type wave_type: str """ if vertical_channel_codes is None: vertical_channel_codes = ['Z'] @@ -577,9 +782,19 @@ def _build_H(spec_st, specnoise_st=None, vertical_channel_codes=None, specnoise_st.append(specnoise_h) -def _check_spectral_sn_ratio(config, spec, specnoise): +def _check_spectral_sn_ratio(spec, specnoise): + """ + Check spectral signal-to-noise ratio. + + :param spec: Signal spectrum + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param specnoise: Noise spectrum + :type specnoise: :class:`~sourcespec.spectrum.Spectrum` + + :raises SpectrumIgnored: If the spectrum is to be ignored + """ spec_id = spec.get_id() - weight = _build_weight_from_noise(config, spec, specnoise) + weight = _build_weight_from_noise(spec, specnoise) freqs = weight.freq # if no noise window is available, snratio is not computed if weight.snratio is None: @@ -621,7 +836,15 @@ def _check_spectral_sn_ratio(config, spec, specnoise): def _ignore_spectrum(msg, spec, specnoise): - """Ignore spectrum. Set ignore flag and reason.""" + """ + Ignore spectrum. Set ignore flag and reason. + + :param msg: Ignore message + :type msg: str + :param spec: Signal spectrum to ignore + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param specnoise: Noise spectrum object to ignore + """ logger.warning(msg) spec.stats.ignore = True spec.stats.ignore_reason = msg.reason @@ -630,24 +853,39 @@ def _ignore_spectrum(msg, spec, specnoise): def _ignore_trace(msg, trace): - """Ignore trace. Set ignore flag and reason.""" + """ + Ignore trace. Set ignore flag and reason. + + :param msg: Ignore message + :type msg: str + :param trace: Trace to ignore + :type trace: :class:`obspy.core.trace.Trace` + """ # NOTE: no logger.warning here, because it is already done in # _ignore_spectrum() trace.stats.ignore = True trace.stats.ignore_reason = msg.reason -def _build_signal_and_noise_streams(config, st): - """Build signal and noise streams.""" +def _build_signal_and_noise_streams(st): + """ + Build signal and noise streams. + + :param st: ObsPy Stream object + :type st: :class:`obspy.core.stream.Stream` + + :return: Signal and noise streams + :rtype: tuple of :class:`obspy.core.stream.Stream` + """ # remove traces with ignore flag traces = Stream([tr for tr in st if not tr.stats.ignore]) signal_st = Stream() noise_st = Stream() for trace in sorted(traces, key=lambda tr: tr.id): try: - _check_data_len(config, trace) - trace_signal, trace_noise = _cut_signal_noise(config, trace) - _check_noise_level(trace_signal, trace_noise, config) + _check_data_len(trace) + trace_signal, trace_noise = _cut_signal_noise(trace) + _check_noise_level(trace_signal, trace_noise) signal_st.append(trace_signal) noise_st.append(trace_noise) except RuntimeError as msg: @@ -657,11 +895,18 @@ def _build_signal_and_noise_streams(config, st): return signal_st, noise_st -def _trim_components(config, signal_st, noise_st, st): +def _trim_components(signal_st, noise_st, original_st): """ Trim components of the same instrument to the same number of samples. Recompute time window of the signal and noise traces for correct plotting. + + :param signal_st: Stream of signal traces + :type signal_st: :class:`obspy.core + :param noise_st: Stream of noise traces + :type noise_st: :class:`obspy.core + :param original_st: Original stream + :type original_st: :class:`obspy.core.stream.Stream` """ for traceid in sorted({tr.id[:-1] for tr in signal_st}): st_sel = signal_st.select(id=f'{traceid}*') +\ @@ -678,26 +923,35 @@ def _trim_components(config, signal_st, noise_st, st): tr.data = tr.data[:npts] elif tr.stats.type == 'noise': tr.data = tr.data[-npts:] - for tr in st.select(id=f'{traceid}*'): + for tr in original_st.select(id=f'{traceid}*'): _recompute_time_window(tr, config.wave_type[0], npts, keep='start') _recompute_time_window(tr, 'N', npts, keep='end') -def _build_signal_and_noise_spectral_streams( - config, signal_st, noise_st, original_st): +def _build_signal_and_noise_spectral_streams(signal_st, noise_st, original_st): """ Build signal and noise spectral streams. Note: original_st is only used to keep track of ignored traces. + + :param signal_st: Stream of signal traces + :type signal_st: :class:`obspy.core + :param noise_st: Stream of noise traces + :type noise_st: :class:`obspy.core + :param original_st: Original stream + :type original_st: :class:`obspy.core + + :return: Signal and noise spectral streams + :rtype: tuple of :class:`~sourcespec.spectrum.SpectrumStream` """ spec_st = SpectrumStream() specnoise_st = SpectrumStream() for trace_signal in sorted(signal_st, key=lambda tr: tr.id): trace_noise = noise_st.select(id=trace_signal.id)[0] try: - spec = _build_spectrum(config, trace_signal) - specnoise = _build_spectrum(config, trace_noise) - _check_spectral_sn_ratio(config, spec, specnoise) + spec = _build_spectrum(trace_signal) + specnoise = _build_spectrum(trace_noise) + _check_spectral_sn_ratio(spec, specnoise) except RuntimeError as msg: # RuntimeError is for skipped spectra logger.warning(msg) @@ -721,40 +975,51 @@ def _build_signal_and_noise_spectral_streams( for specnoise in specnoise_st: specnoise.data_mag = moment_to_mag(specnoise.data) # apply station correction if a residual file is specified in config - spec_st = station_correction(spec_st, config) + spec_st = station_correction(spec_st) return spec_st, specnoise_st -def _zero_pad(config, trace): - """Zero-pad trace to spectral_win_length""" +def _zero_pad(trace): + """ + Zero-pad trace to spectral_win_length + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + """ spec_win_len = config.spectral_win_length if spec_win_len is None: return - wtype = config.wave_type[0] if trace.stats.type == 'signal': + wtype = config.wave_type[0] t1 = trace.stats.arrivals[f'{wtype}1'][1] elif trace.stats.type == 'noise': t1 = trace.stats.arrivals['N1'][1] trace.trim(starttime=t1, endtime=t1 + spec_win_len, pad=True, fill_value=0) -def build_spectra(config, st): +def build_spectra(st): """ Build spectra and the ``spec_st`` object. Computes P- or S-wave (displacement) spectra from accelerometers and velocimeters, uncorrected for anelastic attenuation, corrected for instrumental constants, normalized by geometrical spreading. + + :param st: ObsPy Stream object + :type st: :class:`obspy.core.stream.Stream` + + :return: spectra, noise spectra, and weights + :rtype: tuple of :class:`~sourcespec.spectrum.Spectrum` """ wave_type = config.wave_type logger.info(f'Building {wave_type}-wave spectra...') - signal_st, noise_st = _build_signal_and_noise_streams(config, st) - _trim_components(config, signal_st, noise_st, st) + signal_st, noise_st = _build_signal_and_noise_streams(st) + _trim_components(signal_st, noise_st, st) for trace in signal_st + noise_st: - _zero_pad(config, trace) + _zero_pad(trace) spec_st, specnoise_st = _build_signal_and_noise_spectral_streams( - config, signal_st, noise_st, st) - weight_st = _build_weight_spectral_stream(config, spec_st, specnoise_st) + signal_st, noise_st, st) + weight_st = _build_weight_spectral_stream(spec_st, specnoise_st) logger.info(f'Building {wave_type}-wave spectra: done') logger.info('---------------------------------------------------') return spec_st, specnoise_st, weight_st diff --git a/sourcespec2/ssp_correction.py b/sourcespec2/ssp_correction.py index 1bb9a3ab..50175fa2 100644 --- a/sourcespec2/ssp_correction.py +++ b/sourcespec2/ssp_correction.py @@ -14,17 +14,24 @@ """ import logging from scipy.interpolate import interp1d +from .config import config from .spectrum import read_spectra from .ssp_util import moment_to_mag, mag_to_moment from .ssp_setup import ssp_exit logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def station_correction(spec_st, config): +def station_correction(spec_st): """ Correct spectra using station-average residuals. Residuals are obtained from a previous run. + + :param spec_st: Spectra to correct. + :type spec_st: :class:`~sourcespec.spectrum.Spectrum` + + :return: Corrected spectra. + :rtype: :class:`~sourcespec.spectrum.Spectrum` """ res_filepath = config.residuals_filepath if res_filepath is None: diff --git a/sourcespec2/ssp_radiation_pattern.py b/sourcespec2/ssp_radiation_pattern.py index f3630a3e..6e99112e 100644 --- a/sourcespec2/ssp_radiation_pattern.py +++ b/sourcespec2/ssp_radiation_pattern.py @@ -13,12 +13,21 @@ import logging from math import pi, sin, cos from obspy.taup import TauPyModel +from .config import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) model = TauPyModel(model='iasp91') def toRad(angle): - """Convert angle from degrees to radians.""" + """ + Convert angle from degrees to radians. + + :param angle: Angle in degrees. + :type angle: float + + :return: Angle in radians. + :rtype: float + """ return angle / 180. * pi @@ -27,6 +36,24 @@ def radiation_pattern(strike, dip, rake, takeoff_angle, azimuth, wave): Body wave radiation pattern. From Lay-Wallace, page 340. + + :param strike: Strike of the fault plane, in degrees. + :type strike: float + :param dip: Dip of the fault plane, in degrees. + :type dip: float + :param rake: Rake of the fault plane, in degrees. + :type rake: float + :param takeoff_angle: Takeoff angle of the seismic ray, in degrees. + :type takeoff_angle: float + :param azimuth: Azimuth of the receiver, in degrees. + :type azimuth: float + :param wave: Wave type: 'P', 'S', 'SV' or 'SH'. + :type wave: str + + :return: Radiation pattern coefficient. + :rtype: float + + :raises ValueError: If wave type is unknown. """ strike = toRad(strike) dip = toRad(dip) @@ -49,6 +76,24 @@ def radiation_pattern(strike, dip, rake, takeoff_angle, azimuth, wave): def _rad_patt_P(phi, dip, rake, takeoff): + """ + P-wave radiation pattern. + + From Lay-Wallace, page 340. + + :param phi: Difference between the receiver's azimuth and + the faut strike, in radians. + :type phi: float + :param dip: Dip of the fault plane, in radians. + :type dip: float + :param rake: Rake of the fault plane, in radians. + :type rake: float + :param takeoff: Takeoff angle of the seismic ray, in radians. + :type takeoff: float + + :return: Radiation pattern coefficient. + :rtype: float + """ return ( cos(rake) * sin(dip) * sin(takeoff)**2 * sin(2 * phi) - cos(rake) * cos(dip) * sin(2 * takeoff) * cos(phi) + @@ -59,12 +104,48 @@ def _rad_patt_P(phi, dip, rake, takeoff): def _rad_patt_S(phi, dip, rake, takeoff): + """ + S-wave radiation pattern. + + From Lay-Wallace, page 340. + + :param phi: Difference between the receiver's azimuth and + the faut strike, in radians. + :type phi: float + :param dip: Dip of the fault plane, in radians. + :type dip: float + :param rake: Rake of the fault plane, in radians. + :type rake: float + :param takeoff: Takeoff angle of the seismic ray, in radians. + :type takeoff: float + + :return: Radiation pattern coefficient. + :rtype: float + """ RSV = _rad_patt_SV(phi, dip, rake, takeoff) RSH = _rad_patt_SH(phi, dip, rake, takeoff) return (RSV**2. + RSH**2.)**(1. / 2) def _rad_patt_SV(phi, dip, rake, takeoff): + """ + SV-wave radiation pattern. + + From Lay-Wallace, page 340. + + :param phi: Difference between the receiver's azimuth and + the faut strike, in radians. + :type phi: float + :param dip: Dip of the fault plane, in radians. + :type dip: float + :param rake: Rake of the fault plane, in radians. + :type rake: float + :param takeoff: Takeoff angle of the seismic ray, in radians. + :type takeoff: float + + :return: Radiation pattern coefficient. + :rtype: float + """ return ( sin(rake) * cos(2 * dip) * cos(2 * takeoff) * sin(phi) - cos(rake) * cos(dip) * cos(2 * takeoff) * cos(phi) + @@ -75,6 +156,24 @@ def _rad_patt_SV(phi, dip, rake, takeoff): def _rad_patt_SH(phi, dip, rake, takeoff): + """ + SH-wave radiation pattern. + + From Lay-Wallace, page 340. + + :param phi: Difference between the receiver's azimuth and + the faut strike, in radians. + :type phi: float + :param dip: Dip of the fault plane, in radians. + :type dip: float + :param rake: Rake of the fault plane, in radians. + :type rake: float + :param takeoff: Takeoff angle of the seismic ray, in radians. + :type takeoff: float + + :return: Radiation pattern coefficient. + :rtype: float + """ return ( cos(rake) * cos(dip) * cos(takeoff) * sin(phi) + cos(rake) * sin(dip) * sin(takeoff) * cos(2 * phi) + @@ -89,8 +188,16 @@ def _rad_patt_SH(phi, dip, rake, takeoff): RP_MSG_CACHE = [] -def get_radiation_pattern_coefficient(stats, config): - """Get radiation pattern coefficient.""" +def get_radiation_pattern_coefficient(stats): + """ + Get radiation pattern coefficient. + + :param stats: Spectrum stats. + :type stats: :class:`~sourcespec.spectrum.AttributeDict` + + :return: Radiation pattern coefficient. + :rtype: float + """ wave_type = config.wave_type # P, S, SV, SH simple_wave_type = wave_type[0].lower() # p or s if not config.rp_from_focal_mechanism: From ef62e073cada5e99458fcbe8327dc937094ccdae Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 17:08:57 +0200 Subject: [PATCH 16/73] Use the global `config` object in ssp_plot_traces.py --- sourcespec2/source_model.py | 2 +- sourcespec2/source_spec.py | 4 +- sourcespec2/ssp_plot_traces.py | 153 ++++++++++++++++++++++++++++----- 3 files changed, 136 insertions(+), 23 deletions(-) diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 19eb766f..1b22c98b 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -124,7 +124,7 @@ def main(): from .ssp_plot_spectra import plot_spectra from .ssp_plot_traces import plot_traces - plot_traces(config, proc_st, ncols=2, block=False) + plot_traces(proc_st, ncols=2, block=False) plot_spectra(config, spec_st, ncols=1, stack_plots=True) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 9fb54f71..43a7ae39 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -52,8 +52,8 @@ def main(): spec_st, specnoise_st, weight_st = build_spectra(proc_st) from .ssp_plot_traces import plot_traces - plot_traces(config, st, suffix='raw') - plot_traces(config, proc_st) + plot_traces(st, suffix='raw') + plot_traces(proc_st) # Spectral inversion from .ssp_inversion import spectral_inversion diff --git a/sourcespec2/ssp_plot_traces.py b/sourcespec2/ssp_plot_traces.py index 06ff3cd0..0a576727 100644 --- a/sourcespec2/ssp_plot_traces.py +++ b/sourcespec2/ssp_plot_traces.py @@ -21,6 +21,7 @@ from matplotlib import patches import matplotlib.patheffects as PathEffects from matplotlib.ticker import ScalarFormatter as sf +from .config import config from .savefig import savefig from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -40,8 +41,20 @@ def _set_format(self, vmin=None, vmax=None): phase_label_color = {'P': 'black', 'S': 'black'} -def _nplots(config, st, maxlines, ncols): - """Determine the number of lines and columns of the plot.""" +def _nplots(st, maxlines, ncols): + """ + Determine the number of lines and columns of the plot. + + :param st: Stream of traces. + :type st: :class:`obspy.core.stream.Stream` + :param maxlines: Maximum number of lines. + :type maxlines: int + :param ncols: Number of columns. + :type ncols: int + + :return: Number of lines and columns. + :rtype: tuple of int + """ # Remove the channel letter to determine the number of plots if config.plot_traces_ignored: nplots = len({tr.id[:-1] for tr in st}) @@ -55,7 +68,19 @@ def _nplots(config, st, maxlines, ncols): return nlines, ncols -def _make_fig(config, nlines, ncols): +def _make_fig(nlines, ncols): + """ + Create a figure with a number of subplots. + + :param nlines: Number of lines. + :type nlines: int + :param ncols: Number of columns. + :type ncols: int + + :return: Figure and axes. + :rtype: tuple of :class:`matplotlib.figure.Figure` and list of + :class:`matplotlib.axes.Axes` + """ figsize = (16, 9) if nlines <= 3 else (16, 18) # high dpi needed to rasterize png # vector formats (pdf, svg) do not have rasters @@ -127,7 +152,15 @@ def _make_fig(config, nlines, ncols): BBOX = None -def _savefig(config, figures, suffix, force_numbering=False): +def _savefig(figures, suffix, force_numbering=False): + """ + Save figures to file. + + :param figures: Figures to save. + :type figures: list of :class:`matplotlib.figure.Figure` + :param force_numbering: Force figure numbering. + :type force_numbering: bool + """ global BBOX # pylint: disable=global-statement evid = config.event.event_id figfile_base = os.path.join(config.options.outdir, f'{evid}.traces.') @@ -169,7 +202,24 @@ def _savefig(config, figures, suffix, force_numbering=False): def _plot_min_max(ax, x_vals, y_vals, linewidth, color, alpha, zorder): - """Quick and dirty plot using less points. Useful for vector plotting.""" + """ + Quick and dirty plot using less points. Useful for vector plotting. + + :param ax: Axes object. + :type ax: :class:`matplotlib.axes.Axes` + :param x_vals: X values. + :type x_vals: :class:`numpy.ndarray` + :param y_vals: Y values. + :type y_vals: :class:`numpy.ndarray` + :param linewidth: Line width. + :type linewidth: float + :param color: Line color. + :type color: str + :param alpha: Line alpha. + :type alpha: float + :param zorder: Z-order. + :type zorder: int + """ ax_width_in_pixels = int(np.ceil(ax.bbox.width)) nsamples = len(x_vals) samples_per_pixel = int(np.ceil(nsamples / ax_width_in_pixels)) @@ -194,7 +244,15 @@ def _plot_min_max(ax, x_vals, y_vals, linewidth, color, alpha, zorder): def _freq_string(freq): - """Return a string representing the rounded frequency.""" + """ + Return a string representing the rounded frequency. + + :param freq: Frequency. + :type freq: float + + :return: Frequency string. + :rtype: str + """ # int or float notation for frequencies between 0.01 and 100 if 1e-2 <= freq <= 1e2: int_freq = int(round(freq)) @@ -213,7 +271,25 @@ def _freq_string(freq): ) -def _plot_trace(config, trace, ntraces, tmax, ax, trans, trans3, path_effects): +def _plot_trace(trace, ntraces, tmax, ax, trans, trans3, path_effects): + """ + Plot a trace. + + :param trace: Trace to plot. + :type trace: :class:`obspy.core.trace.Trace` + :param ntraces: Number of traces. + :type ntraces: int + :param tmax: Maximum value of the trace. + :type tmax: float + :param ax: Axes object. + :type ax: :class:`matplotlib.axes.Axes` + :param trans: Transformation for plotting phase labels. + :type trans: :class:`matplotlib.transforms.BboxTransformTo` + :param trans3: Transformation for plotting station info. + :type trans3: :class:`matplotlib.transforms.BboxTransformTo` + :param path_effects: Path effects for text. + :type path_effects: :class:`matplotlib + """ # Origin and height to draw vertical patches for noise and signal windows rectangle_patch_origin = 0 rectangle_patch_height = 1 @@ -302,6 +378,16 @@ def _plot_trace(config, trace, ntraces, tmax, ax, trans, trans3, path_effects): def _add_station_info_text(trace, ax, path_effects): + """ + Add station information text to the plot. + + :param trace: Trace. + :type trace: :class:`obspy.core.trace.Trace` + :param ax: Axes object. + :type ax: :class:`matplotlib.axes.Axes` + :param path_effects: Path effects for text. + :type path_effects: :class:`matplotlib.patheffects` + """ with contextlib.suppress(AttributeError): if ax.has_station_info_text: return @@ -326,7 +412,16 @@ def _add_station_info_text(trace, ax, path_effects): def _add_labels(axes, plotn, ncols): - """Add xlabels to the last row of plots.""" + """ + Add xlabels to the last row of plots. + + :param axes: Axes objects. + :type axes: list of :class:`matplotlib.axes.Axes` + :param plotn: Number of plots. + :type plotn: int + :param ncols: Number of columns. + :type ncols: int + """ # A row has "ncols" plots: the last row is from `plotn-ncols` to `plotn` n0 = max(plotn - ncols, 0) for ax in axes[n0:plotn]: @@ -335,14 +430,25 @@ def _add_labels(axes, plotn, ncols): def _set_ylim(axes): - """Set symmetric ylim.""" + """ + Set symmetric ylim. + + :param axes: Axes objects. + :type axes: list of :class:`matplotlib + """ for ax in axes: ylim = ax.get_ylim() ymax = np.max(np.abs(ylim)) ax.set_ylim(-ymax, ymax) -def _trim_traces(config, st): +def _trim_traces(st): + """ + Trim traces to the time window of interest. + + :param st: Stream of traces. + :type st: :class:`obspy.core.stream.Stream` + """ for trace in st: try: t1 = trace.stats.arrivals['N1'][1] @@ -360,7 +466,7 @@ def _trim_traces(config, st): trace.stats.time_offset = trace.stats.starttime - min_starttime -def _get_ylabel(config, st_sel, processed): +def _get_ylabel(st_sel, processed): if config.correct_instrumental_response and not processed: return 'Counts' if config.trace_units == 'auto': @@ -381,11 +487,18 @@ def _get_ylabel(config, st_sel, processed): raise ValueError(f'Unknown instrument type: {instrtype}') -def plot_traces(config, st, ncols=None, block=True, suffix=None): +def plot_traces(st, ncols=None, block=True, suffix=None): """ Plot raw (counts) or processed traces (instrument units and filtered). Display to screen and/or save to file. + + :param st: Stream of traces. + :type st: :class:`obspy.core.stream.Stream` + :param ncols: Number of columns in the plot (autoset if None). + :type ncols: int + :param block: If True, block execution until the plot window is closed. + :type block: bool """ # Check config, if we need to plot at all if not config.plot_show and not config.plot_save: @@ -397,8 +510,8 @@ def plot_traces(config, st, ncols=None, block=True, suffix=None): ntr = len({t.id[:-1] for t in st}) ncols = 4 if ntr > 6 else 3 - nlines, ncols = _nplots(config, st, config.plot_traces_maxrows, ncols) - fig, axes = _make_fig(config, nlines, ncols) + nlines, ncols = _nplots(st, config.plot_traces_maxrows, ncols) + fig, axes = _make_fig(nlines, ncols) figures = [fig] # Path effect to contour text in white path_effects = [PathEffects.withStroke(linewidth=3, foreground='white')] @@ -442,12 +555,12 @@ def plot_traces(config, st, ncols=None, block=True, suffix=None): config.plot_save_format != 'pdf_multipage' ): # save figure here to free up memory - _savefig(config, figures, suffix, force_numbering=True) - fig, axes = _make_fig(config, nlines, ncols) + _savefig(figures, suffix, force_numbering=True) + fig, axes = _make_fig(nlines, ncols) figures.append(fig) plotn = 1 ax = axes[plotn - 1] - ylabel = _get_ylabel(config, st_sel, processed) + ylabel = _get_ylabel(st_sel, processed) ax.set_ylabel(ylabel, fontsize=8, labelpad=0) # Custom transformation for plotting phase labels: # x coords are data, y coords are axes @@ -457,7 +570,7 @@ def plot_traces(config, st, ncols=None, block=True, suffix=None): transforms.blended_transform_factory(ax.transAxes, ax.transData) trans3 = transforms.offset_copy(trans2, fig=fig, x=0, y=0.1) - _trim_traces(config, st_sel) + _trim_traces(st_sel) max_values = [abs(tr.max()) for tr in st_sel if len(tr.data)] ntraces = len(max_values) if ntraces == 0: @@ -467,7 +580,7 @@ def plot_traces(config, st, ncols=None, block=True, suffix=None): if len(trace.data) == 0: continue _plot_trace( - config, trace, ntraces, tmax, ax, trans, trans3, path_effects) + trace, ntraces, tmax, ax, trans, trans3, path_effects) _set_ylim(axes) # Add labels for the last figure @@ -479,4 +592,4 @@ def plot_traces(config, st, ncols=None, block=True, suffix=None): if config.plot_show: plt.show(block=block) if config.plot_save: - _savefig(config, figures, suffix) + _savefig(figures, suffix) From cc8720d0ca97603d09b8f17c82fea6edc8753190 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 18:46:18 +0200 Subject: [PATCH 17/73] Use the global `config` object in ssp_inversion.py, ssp_grid_sampling.py and ssp_data_types.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_data_types.py | 12 ++-- sourcespec2/ssp_grid_sampling.py | 11 ++-- sourcespec2/ssp_inversion.py | 96 ++++++++++++++++++++++++++------ 4 files changed, 91 insertions(+), 30 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 43a7ae39..c65f7a07 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -57,7 +57,7 @@ def main(): # Spectral inversion from .ssp_inversion import spectral_inversion - sspec_output = spectral_inversion(config, spec_st, weight_st) + sspec_output = spectral_inversion(spec_st, weight_st) # Radiated energy and apparent stress from .ssp_radiated_energy import ( diff --git a/sourcespec2/ssp_data_types.py b/sourcespec2/ssp_data_types.py index 0259799a..e83f067d 100644 --- a/sourcespec2/ssp_data_types.py +++ b/sourcespec2/ssp_data_types.py @@ -12,6 +12,7 @@ import logging from collections import OrderedDict import numpy as np +from .config import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -39,13 +40,12 @@ def get_params0(self): class Bounds(): """Bounds for bounded spectral inversion.""" - def __init__(self, config, spec, initial_values): - self.config = config + def __init__(self, spec, initial_values): self.spec = spec self.hd = spec.stats.hypo_dist self.ini_values = initial_values self.Mw_min = self.Mw_max = None - self._set_fc_min_max(config) + self._set_fc_min_max() if config.Qo_min_max is None: self.t_star_min, self.t_star_max =\ self._check_minmax(config.t_star_min_max) @@ -64,7 +64,7 @@ def __str__(self): *[round(x, 4) if x is not None else x for x in self.bounds[2]]) return s - def _set_fc_min_max(self, config): + def _set_fc_min_max(self): fc_0 = self.ini_values.fc_0 if config.fc_min_max is None: # If no bound is given, set it to fc_0 +/- a decade @@ -94,9 +94,9 @@ def _check_minmax(self, minmax): return (None, None) if minmax is None else minmax def _Qo_to_t_star(self): - phase = self.config.wave_type[0] + phase = config.wave_type[0] travel_time = self.spec.stats.travel_times[phase] - t_star_bounds = travel_time / np.array(self.config.Qo_min_max) + t_star_bounds = travel_time / np.array(config.Qo_min_max) return sorted(t_star_bounds) def _fix_initial_values_t_star(self): diff --git a/sourcespec2/ssp_grid_sampling.py b/sourcespec2/ssp_grid_sampling.py index 301e4677..76aabe23 100644 --- a/sourcespec2/ssp_grid_sampling.py +++ b/sourcespec2/ssp_grid_sampling.py @@ -21,6 +21,7 @@ # pylint: disable=no-name-in-module from scipy.signal._peak_finding_utils import PeakPropertyWarning import matplotlib.pyplot as plt +from .config import config from .kdtree import KDTree from .savefig import savefig logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -252,7 +253,7 @@ def mf(args): self.nsteps = self.misfit.shape self.extent = extent - def plot_conditional_misfit(self, config, label): + def plot_conditional_misfit(self, label): """Plot conditional misfit for each parameter.""" # Check config, if we need to plot at all if not config.plot_show and not config.plot_save: @@ -290,7 +291,7 @@ def plot_conditional_misfit(self, config, label): ax[dim].set_ylabel('misfit') ax[0].set_title(label) plt.tight_layout() - figfile_base = self._get_figfile_base(config) + figfile_base = self._get_figfile_base() figfile_base += f".cond_misfit_{label.replace(' ', '_')}." fmt = config.plot_save_format if fmt == 'pdf_multipage': @@ -306,7 +307,7 @@ def plot_conditional_misfit(self, config, label): f'{label}: conditional misfit plot saved to: {figfile}') config.figures['misfit_1d'].append(figfile) - def plot_misfit_2d(self, config, plot_par_idx, label): + def plot_misfit_2d(self, plot_par_idx, label): """Plot a 2D conditional misfit map.""" # Check config, if we need to plot at all if not config.plot_show and not config.plot_save: @@ -377,7 +378,7 @@ def plot_misfit_2d(self, config, plot_par_idx, label): ax.set_ylabel(ylabel) cbar = plt.colorbar(mmap, ax=ax, extend='max') cbar.set_label('misfit') - figfile_base = self._get_figfile_base(config) + figfile_base = self._get_figfile_base() params_string = f'{params_name[0]}-{params_name[1]}' figfile_base += f".misfit_{params_string}_{label.replace(' ', '_')}." fmt = config.plot_save_format @@ -393,7 +394,7 @@ def plot_misfit_2d(self, config, plot_par_idx, label): logger.info(f'{label}: conditional misfit map saved to: {figfile}') config.figures[f'misfit_{params_string}'].append(figfile) - def _get_figfile_base(self, config): + def _get_figfile_base(self): outdir = os.path.join(config.options.outdir, 'misfit') if not os.path.exists(outdir): os.makedirs(outdir) diff --git a/sourcespec2/ssp_inversion.py b/sourcespec2/ssp_inversion.py index 37706b5f..1908f933 100644 --- a/sourcespec2/ssp_inversion.py +++ b/sourcespec2/ssp_inversion.py @@ -20,6 +20,7 @@ from scipy.optimize import curve_fit, minimize, basinhopping from scipy.signal import argrelmax from obspy.geodetics import gps2dist_azimuth +from .config import config from .spectrum import SpectrumStream from .ssp_spectral_model import ( spectral_model, objective_func, callback) @@ -33,7 +34,7 @@ logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def _curve_fit(config, spec, weight, yerr, initial_values, bounds): +def _curve_fit(spec, weight, yerr, initial_values, bounds): """ Curve fitting. @@ -43,6 +44,20 @@ def _curve_fit(config, spec, weight, yerr, initial_values, bounds): - Truncated Newton algorithm (TNC) with bounds. - Basin-hopping (BH) - Grid search (GS) + + :param spec: Spectrum object. + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param weight: Weight array. + :type weight: :class:`numpy.ndarray` + :param yerr: Error array. + :type yerr: :class:`numpy.ndarray` + :param initial_values: Initial values for the inversion. + :type initial_values: :class:`~sourcespec.ssp_data_types.InitialValues` + :param bounds: Bounds for the inversion. + :type bounds: :class:`~sourcespec.ssp_data_types.Bounds` + + :return: Optimal parameters, parameter errors, misfit. + :rtype: tuple, tuple, float """ freq_logspaced = spec.freq_logspaced ydata = spec.data_mag_logspaced @@ -114,22 +129,34 @@ def _curve_fit(config, spec, weight, yerr, initial_values, bounds): params_opt = grid_sampling.params_opt params_err = grid_sampling.params_err spec_label = f'{spec.id} {spec.stats.instrtype}' - grid_sampling.plot_conditional_misfit(config, spec_label) + grid_sampling.plot_conditional_misfit(spec_label) # fc-t_star plot_par_idx = (1, 2) - grid_sampling.plot_misfit_2d(config, plot_par_idx, spec_label) + grid_sampling.plot_misfit_2d(plot_par_idx, spec_label) # fc-Mw plot_par_idx = (1, 0) - grid_sampling.plot_misfit_2d(config, plot_par_idx, spec_label) + grid_sampling.plot_misfit_2d(plot_par_idx, spec_label) # tstar-Mw plot_par_idx = (2, 0) - grid_sampling.plot_misfit_2d(config, plot_par_idx, spec_label) + grid_sampling.plot_misfit_2d(plot_par_idx, spec_label) misfit = minimize_func(params_opt) return params_opt, params_err, misfit -def _freq_ranges_for_Mw0_and_tstar0(config, weight, freq_logspaced, statId): - """Find the frequency range to compute Mw_0 and, possibly, t_star_0.""" +def _freq_ranges_for_Mw0_and_tstar0(weight, freq_logspaced, statId): + """ + Find the frequency range to compute Mw_0 and, possibly, t_star_0. + + :param weight: Weight array. + :type weight: :class:`numpy.ndarray` + :param freq_logspaced: Log-spaced frequency array. + :type freq_logspaced: :class:`numpy.ndarray` + :param statId: Station ID. + :type statId: str + + :return: Indexes for the frequency range. + :rtype: int, int + """ if config.weighting == 'noise': # we start where signal-to-noise becomes strong idx0 = np.where(weight > 0.5)[0][0] @@ -172,8 +199,21 @@ def _freq_ranges_for_Mw0_and_tstar0(config, weight, freq_logspaced, statId): return idx0, idx1 -def _spec_inversion(config, spec, spec_weight): - """Invert one spectrum, return a StationParameters() object.""" +def _spec_inversion(spec, spec_weight): + """ + Invert one spectrum, return a StationParameters() object. + + :param spec: Spectrum object. + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param spec_weight: Spectrum object with the noise spectrum. + :type spec_weight: :class:`~sourcespec.spectrum.Spectrum` + + :return: Station parameters. + :rtype: :class:`~sourcespec.ssp_data_types.StationParameters` + + :raises RuntimeError: If the inversion fails. + :raises ValueError: If the inversion results are not acceptable. + """ # azimuth computation coords = spec.stats.coords stla = coords.latitude @@ -200,7 +240,7 @@ def _spec_inversion(config, spec, spec_weight): # signal-to-noise ratio try: idx0, idx1 = _freq_ranges_for_Mw0_and_tstar0( - config, weight, freq_logspaced, statId) + weight, freq_logspaced, statId) except RuntimeError: spec.stats.ignore = True spec.stats.ignore_reason = 'fit failed' @@ -237,7 +277,7 @@ def _spec_inversion(config, spec, spec_weight): Mw_0_min = Mw_0_max = Mw_0 initial_values = InitialValues(Mw_0, fc_0, t_star_0) - bounds = Bounds(config, spec, initial_values) + bounds = Bounds(spec, initial_values) Mw_0_variability =\ config.Mw_0_variability if config.Mw_0_variability > 0 else 1e-6 bounds.Mw_min = Mw_0_min * (1 - Mw_0_variability) @@ -251,7 +291,7 @@ def _spec_inversion(config, spec, spec_weight): logger.info(f'{statId}: bounds: {bounds}') try: params_opt, params_err, misfit = _curve_fit( - config, spec, weight, yerr, initial_values, bounds) + spec, weight, yerr, initial_values, bounds) except (RuntimeError, ValueError) as m: spec.stats.ignore = True spec.stats.ignore_reason = 'fit failed' @@ -404,8 +444,18 @@ def _spec_inversion(config, spec, spec_weight): return station_pars -def _synth_spec(config, spec, station_pars): - """Return a stream with one or more synthetic spectra.""" +def _synth_spec(spec, station_pars): + """ + Return a stream with one or more synthetic spectra. + + :param spec: Spectrum object. + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param station_pars: Station parameters. + :type station_pars: :class:`~sourcespec.ssp_data_types.StationParameters` + + :return: Stream with synthetic spectra. + :rtype: :class:`~sourcespec.spectrum.SpectrumStream` + """ par = { x.param_id: x.value for x in station_pars.get_spectral_parameters().values() @@ -460,8 +510,18 @@ def _synth_spec(config, spec, station_pars): return spec_st -def spectral_inversion(config, spec_st, weight_st): - """Inversion of displacement spectra.""" +def spectral_inversion(spec_st, weight_st): + """ + Inversion of displacement spectra. + + :param spec_st: Stream of displacement spectra. + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param weight_st: Stream of noise spectra. + :type weight_st: :class:`~sourcespec.spectrum.SpectrumStream` + + :return: Inversion results. + :rtype: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ logger.info('Inverting spectra...') weighting_messages = { 'noise': 'Using noise weighting for inversion.', @@ -522,11 +582,11 @@ def spectral_inversion(config, spec_st, weight_st): continue spec_weight = select_trace(weight_st, spec.id, spec.stats.instrtype) try: - station_pars = _spec_inversion(config, spec, spec_weight) + station_pars = _spec_inversion(spec, spec_weight) except (RuntimeError, ValueError) as msg: logger.warning(msg) continue - spec_st += _synth_spec(config, spec, station_pars) + spec_st += _synth_spec(spec, station_pars) sspec_output.station_parameters[station_pars.station_id] = station_pars logger.info('Inverting spectra: done') From f2cebc97eeb5abc0ab212ecaf107462691b8a96c Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 18:53:21 +0200 Subject: [PATCH 18/73] Use the global `config` object in ssp_radiated_energy.py --- sourcespec2/source_spec.py | 6 ++-- sourcespec2/ssp_radiated_energy.py | 52 +++++++++++++++++++++++------- 2 files changed, 43 insertions(+), 15 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index c65f7a07..3305d17f 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -60,10 +60,8 @@ def main(): sspec_output = spectral_inversion(spec_st, weight_st) # Radiated energy and apparent stress - from .ssp_radiated_energy import ( - radiated_energy_and_apparent_stress) - radiated_energy_and_apparent_stress( - config, spec_st, specnoise_st, sspec_output) + from .ssp_radiated_energy import radiated_energy_and_apparent_stress + radiated_energy_and_apparent_stress(spec_st, specnoise_st, sspec_output) # Local magnitude if config.compute_local_magnitude: diff --git a/sourcespec2/ssp_radiated_energy.py b/sourcespec2/ssp_radiated_energy.py index 337f1a81..c664b297 100644 --- a/sourcespec2/ssp_radiated_energy.py +++ b/sourcespec2/ssp_radiated_energy.py @@ -18,12 +18,27 @@ import contextlib import logging import numpy as np +from .config import config from .ssp_data_types import SpectralParameter logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) def _spectral_integral(spec, t_star, fmin=None, fmax=None): - """Compute spectral integral in eq. (3) from Lancieri et al. (2012).""" + """ + Compute spectral integral in eq. (3) from Lancieri et al. (2012). + + :param spec: Spectrum object + :param spec type: :class:`~sourcespec.spectrum.Spectrum` + :param t_star: t* value + :param t_star type: float + :param fmin: Minimum frequency for integration (optional) + :param fmin type: float + :param fmax: Maximum frequency for integration (optional) + :param fmax type: float + + :return: Spectral integral + :rtype: float + """ # Note: eq. (3) from Lancieri et al. (2012) is the same as # eq. (1) in Boatwright et al. (2002), but expressed in frequency, # instead of angular frequency (2pi factor). @@ -116,6 +131,16 @@ def _finite_bandwidth_correction(spec, fc, fmax): - Di Bona & Rovelli (1988), eq. 13 - Ide & Beroza (2001), eq. 5 - Lancieri et al. (2012), eq. 4 (note, missing parenthesis in the paper) + + :param spec: Spectrum object + :param spec type: :class:`~sourcespec.spectrum.Spectrum` + :param fc: Corner frequency + :param fc type: float + :param fmax: Maximum frequency for integration + :param fmax type: float + + :return: Finite bandwidth correction + :rtype: float """ if fmax is None: fmax = spec.freq[-1] @@ -125,8 +150,16 @@ def _finite_bandwidth_correction(spec, fc, fmax): ) -def _get_frequency_range(config, spec): - """Get frequency range for spectral integration.""" +def _get_frequency_range(spec): + """ + Get frequency range for spectral integration. + + :param spec: Spectrum object + :param spec type: :class:`~sourcespec.spectrum.Spectrum` + + :return: Frequency range for spectral integration + :rtype: float, float + """ fmin, fmax = config.Er_freq_range if fmin == 'noise': fmin = spec.stats.spectral_snratio_fmin @@ -153,20 +186,17 @@ def _get_frequency_range(config, spec): return fmin, fmax -def radiated_energy_and_apparent_stress( - config, spec_st, specnoise_st, sspec_output): +def radiated_energy_and_apparent_stress(spec_st, specnoise_st, sspec_output): """ Compute radiated energy (in N.m) and apparent stress (in MPa). - :param config: Config object - :param config type: :class:`sourcespec.config.Config` :param spec_st: Stream of spectra - :param spec_st type: :class:`obspy.core.stream.Stream` + :param spec_st type: :class:`~sourcespec.spectrum.SpectrumStream` :param specnoise_st: Stream of noise spectra - :param specnoise_st type: :class:`obspy.core.stream.Stream` + :param specnoise_st type: :class:`~sourcespec.spectrum.SpectrumStream` :param sspec_output: Output of spectral inversion :param sspec_output type: - :class:`sourcespec.ssp_data_types.SourceSpecOutput` + :class:`~sourcespec.ssp_data_types.SourceSpecOutput` """ logger.info('Computing radiated energy and apparent stress...') rho = config.event.hypocenter.rho @@ -206,7 +236,7 @@ def radiated_energy_and_apparent_stress( # Compute signal and noise integrals and subtract noise from signal, # under the hypothesis that energy is additive and noise is stationary - fmin, fmax = _get_frequency_range(config, spec) + fmin, fmax = _get_frequency_range(spec) signal_integral = _spectral_integral(spec, t_star, fmin, fmax) noise_integral = _spectral_integral(specnoise, t_star, fmin, fmax) rho = spec.stats.rho_station From 644d275eb44f9428f9f9973b5e28dc1c3acbc455 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 18:59:08 +0200 Subject: [PATCH 19/73] Use the global `config` object in ssp_local_magnitude.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_local_magnitude.py | 75 +++++++++++++++++++++++++----- 2 files changed, 64 insertions(+), 13 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 3305d17f..d701e72c 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -66,7 +66,7 @@ def main(): # Local magnitude if config.compute_local_magnitude: from .ssp_local_magnitude import local_magnitude - local_magnitude(config, st, proc_st, sspec_output) + local_magnitude(st, proc_st, sspec_output) # Compute summary statistics from station spectral parameters from .ssp_summary_statistics import compute_summary_statistics diff --git a/sourcespec2/ssp_local_magnitude.py b/sourcespec2/ssp_local_magnitude.py index 0fef8164..dc818320 100644 --- a/sourcespec2/ssp_local_magnitude.py +++ b/sourcespec2/ssp_local_magnitude.py @@ -23,6 +23,7 @@ from obspy.signal.invsim import WOODANDERSON from obspy.signal.util import smooth from obspy.signal.trigger import trigger_onset +from .config import config from .ssp_data_types import SpectralParameter from .ssp_util import cosine_taper from .ssp_util import remove_instr_response @@ -30,7 +31,18 @@ def _check_nyquist(freqmax, trace): - """Check if freqmax is smaller than Nyquist frequency.""" + """ + Check if freqmax is smaller than Nyquist frequency. + + :param freqmax: Maximum frequency for bandpass filtering. + :type freqmax: float + + :param trace: Trace to process. + :type trace: :class:`obspy.core.trace.Trace` + + :return: Corrected maximum frequency. + :rtype: float + """ nyquist = 1. / (2. * trace.stats.delta) if freqmax >= nyquist: freqmax = nyquist * 0.999 @@ -46,8 +58,16 @@ def _check_nyquist(freqmax, trace): _check_nyquist.messages = [] # noqa -def _get_cut_times(config, tr): - """Get trace cut times between P arrival and end of envelope coda.""" +def _get_cut_times(tr): + """ + Get trace cut times between P arrival and end of envelope coda. + + :param tr: Trace to process. + :type tr: :class:`obspy.core.trace.Trace` + + :return: Start and end times. + :rtype: tuple of :class:`obspy.core.UTCDateTime` + """ tr_env = tr.copy() # remove the mean... tr_env.detrend(type='constant') @@ -93,8 +113,20 @@ def _get_cut_times(config, tr): return t0, t1 -def _process_trace(config, tr, t0, t1): - """Convert to Wood-Anderson, filter, trim.""" +def _process_trace(tr, t0, t1): + """ + Convert to Wood-Anderson, filter, trim. + + :param tr: Trace to process. + :type tr: :class:`obspy.core.trace.Trace` + :param t0: Start time for trace cut. + :type t0: :class:`obspy.core.utcdatetime.UTCDateTime` + :param t1: End time for trace cut. + :type t1: :class:`obspy.core.utcdatetime.UTCDateTime` + + :return: Processed trace. + :rtype: :class:`obspy.core.trace.Trace` + """ tr_process = tr.copy() # Do a preliminary trim, in order to check if there is enough # data within the selected time window @@ -142,8 +174,18 @@ def _process_trace(config, tr, t0, t1): return tr_process -def _compute_local_magnitude(config, amp, h_dist): - """Compute local magnitude using the Richter formula.""" +def _compute_local_magnitude(amp, h_dist): + """ + Compute local magnitude using the Richter formula. + + :param amp: Maximum amplitude in millimeters. + :type amp: float + :param h_dist: Hypocentral distance in kilometers. + :type h_dist: float + + :return: Local magnitude. + :rtype: float + """ a = config.a b = config.b c = config.c @@ -152,8 +194,17 @@ def _compute_local_magnitude(config, amp, h_dist): ) -def local_magnitude(config, st, proc_st, sspec_output): - """Compute local magnitude from max absolute W-A amplitude.""" +def local_magnitude(st, proc_st, sspec_output): + """ + Compute local magnitude from max absolute W-A amplitude. + + :param st: Stream object with all traces. + :type st: :class:`obspy.core.stream.Stream` + :param proc_st: Stream object with processed traces. + :type proc_st: :class:`obspy.core.stream.Stream` + :param sspec_output: Output of the spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ logger.info('Computing local magnitude...') # We only use traces selected for proc_st trace_ids = {tr.id for tr in proc_st} @@ -183,8 +234,8 @@ def local_magnitude(config, st, proc_st, sspec_output): continue try: - t0, t1 = _get_cut_times(config, tr) - tr_process = _process_trace(config, tr, t0, t1) + t0, t1 = _get_cut_times(tr) + tr_process = _process_trace(tr, t0, t1) except RuntimeError as msg: logger.warning(msg) continue @@ -193,7 +244,7 @@ def local_magnitude(config, st, proc_st, sspec_output): # amp must be in millimeters for local magnitude computation amp = np.abs(tr_process.max()) * 1e3 h_dist = tr_process.stats.hypo_dist - ml = _compute_local_magnitude(config, amp, h_dist) + ml = _compute_local_magnitude(amp, h_dist) statId = f'{tr_id} {tr.stats.instrtype}' logger.info(f'{statId}: Ml {ml:.1f}') # compute average with the other component, if available From d950d50a7a3aee9915ca6432cd0351dcf48b536f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 19:07:55 +0200 Subject: [PATCH 20/73] Use the global `config` object in ssp_summary_statistics.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_summary_statistics.py | 101 ++++++++++++++++++++++---- 2 files changed, 86 insertions(+), 17 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index d701e72c..cedc7938 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -70,7 +70,7 @@ def main(): # Compute summary statistics from station spectral parameters from .ssp_summary_statistics import compute_summary_statistics - compute_summary_statistics(config, sspec_output) + compute_summary_statistics(sspec_output) # Save output from .ssp_output import write_output, save_spectra diff --git a/sourcespec2/ssp_summary_statistics.py b/sourcespec2/ssp_summary_statistics.py index c708f696..27ea3570 100644 --- a/sourcespec2/ssp_summary_statistics.py +++ b/sourcespec2/ssp_summary_statistics.py @@ -13,6 +13,7 @@ import numpy as np from scipy.stats import norm from scipy.integrate import quad +from .config import config from .ssp_setup import ssp_exit from .ssp_data_types import ( SummarySpectralParameter, SummaryStatistics) @@ -26,6 +27,17 @@ def _avg_and_std(values, errors=None, logarithmic=False): Optionally: - errors can be specified for weighted statistics - logarithmic average and standard deviation + + :param values: Values to compute the average and standard deviation. + :type values: :class:`numpy.ndarray` + :param errors: Errors on the values (optional). + :type errors: :class:`numpy.ndarray` + :param logarithmic: Compute the average and standard deviation in + logarithmic space (default: False). + :type logarithmic: bool + + :return: Average and standard deviation. + :rtype: tuple """ average = std = np.nan if len(values) == 0: @@ -54,7 +66,20 @@ def _avg_and_std(values, errors=None, logarithmic=False): def _weights(values, errors=None, logarithmic=False): - """Compute weights for weighted statistics.""" + """ + Compute weights for weighted statistics. + + :param values: Values to compute the weights for. + :type values: :class:`numpy.ndarray` + :param errors: Errors on the values (optional). + :type errors: :class:`numpy.ndarray` + :param logarithmic: Compute the weights in logarithmic space + (default: False). + :type logarithmic: bool + + :return: Weights. + :rtype: :class:`numpy.ndarray` + """ if errors is None: return None # negative errors should not happen @@ -85,6 +110,12 @@ def _normal_confidence_level(n_sigma): """ Compute the confidence level of a normal (Gaussian) distribution between -n_sigma and +n_sigma. + + :param n_sigma: Number of standard deviations. + :type n_sigma: float + + :return: Confidence level. + :rtype: float """ def gauss(x): return norm.pdf(x, 0, 1) @@ -94,7 +125,21 @@ def gauss(x): def _percentiles( values, low_percentage=25, mid_percentage=50, up_percentage=75): - """Compute lower, mid and upper percentiles.""" + """ + Compute lower, mid and upper percentiles. + + :param values: Values to compute the percentiles for. + :type values: :class:`numpy.ndarray` + :param low_percentage: Lower percentile (default: 25). + :type low_percentage: float + :param mid_percentage: Mid percentile (default: 50). + :type mid_percentage: float + :param up_percentage: Upper percentile (default: 75). + :type up_percentage: float + + :return: Lower, mid and upper percentiles. + :rtype: tuple + """ if len(values) == 0: return np.nan, np.nan, np.nan low_percentile, mid_percentile, up_percentile =\ @@ -104,9 +149,28 @@ def _percentiles( def _param_summary_statistics( - config, sspec_output, param_id, name, format_spec, units=None, + sspec_output, param_id, name, format_spec, units=None, logarithmic=False): - """Compute summary statistics for one spectral parameter.""" + """ + Compute summary statistics for one spectral parameter. + + :param sspec_output: Output of the spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param param_id: Parameter ID. + :type param_id: str + :param name: Parameter name. + :type name: str + :param format_spec: Format specification for the parameter value. + :type format_spec: str + :param units: Units of the parameter (optional). + :type units: str + :param logarithmic: Compute the statistics in logarithmic space + (default: False). + :type logarithmic: bool + + :return: Summary statistics. + :rtype: :class:`~sourcespec.ssp_data_types.SummarySpectralParameter` + """ nIQR = config.nIQR summary = SummarySpectralParameter( param_id=param_id, name=name, format_spec=format_spec, units=units) @@ -162,8 +226,13 @@ def _param_summary_statistics( return summary -def compute_summary_statistics(config, sspec_output): - """Compute summary statistics from station spectral parameters.""" +def compute_summary_statistics(sspec_output): + """ + Compute summary statistics from station spectral parameters. + + :param sspec_output: Output of the spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ logger.info('Computing summary statistics...') if len(sspec_output.station_parameters) == 0: logger.info('No source parameter calculated') @@ -175,7 +244,7 @@ def compute_summary_statistics(config, sspec_output): # Mw sspec_output.summary_spectral_parameters.Mw =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='Mw', name='moment magnitude', format_spec='{:.2f}', logarithmic=False ) @@ -183,7 +252,7 @@ def compute_summary_statistics(config, sspec_output): # Mo (N·m) sspec_output.summary_spectral_parameters.Mo =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='Mo', name='seismic moment', units='N·m', format_spec='{:.3e}', logarithmic=True ) @@ -191,7 +260,7 @@ def compute_summary_statistics(config, sspec_output): # fc (Hz) sspec_output.summary_spectral_parameters.fc =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='fc', name='corner frequency', units='Hz', format_spec='{:.3f}', logarithmic=True ) @@ -199,7 +268,7 @@ def compute_summary_statistics(config, sspec_output): # t_star (s) sspec_output.summary_spectral_parameters.t_star =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='t_star', name='t-star', units='s', format_spec='{:.3f}', logarithmic=False ) @@ -207,7 +276,7 @@ def compute_summary_statistics(config, sspec_output): # radius (meters) sspec_output.summary_spectral_parameters.radius =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='radius', name='source radius', units='m', format_spec='{:.3f}', logarithmic=True ) @@ -215,7 +284,7 @@ def compute_summary_statistics(config, sspec_output): # static stress drop (MPa) sspec_output.summary_spectral_parameters.ssd =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='ssd', name='static stress drop', units='MPa', format_spec='{:.3e}', logarithmic=True @@ -224,7 +293,7 @@ def compute_summary_statistics(config, sspec_output): # Quality factor sspec_output.summary_spectral_parameters.Qo =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='Qo', name='quality factor', format_spec='{:.1f}', logarithmic=False ) @@ -232,7 +301,7 @@ def compute_summary_statistics(config, sspec_output): # Er (N·m) sspec_output.summary_spectral_parameters.Er =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='Er', name='radiated energy', units='N·m', format_spec='{:.3e}', logarithmic=True ) @@ -240,7 +309,7 @@ def compute_summary_statistics(config, sspec_output): # Apparent stress (MPa) sspec_output.summary_spectral_parameters.sigma_a =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='sigma_a', name='apparent stress', units='MPa', format_spec='{:.3e}', logarithmic=True ) @@ -249,7 +318,7 @@ def compute_summary_statistics(config, sspec_output): if config.compute_local_magnitude: sspec_output.summary_spectral_parameters.Ml =\ _param_summary_statistics( - config, sspec_output, + sspec_output, param_id='Ml', name='local magnitude', format_spec='{:.2f}', logarithmic=False ) From fc86bef7f98cad52e1a0bfdda753f7773a7ceff3 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 9 Jul 2024 19:18:46 +0200 Subject: [PATCH 21/73] Use the global `config` object in ssp_output.py, ssp_qml_output.py, and ssp_sqlite_output.py --- sourcespec2/source_spec.py | 4 +- sourcespec2/ssp_output.py | 68 +++++++++++++++++++++++--------- sourcespec2/ssp_qml_output.py | 10 ++++- sourcespec2/ssp_sqlite_output.py | 27 ++++++------- 4 files changed, 73 insertions(+), 36 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index cedc7938..3234b1a1 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -74,8 +74,8 @@ def main(): # Save output from .ssp_output import write_output, save_spectra - write_output(config, sspec_output) - save_spectra(config, spec_st) + write_output(sspec_output) + save_spectra(spec_st) # Save residuals from .ssp_residuals import spectral_residuals diff --git a/sourcespec2/ssp_output.py b/sourcespec2/ssp_output.py index 7a56a437..6aebd988 100644 --- a/sourcespec2/ssp_output.py +++ b/sourcespec2/ssp_output.py @@ -22,13 +22,14 @@ from datetime import datetime from tzlocal import get_localzone import numpy as np +from .config import config from .ssp_qml_output import write_qml from .ssp_sqlite_output import write_sqlite from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def _write_author_and_agency_to_parfile(config, parfile): +def _write_author_and_agency_to_parfile(parfile): author_str = empty_author_str = '\n*** Author:' if config.author_name is not None: author_str += f' {config.author_name}' @@ -65,7 +66,7 @@ def _value_error_str(value, error, fmt): return s -def _write_parfile(config, sspec_output): +def _write_parfile(sspec_output): """ Write station source parameters to file. @@ -286,13 +287,23 @@ def _write_parfile(config, sspec_output): f'{config.end_of_run_tz}') if config.options.run_id: parfile.write(f'\n*** Run ID: {config.options.run_id}') - _write_author_and_agency_to_parfile(config, parfile) + _write_author_and_agency_to_parfile(parfile) logger.info(f'Output written to file: {parfilename}') def _dict2yaml(dict_like, level=0): - """Serialize a dict-like object into YAML format.""" + """ + Serialize a dict-like object into YAML format. + + :param dict_like: Dict-like object to serialize. + :type dict_like: dict + :param level: Indentation level. + :type level: int + + :return: YAML-formatted string. + :rtype: str + """ if not isinstance(dict_like, Mapping): raise TypeError('dict_like must be a dict-like object') comments = dict_like.get('_comments', {}) @@ -331,8 +342,13 @@ def _dict2yaml(dict_like, level=0): return lines -def _write_yaml(config, sspec_output): - """Write sspec output in a YAML file.""" +def _write_yaml(sspec_output): + """ + Write sspec output in a YAML file. + + :param sspec_output: Output of the spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ if not os.path.exists(config.options.outdir): os.makedirs(config.options.outdir) evid = config.event.event_id @@ -348,7 +364,13 @@ def _write_yaml(config, sspec_output): logger.info(f'Output written to file: {yamlfilename}') -def _write_hypo71(config, sspec_output): +def _write_hypo71(sspec_output): + """ + Write source parameters to hypo71 file. + + :param sspec_output: Output of the spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ if not config.options.hypo_file: return if config.hypo_file_format != 'hypo71': @@ -378,7 +400,7 @@ def _write_hypo71(config, sspec_output): logger.info(f'Hypo file written to: {hypo_file_out}') -def _make_symlinks(config): +def _make_symlinks(): """Make symlinks to input files into output directory.""" # Windows does not support symlinks if os.name == 'nt': @@ -408,8 +430,13 @@ def _make_symlinks(config): os.symlink(filename, linkname) -def write_output(config, sspec_output): - """Write results into different formats.""" +def write_output(sspec_output): + """ + Write results into different formats. + + :param sspec_output: Output of the spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ # Add run info to output object run_info = sspec_output.run_info run_info.SourceSpec_version = get_versions()['version'] @@ -425,21 +452,26 @@ def write_output(config, sspec_output): run_info.agency_short_name = config.agency_short_name run_info.agency_url = config.agency_url # Symlink input files into output directory - _make_symlinks(config) + _make_symlinks() # Write to parfile (deprecated) - _write_parfile(config, sspec_output) + _write_parfile(sspec_output) # Write to YAML file - _write_yaml(config, sspec_output) + _write_yaml(sspec_output) # Write to SQLite database, if requested - write_sqlite(config, sspec_output) + write_sqlite(sspec_output) # Write to hypo file, if requested - _write_hypo71(config, sspec_output) + _write_hypo71(sspec_output) # Write to quakeml file, if requested - write_qml(config, sspec_output) + write_qml(sspec_output) + +def save_spectra(spec_st): + """ + Save spectra to file. -def save_spectra(config, spec_st): - """Save spectra to file.""" + :param spec_st: Stream of spectra. + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + """ if not config.save_spectra: return outfile = os.path.join( diff --git a/sourcespec2/ssp_qml_output.py b/sourcespec2/ssp_qml_output.py index bb22ea72..c273e27b 100644 --- a/sourcespec2/ssp_qml_output.py +++ b/sourcespec2/ssp_qml_output.py @@ -19,6 +19,7 @@ MomentTensor, QuantityError, ResourceIdentifier, StationMagnitude, StationMagnitudeContribution, WaveformStreamID) +from .config import config from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -63,8 +64,13 @@ def __init__(self, value=None): self.value = value -def write_qml(config, sspec_output): - """Write QuakeML output.""" +def write_qml(sspec_output): + """ + Write QuakeML output. + + :param sspec_output: Output from spectral inversion. + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + """ if not config.options.qml_file: config.qml_file_out = None return diff --git a/sourcespec2/ssp_sqlite_output.py b/sourcespec2/ssp_sqlite_output.py index d659f669..4b93785d 100644 --- a/sourcespec2/ssp_sqlite_output.py +++ b/sourcespec2/ssp_sqlite_output.py @@ -12,6 +12,7 @@ import os.path import logging import sqlite3 +from .config import config from .ssp_setup import ssp_exit from .ssp_db_definitions import ( DB_VERSION, @@ -38,6 +39,7 @@ def _open_sqlite_db(db_file): :param db_file: SQLite database file :type db_file: str + :return: SQLite connection and cursor :rtype: tuple """ @@ -131,7 +133,7 @@ def _create_stations_table(cursor, db_file): _log_db_write_error(db_err, db_file) -def _write_stations_table(cursor, db_file, sspec_output, config): +def _write_stations_table(cursor, db_file, sspec_output): """ Write station source parameters to database. @@ -141,8 +143,9 @@ def _write_stations_table(cursor, db_file, sspec_output, config): :type db_file: str :param sspec_output: sspec output object :type sspec_output: ssp_data_types.SourceSpecOutput - :param config: sspec configuration object - :type config: config.Config + + :return: Number of observations + :rtype: int """ event = config.event evid = event.event_id @@ -218,7 +221,7 @@ def _create_events_table(cursor, db_file): _log_db_write_error(db_err, db_file) -def _write_events_table(cursor, db_file, sspec_output, config, nobs): +def _write_events_table(cursor, db_file, sspec_output, nobs): """ Write Events table. @@ -228,8 +231,6 @@ def _write_events_table(cursor, db_file, sspec_output, config, nobs): :type db_file: str :param sspec_output: SSP output object :type sspec_output: ssp_data_types.SourceSpecOutput - :param config: SSP configuration object - :type config: config.Config :param nobs: Number of observations :type nobs: int """ @@ -396,14 +397,12 @@ def _write_events_table(cursor, db_file, sspec_output, config, nobs): ssp_exit(1) -def write_sqlite(config, sspec_output): +def write_sqlite(sspec_output): """ - Write SSP output to SQLite database. + Write SourceSpec output to SQLite database. - :param config: SSP configuration object - :type config: config.Config - :param sspec_output: SSP output object - :type sspec_output: ssp_data_types.SourceSpecOutput + :param sspec_output: SourceSpec output object + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` """ db_file = config.get('database_file', None) if not db_file: @@ -419,13 +418,13 @@ def write_sqlite(config, sspec_output): # Create Stations table _create_stations_table(cursor, db_file) # Write station source parameters to database - nobs = _write_stations_table(cursor, db_file, sspec_output, config) + nobs = _write_stations_table(cursor, db_file, sspec_output) # Commit changes conn.commit() # Create Events table _create_events_table(cursor, db_file) # Write event source parameters to database - _write_events_table(cursor, db_file, sspec_output, config, nobs) + _write_events_table(cursor, db_file, sspec_output, nobs) # Commit changes and close database conn.commit() conn.close() From 8367bf3dd8739e5f566d9455ee5ea6b43c114a12 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 08:36:39 +0200 Subject: [PATCH 22/73] Use the global `config` object in ssp_residuals.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_residuals.py | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 3234b1a1..88e1c4c8 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -79,7 +79,7 @@ def main(): # Save residuals from .ssp_residuals import spectral_residuals - spectral_residuals(config, spec_st, sspec_output) + spectral_residuals(spec_st, sspec_output) # Plotting from .ssp_plot_spectra import plot_spectra diff --git a/sourcespec2/ssp_residuals.py b/sourcespec2/ssp_residuals.py index 7a105d0d..bee362b2 100644 --- a/sourcespec2/ssp_residuals.py +++ b/sourcespec2/ssp_residuals.py @@ -14,6 +14,7 @@ """ import os import logging +from .config import config from ._version import get_versions from .spectrum import SpectrumStream from .ssp_spectral_model import spectral_model @@ -21,13 +22,11 @@ logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def spectral_residuals(config, spec_st, sspec_output): +def spectral_residuals(spec_st, sspec_output): """ Compute spectral residuals with respect to an average spectral model. Saves a stream of residuals to disk in HDF5 format. - :param config: Configuration object - :type config: :class:`~sourcespec.config.Config` :param spec_st: Stream of spectra :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` :param sspec_output: Output of the source spectral parameter estimation From 840309a9ff09243e34177146f8a1a0b0ff817ac0 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 09:00:55 +0200 Subject: [PATCH 23/73] Use the global `config` object in ssp_plot_spectra.py --- sourcespec2/source_model.py | 2 +- sourcespec2/source_spec.py | 4 +- sourcespec2/ssp_plot_spectra.py | 212 ++++++++++++++++++++++++++------ 3 files changed, 178 insertions(+), 40 deletions(-) diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 1b22c98b..3dec065a 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -125,7 +125,7 @@ def main(): from .ssp_plot_spectra import plot_spectra from .ssp_plot_traces import plot_traces plot_traces(proc_st, ncols=2, block=False) - plot_spectra(config, spec_st, ncols=1, stack_plots=True) + plot_spectra(spec_st, ncols=1, stack_plots=True) if __name__ == '__main__': diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 88e1c4c8..b27a3af8 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -83,8 +83,8 @@ def main(): # Plotting from .ssp_plot_spectra import plot_spectra - plot_spectra(config, spec_st, specnoise_st, plot_type='regular') - plot_spectra(config, weight_st, plot_type='weight') + plot_spectra(spec_st, specnoise_st, plot_type='regular') + plot_spectra(weight_st, plot_type='weight') from .ssp_plot_stacked_spectra import plot_stacked_spectra plot_stacked_spectra(config, spec_st, weight_st, sspec_output) from .ssp_plot_params_stats import box_plots diff --git a/sourcespec2/ssp_plot_spectra.py b/sourcespec2/ssp_plot_spectra.py index 3ad10d21..00354df7 100644 --- a/sourcespec2/ssp_plot_spectra.py +++ b/sourcespec2/ssp_plot_spectra.py @@ -25,6 +25,7 @@ import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages import matplotlib.patheffects as PathEffects +from .config import config from .ssp_util import spec_minmax, moment_to_mag, mag_to_moment from .savefig import savefig from ._version import get_versions @@ -62,7 +63,7 @@ def __init__(self): self.axes = [] self.ax0 = None - def set_plot_params(self, config, spec_st, specnoise_st): + def set_plot_params(self, spec_st, specnoise_st): """Determine the number of plots and axes min and max.""" nplots = 0 moment_minmax = None @@ -98,7 +99,18 @@ def set_plot_params(self, config, spec_st, specnoise_st): self.moment_minmax = moment_minmax -def _make_fig(config, plot_params): +def _make_fig(plot_params): + """ + Create a new figure and axes for plotting spectra. + + The number of lines and columns is determined by the number of spectra. + The number of lines is limited to `config.plot_spectra_maxrows`. + + Figure is appended to `plot_params.figures`. + + :param plot_params: PlotParams object + :type plot_params: PlotParams + """ nlines = plot_params.nlines ncols = plot_params.ncols stack_plots = plot_params.stack_plots @@ -198,7 +210,17 @@ def _make_fig(config, plot_params): BBOX = None -def _savefig(config, plottype, figures, force_numbering=False): +def _savefig(plottype, figures, force_numbering=False): + """ + Save the figure(s) to a file. + + :param plottype: Type of plot ('regular' or 'weight') + :type plottype: str + :param figures: List of figures + :type figures: list + :param force_numbering: Force numbering of the figures + :type force_numbering: bool + """ global BBOX # pylint: disable=global-statement evid = config.event.event_id if plottype == 'regular': @@ -247,6 +269,9 @@ def _add_labels(plot_params): Add xlabels to the last row plots. Add ylabels to the first and last columns. + + :param plot_params: PlotParams object + :type plot_params: PlotParams """ plotn = plot_params.plotn ncols = plot_params.ncols @@ -284,7 +309,19 @@ def _add_labels(plot_params): ax2.set_ylabel('Magnitude') -def _color_lines(config, orientation, plotn, stack_plots): +def _color_lines(orientation, plotn, stack_plots): + """ + Determine the color, linestyle, and linewidth for the plot. + + :param orientation: Channel code + :type orientation: str + :param plotn: Plot number + :type plotn: int + :param stack_plots: Stack plots + :type stack_plots: bool + :return: Color, linestyle, linewidth + :rtype: tuple + """ if orientation in config.vertical_channel_codes: color = 'purple' linestyle = 'solid' @@ -329,7 +366,17 @@ def _color_lines(config, orientation, plotn, stack_plots): return color, linestyle, linewidth -def _add_legend(config, plot_params, spec_st, specnoise_st): +def _add_legend(plot_params, spec_st, specnoise_st): + """ + Add a legend to the plot. + + :param plot_params: PlotParams object + :type plot_params: PlotParams + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param specnoise_st: Stream of noise spectra + :type specnoise_st: :class:`~sourcespec.spectrum.SpectrumStream` + """ stack_plots = plot_params.stack_plots plot_type = plot_params.plot_type ax0 = plot_params.ax0 @@ -340,8 +387,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if 'H' in channel_codes: ncol0 += 1 orientation = 'H' - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 if plot_type == 'weight': label = 'Weight' @@ -355,8 +401,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if 'h' in channel_codes: ncol0 += 1 orientation = 'h' - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = 'RSS, uncorr.' _h, = ax0.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -367,8 +412,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if len(Z_codes) > 0: ncol0 += 1 orientation = Z_codes[0] - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = ', '.join(Z_codes) _h, = ax0.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -379,8 +423,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if len(H1_codes) > 0: ncol0 += 1 orientation = H1_codes[0] - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = ', '.join(H1_codes) _h, = ax0.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -391,8 +434,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if len(H2_codes) > 0: ncol0 += 1 orientation = H2_codes[0] - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = ', '.join(H2_codes) _h, = ax0.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -415,8 +457,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if 'S' in channel_codes: ncol1 += 1 orientation = 'S' - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = 'Brune fit' _h, = ax1.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -425,8 +466,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if 's' in channel_codes: ncol1 += 1 orientation = 's' - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = 'Brune fit no attenuation' _h, = ax1.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -435,8 +475,7 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): if 't' in channel_codes: ncol1 += 1 orientation = 't' - color, linestyle, linewidth =\ - _color_lines(config, orientation, 0, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, 0, stack_plots) linewidth = 2 label = 'Brune fit no fc' _h, = ax1.plot(range(2), linestyle=linestyle, linewidth=linewidth, @@ -457,6 +496,19 @@ def _add_legend(config, plot_params, spec_st, specnoise_st): def _snratio_text(spec, ax, color, path_effects): + """ + Add the spectral S/N ratio to the plot. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param ax: Axes object + :type ax: :class:`matplotlib.axes.Axes` + :param color: Color + :type color: str + :param path_effects: Path effects + :type path_effects: list of + :class:`matplotlib.patheffects.AbstractPathEffect` + """ global SNRATIO_TEXT_YPOS # pylint: disable=global-statement if spec.stats.spectral_snratio is None: return @@ -469,6 +521,21 @@ def _snratio_text(spec, ax, color, path_effects): def _station_text(spec, ax, color, path_effects, stack_plots): + """ + Add station information to the plot. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param ax: Axes object + :type ax: :class:`matplotlib.axes.Axes` + :param color: Color + :type color: str + :param path_effects: Path effects + :type path_effects: list of + :class:`matplotlib.patheffects.AbstractPathEffect` + :param stack_plots: Stack plots + :type stack_plots: bool + """ station_text = f'{spec.id[:-1]} {spec.stats.instrtype}' if not stack_plots: color = 'black' @@ -487,6 +554,21 @@ def _station_text(spec, ax, color, path_effects, stack_plots): def _ignore_text(spec, ax, color, path_effects, stack_plots): + """ + Add ignore reason to the plot. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param ax: Axes object + :type ax: :class:`matplotlib.axes.Axes` + :param color: Color + :type color: str + :param path_effects: Path effects + :type path_effects: list of + :class:`matplotlib.patheffects.AbstractPathEffect` + :param stack_plots: Stack plots + :type stack_plots: bool + """ ignore_text = spec.stats.ignore_reason if not stack_plots: color = 'black' @@ -503,6 +585,21 @@ def _ignore_text(spec, ax, color, path_effects, stack_plots): def _params_text(spec, ax, color, path_effects, stack_plots): + """ + Add spectral parameters to the plot. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param ax: Axes object + :type ax: :class:`matplotlib.axes.Axes` + :param color: Color + :type color: str + :param path_effects: Path effects + :type path_effects: list of + :class:`matplotlib.patheffects.AbstractPathEffect` + :param stack_plots: Stack plots + :type stack_plots: bool + """ global STATION_TEXT_YPOS # pylint: disable=global-statement if stack_plots: params_text_ypos = STATION_TEXT_YPOS - 0.04 @@ -572,6 +669,16 @@ def _params_text(spec, ax, color, path_effects, stack_plots): def _plot_fc_and_mw(spec, ax, ax2): + """ + Plot corner frequency and moment magnitude lines. + + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param ax: Axes object (moment units) + :type ax: :class:`matplotlib.axes.Axes` + :param ax2: Axes object (magnitude units) + :type ax2: :class:`matplotlib.axes.Axes` + """ fc = spec.stats.par['fc'] Mw = spec.stats.par['Mw'] if 'par_err' in spec.stats.keys(): @@ -588,8 +695,17 @@ def _plot_fc_and_mw(spec, ax, ax2): ax.axvline(fc, color='#999999', linewidth=2., zorder=1) -def _plot_spec(config, plot_params, spec, spec_noise): - """Plot one spectrum (and its associated noise).""" +def _plot_spec(plot_params, spec, spec_noise): + """ + Plot one spectrum (and its associated noise). + + :param plot_params: PlotParams object + :type plot_params: PlotParams + :param spec: Spectrum object + :type spec: :class:`~sourcespec.spectrum.Spectrum` + :param spec_noise: Noise spectrum object + :type spec_noise: :class:`~sourcespec.spectrum.Spectrum` + """ plotn = plot_params.plotn plot_type = plot_params.plot_type stack_plots = plot_params.stack_plots @@ -598,8 +714,7 @@ def _plot_spec(config, plot_params, spec, spec_noise): orientation = spec.stats.channel[-1] # Path effect to contour text in white path_effects = [PathEffects.withStroke(linewidth=3, foreground='white')] - color, linestyle, linewidth =\ - _color_lines(config, orientation, plotn, stack_plots) + color, linestyle, linewidth = _color_lines(orientation, plotn, stack_plots) # dim out ignored spectra, make invisible spectra not to be plotted alpha = 1.0 if spec.stats.ignore: @@ -641,8 +756,19 @@ def _plot_spec(config, plot_params, spec, spec_noise): _station_text(spec, ax, color, path_effects, stack_plots) -def _plot_specid(config, plot_params, specid, spec_st, specnoise_st): - """Plot all spectra having the same specid.""" +def _plot_specid(plot_params, specid, spec_st, specnoise_st): + """ + Plot all spectra having the same specid. + + :param plot_params: PlotParams object + :type plot_params: PlotParams + :param specid: Band+instrument code + :type specid: str + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param specnoise_st: Stream of noise spectra + :type specnoise_st: :class:`~sourcespec.spectrum.SpectrumStream` + """ plotn = plot_params.plotn + 1 nlines = plot_params.nlines ncols = plot_params.ncols @@ -653,7 +779,7 @@ def _plot_specid(config, plot_params, specid, spec_st, specnoise_st): if plotn > nlines * ncols: # Add labels and legend before making a new figure _add_labels(plot_params) - _add_legend(config, plot_params, spec_st, specnoise_st) + _add_legend(plot_params, spec_st, specnoise_st) if ( config.plot_save_asap and config.plot_save and not config.plot_show and @@ -661,9 +787,9 @@ def _plot_specid(config, plot_params, specid, spec_st, specnoise_st): ): # save figure here to free up memory _savefig( - config, plot_params.plot_type, plot_params.figures, + plot_params.plot_type, plot_params.figures, force_numbering=True) - _make_fig(config, plot_params) + _make_fig(plot_params) plotn = 1 plot_params.plotn = plotn special_orientations = ['S', 's', 't', 'H', 'h'] @@ -702,15 +828,27 @@ def _plot_specid(config, plot_params, specid, spec_st, specnoise_st): specid = spec.get_id() with contextlib.suppress(Exception): spec_noise = specnoise_st.select(id=specid)[0] - _plot_spec(config, plot_params, spec, spec_noise) + _plot_spec(plot_params, spec, spec_noise) -def plot_spectra(config, spec_st, specnoise_st=None, ncols=None, +def plot_spectra(spec_st, specnoise_st=None, ncols=None, stack_plots=False, plot_type='regular'): """ Plot spectra for signal and noise. Display to screen and/or save to file. + + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param specnoise_st: Stream of noise spectra (optional) + :type specnoise_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param ncols: Number of columns in the plot. If None, it is automatically + determined based on the number of spectra. + :type ncols: int + :param stack_plots: If True, stack the plots vertically in the same figure. + :type stack_plots: bool + :param plot_type: Type of plot: 'regular' (default), 'weight'. + :type plot_type: str """ # Check config, if we need to plot at all if not config.plot_show and not config.plot_save: @@ -726,8 +864,8 @@ def plot_spectra(config, spec_st, specnoise_st=None, ncols=None, plot_params.plot_type = plot_type plot_params.stack_plots = stack_plots plot_params.ncols = ncols - plot_params.set_plot_params(config, spec_st, specnoise_st) - _make_fig(config, plot_params) + plot_params.set_plot_params(spec_st, specnoise_st) + _make_fig(plot_params) # Plot! if config.plot_spectra_ignored: @@ -739,11 +877,11 @@ def plot_spectra(config, spec_st, specnoise_st=None, ncols=None, if not sp.stats.ignore }) for _, specid in stalist: - _plot_specid(config, plot_params, specid, spec_st, specnoise_st) + _plot_specid(plot_params, specid, spec_st, specnoise_st) # Add labels and legend for the last figure _add_labels(plot_params) - _add_legend(config, plot_params, spec_st, specnoise_st) + _add_legend(plot_params, spec_st, specnoise_st) # Turn off the unused axes for ax, ax2 in plot_params.axes[plot_params.plotn:]: ax.set_axis_off() @@ -753,4 +891,4 @@ def plot_spectra(config, spec_st, specnoise_st=None, ncols=None, if config.plot_show: plt.show() if config.plot_save: - _savefig(config, plot_type, plot_params.figures) + _savefig(plot_type, plot_params.figures) From f8612b2dab01468d8e183b28445ed22dcec5b7b4 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 09:08:44 +0200 Subject: [PATCH 24/73] Use the global `config` object in ssp_plot_stacked_spectra.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_plot_stacked_spectra.py | 117 ++++++++++++++++++++++-- 2 files changed, 108 insertions(+), 11 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index b27a3af8..87dceb61 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -86,7 +86,7 @@ def main(): plot_spectra(spec_st, specnoise_st, plot_type='regular') plot_spectra(weight_st, plot_type='weight') from .ssp_plot_stacked_spectra import plot_stacked_spectra - plot_stacked_spectra(config, spec_st, weight_st, sspec_output) + plot_stacked_spectra(spec_st, weight_st, sspec_output) from .ssp_plot_params_stats import box_plots box_plots(config, sspec_output) if config.plot_station_map: diff --git a/sourcespec2/ssp_plot_stacked_spectra.py b/sourcespec2/ssp_plot_stacked_spectra.py index 73156ca8..1f73333b 100644 --- a/sourcespec2/ssp_plot_stacked_spectra.py +++ b/sourcespec2/ssp_plot_stacked_spectra.py @@ -17,6 +17,7 @@ import matplotlib.pyplot as plt import matplotlib.patheffects as PathEffects from matplotlib.collections import LineCollection +from .config import config from .ssp_util import moment_to_mag, mag_to_moment from .ssp_spectral_model import spectral_model from .savefig import savefig @@ -28,10 +29,39 @@ def _spectral_model_moment(freq, Mw, fc, t_star): + """ + Compute spectral model in moment units. + + :param freq: Frequency array + :type freq: :class:`numpy.ndarray` + :param Mw: Moment magnitude + :type Mw: float + :param fc: Corner frequency + :type fc: float + :param t_star: Attenuation parameter + :type t_star: float + + :return: Spectral model in moment units + :rtype: :class:`numpy.ndarray` + """ return mag_to_moment(spectral_model(freq, Mw=Mw, fc=fc, t_star=t_star)) def _summary_synth_spec(sspec_output, fmin, fmax): + """ + Compute synthetic spectrum for summary parameters. + + :param sspec_output: Output of spectral inversion + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param fmin: Minimum frequency + :type fmin: float + :param fmax: Maximum frequency + :type fmax: float + + :return: Frequency array, synthetic spectrum, synthetic spectrum without + attenuation, synthetic spectrum without corner frequency + :rtype: tuple of :class:`numpy.ndarray` + """ npts = 100 freq = np.logspace(np.log10(fmin), np.log10(fmax), npts) summary_values = sspec_output.reference_values() @@ -44,7 +74,14 @@ def _summary_synth_spec(sspec_output, fmin, fmax): return freq, synth_model, synth_model_no_att, synth_model_no_fc -def _make_fig(config): +def _make_fig(): + """ + Create figure and axes for stacked spectra plot. + + :return: Figure and axes + :rtype: tuple of :class:`matplotlib.figure.Figure`, + :class:`matplotlib.axes.Axes` + """ matplotlib.rcParams['pdf.fonttype'] = 42 # to edit text in Illustrator figsize = (7, 7) dpi = ( @@ -59,6 +96,16 @@ def _make_fig(config): def _plot_fc_and_mw(sspec_output, ax, ax2): + """ + Plot corner frequency and moment magnitude on the axes. + + :param sspec_output: Output of spectral inversion + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param ax: Axes for stacked spectra plot + :type ax: :class:`matplotlib.axes.Axes` + :param ax2: Axes for magnitude plot + :type ax2: :class:`matplotlib.axes.Axes` + """ fc = sspec_output.reference_values()['fc'] fc_err_left, fc_err_right = sspec_output.reference_uncertainties()['fc'] fc_min = fc - fc_err_left @@ -75,6 +122,15 @@ def _plot_fc_and_mw(sspec_output, ax, ax2): def _make_ax2(ax): + """ + Create a second y-axis for magnitude units. + + :param ax: Axes for stacked spectra plot + :type ax: :class:`matplotlib.axes.Axes` + + :return: Axes for magnitude units + :rtype: :class:`matplotlib.axes.Axes` + """ ax2 = ax.twinx() # Move ax2 below ax ax.zorder = 2 @@ -90,6 +146,15 @@ def _make_ax2(ax): def _nspectra_text(spec_list): + """ + Return text for the number of spectra. + + :param spec_list: List of spectra + :type spec_list: list + + :return: Text for the number of spectra + :rtype: str + """ nspectra = len(spec_list) return ( 'Inverted spectrum' @@ -99,6 +164,14 @@ def _nspectra_text(spec_list): def _summary_params_text(sspec_output, ax): + """ + Add summary parameters to the plot. + + :param sspec_output: Output of spectral inversion + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param ax: Axes for stacked spectra plot + :type ax: :class:`matplotlib.axes.Axes` + """ summary_values = sspec_output.reference_values() summary_uncertainties = sspec_output.reference_uncertainties() Mo_value = summary_values['Mo'] @@ -141,7 +214,13 @@ def _summary_params_text(sspec_output, ax): path_effects=path_effects) -def _add_title(config, ax): +def _add_title(ax): + """ + Add event information as a title to the plot. + + :param ax: Axes for stacked spectra plot + :type ax: :class:`matplotlib.axes.Axes` + """ # Add event information as a title evid = config.event.event_id hypo = config.event.hypocenter @@ -160,8 +239,13 @@ def _add_title(config, ax): ha='left', va='top', transform=ax.transAxes) -def _add_code_author(config, ax): - # Add code and author information to the figure bottom +def _add_code_author(ax): + """ + Add code and author information to the figure bottom. + + :param ax: Axes for stacked spectra plot + :type ax: :class:`matplotlib.axes.Axes` + """ textstr = ( f'SourceSpec v{get_versions()["version"]} ' f'- {config.end_of_run.strftime("%Y-%m-%d %H:%M:%S")} ' @@ -188,7 +272,13 @@ def _add_code_author(config, ax): ha='right', va='top', transform=ax.transAxes) -def _savefig(config, fig): +def _savefig(fig): + """ + Save figure to file. + + :param fig: Figure to save + :type fig: :class:`matplotlib.figure.Figure` + """ evid = config.event.event_id figfile_base = os.path.join(config.options.outdir, evid) figfile_base += '.stacked_spectra.' @@ -214,9 +304,16 @@ def _truncate_colormap(cmap, minval=0.0, maxval=1.0, n=100): ) -def plot_stacked_spectra(config, spec_st, weight_st, sspec_output): +def plot_stacked_spectra(spec_st, weight_st, sspec_output): """ Plot stacked spectra, along with summary inverted spectrum. + + :param spec_st: Stream of spectra + :type spec_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param weight_st: Stream of spectral weights + :type weight_st: :class:`~sourcespec.spectrum.SpectrumStream` + :param sspec_output: Output of spectral inversion + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` """ # Check config, if we need to plot at all if not config.plot_show and not config.plot_save: @@ -229,7 +326,7 @@ def plot_stacked_spectra(config, spec_st, weight_st, sspec_output): and not spec.stats.ignore ] # plotting - fig, ax = _make_fig(config) + fig, ax = _make_fig() ax.set_xscale('log') ax.set_yscale('log') cmap = matplotlib.colors.LinearSegmentedColormap.from_list( @@ -324,6 +421,6 @@ def plot_stacked_spectra(config, spec_st, weight_st, sspec_output): ax2 = _make_ax2(ax) _plot_fc_and_mw(sspec_output, ax, ax2) _summary_params_text(sspec_output, ax) - _add_title(config, ax) - _add_code_author(config, ax) - _savefig(config, fig) + _add_title(ax) + _add_code_author(ax) + _savefig(fig) From 7d0fd3d430122f22d4095c46a9e9b045a85db0c1 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 09:11:39 +0200 Subject: [PATCH 25/73] Use the global `config` object in ssp_plot_params_stats.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_plot_params_stats.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 87dceb61..a0dd54f9 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -88,7 +88,7 @@ def main(): from .ssp_plot_stacked_spectra import plot_stacked_spectra plot_stacked_spectra(spec_st, weight_st, sspec_output) from .ssp_plot_params_stats import box_plots - box_plots(config, sspec_output) + box_plots(sspec_output) if config.plot_station_map: from .ssp_plot_stations import plot_stations plot_stations(config, sspec_output) diff --git a/sourcespec2/ssp_plot_params_stats.py b/sourcespec2/ssp_plot_params_stats.py index 6b819241..1e0fa4b5 100644 --- a/sourcespec2/ssp_plot_params_stats.py +++ b/sourcespec2/ssp_plot_params_stats.py @@ -16,7 +16,7 @@ import matplotlib import matplotlib.pyplot as plt import matplotlib.patheffects as mpe - +from .config import config from ._version import get_versions from .savefig import savefig logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -33,8 +33,13 @@ def __init__(self, name, unit, color): self.color = color -def box_plots(config, sspec_output): - """Show parameter statistics through box plots.""" +def box_plots(sspec_output): + """ + Show parameter statistics through box plots. + + :param sspec_output: SourceSpec output. + :type sspec_output: :class:`sourcespec.ssp_data_types.SourceSpecOutput` + """ # Check config, if we need to plot at all if not config.plot_show and not config.plot_save: return @@ -159,10 +164,16 @@ def box_plots(config, sspec_output): if config.plot_show: plt.show() if config.plot_save: - _savefig(config, fig) + _savefig(fig) + +def _savefig(fig): + """ + Save the figure to a file. -def _savefig(config, fig): + :param fig: Figure to save. + :type fig: :class:`matplotlib.figure.Figure` + """ evid = config.event.event_id figfile_base = os.path.join(config.options.outdir, f'{evid}.boxplot.') fmt = config.plot_save_format From fc07b448e57d69657240a1eaf56fca512b64c2bf Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 09:29:04 +0200 Subject: [PATCH 26/73] Use the global `config` object in ssp_plot_stations.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_plot_stations.py | 247 +++++++++++++++++++++++++++---- 2 files changed, 222 insertions(+), 27 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index a0dd54f9..50db06a0 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -91,7 +91,7 @@ def main(): box_plots(sspec_output) if config.plot_station_map: from .ssp_plot_stations import plot_stations - plot_stations(config, sspec_output) + plot_stations(sspec_output) if config.html_report: from .ssp_html_report import html_report diff --git a/sourcespec2/ssp_plot_stations.py b/sourcespec2/ssp_plot_stations.py index 34641ffe..b4dbc1b9 100644 --- a/sourcespec2/ssp_plot_stations.py +++ b/sourcespec2/ssp_plot_stations.py @@ -25,6 +25,7 @@ from matplotlib import colors import matplotlib.patheffects as PathEffects from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +from .config import config from .adjustText import adjust_text from .cached_tiler import CachedTiler from .map_tiles import ( @@ -132,6 +133,14 @@ def _shiftedColorMap(cmap, start=0, midpoint=0.5, stop=1.0, name='shifted'): def _get_circles_step(maxdist, ncircles): """ Return the step for the circles to be plotted. + + :param maxdist: maximum distance from the epicenter. + :type maxdist: float + :param ncircles: number of circles to plot. + :type ncircles: int + + :return: step for the circles. + :rtype: float """ step = _round_to_base(maxdist / ncircles, base=5) if step == 0: @@ -142,6 +151,21 @@ def _get_circles_step(maxdist, ncircles): def _plot_circles(ax, evlo, evla, distances): + """ + Plot circles around the epicenter. + + :param ax: axes to plot the circles. + :type ax: matplotlib.axes.Axes + :param evlo: epicenter longitude. + :type evlo: float + :param evla: epicenter latitude. + :type evla: float + :param distances: distances from the epicenter. + :type distances: list of float + + :return: list of texts with circle labels. + :rtype: list of matplotlib.text.Text + """ geodetic_transform = ccrs.PlateCarree() g = Geod(ellps='WGS84') texts = [] @@ -177,6 +201,14 @@ def _plot_circles(ax, evlo, evla, distances): def _plot_epicenter_as_beachball(ax, event): + """ + Plot the epicenter as a beachball. + + :param ax: axes to plot the beachball. + :type ax: matplotlib.axes.Axes + :param event: event object. + :type event: :class:`~sourcespec.ssp_event.SSPEvent` + """ geodetic_transform = ccrs.PlateCarree() fm = event.focal_mechanism # TODO: draw full moment tensor, if available @@ -214,6 +246,14 @@ def _plot_epicenter_as_beachball(ax, event): def _plot_epicenter_as_star(ax, event): + """ + Plot the epicenter as a star. + + :param ax: axes to plot the star. + :type ax: matplotlib.axes.Axes + :param event: event object. + :type event: :class:`~sourcespec.ssp_event.SSPEvent` + """ geodetic_transform = ccrs.PlateCarree() hypo = event.hypocenter evlo = hypo.longitude.value_in_deg @@ -227,7 +267,14 @@ def _plot_epicenter_as_star(ax, event): def _add_event_info(event, ax): - """Add event information as plot title.""" + """ + Add event information as plot title. + + :param event: event object. + :type event: :class:`~sourcespec.ssp_event.SSPEvent` + :param ax: axes to plot the event information. + :type ax: matplotlib.axes.Axes + """ evid = event.event_id hypo = event.hypocenter evlo = hypo.longitude.value_in_deg @@ -245,8 +292,17 @@ def _add_event_info(event, ax): ha='left', va='top', linespacing=1.5, transform=ax.transAxes) -def _add_tiles(config, ax, tiler, alpha=1): - """Add map tiles to basemap.""" +def _add_tiles(ax, tiler, alpha=1): + """ + Add map tiles to basemap. + + :param ax: axes to plot the tiles. + :type ax: matplotlib.axes.Axes + :param tiler: tiler object. + :type tiler: :class:`~sourcespec.cached_tiler.CachedTiler` + :param alpha: transparency of the tiles. + :type alpha: float + """ if config.plot_map_tiles_zoom_level: tile_zoom_level = config.plot_map_tiles_zoom_level else: @@ -269,8 +325,13 @@ def _add_tiles(config, ax, tiler, alpha=1): tile_zoom_level -= 1 -def _add_coastlines(config, ax): - """Add coastlines and borders to basemap.""" +def _add_coastlines(ax): + """ + Add coastlines and borders to basemap. + + :param ax: axes to plot the coastlines. + :type ax: matplotlib.axes.Axes + """ if config.plot_coastline_resolution == 'no_coastline': return # add coastlines from GSHHS @@ -311,6 +372,12 @@ def _add_gridlines_to_orthographic_axes(ax, bounding_box): """ Add gridlines to global orthographic GeoAxes. + :param ax: axes to plot the gridlines. + :type ax: matplotlib.axes.Axes + :param bounding_box: bounding box of the map. + :type bounding_box: list of float + + .. note:: We need to compute gridlines manually, since Cartopy has occasional bugs with gridlines in global projections. @@ -368,7 +435,7 @@ def _get_ax_projection(maxdiagonal, max_map_diagonal=100, tiler=None): return ccrs.Mercator() -def _make_geoaxes_planar(config, fig, buonding_box, maxdiagonal): +def _make_geoaxes_planar(fig, buonding_box, maxdiagonal): """ Create a GeoAxes with a planar projection and optionally add map tiles. @@ -405,7 +472,7 @@ def _make_geoaxes_planar(config, fig, buonding_box, maxdiagonal): if map_style in ['hillshade', 'hillshade_dark']: # add a sea mask to the hillshade map ax.add_feature(ocean_10m, zorder=ZORDER_TILES+1) - _add_tiles(config, ax, tiler) + _add_tiles(ax, tiler) if map_style in ['hillshade', 'hillshade_dark', 'ocean', 'satellite']: ax.attribution_text = 'Map powered by Esri and Natural Earth' elif map_style == 'stamen_terrain': @@ -421,6 +488,18 @@ def _make_geoaxes_orthographic(fig, evlo, evla, bounding_box): Create a GeoAxes with global Orthographic projection. The basemap is the Earth stock image from Cartopy. + + :param fig: figure to add the axes. + :type fig: matplotlib.figure.Figure + :param evlo: epicenter longitude. + :type evlo: float + :param evla: epicenter latitude. + :type evla: float + :param bounding_box: bounding box of the map. + :type bounding_box: list of float + + :return: axes with Orthographic projection. + :rtype: matplotlib.axes.Axes """ _projection = ccrs.Orthographic( central_longitude=evlo, central_latitude=evla) @@ -432,10 +511,19 @@ def _make_geoaxes_orthographic(fig, evlo, evla, bounding_box): return ax -def _make_basemap(config, maxdist): +def _make_basemap(maxdist): """ Create basemap with tiles, coastlines, hypocenter and distance circles. + + :param maxdist: maximum distance from the epicenter. + :type maxdist: float + + :return: ax0 (invisible axis for title and footer), + ax (GeoAxes with basemap), + circle_texts (list of texts with circle labels). + :rtype: tuple of matplotlib.axes.Axes, matplotlib.axes.Axes, + list of matplotlib.text.Text """ g = Geod(ellps='WGS84') event = config.event @@ -466,10 +554,10 @@ def _make_basemap(config, maxdist): ax0.set_axis_off() _add_event_info(config.event, ax0) if maxdist < 3000: # km - ax = _make_geoaxes_planar(config, fig, bounding_box, maxdiagonal) + ax = _make_geoaxes_planar(fig, bounding_box, maxdiagonal) else: ax = _make_geoaxes_orthographic(fig, evlo, evla, bounding_box) - _add_coastlines(config, ax) + _add_coastlines(ax) circles_distances = np.arange(1, ncircles + 1) * circles_step circle_texts = _plot_circles(ax, evlo, evla, circles_distances) try: @@ -479,9 +567,14 @@ def _make_basemap(config, maxdist): return ax0, ax, circle_texts -def _add_footer(config, ax, attribution_text=None): +def _add_footer(ax, attribution_text=None): """ Add code and author information at the figure footer. + + :param ax: axes to plot the footer. + :type ax: matplotlib.axes.Axes + :param attribution_text: additional attribution text. + :type attribution_text: str """ textstr = ( f'SourceSpec v{get_versions()["version"]} ' @@ -515,6 +608,15 @@ def _add_footer(config, ax, attribution_text=None): def _add_main_title(ax, vname, vmean, verr): """ Add to the figure the main title with the value and its error. + + :param ax: axes to plot the title. + :type ax: matplotlib.axes.Axes + :param vname: name of the value. + :type vname: str + :param vmean: mean value. + :type vmean: float + :param verr: error value. + :type verr: float or tuple of float """ verr_minus, verr_plus = _get_verr_minus_plus(verr) if vname == 'fc': @@ -539,6 +641,12 @@ def _contrast_color(color): Return the best contrasting color, either black or white. Source: https://stackoverflow.com/a/3943023/2021880 + + :param color: color in RGB format. + :type color: tuple of float + + :return: 'black' or 'white'. + :rtype: str """ R, G, B = [ c / 12.92 if c <= 0.03928 @@ -551,6 +659,12 @@ def _contrast_color(color): def _get_verr_minus_plus(verr): """ Return the minus and plus error values for the given error value. + + :param verr: error value. + :type verr: float or tuple of float + + :return: minus and plus error values. + :rtype: tuple of float """ if isinstance(verr, tuple): verr_minus, verr_plus = verr @@ -562,6 +676,20 @@ def _get_verr_minus_plus(verr): def _get_cmap_and_norm(values, outliers, vname, vmean, verr): """ Return the colormap and normalization for the given values. + + :param values: values to plot. + :type values: numpy.ndarray + :param outliers: boolean array with the outliers. + :type outliers: numpy.ndarray + :param vname: name of the value. + :type vname: str + :param vmean: mean value. + :type vmean: float + :param verr: error value. + :type verr: float or tuple of float + + :return: colormap, normalization and colorbar extension. + :rtype: matplotlib.colors.Colormap, matplotlib.colors.Normalize, str """ verr_minus, verr_plus = _get_verr_minus_plus(verr) values_no_outliers = values[~outliers] @@ -601,10 +729,25 @@ def _get_cmap_and_norm(values, outliers, vname, vmean, verr): return cmap, norm, cbar_extend -def _plot_stations_scatter( - config, ax, lonlat_dist, st_ids, values, cmap, norm): +def _plot_stations_scatter(ax, lonlat_dist, st_ids, values, cmap, norm): """ Plot the stations as scatter points on the map. + + :param ax: axes to plot the stations. + :type ax: matplotlib.axes.Axes + :param lonlat_dist: array with the station coordinates and distances. + :type lonlat_dist: numpy.ndarray + :param st_ids: list with the station IDs. + :type st_ids: list of str + :param values: values to plot. + :type values: numpy.ndarray + :param cmap: colormap. + :type cmap: matplotlib.colors.Colormap + :param norm: normalization. + :type norm: matplotlib.colors.Normalize + + :return: list of texts with station labels. + :rtype: list of matplotlib.text.Text """ trans = ccrs.PlateCarree() lonlat = lonlat_dist[:, :2] @@ -634,6 +777,13 @@ def _plot_stations_scatter( def _adjust_text_labels(station_texts, circle_texts, ax): """ Adjust the text labels so that they do not overlap. + + :param station_texts: list of texts with station labels. + :type station_texts: list of matplotlib.text.Text + :param circle_texts: list of texts with circle labels. + :type circle_texts: list of matplotlib.text.Text + :param ax: axes to plot the text labels. + :type ax: matplotlib.axes.Axes """ if not station_texts: return @@ -658,10 +808,24 @@ def _adjust_text_labels(station_texts, circle_texts, ax): t.set_position((x_pos, y_pos)) -def _add_colorbar( - ax, cmap, norm, vmean, verr, vname, cbar_extend): +def _add_colorbar(ax, cmap, norm, vmean, verr, vname, cbar_extend): """ Add a colorbar to the given axes. + + :param ax: axes to plot the colorbar. + :type ax: matplotlib.axes.Axes + :param cmap: colormap. + :type cmap: matplotlib.colors.Colormap + :param norm: normalization. + :type norm: matplotlib.colors.Normalize + :param vmean: mean value. + :type vmean: float + :param verr: error value. + :type verr: float or tuple of float + :param vname: name of the value. + :type vname: str + :param cbar_extend: colorbar extension. + :type cbar_extend: str """ ax_divider = make_axes_locatable(ax) cax = ax_divider.append_axes( @@ -688,9 +852,14 @@ def _add_colorbar( cax.set_ylabel(cm_label) -def _savefig(config, fig, vname): +def _savefig(fig, vname): """ Save the figure to a file. + + :param fig: figure to save. + :type fig: matplotlib.figure.Figure + :param vname: name of the value. + :type vname: str """ evid = config.event.event_id figfile_base = os.path.join(config.options.outdir, evid) @@ -711,24 +880,52 @@ def _savefig(config, fig, vname): def _make_station_map( - config, lonlat_dist, st_ids, values, outliers, vmean, verr, vname): + lonlat_dist, st_ids, values, outliers, vmean, verr, vname): """ Make a map of stations with the given values. + + :param lonlat_dist: array with the station coordinates and distances. + :type lonlat_dist: numpy.ndarray + :param st_ids: list with the station IDs. + :type st_ids: list of str + :param values: values to plot. + :type values: numpy.ndarray + :param outliers: boolean array with the outliers. + :type outliers: numpy.ndarray + :param vmean: mean value. + :type vmean: float + :param verr: error value. + :type verr: float or tuple of float + :param vname: name of the value. + :type vname: str """ maxdist = np.max(lonlat_dist[:, 2]) - ax0, ax, circle_texts = _make_basemap(config, maxdist) + ax0, ax, circle_texts = _make_basemap(maxdist) _add_main_title(ax0, vname, vmean, verr) cmap, norm, cbar_extend = _get_cmap_and_norm( values, outliers, vname, vmean, verr) station_texts = _plot_stations_scatter( - config, ax, lonlat_dist, st_ids, values, cmap, norm) + ax, lonlat_dist, st_ids, values, cmap, norm) _add_colorbar(ax, cmap, norm, vmean, verr, vname, cbar_extend) - _add_footer(config, ax0, ax.attribution_text) + _add_footer(ax0, ax.attribution_text) _adjust_text_labels(station_texts, circle_texts, ax) - _savefig(config, ax0.get_figure(), vname) + _savefig(ax0.get_figure(), vname) def _spread_overlapping_stations(lonlat_dist, min_dlonlat=1e-3, spread=0.03): + """ + Spread overlapping stations horizontally and vertically. + + :param lonlat_dist: array with the station coordinates and distances. + :type lonlat_dist: numpy.ndarray + :param min_dlonlat: minimum distance between stations. + :type min_dlonlat: float + :param spread: spread distance. + :type spread: float + + :return: array with the station coordinates and distances. + :rtype: numpy.ndarray + """ dlonlat = np.diff(lonlat_dist[:, :2], axis=0) dlonlat = np.sum(dlonlat**2, axis=1)**(0.5) # find indexes of overlapping stations @@ -745,12 +942,10 @@ def _spread_overlapping_stations(lonlat_dist, min_dlonlat=1e-3, spread=0.03): return lonlat_dist -def plot_stations(config, sspec_output): +def plot_stations(sspec_output): """ Plot station map, color coded by magnitude or fc. - :param config: Configuration object - :type config: config.Config :param sspec_output: SourceSpecOutput object :type sspec_output: ssp_data_types.SourceSpecOutput """ @@ -775,12 +970,12 @@ def plot_stations(config, sspec_output): summary_mag = summary_values['Mw'] summary_mag_err = summary_uncertainties['Mw'] _make_station_map( - config, lonlat_dist, st_ids, + lonlat_dist, st_ids, mag, mag_outliers, summary_mag, summary_mag_err, 'mag') fc = np.array([stationpar[k]['fc'].value for k in st_ids]) fc_outliers = np.array([stationpar[k]['fc'].outlier for k in st_ids]) summary_fc = summary_values['fc'] summary_fc_err = summary_uncertainties['fc'] _make_station_map( - config, lonlat_dist, st_ids, + lonlat_dist, st_ids, fc, fc_outliers, summary_fc, summary_fc_err, 'fc') From 652e3673b239c8fac3061dc41eb4622db8ceb4c9 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 09:48:46 +0200 Subject: [PATCH 27/73] Use the global `config` object in ssp_html_report.py --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_html_report.py | 321 ++++++++++++++++++++++++++------- 2 files changed, 258 insertions(+), 65 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 50db06a0..f46164bc 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -95,7 +95,7 @@ def main(): if config.html_report: from .ssp_html_report import html_report - html_report(config, sspec_output) + html_report(sspec_output) ssp_exit() diff --git a/sourcespec2/ssp_html_report.py b/sourcespec2/ssp_html_report.py index 72e6cd30..e60a6cde 100644 --- a/sourcespec2/ssp_html_report.py +++ b/sourcespec2/ssp_html_report.py @@ -16,6 +16,7 @@ import contextlib from urllib.parse import urlparse import numpy as np +from .config import config from ._version import get_versions from .ssp_data_types import SpectralParameter logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -26,13 +27,21 @@ def _multireplace(string, replacements, ignore_case=False): """ Given a string and a replacement map, it returns the replaced string. - :param str string: string to execute replacements on - :param dict replacements: replacement dictionary - {value to find: value to replace} - :param bool ignore_case: whether the match should be case insensitive + :param tring: string to execute replacements on + :type string: str + :param replacements: replacement dictionary + {value to find: value to replace} + :type replacements: dict + :param ignore_case: whether the match should be case insensitive + :type ignore_case: bool + + :return: replaced string :rtype: str - Source: https://gist.github.com/bgusach/a967e0587d6e01e889fd1d776c5f3729 + .. note:: + + Source: + https://gist.github.com/bgusach/a967e0587d6e01e889fd1d776c5f3729 """ if not replacements: # Edge case that'd produce a funny regex and cause a KeyError @@ -74,7 +83,13 @@ def normalize_old(s): lambda match: replacements[normalize_old(match.group(0))], string) -def _agency_logo_path(config): +def _agency_logo_path(): + """ + Return the path to the agency logo file. + + :return: path to the agency logo file + :rtype: str + """ agency_logo_path = config.agency_logo if agency_logo_path is None: return None @@ -94,8 +109,14 @@ def _agency_logo_path(config): return agency_logo_path -def _agency_logo(config): - agency_logo_path = _agency_logo_path(config) +def _agency_logo(): + """ + Return the HTML code for the agency logo. + + :return: HTML code for the agency logo + :rtype: str + """ + agency_logo_path = _agency_logo_path() if agency_logo_path is None: return '' agency_logo_img = f'' @@ -114,11 +135,23 @@ def _agency_logo(config): def _logo_file_url(): + """ + Return the URL of the SourceSpec logo file. + + :return: URL of the SourceSpec logo file + :rtype: str + """ cdn_baseurl = 'https://cdn.jsdelivr.net/gh/SeismicSource/sourcespec@1.6' return f'{cdn_baseurl}/imgs/SourceSpec_logo.svg' -def _version_and_run_completed(config): +def _version_and_run_completed(): + """ + Return the SourceSpec version and the run completion date. + + :return: SourceSpec version and run completion date + :rtype: tuple of str + """ ssp_version = get_versions()['version'] run_completed = ( f'{config.end_of_run.strftime("%Y-%m-%d %H:%M:%S")} ' @@ -127,7 +160,13 @@ def _version_and_run_completed(config): return ssp_version, run_completed -def _author_html(config): +def _author_html(): + """ + Return the HTML code for the author. + + :return: HTML code for the author + :rtype: str + """ author = '' if config.author_name is not None: author = config.author_name @@ -138,7 +177,13 @@ def _author_html(config): return author -def _agency_html(config): +def _agency_html(): + """ + Return the HTML code for the agency. + + :return: HTML code for the agency + :rtype: str + """ agency = '' if config.agency_full_name is not None: agency = config.agency_full_name @@ -154,6 +199,17 @@ def _agency_html(config): def _author_and_agency_html(author, agency): + """ + Return the HTML code for the author and the agency. + + :param author: HTML code for the author + :type author: str + :param agency: HTML code for the agency + :type agency: str + + :return: HTML code for the author and the agency + :rtype: str + """ if author != '': author = f'

{author}' if author == '' and agency != '': @@ -163,13 +219,19 @@ def _author_and_agency_html(author, agency): return author + agency -def _page_footer(config): +def _page_footer(): + """ + Return the HTML code for the page footer. + + :return: HTML code for the page footer + :rtype: str + """ footer_html = '' indent3 = 3 * ' ' indent4 = 4 * ' ' footer_html += f'{indent3}
\n' - author = _author_html(config) - agency = _agency_html(config) + author = _author_html() + agency = _agency_html() auth_agen_text = '' if author != '': if agency != '': @@ -186,7 +248,7 @@ def _page_footer(config): else f'{indent4}{run_completed}\n' ) footer_html += f'{indent3}
\n' - agency_logo_path = _agency_logo_path(config) + agency_logo_path = _agency_logo_path() if agency_logo_path is not None: if config.agency_url is not None: a_agency = f'' @@ -202,7 +264,17 @@ def _page_footer(config): def _format_exponent(value, reference): - """Format `value` to a string having the same exponent than `reference`.""" + """ + Format `value` to a string having the same exponent than `reference`. + + :param value: value to format + :type value: float + :param reference: reference value + :type reference: float + + :return: formatted value + :rtype: str + """ # get the exponent of reference value xp = np.int(np.floor(np.log10(np.abs(reference)))) # format value to print it with the same exponent of reference value @@ -211,7 +283,19 @@ def _format_exponent(value, reference): def _summary_value_and_err_text(value, error, fmt): - """Format summary value and error text.""" + """ + Format summary value and error text. + + :param value: value + :type value: float + :param error: error + :type error: float or tuple of float + :param fmt: format string + :type fmt: str + + :return: formatted text + :rtype: str + """ if error[0] == error[1]: text = f'{fmt}
±{fmt}' text = text.format(value, error[0]) @@ -222,7 +306,19 @@ def _summary_value_and_err_text(value, error, fmt): def _station_value_and_err_text(par, key, fmt): - """Format station value and error text.""" + """ + Format station value and error text. + + :param par: dictionary of station parameters + :type par: dict + :param key: parameter key + :type key: str + :param fmt: format string + :type fmt: str + + :return: formatted value and error text + :rtype: tuple of str + """ # par[key] can be None even if key is in par (e.g., if key is 'Ml' and # local magnitude is not computed) _par = par[key] if key in par else None @@ -253,6 +349,15 @@ def _station_value_and_err_text(par, key, fmt): def _misfit_table_rows(misfit_plot_files): + """ + Return the HTML code for the misfit table rows. + + :param misfit_plot_files: list of misfit plot files + :type misfit_plot_files: list of str + + :return: HTML code for the misfit table rows + :rtype: str + """ template_dir = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'html_report_template' @@ -281,7 +386,7 @@ def _misfit_table_rows(misfit_plot_files): return misfit_table_rows -def _misfit_page(config): +def _misfit_page(): """Generate an HTML page with misfit plots.""" # Read template files template_dir = os.path.join( @@ -295,11 +400,11 @@ def _misfit_page(config): logo_file = _logo_file_url() # Version and run completed - ssp_version, run_completed = _version_and_run_completed(config) + ssp_version, run_completed = _version_and_run_completed() # Author and agency - author = _author_html(config) - agency = _agency_html(config) + author = _author_html() + agency = _agency_html() author_and_agency = _author_and_agency_html(author, agency) # 1d conditional misfit plots @@ -337,20 +442,25 @@ def _misfit_page(config): fp.write(misfit) -def _add_run_info_to_html(config, replacements): - """Add run info to HTML report.""" +def _add_run_info_to_html(replacements): + """ + Add run info to HTML report. + + :param replacements: replacement dictionary + :type replacements: dict + """ ssp_url = 'https://sourcespec.seismicsource.org' logo_file = _logo_file_url() - agency_logo = _agency_logo(config) - ssp_version, run_completed = _version_and_run_completed(config) - author = _author_html(config) + agency_logo = _agency_logo() + ssp_version, run_completed = _version_and_run_completed() + author = _author_html() if not author: author_comment_begin = '' else: author_comment_begin = '' author_comment_end = '' - agency = _agency_html(config) + agency = _agency_html() if not agency: agency_comment_begin = '' @@ -358,7 +468,7 @@ def _add_run_info_to_html(config, replacements): agency_comment_begin = '' agency_comment_end = '' author_and_agency = _author_and_agency_html(author, agency) - page_footer = _page_footer(config) + page_footer = _page_footer() replacements.update({ '{AGENCY_LOGO}': agency_logo, '{LOGO_FILE}': logo_file, @@ -376,8 +486,13 @@ def _add_run_info_to_html(config, replacements): }) -def _add_event_info_to_html(config, replacements): - """Add event info to HTML report.""" +def _add_event_info_to_html(replacements): + """ + Add event info to HTML report. + + :param replacements: replacement dictionary + :type replacements: dict + """ evid = config.event.event_id evname = config.event.name hypo = config.event.hypocenter @@ -434,8 +549,13 @@ def _add_event_info_to_html(config, replacements): }) -def _add_maps_to_html(config, replacements): - """Add maps to HTML report.""" +def _add_maps_to_html(replacements): + """ + Add maps to HTML report. + + :param replacements: replacement dictionary + :type replacements: dict + """ try: station_maps = [ m for m in config.figures['station_maps'] @@ -472,8 +592,15 @@ def _add_maps_to_html(config, replacements): }) -def _add_traces_plots_to_html(config, templates, replacements): - """Add trace plots to HTML report.""" +def _add_traces_plots_to_html(templates, replacements): + """ + Add trace plots to HTML report. + + :param templates: template files + :type templates: :class:`HTMLTemplates` + :param replacements: replacement dictionary + :type replacements: dict + """ with open(templates.traces_plot_html, encoding='utf-8') as fp: traces_plot = fp.read() traces_plot_files = [ @@ -508,8 +635,15 @@ def _add_traces_plots_to_html(config, templates, replacements): }) -def _add_spectra_plots_to_html(config, templates, replacements): - """Add spectra plots to HTML report.""" +def _add_spectra_plots_to_html(templates, replacements): + """ + Add spectra plots to HTML report. + + :param templates: template files + :type templates: :class:`HTMLTemplates` + :param replacements: replacement dictionary + :type replacements: dict + """ with open(templates.spectra_plot_html, encoding='utf-8') as fp: spectra_plot = fp.read() spectra_plot_files = [ @@ -545,6 +679,14 @@ def _add_spectra_plots_to_html(config, templates, replacements): def _add_inversion_info_to_html(sspec_output, replacements): + """ + Add inversion info to HTML report. + + :param sspec_output: SourceSpec output + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param replacements: replacement dictionary + :type replacements: dict + """ # Inversion information inversion_algorithms = { 'TNC': 'Truncated Newton', @@ -596,8 +738,15 @@ def _add_inversion_info_to_html(sspec_output, replacements): }) -def _add_summary_spectral_params_to_html(config, sspec_output, replacements): - """Add summary spectral parameters to HTML report.""" +def _add_summary_spectral_params_to_html(sspec_output, replacements): + """ + Add summary spectral parameters to HTML report. + + :param sspec_output: SourceSpec output + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param replacements: replacement dictionary + :type replacements: dict + """ ref_stat = sspec_output.summary_spectral_parameters.reference_statistics col_mean_highlighted = col_wmean_highlighted = col_perc_highlighted = '' if ref_stat == 'mean': @@ -813,8 +962,13 @@ def _add_summary_spectral_params_to_html(config, sspec_output, replacements): }) -def _add_box_plots_to_html(config, replacements): - """Add box plots to HTML report.""" +def _add_box_plots_to_html(replacements): + """ + Add box plots to HTML report. + + :param replacements: replacement dictionary + :type replacements: dict + """ box_plots = '' with contextlib.suppress(KeyError, IndexError): box_plots = [ @@ -833,8 +987,13 @@ def _add_box_plots_to_html(config, replacements): }) -def _add_stacked_spectra_to_html(config, replacements): - """Add stacked spectra to HTML report.""" +def _add_stacked_spectra_to_html(replacements): + """ + Add stacked spectra to HTML report. + + :param replacements: replacement dictionary + :type replacements: dict + """ stacked_spectra = '' with contextlib.suppress(KeyError, IndexError): stacked_spectra = [ @@ -853,8 +1012,17 @@ def _add_stacked_spectra_to_html(config, replacements): }) -def _add_station_table_to_html(config, sspec_output, templates, replacements): - """Add station table to HTML report.""" +def _add_station_table_to_html(sspec_output, templates, replacements): + """ + Add station table to HTML report. + + :param sspec_output: SourceSpec output + :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` + :param templates: template files + :type templates: :class:`HTMLTemplates` + :param replacements: replacement dictionary + :type replacements: dict + """ with open(templates.station_table_row_html, encoding='utf-8') as fp: station_table_row = fp.read() station_table_rows = '' @@ -921,12 +1089,17 @@ def _add_station_table_to_html(config, sspec_output, templates, replacements): }) -def _add_misfit_plots_to_html(config, replacements): - """Add misfit plots to HTML report.""" +def _add_misfit_plots_to_html(replacements): + """ + Add misfit plots to HTML report. + + :param replacements: replacement dictionary + :type replacements: dict + """ if 'misfit_1d' in config.figures: misfit_plot_comment_begin = '' misfit_plot_comment_end = '' - _misfit_page(config) + _misfit_page() else: misfit_plot_comment_begin = '' @@ -936,8 +1109,15 @@ def _add_misfit_plots_to_html(config, replacements): }) -def _add_downloadable_files_to_html(config, templates, replacements): - """Add links to downloadable files to HTML report.""" +def _add_downloadable_files_to_html(templates, replacements): + """ + Add links to downloadable files to HTML report. + + :param templates: template files + :type templates: :class:`HTMLTemplates` + :param replacements: replacement dictionary + :type replacements: dict + """ # symlink to input files (not supported on Windows) input_files = '' if os.name == 'nt' else 'input_files' input_files_text = '' if os.name == 'nt'\ @@ -1014,7 +1194,15 @@ def __init__(self): def _cleanup_html(text): - """Remove unnecessary comments and whitespace from HTML.""" + """ + Remove unnecessary comments and whitespace from HTML. + + :param text: HTML text + :type text: str + + :returns: cleaned HTML text + :rtype: str + """ # remove HTML-style comments text = re.sub(r'', '', text, flags=re.DOTALL) # strip spaces at the end of lines @@ -1024,23 +1212,28 @@ def _cleanup_html(text): return text -def html_report(config, sspec_output): - """Generate an HTML report.""" +def html_report(sspec_output): + """ + Generate an HTML report. + + :param sspec_output: Output from the SourceSpec inversion. + :type sspec_output: :class:`sourcespec.ssp_data_types.SourceSpecOutput` + """ templates = HTMLtemplates() replacements = {} - _add_run_info_to_html(config, replacements) - _add_event_info_to_html(config, replacements) - _add_maps_to_html(config, replacements) - _add_traces_plots_to_html(config, templates, replacements) - _add_spectra_plots_to_html(config, templates, replacements) + _add_run_info_to_html(replacements) + _add_event_info_to_html(replacements) + _add_maps_to_html(replacements) + _add_traces_plots_to_html(templates, replacements) + _add_spectra_plots_to_html(templates, replacements) _add_inversion_info_to_html(sspec_output, replacements) - _add_summary_spectral_params_to_html(config, sspec_output, replacements) - _add_box_plots_to_html(config, replacements) - _add_stacked_spectra_to_html(config, replacements) - _add_station_table_to_html(config, sspec_output, templates, replacements) - _add_misfit_plots_to_html(config, replacements) - _add_downloadable_files_to_html(config, templates, replacements) + _add_summary_spectral_params_to_html(sspec_output, replacements) + _add_box_plots_to_html(replacements) + _add_stacked_spectra_to_html(replacements) + _add_station_table_to_html(sspec_output, templates, replacements) + _add_misfit_plots_to_html(replacements) + _add_downloadable_files_to_html(templates, replacements) with open(templates.index_html, encoding='utf-8') as fp: index = fp.read() From 61f24296e4799845fcd9289967e3f7f95ba44d58 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 10:14:32 +0200 Subject: [PATCH 28/73] Use the global `config` object in ssp_read_station_metadata.py --- sourcespec2/config/configure_cli.py | 10 ---------- sourcespec2/ssp_read_station_metadata.py | 10 ++++------ 2 files changed, 4 insertions(+), 16 deletions(-) diff --git a/sourcespec2/config/configure_cli.py b/sourcespec2/config/configure_cli.py index 861671eb..cb8783af 100644 --- a/sourcespec2/config/configure_cli.py +++ b/sourcespec2/config/configure_cli.py @@ -26,10 +26,6 @@ from .configobj.validate import Validator from ..ssp_update_db import update_db_file -# TODO: remove these when the global config object will be implemented -INSTR_CODES_VEL = [] -INSTR_CODES_ACC = [] - def _write_sample_config(configspec, progname): """ @@ -326,11 +322,5 @@ def configure_cli(options=None, progname='source_spec', config_overrides=None): except ImportError as err: sys.exit(err) - # TODO: remove these when the global config object will be implemented - global INSTR_CODES_VEL - global INSTR_CODES_ACC - INSTR_CODES_VEL = config.INSTR_CODES_VEL - INSTR_CODES_ACC = config.INSTR_CODES_ACC - _init_traceid_map() _init_plotting() diff --git a/sourcespec2/ssp_read_station_metadata.py b/sourcespec2/ssp_read_station_metadata.py index bc9c5a78..e489ad71 100644 --- a/sourcespec2/ssp_read_station_metadata.py +++ b/sourcespec2/ssp_read_station_metadata.py @@ -15,7 +15,7 @@ import logging from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response -from .config.configure_cli import INSTR_CODES_VEL, INSTR_CODES_ACC +from .config import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -32,7 +32,7 @@ class PAZ(): input_units = None linenum = None - def __init__(self, file=None, INST_CODES_VEL=None, INST_CODES_ACC=None): + def __init__(self, file=None): """ Init PAZ object. @@ -41,8 +41,6 @@ def __init__(self, file=None, INST_CODES_VEL=None, INST_CODES_ACC=None): """ if file is not None: self._read(file) - self.INSTR_CODES_VEL = INST_CODES_VEL - self.INSTR_CODES_ACC = INST_CODES_ACC def __str__(self): return ( @@ -79,13 +77,13 @@ def _guess_input_units(self): return instr_code = self.channel[1] self.input_units = None - if instr_code in INSTR_CODES_VEL: + if instr_code in config.INSTR_CODES_VEL: band_code = self.channel[0] # SEED standard band codes for velocity channels # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming if band_code in ['B', 'C', 'D', 'E', 'F', 'G', 'H', 'S']: self.input_units = 'M/S' - elif instr_code in INSTR_CODES_ACC: + elif instr_code in config.INSTR_CODES_ACC: self.input_units = 'M/S**2' def _read(self, file): From 56c33972da76966c0823f99bc9b0eaff99bac469 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 10 Jul 2024 10:19:45 +0200 Subject: [PATCH 29/73] Use the global `config` object in ssp_util.py --- sourcespec2/ssp_build_spectra.py | 2 +- sourcespec2/ssp_read_traces.py | 2 +- sourcespec2/ssp_util.py | 21 +++++++++------------ 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/sourcespec2/ssp_build_spectra.py b/sourcespec2/ssp_build_spectra.py index 2fb783be..93ab5627 100644 --- a/sourcespec2/ssp_build_spectra.py +++ b/sourcespec2/ssp_build_spectra.py @@ -409,7 +409,7 @@ def _displacement_to_moment(stats): lon = stats.coords.longitude lat = stats.coords.latitude depth = -stats.coords.elevation - medium_properties = MediumProperties(lon, lat, depth, config) + medium_properties = MediumProperties(lon, lat, depth) depth_string = medium_properties.to_string('station depth', depth) v_name = f'v{phase.lower()}' v_source = config.event.hypocenter[v_name] diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index 68d9e302..6a8d3621 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -310,7 +310,7 @@ def _hypo_vel(hypo): :type hypo: :class:`sourcespec.ssp_event.Hypocenter` """ medium_properties = MediumProperties( - hypo.longitude, hypo.latitude, hypo.depth.value_in_km, config) + hypo.longitude, hypo.latitude, hypo.depth.value_in_km) hypo.vp = medium_properties.get(mproperty='vp', where='source') hypo.vs = medium_properties.get(mproperty='vs', where='source') hypo.rho = medium_properties.get(mproperty='rho', where='source') diff --git a/sourcespec2/ssp_util.py b/sourcespec2/ssp_util.py index 9b2cf387..b1dc7e9a 100644 --- a/sourcespec2/ssp_util.py +++ b/sourcespec2/ssp_util.py @@ -17,6 +17,7 @@ from obspy.signal.invsim import cosine_taper as _cos_taper from obspy.geodetics import gps2dist_azimuth, kilometers2degrees from obspy.taup import TauPyModel +from .config import config model = TauPyModel(model='iasp91') v_model = model.model.s_mod.v_mod logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -64,15 +65,12 @@ class MediumProperties(): :type lat: float :param depth_in_km: Depth (km). :type depth_in_km: float - :param config: Configuration object. - :type config: :class:`~sourcespec.config.Config` """ - def __init__(self, lon, lat, depth_in_km, config): + def __init__(self, lon, lat, depth_in_km): self.lon = lon self.lat = lat self.depth_in_km = depth_in_km - self.config = config def get_from_config_param_source(self, mproperty): """ @@ -86,13 +84,13 @@ def get_from_config_param_source(self, mproperty): """ if mproperty not in ['vp', 'vs', 'rho']: raise ValueError(f'Invalid property: {mproperty}') - values = self.config[f'{mproperty}_source'] + values = config[f'{mproperty}_source'] if values is None: return None - if self.config.layer_top_depths is None: + if config.layer_top_depths is None: return values[0] values = np.array(values) - depths = np.array(self.config.layer_top_depths) + depths = np.array(config.layer_top_depths) try: # find the last value that is smaller than the source depth value = values[depths <= self.depth_in_km][-1] @@ -112,10 +110,9 @@ def get_from_config_param_station(self, mproperty): """ if mproperty not in ['vp', 'vs', 'rho']: raise ValueError(f'Invalid property: {mproperty}') - value = self.config[f'{mproperty}_stations'] + value = config[f'{mproperty}_stations'] if value is None: - value = self.get_from_config_param_source( - mproperty) + value = self.get_from_config_param_source(mproperty) return value def get_vel_from_NLL(self, wave): @@ -131,7 +128,7 @@ def get_vel_from_NLL(self, wave): # pylint: disable=import-outside-toplevel from nllgrid import NLLGrid grdfile = f'*.{wave}.mod.hdr' - grdfile = os.path.join(self.config.NLL_model_dir, grdfile) + grdfile = os.path.join(config.NLL_model_dir, grdfile) try: grdfile = glob(grdfile)[0] except IndexError as e: @@ -206,7 +203,7 @@ def get(self, mproperty, where): else: raise ValueError(f'Invalid location: {where}') if ( - self.config.NLL_model_dir is not None and + config.NLL_model_dir is not None and mproperty in ['vp', 'vs'] ): wave = 'P' if mproperty == 'vp' else 'S' From fb3dabea9c3222497f250836677fc23914e2bb5c Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Thu, 9 Jan 2025 20:20:55 +0100 Subject: [PATCH 30/73] Use the global `config` object in ssp_geom_spreading.py --- sourcespec2/ssp_geom_spreading.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sourcespec2/ssp_geom_spreading.py b/sourcespec2/ssp_geom_spreading.py index 642d2178..cbea8978 100644 --- a/sourcespec2/ssp_geom_spreading.py +++ b/sourcespec2/ssp_geom_spreading.py @@ -177,12 +177,11 @@ def geom_spread_teleseismic( :rtype: float """ # Don't need to specify coordinates, since we use a spherically symmetric - # Earth; don't need to specify a config object, since we use the global - # model (iasp91) + # Earth medium_properties_source = MediumProperties( - 0, 0, source_depth_in_km, None) + 0, 0, source_depth_in_km) medium_properties_station = MediumProperties( - 0, 0, station_depth_in_km, None) + 0, 0, station_depth_in_km) if phase == 'P': v_source = medium_properties_source.get_from_taup('vp') v_station = medium_properties_station.get_from_taup('vp') From 463f580bc268b24d96d77624aa067706c2a14427 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 10 Jul 2024 18:21:41 +0200 Subject: [PATCH 31/73] Moved code from move_outdir function to new get_outdir_path function. --- sourcespec2/ssp_setup.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index 420347bb..7e677d1e 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -55,8 +55,8 @@ def save_config(): os.rename(src, dst) -def move_outdir(): - """Move outdir to a new dir named from evid (and optional run_id).""" +def get_outdir_path(): + """Construct full path to output directory""" try: evid = config.event.event_id except Exception: @@ -64,12 +64,21 @@ def move_outdir(): src = config.options.outdir run_id = config.options.run_id run_id_subdir = config.options.run_id_subdir - dst = os.path.split(src)[0] - dst = os.path.join(dst, str(evid)) + # TODO: does next line also work if no tmpdir has been created first? + outdir = os.path.split(src)[0] + outdir = os.path.join(outdir, str(evid)) if run_id and run_id_subdir: - dst = os.path.join(dst, str(run_id)) + outdir = os.path.join(outdir, str(run_id)) elif run_id: - dst += f'_{run_id}' + outdir += f'_{run_id}' + + return outdir + + +def move_outdir(): + """Move outdir to a new dir named from evid (and optional run_id).""" + src = config.options.outdir + dst = get_outdir_path() # Create destination if not os.path.exists(dst): os.makedirs(dst) From 0bf963d29185b8ebd78eb3c8b51fc1fb9adc5483 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 10 Jul 2024 18:32:29 +0200 Subject: [PATCH 32/73] Removed arguments from _read_trace_files function, and moved code to new select_components and augment_traces functions. Moved code from read_traces function to new read_station_inventory, read_event_and_picks and augment_event functions. --- sourcespec2/ssp_read_traces.py | 163 ++++++++++++++++++++++++--------- 1 file changed, 120 insertions(+), 43 deletions(-) diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index 6a8d3621..03923a87 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -363,10 +363,10 @@ def _build_filelist(path, filelist, tmpdir): filelist.append(path) -def _read_trace_files(inventory, ssp_event, picks): +def _read_trace_files(): """ - Read trace files from a given path. Complete trace metadata and - return a stream object. + Read trace files from a given path and return a stream object. + Trace metadata are not yet updated. :param inventory: ObsPy Inventory object :type inventory: :class:`obspy.core.inventory.Inventory` @@ -392,9 +392,6 @@ def _read_trace_files(inventory, ssp_event, picks): fullpath = os.path.join(tmpdir, filename) _build_filelist(fullpath, filelist, None) # phase 2: build a stream object from the file list - orientation_codes = config.vertical_channel_codes +\ - config.horizontal_channel_codes_1 +\ - config.horizontal_channel_codes_2 st = Stream() for filename in sorted(filelist): try: @@ -403,31 +400,14 @@ def _read_trace_files(inventory, ssp_event, picks): logger.warning( f'{filename}: Unable to read file as a trace: skipping') continue + # TODO: optionally we could already call select_components here + #tmpst = select_components(tmpst) for trace in tmpst.traces: - orientation = trace.stats.channel[-1] - if orientation not in orientation_codes: - logger.warning( - f'{trace.id}: Unknown channel orientation: ' - f'"{orientation}": skipping trace' - ) - continue # only use the station specified by the command line option # "--station", if any if (config.options.station is not None and trace.stats.station != config.options.station): continue - _correct_traceid(trace) - try: - _add_instrtype(trace) - _add_inventory(trace, inventory) - _check_instrtype(trace) - _add_coords(trace) - _add_event(trace, ssp_event) - _add_picks(trace, picks) - except Exception as err: - for line in str(err).splitlines(): - logger.warning(line) - continue st.append(trace) shutil.rmtree(tmpdir) return st @@ -446,12 +426,53 @@ def _log_event_info(ssp_event): logger.info('---------------------------------------------------') -# Public interface: -def read_traces(): - """Read traces, store waveforms and metadata.""" - # read station metadata into an ObsPy ``Inventory`` object - inventory = read_station_metadata(config.station_metadata) +def select_components(st): + """ + Select requested components from stream + + :param st: ObsPy Stream object + :type st: :class:`obspy.core.stream.Stream` + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` + """ + orientation_codes = config.vertical_channel_codes +\ + config.horizontal_channel_codes_1 +\ + config.horizontal_channel_codes_2 + + tmpst = st.copy() + st = Stream() + for trace in tmpst.traces: + orientation = trace.stats.channel[-1] + if orientation not in orientation_codes: + logger.warning( + f'{trace.id}: Unknown channel orientation: ' + f'"{orientation}": skipping trace' + ) + continue + # TODO: should we also filter by station here? + # only use the station specified by the command line option + # "--station", if any + #if (config.options.station is not None and + # trace.stats.station != config.options.station): + # continue + st.append(trace) + + return st + + +def read_event_and_picks(trace1=None): + """ + Read event and phase picks + :param trace1: ObsPy Trace object containing event info (optional) + :type trace1: :class:`obspy.core.stream.Stream` + + :return: (ssp_event, picks) + :rtype: tuple of + :class:`sourcespec.ssp_event.SSPEvent`, + list of :class:`sourcespec.ssp_event.Pick` + """ picks = [] ssp_event = None # parse hypocenter file @@ -468,20 +489,10 @@ def read_traces(): if ssp_event is not None: _log_event_info(ssp_event) - # finally, read trace files - logger.info('Reading traces...') - st = _read_trace_files(inventory, ssp_event, picks) - logger.info('Reading traces: done') - logger.info('---------------------------------------------------') - if len(st) == 0: - logger.error('No trace loaded') - ssp_exit(1) - _complete_picks(st) - # if ssp_event is still None, get it from first trace - if ssp_event is None: + if ssp_event is None and trace1 is not None: try: - ssp_event = st[0].stats.event + ssp_event = trace1.stats.event _log_event_info(ssp_event) except AttributeError: logger.error('No hypocenter information found.') @@ -492,6 +503,21 @@ def read_traces(): '(if you use the SAC format).\n' ) ssp_exit(1) + # TODO: log also if trace1 is None? + + return (ssp_event, picks) + + +def augment_event(ssp_event): + """ + Add velocity info to hypocenter + and add event name from/to config.options + + The augmented event is stored in config.event + + :param ssp_event: Evento to be augmented + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + """ # add velocity info to hypocenter try: _hypo_vel(ssp_event.hypocenter) @@ -508,5 +534,56 @@ def read_traces(): # add event to config file config.event = ssp_event - st.sort() + +def augment_traces(st, inventory, ssp_event, picks): + """ + Add all required information to trace headers + + :param st: Traces to be augmented + :type st: :class:`obspy.core.stream.Stream` + :param inventory: Station metadata + :type inventory: :class:`obspy.core.inventory.Inventory` + :param ssp_event: Event information + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + :param picks: list of picks + :type picks: list of :class:`sourcespec.ssp_event.Pick` + """ + for trace in st: + _correct_traceid(trace) + try: + _add_instrtype(trace) + _add_inventory(trace, inventory) + _check_instrtype(trace) + _add_coords(trace) + _add_event(trace, ssp_event) + _add_picks(trace, picks) + except Exception as err: + for line in str(err).splitlines(): + logger.warning(line) + continue + + _complete_picks(st) + + +def read_station_inventory(): + """read station metadata into an ObsPy ``Inventory`` object""" + inventory = read_station_metadata(config.station_metadata) + return inventory + + +# Public interface: +def read_traces(): + """ + Read trace files + + :return: Traces + :rtype: :class:`obspy.core.stream.Stream` + """ + logger.info('Reading traces...') + st = _read_trace_files() + logger.info('Reading traces: done') + logger.info('---------------------------------------------------') + if len(st) == 0: + logger.error('No trace loaded') + ssp_exit(1) return st From 34d2388307148684f2ea032eaa63df0d8c2856a7 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 10 Jul 2024 18:44:44 +0200 Subject: [PATCH 33/73] Moved code from main function to new ssp_run and ssp_output functions. --- sourcespec2/source_spec.py | 147 ++++++++++++++++++++++++++++++------- 1 file changed, 120 insertions(+), 27 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index f46164bc..5a1afb7d 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -17,31 +17,58 @@ """ -def main(): - """Main routine for source_spec.""" +def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): + """ + Run source_spec as function with collected traces, station inventory, + event and picks + + :param st: Traces to be processed + :type st: :class:`obspy.core.stream.Stream` + :param inventory: Station metadata + :type inventory: :class:`obspy.core.inventory.Inventory` + :param ssp_event: Event information + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + :param picks: List of picks + :type picks: list of :class:`sourcespec.ssp_event.Pick` + :param allow_exit: whether to allow hard exit (without returning) + :type allow_exit: bool + + :return: (proc_st, spec_st, specnoise_st, weight_st, sspec_output) + :rtype: tuple of + :class:`obspy.core.stream.Stream`, + :class:`sourcespec.spectrum.SpectrumStream`, + :class:`sourcespec.spectrum.SpectrumStream`, + :class:`sourcespec.spectrum.SpectrumStream`, + :class:`sourcespec.ssp_data_types.SourceSpecOutput` + """ # pylint: disable=import-outside-toplevel # Lazy-import modules for speed - from .ssp_parse_arguments import parse_args - options = parse_args(progname='source_spec') - - # Setup stage - from .config import config, configure_cli - configure_cli(options, progname='source_spec') - from .ssp_setup import ( - move_outdir, remove_old_outdir, setup_logging, - save_config, ssp_exit) - setup_logging() - - from .ssp_read_traces import read_traces - st = read_traces() - - # Now that we have an evid, we can rename the outdir and the log file - move_outdir() - setup_logging(config.event.event_id) - remove_old_outdir() - - # Save config to out dir - save_config() + import os + from .config import config + + # Avoid hard exit when ssp_exit is called somewhere + if not allow_exit: + from . import ssp_setup + ssp_setup.SSP_EXIT_CALLED = True + + # Create output folder if required, save config and setup logging + from .ssp_setup import (get_outdir_path, save_config, setup_logging) + if config.options.outdir: + outdir = get_outdir_path() + if not os.path.exists(outdir): + os.makedirs(outdir) + # Setup logging + # (it is assumed this is already done if outdir exists) + setup_logging(config.event.event_id) + # Save config to out dir + save_config() + + # Preprocessing + from .ssp_read_traces import (augment_event, augment_traces, + select_components) + augment_event(ssp_event) + st = select_components(st) + augment_traces(st, inventory, ssp_event, picks) # Deconvolve, filter, cut traces: from .ssp_process_traces import process_traces @@ -51,10 +78,6 @@ def main(): from .ssp_build_spectra import build_spectra spec_st, specnoise_st, weight_st = build_spectra(proc_st) - from .ssp_plot_traces import plot_traces - plot_traces(st, suffix='raw') - plot_traces(proc_st) - # Spectral inversion from .ssp_inversion import spectral_inversion sspec_output = spectral_inversion(spec_st, weight_st) @@ -72,6 +95,36 @@ def main(): from .ssp_summary_statistics import compute_summary_statistics compute_summary_statistics(sspec_output) + return (proc_st, spec_st, specnoise_st, weight_st, sspec_output) + + +def ssp_output(st, proc_st, spec_st, specnoise_st, weight_st, sspec_output): + """ + Function writing all output to disk. + + If no output directory is specified, no output is written. + + :param st: Original traces + :type st: :class:`obspy.core.stream.Stream` + :param proc_st: Processed traces + :type proc_st: :class:`obspy.core.stream.Stream` + :param spec_st: Signal spectra + :type spec_st: :class:`sourcespec.spectrum.SpectrumStream` + :param specnoise_st: Noise spectra + :type specnoise_st: :class:`sourcespec.spectrum.SpectrumStream` + :param weight_st: Spectral weights + :type weight_st: :class:`sourcespec.spectrum.SpectrumStream` + :param sspec_output: SourceSpec output + :type sspec_output: :class:`sourcespec.ssp_data_types.SourceSpecOutput` + """ + # pylint: disable=import-outside-toplevel + # Lazy-import modules for speed + from .config import config + + # TODO: check (again) if outdir exists + if not config.options.outdir: + return + # Save output from .ssp_output import write_output, save_spectra write_output(sspec_output) @@ -82,6 +135,9 @@ def main(): spectral_residuals(spec_st, sspec_output) # Plotting + from .ssp_plot_traces import plot_traces + plot_traces(st, suffix='raw') + plot_traces(proc_st) from .ssp_plot_spectra import plot_spectra plot_spectra(spec_st, specnoise_st, plot_type='regular') plot_spectra(weight_st, plot_type='weight') @@ -97,6 +153,43 @@ def main(): from .ssp_html_report import html_report html_report(sspec_output) + +def main(): + """Main routine for source_spec.""" + # pylint: disable=import-outside-toplevel + # Lazy-import modules for speed + from .ssp_parse_arguments import parse_args + options = parse_args(progname='source_spec') + + # Setup stage + from .config import config, configure_cli + configure_cli(options, progname='source_spec') + from .ssp_setup import ( + move_outdir, remove_old_outdir, setup_logging, + ssp_exit) + setup_logging() + + # Read all required information from disk + from .ssp_read_traces import (read_traces, read_station_inventory, + read_event_and_picks) + st = read_traces() + trace1 = st[0] if len(st) else None + st.sort() + inventory = read_station_inventory() + ssp_event, picks = read_event_and_picks(trace1) + + # Now that we have an evid, we can rename the outdir and the log file + move_outdir() + setup_logging(config.event.event_id) + remove_old_outdir() + + # Run sourcespec function + result = ssp_run(st, inventory, ssp_event, picks, allow_exit=True) + (proc_st, spec_st, specnoise_st, weight_st, sspec_output) = result + + # Generate output + ssp_output(st, proc_st, spec_st, specnoise_st, weight_st, sspec_output) + ssp_exit() From 1bbed2261962c9d2ec3d178a730f20c9800b9f8f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 15 Jul 2024 13:08:48 +0200 Subject: [PATCH 34/73] Explicitly pass `eventid` to setup functions Since the `event` object might not be yet in the `config` object. --- sourcespec2/source_spec.py | 12 ++++++------ sourcespec2/ssp_setup.py | 31 ++++++++++++++++++++----------- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 5a1afb7d..561c771a 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -54,14 +54,14 @@ def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): # Create output folder if required, save config and setup logging from .ssp_setup import (get_outdir_path, save_config, setup_logging) if config.options.outdir: - outdir = get_outdir_path() + outdir = get_outdir_path(ssp_event.event_id) if not os.path.exists(outdir): os.makedirs(outdir) # Setup logging # (it is assumed this is already done if outdir exists) - setup_logging(config.event.event_id) + setup_logging(ssp_event.event_id) # Save config to out dir - save_config() + save_config(ssp_event.event_id) # Preprocessing from .ssp_read_traces import (augment_event, augment_traces, @@ -162,7 +162,7 @@ def main(): options = parse_args(progname='source_spec') # Setup stage - from .config import config, configure_cli + from .config import configure_cli configure_cli(options, progname='source_spec') from .ssp_setup import ( move_outdir, remove_old_outdir, setup_logging, @@ -179,8 +179,8 @@ def main(): ssp_event, picks = read_event_and_picks(trace1) # Now that we have an evid, we can rename the outdir and the log file - move_outdir() - setup_logging(config.event.event_id) + move_outdir(ssp_event.event_id) + setup_logging(ssp_event.event_id) remove_old_outdir() # Run sourcespec function diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/ssp_setup.py index 7e677d1e..c0f51f27 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/ssp_setup.py @@ -43,30 +43,39 @@ SSP_EXIT_CALLED = False -def save_config(): +def save_config(eventid=None): """Save config file to output dir.""" + if eventid is None: + try: + eventid = config.event.event_id + except AttributeError as err: + raise RuntimeError( + 'No event ID found. Cannot save config file.' + ) from err # Actually, it renames the file already existing. src = os.path.join(config.options.outdir, 'source_spec.conf') - evid = config.event.event_id - dst = os.path.join(config.options.outdir, f'{evid}.ssp.conf') + dst = os.path.join(config.options.outdir, f'{eventid}.ssp.conf') # On Windows, dst file must not exist with contextlib.suppress(Exception): os.remove(dst) os.rename(src, dst) -def get_outdir_path(): +def get_outdir_path(eventid=None): """Construct full path to output directory""" - try: - evid = config.event.event_id - except Exception: - return + if eventid is None: + try: + eventid = config.event.event_id + except AttributeError as err: + raise RuntimeError( + 'No event ID found. Cannot create output directory.' + ) from err src = config.options.outdir run_id = config.options.run_id run_id_subdir = config.options.run_id_subdir # TODO: does next line also work if no tmpdir has been created first? outdir = os.path.split(src)[0] - outdir = os.path.join(outdir, str(evid)) + outdir = os.path.join(outdir, str(eventid)) if run_id and run_id_subdir: outdir = os.path.join(outdir, str(run_id)) elif run_id: @@ -75,10 +84,10 @@ def get_outdir_path(): return outdir -def move_outdir(): +def move_outdir(eventid=None): """Move outdir to a new dir named from evid (and optional run_id).""" src = config.options.outdir - dst = get_outdir_path() + dst = get_outdir_path(eventid) # Create destination if not os.path.exists(dst): os.makedirs(dst) From c2ba6354a3aa3575316b0856be367bd3aa0d374d Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 15 Jul 2024 16:15:24 +0200 Subject: [PATCH 35/73] Add TRACEID_MAP to config object. Take care of 'None' values in lists. --- sourcespec2/config/config.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sourcespec2/config/config.py b/sourcespec2/config/config.py index 739bfb2a..58a7a4df 100644 --- a/sourcespec2/config/config.py +++ b/sourcespec2/config/config.py @@ -80,11 +80,13 @@ class _Config(dict): Import the global config object instead. """ def __init__(self): - # Additional config values. Tey must be defined using the dict syntax. + # Additional config values that must exist for the code to run without + # errors. Tey must be defined using the dict syntax. self['running_from_command_line'] = False self['vertical_channel_codes'] = ['Z'] self['horizontal_channel_codes_1'] = ['N', 'R'] self['horizontal_channel_codes_2'] = ['E', 'T'] + self['TRACEID_MAP'] = None # Empty options object, for compatibility with the command line version self['options'] = types.SimpleNamespace() # A list of warnings to be issued when logger is set up @@ -131,6 +133,8 @@ def update(self, other): for key, value in self.items(): if value == 'None': self[key] = None + if isinstance(value, list): + self[key] = [None if val == 'None' else val for val in value] # Make sure that self['figures'] is still a defaultdict self['figures'] = defaultdict(list, self['figures']) self._update_channel_codes() From 7a1f8b55445146d9da7d7415e82a58c4bb911e4d Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 15 Jul 2024 18:22:34 +0200 Subject: [PATCH 36/73] Make `select_components()` an in-place operation Also, move the trace skipping logic which was previously in `ssp_process_traces.py` to `select_components()` --- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_process_traces.py | 51 ------------------- sourcespec2/ssp_read_traces.py | 82 ++++++++++++++++++++++++------- 3 files changed, 65 insertions(+), 70 deletions(-) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 561c771a..10cd853c 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -67,7 +67,7 @@ def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): from .ssp_read_traces import (augment_event, augment_traces, select_components) augment_event(ssp_event) - st = select_components(st) + select_components(st) augment_traces(st, inventory, ssp_event, picks) # Deconvolve, filter, cut traces: diff --git a/sourcespec2/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py index f1f9d472..eb441092 100644 --- a/sourcespec2/ssp_process_traces.py +++ b/sourcespec2/ssp_process_traces.py @@ -15,7 +15,6 @@ (http://www.cecill.info/licences.en.html) """ import logging -import re import numpy as np from scipy.signal import savgol_filter from obspy.core import Stream @@ -574,53 +573,6 @@ def _merge_stream(st): return st[0] -def _skip_ignored(st): - """ - Skip traces ignored from config. - - :param st: ObsPy Stream object. - :type st: :class:`obspy.core.stream.Stream` - - :raises: RuntimeError if traces are ignored from config file. - """ - traceid = st[0].id - network, station, location, channel = traceid.split('.') - # build a list of all possible ids, from station only - # to full net.sta.loc.chan - ss = [ - station, - '.'.join((network, station)), - '.'.join((network, station, location)), - '.'.join((network, station, location, channel)), - ] - if config.use_traceids is not None: - # - combine all ignore_traceids in a single regex - # - escape the dots, otherwise they are interpreted as any character - # - add a dot before the first asterisk, to avoid a pattern error - combined = ( - "(" + ")|(".join(config.use_traceids) + ")" - ).replace('.', r'\.').replace('(*', '(.*') - if not any(re.match(combined, s) for s in ss): - _skip_stream_and_raise( - st, - reason='ignored from config file', - short_reason='ignored from config' - ) - if config.ignore_traceids is not None: - # - combine all ignore_traceids in a single regex - # - escape the dots, otherwise they are interpreted as any character - # - add a dot before the first asterisk, to avoid a pattern error - combined = ( - "(" + ")|(".join(config.ignore_traceids) + ")" - ).replace('.', r'\.').replace('(*', '(.*') - if any(re.match(combined, s) for s in ss): - _skip_stream_and_raise( - st, - reason='ignored from config file', - short_reason='ignored from config' - ) - - def process_traces(st): """ Remove mean, deconvolve and ignore unwanted components. @@ -643,9 +595,6 @@ def process_traces(st): _check_epicentral_distance(_trace) _add_arrivals( _trace) _define_signal_and_noise_windows(_trace) - # We skip traces ignored from config file here, so that we have - # the metadata needed for the raw plot - _skip_ignored(st_sel) _check_signal_window(st_sel) trace = _merge_stream(st_sel) trace.stats.ignore = False diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index 03923a87..8c8b7642 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -17,6 +17,7 @@ """ import sys import os +import re import logging import shutil import tarfile @@ -426,6 +427,53 @@ def _log_event_info(ssp_event): logger.info('---------------------------------------------------') +def _skip_traces_from_config(traceid): + """ + Skip traces with unknown channel orientation or ignored from config file. + + :param traceid: Trace ID. + :type traceid: str + + :raises: RuntimeError if traceid is ignored from config file. + """ + network, station, location, channel = traceid.split('.') + orientation_codes = config.vertical_channel_codes +\ + config.horizontal_channel_codes_1 +\ + config.horizontal_channel_codes_2 + orientation = channel[-1] + if orientation not in orientation_codes: + raise RuntimeError( + f'{traceid}: Unknown channel orientation: ' + f'"{orientation}": skipping trace' + ) + # build a list of all possible ids, from station only + # to full net.sta.loc.chan + ss = [ + station, + '.'.join((network, station)), + '.'.join((network, station, location)), + '.'.join((network, station, location, channel)), + ] + if config.use_traceids is not None: + # - combine all ignore_traceids in a single regex + # - escape the dots, otherwise they are interpreted as any character + # - add a dot before the first asterisk, to avoid a pattern error + combined = ( + "(" + ")|(".join(config.use_traceids) + ")" + ).replace('.', r'\.').replace('(*', '(.*') + if not any(re.match(combined, s) for s in ss): + raise RuntimeError(f'{traceid}: ignored from config file') + if config.ignore_traceids is not None: + # - combine all ignore_traceids in a single regex + # - escape the dots, otherwise they are interpreted as any character + # - add a dot before the first asterisk, to avoid a pattern error + combined = ( + "(" + ")|(".join(config.ignore_traceids) + ")" + ).replace('.', r'\.').replace('(*', '(.*') + if any(re.match(combined, s) for s in ss): + raise RuntimeError(f'{traceid}: ignored from config file') + + def select_components(st): """ Select requested components from stream @@ -436,19 +484,12 @@ def select_components(st): :return: ObsPy Stream object :rtype: :class:`obspy.core.stream.Stream` """ - orientation_codes = config.vertical_channel_codes +\ - config.horizontal_channel_codes_1 +\ - config.horizontal_channel_codes_2 - - tmpst = st.copy() - st = Stream() - for trace in tmpst.traces: - orientation = trace.stats.channel[-1] - if orientation not in orientation_codes: - logger.warning( - f'{trace.id}: Unknown channel orientation: ' - f'"{orientation}": skipping trace' - ) + traces_to_keep = [] + for trace in st: + try: + _skip_traces_from_config(trace.id) + except RuntimeError as e: + logger.warning(str(e)) continue # TODO: should we also filter by station here? # only use the station specified by the command line option @@ -456,9 +497,9 @@ def select_components(st): #if (config.options.station is not None and # trace.stats.station != config.options.station): # continue - st.append(trace) - - return st + traces_to_keep.append(trace) + # in-place update of st + st.traces[:] = traces_to_keep[:] def read_event_and_picks(trace1=None): @@ -537,7 +578,8 @@ def augment_event(ssp_event): def augment_traces(st, inventory, ssp_event, picks): """ - Add all required information to trace headers + Add all required information to trace headers. + Remove problematic traces. :param st: Traces to be augmented :type st: :class:`obspy.core.stream.Stream` @@ -548,6 +590,7 @@ def augment_traces(st, inventory, ssp_event, picks): :param picks: list of picks :type picks: list of :class:`sourcespec.ssp_event.Pick` """ + traces_to_keep = [] for trace in st: _correct_traceid(trace) try: @@ -557,11 +600,14 @@ def augment_traces(st, inventory, ssp_event, picks): _add_coords(trace) _add_event(trace, ssp_event) _add_picks(trace, picks) + trace.stats.ignore = False except Exception as err: for line in str(err).splitlines(): logger.warning(line) continue - + traces_to_keep.append(trace) + # in-place update of st + st.traces[:] = traces_to_keep[:] _complete_picks(st) From cefd44aaae02e9f8de26f5a58b4bb0ee2f6750ad Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 15 Jul 2024 17:25:49 +0200 Subject: [PATCH 37/73] Rename `config` module to `setup`. Split `ssp_setup.py` into separate modules. `ssp_setup.py` split to: - `logging.py`: logging setup - `exit.py`: exit function - `outdir.py`: functions to handle output directory --- pyproject.toml | 2 +- sourcespec2/{config => setup}/__init__.py | 6 +- sourcespec2/{config => setup}/config.py | 0 .../{config => setup}/configobj/LICENSE | 0 .../{config => setup}/configobj/__init__.py | 0 .../{config => setup}/configobj/_version.py | 0 .../{config => setup}/configobj/validate.py | 0 .../{config => setup}/configobj_helpers.py | 0 sourcespec2/{config => setup}/configspec.conf | 0 .../{config => setup}/configure_cli.py | 12 ++ sourcespec2/setup/exit.py | 49 ++++++ .../{config => setup}/library_versions.py | 0 .../{ssp_setup.py => setup/logging.py} | 155 +++--------------- .../{config => setup}/mandatory_deprecated.py | 0 sourcespec2/setup/outdir.py | 90 ++++++++++ sourcespec2/{config => setup}/ssp_event.yaml | 0 sourcespec2/source_model.py | 3 +- sourcespec2/source_spec.py | 17 +- sourcespec2/ssp_build_spectra.py | 3 +- sourcespec2/ssp_correction.py | 3 +- sourcespec2/ssp_data_types.py | 2 +- sourcespec2/ssp_grid_sampling.py | 2 +- sourcespec2/ssp_html_report.py | 2 +- sourcespec2/ssp_inversion.py | 2 +- sourcespec2/ssp_local_magnitude.py | 2 +- sourcespec2/ssp_output.py | 2 +- sourcespec2/ssp_plot_params_stats.py | 2 +- sourcespec2/ssp_plot_spectra.py | 2 +- sourcespec2/ssp_plot_stacked_spectra.py | 2 +- sourcespec2/ssp_plot_stations.py | 2 +- sourcespec2/ssp_plot_traces.py | 2 +- sourcespec2/ssp_process_traces.py | 3 +- sourcespec2/ssp_qml_output.py | 2 +- sourcespec2/ssp_radiated_energy.py | 2 +- sourcespec2/ssp_radiation_pattern.py | 2 +- sourcespec2/ssp_read_event_metadata.py | 3 +- sourcespec2/ssp_read_sac_header.py | 3 +- sourcespec2/ssp_read_station_metadata.py | 2 +- sourcespec2/ssp_read_traces.py | 6 +- sourcespec2/ssp_residuals.py | 2 +- sourcespec2/ssp_sqlite_output.py | 3 +- sourcespec2/ssp_summary_statistics.py | 3 +- sourcespec2/ssp_util.py | 2 +- sourcespec2/ssp_wave_arrival.py | 2 +- 44 files changed, 217 insertions(+), 180 deletions(-) rename sourcespec2/{config => setup}/__init__.py (67%) rename sourcespec2/{config => setup}/config.py (100%) rename sourcespec2/{config => setup}/configobj/LICENSE (100%) rename sourcespec2/{config => setup}/configobj/__init__.py (100%) rename sourcespec2/{config => setup}/configobj/_version.py (100%) rename sourcespec2/{config => setup}/configobj/validate.py (100%) rename sourcespec2/{config => setup}/configobj_helpers.py (100%) rename sourcespec2/{config => setup}/configspec.conf (100%) rename sourcespec2/{config => setup}/configure_cli.py (96%) create mode 100644 sourcespec2/setup/exit.py rename sourcespec2/{config => setup}/library_versions.py (100%) rename sourcespec2/{ssp_setup.py => setup/logging.py} (59%) rename sourcespec2/{config => setup}/mandatory_deprecated.py (100%) create mode 100644 sourcespec2/setup/outdir.py rename sourcespec2/{config => setup}/ssp_event.yaml (100%) diff --git a/pyproject.toml b/pyproject.toml index 0428d2ab..a7d43297 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ include = ["sourcespec2", "sourcespec2.*"] [tool.setuptools.package-data] "*" = ["LICENSE"] -"sourcespec2.config" = ["*.yaml", "*.conf"] +"sourcespec2.setup" = ["*.yaml", "*.conf"] "sourcespec2.html_report_template" = ["*.html", "*.css"] [tool.setuptools.dynamic] diff --git a/sourcespec2/config/__init__.py b/sourcespec2/setup/__init__.py similarity index 67% rename from sourcespec2/config/__init__.py rename to sourcespec2/setup/__init__.py index 8023baab..3a29dc59 100644 --- a/sourcespec2/config/__init__.py +++ b/sourcespec2/setup/__init__.py @@ -11,4 +11,8 @@ """ from .config import config # noqa from .configure_cli import configure_cli # noqa -from .library_versions import library_versions # noqa +from .logging import setup_logging # noqa +from .exit import ssp_exit # noqa +from .outdir import ( # noqa + get_outdir_path, save_config, move_outdir, remove_old_outdir +) diff --git a/sourcespec2/config/config.py b/sourcespec2/setup/config.py similarity index 100% rename from sourcespec2/config/config.py rename to sourcespec2/setup/config.py diff --git a/sourcespec2/config/configobj/LICENSE b/sourcespec2/setup/configobj/LICENSE similarity index 100% rename from sourcespec2/config/configobj/LICENSE rename to sourcespec2/setup/configobj/LICENSE diff --git a/sourcespec2/config/configobj/__init__.py b/sourcespec2/setup/configobj/__init__.py similarity index 100% rename from sourcespec2/config/configobj/__init__.py rename to sourcespec2/setup/configobj/__init__.py diff --git a/sourcespec2/config/configobj/_version.py b/sourcespec2/setup/configobj/_version.py similarity index 100% rename from sourcespec2/config/configobj/_version.py rename to sourcespec2/setup/configobj/_version.py diff --git a/sourcespec2/config/configobj/validate.py b/sourcespec2/setup/configobj/validate.py similarity index 100% rename from sourcespec2/config/configobj/validate.py rename to sourcespec2/setup/configobj/validate.py diff --git a/sourcespec2/config/configobj_helpers.py b/sourcespec2/setup/configobj_helpers.py similarity index 100% rename from sourcespec2/config/configobj_helpers.py rename to sourcespec2/setup/configobj_helpers.py diff --git a/sourcespec2/config/configspec.conf b/sourcespec2/setup/configspec.conf similarity index 100% rename from sourcespec2/config/configspec.conf rename to sourcespec2/setup/configspec.conf diff --git a/sourcespec2/config/configure_cli.py b/sourcespec2/setup/configure_cli.py similarity index 96% rename from sourcespec2/config/configure_cli.py rename to sourcespec2/setup/configure_cli.py index cb8783af..c6850c5d 100644 --- a/sourcespec2/config/configure_cli.py +++ b/sourcespec2/setup/configure_cli.py @@ -26,6 +26,18 @@ from .configobj.validate import Validator from ..ssp_update_db import update_db_file +# define ipshell(), if possible +# note: ANSI colors do not work on Windows standard terminal +if sys.stdout.isatty() and sys.platform != 'win32': + try: + from IPython.terminal.embed import InteractiveShellEmbed + from traitlets.config import MultipleInstanceError + IPSHELL = InteractiveShellEmbed.instance() + except (ImportError, MultipleInstanceError): + IPSHELL = None +else: + IPSHELL = None + def _write_sample_config(configspec, progname): """ diff --git a/sourcespec2/setup/exit.py b/sourcespec2/setup/exit.py new file mode 100644 index 00000000..2be038ee --- /dev/null +++ b/sourcespec2/setup/exit.py @@ -0,0 +1,49 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +An exit function for SourceSpec. + +:copyright: + 2012 Claudio Satriano + + 2013-2014 Claudio Satriano , + Emanuela Matrullo , + Agnes Chounet + + 2015-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import sys +import logging +import signal +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def ssp_exit(retval=0, abort=False): + """Exit the program.""" + # ssp_exit might have already been called if multiprocessing + if ssp_exit.SSP_EXIT_CALLED: + return + ssp_exit.SSP_EXIT_CALLED = True + if abort: + print('\nAborting.') + if logger is not None: + logger.debug('source_spec ABORTED') + elif logger is not None: + logger.debug('source_spec END') + logging.shutdown() + sys.exit(retval) + + +ssp_exit.SSP_EXIT_CALLED = False + + +def _sigint_handler(sig, frame): + """Handle SIGINT signal.""" + # pylint: disable=unused-argument + ssp_exit(1, abort=True) + + +signal.signal(signal.SIGINT, _sigint_handler) diff --git a/sourcespec2/config/library_versions.py b/sourcespec2/setup/library_versions.py similarity index 100% rename from sourcespec2/config/library_versions.py rename to sourcespec2/setup/library_versions.py diff --git a/sourcespec2/ssp_setup.py b/sourcespec2/setup/logging.py similarity index 59% rename from sourcespec2/ssp_setup.py rename to sourcespec2/setup/logging.py index c0f51f27..f169019a 100644 --- a/sourcespec2/ssp_setup.py +++ b/sourcespec2/setup/logging.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: CECILL-2.1 """ -Setup functions for sourcespec. +Logging setup for SourceSpec. :copyright: 2012 Claudio Satriano @@ -20,96 +20,15 @@ import platform import shutil import logging -import signal import contextlib from datetime import datetime -from . import __version__, __banner__ -from .config import config, library_versions - -# define ipshell(), if possible -# note: ANSI colors do not work on Windows standard terminal -if sys.stdout.isatty() and sys.platform != 'win32': - try: - from IPython.terminal.embed import InteractiveShellEmbed - IPSHELL = InteractiveShellEmbed.instance() - except ImportError: - IPSHELL = None -else: - IPSHELL = None +from .config import config +from .library_versions import library_versions +from .. import __version__, __banner__ # global variables OLDLOGFILE = None LOGGER = None -SSP_EXIT_CALLED = False - - -def save_config(eventid=None): - """Save config file to output dir.""" - if eventid is None: - try: - eventid = config.event.event_id - except AttributeError as err: - raise RuntimeError( - 'No event ID found. Cannot save config file.' - ) from err - # Actually, it renames the file already existing. - src = os.path.join(config.options.outdir, 'source_spec.conf') - dst = os.path.join(config.options.outdir, f'{eventid}.ssp.conf') - # On Windows, dst file must not exist - with contextlib.suppress(Exception): - os.remove(dst) - os.rename(src, dst) - - -def get_outdir_path(eventid=None): - """Construct full path to output directory""" - if eventid is None: - try: - eventid = config.event.event_id - except AttributeError as err: - raise RuntimeError( - 'No event ID found. Cannot create output directory.' - ) from err - src = config.options.outdir - run_id = config.options.run_id - run_id_subdir = config.options.run_id_subdir - # TODO: does next line also work if no tmpdir has been created first? - outdir = os.path.split(src)[0] - outdir = os.path.join(outdir, str(eventid)) - if run_id and run_id_subdir: - outdir = os.path.join(outdir, str(run_id)) - elif run_id: - outdir += f'_{run_id}' - - return outdir - - -def move_outdir(eventid=None): - """Move outdir to a new dir named from evid (and optional run_id).""" - src = config.options.outdir - dst = get_outdir_path(eventid) - # Create destination - if not os.path.exists(dst): - os.makedirs(dst) - # Copy all files into destination - file_names = os.listdir(src) - for file_name in file_names: - shutil.copyfile( - os.path.join(src, file_name), - os.path.join(dst, file_name) - ) - # Old outdir cannot be removed yet, because the log file is still opened - config.options.oldoutdir = src - config.options.outdir = dst - - -def remove_old_outdir(): - """Try to remove the old outdir.""" - try: - oldoutdir = config.options.oldoutdir - shutil.rmtree(oldoutdir) - except Exception: - return def _color_handler_emit(fn): @@ -139,6 +58,26 @@ def new(*args): return new +def _log_debug_information(): + banner = f'\n{__banner__}\nThis is SourceSpec v{__version__}.\n' + LOGGER.info(banner) + LOGGER.debug('source_spec START') + LOGGER.debug(f'SourceSpec version: {__version__}') + uname = platform.uname() + uname_str = f'{uname[0]} {uname[2]} {uname[4]}' + LOGGER.debug(f'Platform: {uname_str}') + lv = library_versions + LOGGER.debug(f'Python version: {lv.PYTHON_VERSION_STR}') + LOGGER.debug(f'ObsPy version: {lv.OBSPY_VERSION_STR}') + LOGGER.debug(f'NumPy version: {lv.NUMPY_VERSION_STR}') + LOGGER.debug(f'SciPy version: {lv.SCIPY_VERSION_STR}') + LOGGER.debug(f'Matplotlib version: {lv.MATPLOTLIB_VERSION_STR}') + if lv.CARTOPY_VERSION_STR is not None: + LOGGER.debug(f'Cartopy version: {lv.CARTOPY_VERSION_STR}') + LOGGER.debug('Running arguments:') + LOGGER.debug(' '.join(sys.argv)) + + def setup_logging(basename=None, progname='source_spec'): """ Set up the logging infrastructure. @@ -215,49 +154,3 @@ def setup_logging(basename=None, progname='source_spec'): for _ in range(len(config.warnings)): msg = config.warnings.pop(0) LOGGER.warning(msg) - - -def _log_debug_information(): - banner = f'\n{__banner__}\nThis is SourceSpec v{__version__}.\n' - LOGGER.info(banner) - LOGGER.debug('source_spec START') - LOGGER.debug(f'SourceSpec version: {__version__}') - uname = platform.uname() - uname_str = f'{uname[0]} {uname[2]} {uname[4]}' - LOGGER.debug(f'Platform: {uname_str}') - lv = library_versions - LOGGER.debug(f'Python version: {lv.PYTHON_VERSION_STR}') - LOGGER.debug(f'ObsPy version: {lv.OBSPY_VERSION_STR}') - LOGGER.debug(f'NumPy version: {lv.NUMPY_VERSION_STR}') - LOGGER.debug(f'SciPy version: {lv.SCIPY_VERSION_STR}') - LOGGER.debug(f'Matplotlib version: {lv.MATPLOTLIB_VERSION_STR}') - if lv.CARTOPY_VERSION_STR is not None: - LOGGER.debug(f'Cartopy version: {lv.CARTOPY_VERSION_STR}') - LOGGER.debug('Running arguments:') - LOGGER.debug(' '.join(sys.argv)) - - -def ssp_exit(retval=0, abort=False): - """Exit the program.""" - # ssp_exit might have already been called if multiprocessing - global SSP_EXIT_CALLED # pylint: disable=global-statement - if SSP_EXIT_CALLED: - return - SSP_EXIT_CALLED = True - if abort: - print('\nAborting.') - if LOGGER is not None: - LOGGER.debug('source_spec ABORTED') - elif LOGGER is not None: - LOGGER.debug('source_spec END') - logging.shutdown() - sys.exit(retval) - - -def sigint_handler(sig, frame): - """Handle SIGINT signal.""" - # pylint: disable=unused-argument - ssp_exit(1, abort=True) - - -signal.signal(signal.SIGINT, sigint_handler) diff --git a/sourcespec2/config/mandatory_deprecated.py b/sourcespec2/setup/mandatory_deprecated.py similarity index 100% rename from sourcespec2/config/mandatory_deprecated.py rename to sourcespec2/setup/mandatory_deprecated.py diff --git a/sourcespec2/setup/outdir.py b/sourcespec2/setup/outdir.py new file mode 100644 index 00000000..012f795b --- /dev/null +++ b/sourcespec2/setup/outdir.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Setup functions for sourcespec. + +:copyright: + 2012 Claudio Satriano + + 2013-2014 Claudio Satriano , + Emanuela Matrullo , + Agnes Chounet + + 2015-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import shutil +import contextlib +from .config import config + + +def save_config(eventid=None): + """Save config file to output dir.""" + if eventid is None: + try: + eventid = config.event.event_id + except AttributeError as err: + raise RuntimeError( + 'No event ID found. Cannot save config file.' + ) from err + # Actually, it renames the file already existing. + src = os.path.join(config.options.outdir, 'source_spec.conf') + dst = os.path.join(config.options.outdir, f'{eventid}.ssp.conf') + # On Windows, dst file must not exist + with contextlib.suppress(Exception): + os.remove(dst) + os.rename(src, dst) + + +def get_outdir_path(eventid=None): + """Construct full path to output directory""" + if eventid is None: + try: + eventid = config.event.event_id + except AttributeError as err: + raise RuntimeError( + 'No event ID found. Cannot create output directory.' + ) from err + src = config.options.outdir + run_id = config.options.run_id + run_id_subdir = config.options.run_id_subdir + # TODO: does next line also work if no tmpdir has been created first? + outdir = os.path.split(src)[0] + outdir = os.path.join(outdir, str(eventid)) + if run_id and run_id_subdir: + outdir = os.path.join(outdir, str(run_id)) + elif run_id: + outdir += f'_{run_id}' + + return outdir + + +def move_outdir(eventid=None): + """Move outdir to a new dir named from evid (and optional run_id).""" + src = config.options.outdir + dst = get_outdir_path(eventid) + # Create destination + if not os.path.exists(dst): + os.makedirs(dst) + # Copy all files into destination + file_names = os.listdir(src) + for file_name in file_names: + shutil.copyfile( + os.path.join(src, file_name), + os.path.join(dst, file_name) + ) + # Old outdir cannot be removed yet, because the log file is still opened + config.options.oldoutdir = src + config.options.outdir = dst + + +def remove_old_outdir(): + """Try to remove the old outdir.""" + try: + oldoutdir = config.options.oldoutdir + shutil.rmtree(oldoutdir) + except Exception: + return diff --git a/sourcespec2/config/ssp_event.yaml b/sourcespec2/setup/ssp_event.yaml similarity index 100% rename from sourcespec2/config/ssp_event.yaml rename to sourcespec2/setup/ssp_event.yaml diff --git a/sourcespec2/source_model.py b/sourcespec2/source_model.py index 3dec065a..d31617ae 100644 --- a/sourcespec2/source_model.py +++ b/sourcespec2/source_model.py @@ -84,8 +84,7 @@ def main(): # Lazy-import modules for speed from .ssp_parse_arguments import parse_args options = parse_args(progname='source_model') - from .config import config, configure_cli - from .ssp_setup import ssp_exit + from .setup import config, configure_cli, ssp_exit plot_show = bool(options.plot) conf_overrides = { 'plot_show': plot_show, diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 10cd853c..7ae755bd 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -44,15 +44,14 @@ def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): # pylint: disable=import-outside-toplevel # Lazy-import modules for speed import os - from .config import config + from .setup import config, ssp_exit # Avoid hard exit when ssp_exit is called somewhere if not allow_exit: - from . import ssp_setup - ssp_setup.SSP_EXIT_CALLED = True + ssp_exit.SSP_EXIT_CALLED = True # Create output folder if required, save config and setup logging - from .ssp_setup import (get_outdir_path, save_config, setup_logging) + from .setup import get_outdir_path, save_config, setup_logging if config.options.outdir: outdir = get_outdir_path(ssp_event.event_id) if not os.path.exists(outdir): @@ -119,7 +118,7 @@ def ssp_output(st, proc_st, spec_st, specnoise_st, weight_st, sspec_output): """ # pylint: disable=import-outside-toplevel # Lazy-import modules for speed - from .config import config + from .setup import config # TODO: check (again) if outdir exists if not config.options.outdir: @@ -162,11 +161,9 @@ def main(): options = parse_args(progname='source_spec') # Setup stage - from .config import configure_cli + from .setup import configure_cli configure_cli(options, progname='source_spec') - from .ssp_setup import ( - move_outdir, remove_old_outdir, setup_logging, - ssp_exit) + from .setup import setup_logging setup_logging() # Read all required information from disk @@ -179,6 +176,7 @@ def main(): ssp_event, picks = read_event_and_picks(trace1) # Now that we have an evid, we can rename the outdir and the log file + from .setup import move_outdir, remove_old_outdir move_outdir(ssp_event.event_id) setup_logging(ssp_event.event_id) remove_old_outdir() @@ -190,6 +188,7 @@ def main(): # Generate output ssp_output(st, proc_st, spec_st, specnoise_st, weight_st, sspec_output) + from .setup import ssp_exit ssp_exit() diff --git a/sourcespec2/ssp_build_spectra.py b/sourcespec2/ssp_build_spectra.py index 93ab5627..7605c044 100644 --- a/sourcespec2/ssp_build_spectra.py +++ b/sourcespec2/ssp_build_spectra.py @@ -25,9 +25,8 @@ from scipy.integrate import cumulative_trapezoid as cumtrapz from scipy.interpolate import interp1d from obspy.core import Stream -from .config import config +from .setup import config, ssp_exit from .spectrum import Spectrum, SpectrumStream -from .ssp_setup import ssp_exit from .ssp_util import ( smooth, cosine_taper, moment_to_mag, MediumProperties) from .ssp_geom_spreading import ( diff --git a/sourcespec2/ssp_correction.py b/sourcespec2/ssp_correction.py index 50175fa2..40e091c3 100644 --- a/sourcespec2/ssp_correction.py +++ b/sourcespec2/ssp_correction.py @@ -14,10 +14,9 @@ """ import logging from scipy.interpolate import interp1d -from .config import config +from .setup import config, ssp_exit from .spectrum import read_spectra from .ssp_util import moment_to_mag, mag_to_moment -from .ssp_setup import ssp_exit logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_data_types.py b/sourcespec2/ssp_data_types.py index e83f067d..41473cb5 100644 --- a/sourcespec2/ssp_data_types.py +++ b/sourcespec2/ssp_data_types.py @@ -12,7 +12,7 @@ import logging from collections import OrderedDict import numpy as np -from .config import config +from .setup import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_grid_sampling.py b/sourcespec2/ssp_grid_sampling.py index 76aabe23..3c3b8b1d 100644 --- a/sourcespec2/ssp_grid_sampling.py +++ b/sourcespec2/ssp_grid_sampling.py @@ -21,7 +21,7 @@ # pylint: disable=no-name-in-module from scipy.signal._peak_finding_utils import PeakPropertyWarning import matplotlib.pyplot as plt -from .config import config +from .setup import config from .kdtree import KDTree from .savefig import savefig logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_html_report.py b/sourcespec2/ssp_html_report.py index e60a6cde..7b94f947 100644 --- a/sourcespec2/ssp_html_report.py +++ b/sourcespec2/ssp_html_report.py @@ -16,7 +16,7 @@ import contextlib from urllib.parse import urlparse import numpy as np -from .config import config +from .setup import config from ._version import get_versions from .ssp_data_types import SpectralParameter logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_inversion.py b/sourcespec2/ssp_inversion.py index 1908f933..54b44a26 100644 --- a/sourcespec2/ssp_inversion.py +++ b/sourcespec2/ssp_inversion.py @@ -20,7 +20,7 @@ from scipy.optimize import curve_fit, minimize, basinhopping from scipy.signal import argrelmax from obspy.geodetics import gps2dist_azimuth -from .config import config +from .setup import config from .spectrum import SpectrumStream from .ssp_spectral_model import ( spectral_model, objective_func, callback) diff --git a/sourcespec2/ssp_local_magnitude.py b/sourcespec2/ssp_local_magnitude.py index dc818320..d41c618b 100644 --- a/sourcespec2/ssp_local_magnitude.py +++ b/sourcespec2/ssp_local_magnitude.py @@ -23,7 +23,7 @@ from obspy.signal.invsim import WOODANDERSON from obspy.signal.util import smooth from obspy.signal.trigger import trigger_onset -from .config import config +from .setup import config from .ssp_data_types import SpectralParameter from .ssp_util import cosine_taper from .ssp_util import remove_instr_response diff --git a/sourcespec2/ssp_output.py b/sourcespec2/ssp_output.py index 6aebd988..c3622aa3 100644 --- a/sourcespec2/ssp_output.py +++ b/sourcespec2/ssp_output.py @@ -22,7 +22,7 @@ from datetime import datetime from tzlocal import get_localzone import numpy as np -from .config import config +from .setup import config from .ssp_qml_output import write_qml from .ssp_sqlite_output import write_sqlite from ._version import get_versions diff --git a/sourcespec2/ssp_plot_params_stats.py b/sourcespec2/ssp_plot_params_stats.py index 1e0fa4b5..fa960733 100644 --- a/sourcespec2/ssp_plot_params_stats.py +++ b/sourcespec2/ssp_plot_params_stats.py @@ -16,7 +16,7 @@ import matplotlib import matplotlib.pyplot as plt import matplotlib.patheffects as mpe -from .config import config +from .setup import config from ._version import get_versions from .savefig import savefig logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_plot_spectra.py b/sourcespec2/ssp_plot_spectra.py index 00354df7..e7975be7 100644 --- a/sourcespec2/ssp_plot_spectra.py +++ b/sourcespec2/ssp_plot_spectra.py @@ -25,7 +25,7 @@ import matplotlib.pyplot as plt from matplotlib.backends.backend_pdf import PdfPages import matplotlib.patheffects as PathEffects -from .config import config +from .setup import config from .ssp_util import spec_minmax, moment_to_mag, mag_to_moment from .savefig import savefig from ._version import get_versions diff --git a/sourcespec2/ssp_plot_stacked_spectra.py b/sourcespec2/ssp_plot_stacked_spectra.py index 1f73333b..10be914b 100644 --- a/sourcespec2/ssp_plot_stacked_spectra.py +++ b/sourcespec2/ssp_plot_stacked_spectra.py @@ -17,7 +17,7 @@ import matplotlib.pyplot as plt import matplotlib.patheffects as PathEffects from matplotlib.collections import LineCollection -from .config import config +from .setup import config from .ssp_util import moment_to_mag, mag_to_moment from .ssp_spectral_model import spectral_model from .savefig import savefig diff --git a/sourcespec2/ssp_plot_stations.py b/sourcespec2/ssp_plot_stations.py index b4dbc1b9..7d9c6535 100644 --- a/sourcespec2/ssp_plot_stations.py +++ b/sourcespec2/ssp_plot_stations.py @@ -25,7 +25,7 @@ from matplotlib import colors import matplotlib.patheffects as PathEffects from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable -from .config import config +from .setup import config from .adjustText import adjust_text from .cached_tiler import CachedTiler from .map_tiles import ( diff --git a/sourcespec2/ssp_plot_traces.py b/sourcespec2/ssp_plot_traces.py index 0a576727..1381b869 100644 --- a/sourcespec2/ssp_plot_traces.py +++ b/sourcespec2/ssp_plot_traces.py @@ -21,7 +21,7 @@ from matplotlib import patches import matplotlib.patheffects as PathEffects from matplotlib.ticker import ScalarFormatter as sf -from .config import config +from .setup import config from .savefig import savefig from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py index eb441092..693b1b24 100644 --- a/sourcespec2/ssp_process_traces.py +++ b/sourcespec2/ssp_process_traces.py @@ -19,8 +19,7 @@ from scipy.signal import savgol_filter from obspy.core import Stream from obspy.core.util import AttribDict -from .config import config -from .ssp_setup import ssp_exit +from .setup import config, ssp_exit from .ssp_util import ( remove_instr_response, station_to_event_position) from .ssp_wave_arrival import add_arrival_to_trace diff --git a/sourcespec2/ssp_qml_output.py b/sourcespec2/ssp_qml_output.py index c273e27b..b810660a 100644 --- a/sourcespec2/ssp_qml_output.py +++ b/sourcespec2/ssp_qml_output.py @@ -19,7 +19,7 @@ MomentTensor, QuantityError, ResourceIdentifier, StationMagnitude, StationMagnitudeContribution, WaveformStreamID) -from .config import config +from .setup import config from ._version import get_versions logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_radiated_energy.py b/sourcespec2/ssp_radiated_energy.py index c664b297..54faada1 100644 --- a/sourcespec2/ssp_radiated_energy.py +++ b/sourcespec2/ssp_radiated_energy.py @@ -18,7 +18,7 @@ import contextlib import logging import numpy as np -from .config import config +from .setup import config from .ssp_data_types import SpectralParameter logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_radiation_pattern.py b/sourcespec2/ssp_radiation_pattern.py index 6e99112e..58965d62 100644 --- a/sourcespec2/ssp_radiation_pattern.py +++ b/sourcespec2/ssp_radiation_pattern.py @@ -13,7 +13,7 @@ import logging from math import pi, sin, cos from obspy.taup import TauPyModel -from .config import config +from .setup import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) model = TauPyModel(model='iasp91') diff --git a/sourcespec2/ssp_read_event_metadata.py b/sourcespec2/ssp_read_event_metadata.py index 4037d8a2..5b365bf9 100644 --- a/sourcespec2/ssp_read_event_metadata.py +++ b/sourcespec2/ssp_read_event_metadata.py @@ -20,8 +20,7 @@ import yaml from obspy import UTCDateTime from obspy import read_events -from .config import config -from .ssp_setup import ssp_exit +from .setup import config, ssp_exit from .ssp_event import SSPEvent from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_sac_header.py b/sourcespec2/ssp_read_sac_header.py index 41848737..1a057d41 100644 --- a/sourcespec2/ssp_read_sac_header.py +++ b/sourcespec2/ssp_read_sac_header.py @@ -13,8 +13,7 @@ import logging import contextlib from obspy.core.util import AttribDict -from .config import config -from .ssp_setup import ssp_exit +from .setup import config, ssp_exit from .ssp_event import SSPEvent from .ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_station_metadata.py b/sourcespec2/ssp_read_station_metadata.py index e489ad71..dfac62fb 100644 --- a/sourcespec2/ssp_read_station_metadata.py +++ b/sourcespec2/ssp_read_station_metadata.py @@ -15,7 +15,7 @@ import logging from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response -from .config import config +from .setup import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/ssp_read_traces.py index 8c8b7642..077eeb80 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/ssp_read_traces.py @@ -27,11 +27,9 @@ from obspy import read from obspy.core import Stream from obspy.core.util import AttribDict -from .config import config -from .ssp_setup import ssp_exit +from .setup import config, ssp_exit from .ssp_util import MediumProperties -from .ssp_read_station_metadata import ( - read_station_metadata, PAZ) +from .ssp_read_station_metadata import read_station_metadata, PAZ from .ssp_read_event_metadata import ( parse_qml, parse_hypo_file, parse_hypo71_picks) from .ssp_read_sac_header import ( diff --git a/sourcespec2/ssp_residuals.py b/sourcespec2/ssp_residuals.py index bee362b2..9aabdf7a 100644 --- a/sourcespec2/ssp_residuals.py +++ b/sourcespec2/ssp_residuals.py @@ -14,7 +14,7 @@ """ import os import logging -from .config import config +from .setup import config from ._version import get_versions from .spectrum import SpectrumStream from .ssp_spectral_model import spectral_model diff --git a/sourcespec2/ssp_sqlite_output.py b/sourcespec2/ssp_sqlite_output.py index 4b93785d..b2355a9d 100644 --- a/sourcespec2/ssp_sqlite_output.py +++ b/sourcespec2/ssp_sqlite_output.py @@ -12,8 +12,7 @@ import os.path import logging import sqlite3 -from .config import config -from .ssp_setup import ssp_exit +from .setup import config, ssp_exit from .ssp_db_definitions import ( DB_VERSION, STATIONS_TABLE, STATIONS_PRIMARY_KEYS, EVENTS_TABLE, EVENTS_PRIMARY_KEYS) diff --git a/sourcespec2/ssp_summary_statistics.py b/sourcespec2/ssp_summary_statistics.py index 27ea3570..a09cd9ce 100644 --- a/sourcespec2/ssp_summary_statistics.py +++ b/sourcespec2/ssp_summary_statistics.py @@ -13,8 +13,7 @@ import numpy as np from scipy.stats import norm from scipy.integrate import quad -from .config import config -from .ssp_setup import ssp_exit +from .setup import config, ssp_exit from .ssp_data_types import ( SummarySpectralParameter, SummaryStatistics) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_util.py b/sourcespec2/ssp_util.py index b1dc7e9a..08bba742 100644 --- a/sourcespec2/ssp_util.py +++ b/sourcespec2/ssp_util.py @@ -17,7 +17,7 @@ from obspy.signal.invsim import cosine_taper as _cos_taper from obspy.geodetics import gps2dist_azimuth, kilometers2degrees from obspy.taup import TauPyModel -from .config import config +from .setup import config model = TauPyModel(model='iasp91') v_model = model.model.s_mod.v_mod logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_wave_arrival.py b/sourcespec2/ssp_wave_arrival.py index f559305d..ea931db5 100644 --- a/sourcespec2/ssp_wave_arrival.py +++ b/sourcespec2/ssp_wave_arrival.py @@ -16,7 +16,7 @@ import warnings from math import asin, degrees from obspy.taup import TauPyModel -from .config import config +from .setup import config model = TauPyModel(model='iasp91') logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) From 019c38eaf77cbbe482baa5900168de4e114c3698 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 16 Jul 2024 12:27:17 +0200 Subject: [PATCH 38/73] Move input functions into `input` submodule --- sourcespec2/input/__init__.py | 16 + sourcespec2/input/augment_event.py | 68 ++ .../augment_traces.py} | 409 +++------- sourcespec2/input/event_and_picks.py | 81 ++ .../sac_header.py} | 6 +- .../station_metadata.py} | 19 +- sourcespec2/input/traces.py | 125 +++ sourcespec2/source_spec.py | 11 +- sourcespec2/ssp_read_event_metadata.py | 720 ------------------ 9 files changed, 404 insertions(+), 1051 deletions(-) create mode 100644 sourcespec2/input/__init__.py create mode 100644 sourcespec2/input/augment_event.py rename sourcespec2/{ssp_read_traces.py => input/augment_traces.py} (63%) create mode 100644 sourcespec2/input/event_and_picks.py rename sourcespec2/{ssp_read_sac_header.py => input/sac_header.py} (98%) rename sourcespec2/{ssp_read_station_metadata.py => input/station_metadata.py} (95%) create mode 100644 sourcespec2/input/traces.py delete mode 100644 sourcespec2/ssp_read_event_metadata.py diff --git a/sourcespec2/input/__init__.py b/sourcespec2/input/__init__.py new file mode 100644 index 00000000..244f8be8 --- /dev/null +++ b/sourcespec2/input/__init__.py @@ -0,0 +1,16 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Input functions for SourceSpec. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +from .traces import read_traces # noqa +from .augment_traces import augment_traces # noqa +from .event_and_picks import read_event_and_picks # noqa +from .augment_event import augment_event # noqa +from .station_metadata import read_station_metadata # noqa diff --git a/sourcespec2/input/augment_event.py b/sourcespec2/input/augment_event.py new file mode 100644 index 00000000..efa3d7df --- /dev/null +++ b/sourcespec2/input/augment_event.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Augment event with velocity info and event name. + +:copyright: + 2012 Claudio Satriano + + 2013-2014 Claudio Satriano , + Emanuela Matrullo + + 2015-2024 Claudio Satriano , + Sophie Lambotte +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import logging +from ..setup import config, ssp_exit +from ..ssp_util import MediumProperties +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _hypo_vel(hypo): + """ + Compute velocity at hypocenter. + + :param hypo: Hypocenter object + :type hypo: :class:`sourcespec.ssp_event.Hypocenter` + """ + medium_properties = MediumProperties( + hypo.longitude, hypo.latitude, hypo.depth.value_in_km) + hypo.vp = medium_properties.get(mproperty='vp', where='source') + hypo.vs = medium_properties.get(mproperty='vs', where='source') + hypo.rho = medium_properties.get(mproperty='rho', where='source') + depth_string = medium_properties.to_string( + 'source depth', hypo.depth.value_in_km) + vp_string = medium_properties.to_string('vp_source', hypo.vp) + vs_string = medium_properties.to_string('vs_source', hypo.vs) + rho_string = medium_properties.to_string('rho_source', hypo.rho) + logger.info(f'{depth_string}, {vp_string}, {vs_string}, {rho_string}') + + +def augment_event(ssp_event): + """ + Add velocity info to hypocenter + and add event name from/to config.options + + The augmented event is stored in config.event + + :param ssp_event: Evento to be augmented + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + """ + # add velocity info to hypocenter + try: + _hypo_vel(ssp_event.hypocenter) + except Exception as e: + logger.error( + f'Unable to compute velocity at hypocenter: {e}\n') + ssp_exit(1) + if config.options.evname is not None: + # add evname from command line, if any, overriding the one in ssp_event + ssp_event.name = config.options.evname + else: + # add evname from ssp_event, if any, to config file + config.options.evname = ssp_event.name + # add event to config file + config.event = ssp_event diff --git a/sourcespec2/ssp_read_traces.py b/sourcespec2/input/augment_traces.py similarity index 63% rename from sourcespec2/ssp_read_traces.py rename to sourcespec2/input/augment_traces.py index 077eeb80..0a573a94 100644 --- a/sourcespec2/ssp_read_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: CECILL-2.1 """ -Read traces in multiple formats of data and metadata. +Augment traces with station and event metadata. :copyright: 2012 Claudio Satriano @@ -15,31 +15,94 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -import sys -import os import re import logging -import shutil -import tarfile -import zipfile -import tempfile import contextlib -from obspy import read -from obspy.core import Stream from obspy.core.util import AttribDict -from .setup import config, ssp_exit -from .ssp_util import MediumProperties -from .ssp_read_station_metadata import read_station_metadata, PAZ -from .ssp_read_event_metadata import ( - parse_qml, parse_hypo_file, parse_hypo71_picks) -from .ssp_read_sac_header import ( +from ..setup import config +from .station_metadata import PAZ +from .sac_header import ( compute_sensitivity_from_SAC, get_instrument_from_SAC, get_station_coordinates_from_SAC, get_event_from_SAC, get_picks_from_SAC) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -# TRACE MANIPULATION ---------------------------------------------------------- +def _skip_traces_from_config(traceid): + """ + Skip traces with unknown channel orientation or ignored from config file. + + :param traceid: Trace ID. + :type traceid: str + + :raises: RuntimeError if traceid is ignored from config file. + """ + network, station, location, channel = traceid.split('.') + orientation_codes = config.vertical_channel_codes +\ + config.horizontal_channel_codes_1 +\ + config.horizontal_channel_codes_2 + orientation = channel[-1] + if orientation not in orientation_codes: + raise RuntimeError( + f'{traceid}: Unknown channel orientation: ' + f'"{orientation}": skipping trace' + ) + # build a list of all possible ids, from station only + # to full net.sta.loc.chan + ss = [ + station, + '.'.join((network, station)), + '.'.join((network, station, location)), + '.'.join((network, station, location, channel)), + ] + if config.use_traceids is not None: + # - combine all use_traceids in a single regex + # - escape the dots, otherwise they are interpreted as any character + # - add a dot before the first asterisk, to avoid a pattern error + combined = ( + "(" + ")|(".join(config.use_traceids) + ")" + ).replace('.', r'\.').replace('(*', '(.*') + if not any(re.match(combined, s) for s in ss): + raise RuntimeError(f'{traceid}: ignored from config file') + if config.ignore_traceids is not None: + # - combine all ignore_traceids in a single regex + # - escape the dots, otherwise they are interpreted as any character + # - add a dot before the first asterisk, to avoid a pattern error + combined = ( + "(" + ")|(".join(config.ignore_traceids) + ")" + ).replace('.', r'\.').replace('(*', '(.*') + if any(re.match(combined, s) for s in ss): + raise RuntimeError(f'{traceid}: ignored from config file') + + +def _select_components(st): + """ + Select requested components from stream + + :param st: ObsPy Stream object + :type st: :class:`obspy.core.stream.Stream` + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` + """ + traces_to_keep = [] + for trace in st: + try: + _skip_traces_from_config(trace.id) + except RuntimeError as e: + logger.warning(str(e)) + continue + # TODO: should we also filter by station here? + # only use the station specified by the command line option + # "--station", if any + #if (config.options.station is not None and + # trace.stats.station != config.options.station): + # continue + traces_to_keep.append(trace) + # in-place update of st + st.traces[:] = traces_to_keep[:] + + def _correct_traceid(trace): """ Correct traceid from config.TRACEID_MAP, if available. @@ -297,287 +360,36 @@ def _complete_picks(st): tr.stats.picks += default_P_pick if not [p for p in tr.stats.picks if p.phase == 'S']: tr.stats.picks += default_S_pick -# ----------------------------------------------------------------------------- - -# FILE PARSING ---------------------------------------------------------------- -def _hypo_vel(hypo): - """ - Compute velocity at hypocenter. - :param hypo: Hypocenter object - :type hypo: :class:`sourcespec.ssp_event.Hypocenter` - """ - medium_properties = MediumProperties( - hypo.longitude, hypo.latitude, hypo.depth.value_in_km) - hypo.vp = medium_properties.get(mproperty='vp', where='source') - hypo.vs = medium_properties.get(mproperty='vs', where='source') - hypo.rho = medium_properties.get(mproperty='rho', where='source') - depth_string = medium_properties.to_string( - 'source depth', hypo.depth.value_in_km) - vp_string = medium_properties.to_string('vp_source', hypo.vp) - vs_string = medium_properties.to_string('vs_source', hypo.vs) - rho_string = medium_properties.to_string('rho_source', hypo.rho) - logger.info(f'{depth_string}, {vp_string}, {vs_string}, {rho_string}') - - -def _build_filelist(path, filelist, tmpdir): - """ - Build a list of files to read. - - :param path: Path to a file or directory - :type path: str - :param filelist: List of files to read - :type filelist: list - :param tmpdir: Temporary directory - :type tmpdir: str - """ - if os.path.isdir(path): - listing = os.listdir(path) - for filename in listing: - fullpath = os.path.join(path, filename) - _build_filelist(fullpath, filelist, tmpdir) - else: - try: - # pylint: disable=unspecified-encoding consider-using-with - open(path) - except IOError as err: - logger.error(err) - return - if tarfile.is_tarfile(path) and tmpdir is not None: - with tarfile.open(path) as tar: - try: - tar.extractall(path=tmpdir) - except Exception as msg: - logger.warning( - f'{path}: Unable to fully extract tar archive: {msg}') - elif zipfile.is_zipfile(path) and tmpdir is not None: - with zipfile.ZipFile(path) as zipf: - try: - zipf.extractall(path=tmpdir) - except Exception as msg: - logger.warning( - f'{path}: Unable to fully extract zip archive: {msg}') - else: - filelist.append(path) - - -def _read_trace_files(): +def _augment_trace(trace, inventory, ssp_event, picks): """ - Read trace files from a given path and return a stream object. - Trace metadata are not yet updated. + Augment trace with station and event metadata. + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` :param inventory: ObsPy Inventory object :type inventory: :class:`obspy.core.inventory.Inventory` :param ssp_event: SSPEvent object :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` :param picks: list of picks :type picks: list of :class:`sourcespec.ssp_event.Pick` - - :return: ObsPy Stream object - :rtype: :class:`obspy.core.stream.Stream` - """ - # phase 1: build a file list - # ph 1.1: create a temporary dir and run '_build_filelist()' - # to move files to it and extract all tar archives - tmpdir = tempfile.mkdtemp() - filelist = [] - for trace_path in config.options.trace_path: - _build_filelist(trace_path, filelist, tmpdir) - # ph 1.2: rerun '_build_filelist()' in tmpdir to add to the - # filelist all the extraceted files - listing = os.listdir(tmpdir) - for filename in listing: - fullpath = os.path.join(tmpdir, filename) - _build_filelist(fullpath, filelist, None) - # phase 2: build a stream object from the file list - st = Stream() - for filename in sorted(filelist): - try: - tmpst = read(filename, fsize=False) - except Exception: - logger.warning( - f'{filename}: Unable to read file as a trace: skipping') - continue - # TODO: optionally we could already call select_components here - #tmpst = select_components(tmpst) - for trace in tmpst.traces: - # only use the station specified by the command line option - # "--station", if any - if (config.options.station is not None and - trace.stats.station != config.options.station): - continue - st.append(trace) - shutil.rmtree(tmpdir) - return st -# ----------------------------------------------------------------------------- - - -def _log_event_info(ssp_event): """ - Log event information. - - :param ssp_event: SSPEvent object - :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` - """ - for line in str(ssp_event).splitlines(): - logger.info(line) - logger.info('---------------------------------------------------') - - -def _skip_traces_from_config(traceid): - """ - Skip traces with unknown channel orientation or ignored from config file. - - :param traceid: Trace ID. - :type traceid: str - - :raises: RuntimeError if traceid is ignored from config file. - """ - network, station, location, channel = traceid.split('.') - orientation_codes = config.vertical_channel_codes +\ - config.horizontal_channel_codes_1 +\ - config.horizontal_channel_codes_2 - orientation = channel[-1] - if orientation not in orientation_codes: - raise RuntimeError( - f'{traceid}: Unknown channel orientation: ' - f'"{orientation}": skipping trace' - ) - # build a list of all possible ids, from station only - # to full net.sta.loc.chan - ss = [ - station, - '.'.join((network, station)), - '.'.join((network, station, location)), - '.'.join((network, station, location, channel)), - ] - if config.use_traceids is not None: - # - combine all ignore_traceids in a single regex - # - escape the dots, otherwise they are interpreted as any character - # - add a dot before the first asterisk, to avoid a pattern error - combined = ( - "(" + ")|(".join(config.use_traceids) + ")" - ).replace('.', r'\.').replace('(*', '(.*') - if not any(re.match(combined, s) for s in ss): - raise RuntimeError(f'{traceid}: ignored from config file') - if config.ignore_traceids is not None: - # - combine all ignore_traceids in a single regex - # - escape the dots, otherwise they are interpreted as any character - # - add a dot before the first asterisk, to avoid a pattern error - combined = ( - "(" + ")|(".join(config.ignore_traceids) + ")" - ).replace('.', r'\.').replace('(*', '(.*') - if any(re.match(combined, s) for s in ss): - raise RuntimeError(f'{traceid}: ignored from config file') - - -def select_components(st): - """ - Select requested components from stream - - :param st: ObsPy Stream object - :type st: :class:`obspy.core.stream.Stream` - - :return: ObsPy Stream object - :rtype: :class:`obspy.core.stream.Stream` - """ - traces_to_keep = [] - for trace in st: - try: - _skip_traces_from_config(trace.id) - except RuntimeError as e: - logger.warning(str(e)) - continue - # TODO: should we also filter by station here? - # only use the station specified by the command line option - # "--station", if any - #if (config.options.station is not None and - # trace.stats.station != config.options.station): - # continue - traces_to_keep.append(trace) - # in-place update of st - st.traces[:] = traces_to_keep[:] - - -def read_event_and_picks(trace1=None): - """ - Read event and phase picks - - :param trace1: ObsPy Trace object containing event info (optional) - :type trace1: :class:`obspy.core.stream.Stream` - - :return: (ssp_event, picks) - :rtype: tuple of - :class:`sourcespec.ssp_event.SSPEvent`, - list of :class:`sourcespec.ssp_event.Pick` - """ - picks = [] - ssp_event = None - # parse hypocenter file - if config.options.hypo_file is not None: - ssp_event, picks, file_format = parse_hypo_file( - config.options.hypo_file, config.options.evid) - config.hypo_file_format = file_format - # parse pick file - if config.options.pick_file is not None: - picks = parse_hypo71_picks() - # parse QML file - if config.options.qml_file is not None: - ssp_event, picks = parse_qml() - if ssp_event is not None: - _log_event_info(ssp_event) - - # if ssp_event is still None, get it from first trace - if ssp_event is None and trace1 is not None: - try: - ssp_event = trace1.stats.event - _log_event_info(ssp_event) - except AttributeError: - logger.error('No hypocenter information found.') - sys.stderr.write( - '\n' - 'Use "-q" or "-H" options to provide hypocenter information\n' - 'or add hypocenter information to the SAC file header\n' - '(if you use the SAC format).\n' - ) - ssp_exit(1) - # TODO: log also if trace1 is None? - - return (ssp_event, picks) - - -def augment_event(ssp_event): - """ - Add velocity info to hypocenter - and add event name from/to config.options - - The augmented event is stored in config.event - - :param ssp_event: Evento to be augmented - :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` - """ - # add velocity info to hypocenter - try: - _hypo_vel(ssp_event.hypocenter) - except Exception as e: - logger.error( - f'Unable to compute velocity at hypocenter: {e}\n') - ssp_exit(1) - if config.options.evname is not None: - # add evname from command line, if any, overriding the one in ssp_event - ssp_event.name = config.options.evname - else: - # add evname from ssp_event, if any, to config file - config.options.evname = ssp_event.name - # add event to config file - config.event = ssp_event + _add_instrtype(trace) + _add_inventory(trace, inventory) + _check_instrtype(trace) + _add_coords(trace) + _add_event(trace, ssp_event) + _add_picks(trace, picks) + trace.stats.ignore = False def augment_traces(st, inventory, ssp_event, picks): """ Add all required information to trace headers. - Remove problematic traces. + + Only the traces that satisfy the conditions in the config file are kept. + Problematic traces are also removed. :param st: Traces to be augmented :type st: :class:`obspy.core.stream.Stream` @@ -588,17 +400,14 @@ def augment_traces(st, inventory, ssp_event, picks): :param picks: list of picks :type picks: list of :class:`sourcespec.ssp_event.Pick` """ + # First, select the components based on the config options + _select_components(st) + # Then, augment the traces and remove the problematic ones traces_to_keep = [] for trace in st: _correct_traceid(trace) try: - _add_instrtype(trace) - _add_inventory(trace, inventory) - _check_instrtype(trace) - _add_coords(trace) - _add_event(trace, ssp_event) - _add_picks(trace, picks) - trace.stats.ignore = False + _augment_trace(trace, inventory, ssp_event, picks) except Exception as err: for line in str(err).splitlines(): logger.warning(line) @@ -607,27 +416,3 @@ def augment_traces(st, inventory, ssp_event, picks): # in-place update of st st.traces[:] = traces_to_keep[:] _complete_picks(st) - - -def read_station_inventory(): - """read station metadata into an ObsPy ``Inventory`` object""" - inventory = read_station_metadata(config.station_metadata) - return inventory - - -# Public interface: -def read_traces(): - """ - Read trace files - - :return: Traces - :rtype: :class:`obspy.core.stream.Stream` - """ - logger.info('Reading traces...') - st = _read_trace_files() - logger.info('Reading traces: done') - logger.info('---------------------------------------------------') - if len(st) == 0: - logger.error('No trace loaded') - ssp_exit(1) - return st diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py new file mode 100644 index 00000000..d29c18a6 --- /dev/null +++ b/sourcespec2/input/event_and_picks.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read event and phase picks. + +:copyright: + 2012 Claudio Satriano + + 2013-2014 Claudio Satriano , + Emanuela Matrullo + + 2015-2024 Claudio Satriano , + Sophie Lambotte +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import sys +import logging +from ..setup import config, ssp_exit +from .event_metadata import parse_qml, parse_hypo_file, parse_hypo71_picks +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _log_event_info(ssp_event): + """ + Log event information. + + :param ssp_event: SSPEvent object + :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + """ + for line in str(ssp_event).splitlines(): + logger.info(line) + logger.info('---------------------------------------------------') + + +def read_event_and_picks(trace1=None): + """ + Read event and phase picks + + :param trace1: ObsPy Trace object containing event info (optional) + :type trace1: :class:`obspy.core.stream.Stream` + + :return: (ssp_event, picks) + :rtype: tuple of + :class:`sourcespec.ssp_event.SSPEvent`, + list of :class:`sourcespec.ssp_event.Pick` + """ + picks = [] + ssp_event = None + # parse hypocenter file + if config.options.hypo_file is not None: + ssp_event, picks, file_format = parse_hypo_file( + config.options.hypo_file, config.options.evid) + config.hypo_file_format = file_format + # parse pick file + if config.options.pick_file is not None: + picks = parse_hypo71_picks() + # parse QML file + if config.options.qml_file is not None: + ssp_event, picks = parse_qml() + if ssp_event is not None: + _log_event_info(ssp_event) + + # if ssp_event is still None, get it from first trace + if ssp_event is None and trace1 is not None: + try: + ssp_event = trace1.stats.event + _log_event_info(ssp_event) + except AttributeError: + logger.error('No hypocenter information found.') + sys.stderr.write( + '\n' + 'Use "-q" or "-H" options to provide hypocenter information\n' + 'or add hypocenter information to the SAC file header\n' + '(if you use the SAC format).\n' + ) + ssp_exit(1) + # TODO: log also if trace1 is None? + + return (ssp_event, picks) diff --git a/sourcespec2/ssp_read_sac_header.py b/sourcespec2/input/sac_header.py similarity index 98% rename from sourcespec2/ssp_read_sac_header.py rename to sourcespec2/input/sac_header.py index 1a057d41..ec26c4ad 100644 --- a/sourcespec2/ssp_read_sac_header.py +++ b/sourcespec2/input/sac_header.py @@ -13,9 +13,9 @@ import logging import contextlib from obspy.core.util import AttribDict -from .setup import config, ssp_exit -from .ssp_event import SSPEvent -from .ssp_pick import SSPPick +from ..setup import config, ssp_exit +from ..ssp_event import SSPEvent +from ..ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/ssp_read_station_metadata.py b/sourcespec2/input/station_metadata.py similarity index 95% rename from sourcespec2/ssp_read_station_metadata.py rename to sourcespec2/input/station_metadata.py index dfac62fb..bd9bf442 100644 --- a/sourcespec2/ssp_read_station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -15,7 +15,7 @@ import logging from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response -from .setup import config +from ..setup import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -182,24 +182,23 @@ def _read_paz_file(file): return paz.to_inventory() -def read_station_metadata(path): +def read_station_metadata(): """ Read station metadata into an ObsPy ``Inventory`` object. - :param path: path to the station metadata file or directory - :type path: str - :return: inventory :rtype: :class:`~obspy.core.inventory.inventory.Inventory` """ inventory = Inventory() - if path is None: - return inventory logger.info('Reading station metadata...') - if os.path.isdir(path): - filelist = [os.path.join(path, file) for file in os.listdir(path)] + metadata_path = config.station_metadata + if os.path.isdir(metadata_path): + filelist = [ + os.path.join(metadata_path, file) + for file in os.listdir(metadata_path) + ] else: - filelist = [path, ] + filelist = [metadata_path, ] for file in sorted(filelist): if os.path.isdir(file): # we do not enter into subdirs of "path" diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py new file mode 100644 index 00000000..ec4638b1 --- /dev/null +++ b/sourcespec2/input/traces.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read traces in multiple formats. + +:copyright: + 2012 Claudio Satriano + + 2013-2014 Claudio Satriano , + Emanuela Matrullo + + 2015-2024 Claudio Satriano , + Sophie Lambotte +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import logging +import shutil +import tarfile +import zipfile +import tempfile +from obspy import read +from obspy.core import Stream +from ..setup import config, ssp_exit +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _build_filelist(path, filelist, tmpdir): + """ + Build a list of files to read. + + :param path: Path to a file or directory + :type path: str + :param filelist: List of files to read + :type filelist: list + :param tmpdir: Temporary directory + :type tmpdir: str + """ + if os.path.isdir(path): + listing = os.listdir(path) + for filename in listing: + fullpath = os.path.join(path, filename) + _build_filelist(fullpath, filelist, tmpdir) + else: + try: + # pylint: disable=unspecified-encoding consider-using-with + open(path) + except IOError as err: + logger.error(err) + return + if tarfile.is_tarfile(path) and tmpdir is not None: + with tarfile.open(path) as tar: + try: + tar.extractall(path=tmpdir) + except Exception as msg: + logger.warning( + f'{path}: Unable to fully extract tar archive: {msg}') + elif zipfile.is_zipfile(path) and tmpdir is not None: + with zipfile.ZipFile(path) as zipf: + try: + zipf.extractall(path=tmpdir) + except Exception as msg: + logger.warning( + f'{path}: Unable to fully extract zip archive: {msg}') + else: + filelist.append(path) + + +def _read_trace_files(): + """ + Read trace files from the path specified in the configuration file. + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` + """ + # phase 1: build a file list + # ph 1.1: create a temporary dir and run '_build_filelist()' + # to move files to it and extract all tar archives + tmpdir = tempfile.mkdtemp() + filelist = [] + for trace_path in config.options.trace_path: + _build_filelist(trace_path, filelist, tmpdir) + # ph 1.2: rerun '_build_filelist()' in tmpdir to add to the + # filelist all the extraceted files + listing = os.listdir(tmpdir) + for filename in listing: + fullpath = os.path.join(tmpdir, filename) + _build_filelist(fullpath, filelist, None) + # phase 2: build a stream object from the file list + st = Stream() + for filename in sorted(filelist): + try: + tmpst = read(filename, fsize=False) + except Exception: + logger.warning( + f'{filename}: Unable to read file as a trace: skipping') + continue + for trace in tmpst.traces: + # only use the station specified by the command line option + # "--station", if any + if (config.options.station is not None and + trace.stats.station != config.options.station): + continue + st.append(trace) + shutil.rmtree(tmpdir) + return st + + +def read_traces(): + """ + Read trace files + + :return: Traces + :rtype: :class:`obspy.core.stream.Stream` + """ + logger.info('Reading traces...') + st = _read_trace_files() + logger.info('Reading traces: done') + logger.info('---------------------------------------------------') + if len(st) == 0: + logger.error('No trace loaded') + ssp_exit(1) + return st diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 7ae755bd..2580e62e 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -63,10 +63,8 @@ def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): save_config(ssp_event.event_id) # Preprocessing - from .ssp_read_traces import (augment_event, augment_traces, - select_components) + from .input import augment_event, augment_traces augment_event(ssp_event) - select_components(st) augment_traces(st, inventory, ssp_event, picks) # Deconvolve, filter, cut traces: @@ -167,12 +165,13 @@ def main(): setup_logging() # Read all required information from disk - from .ssp_read_traces import (read_traces, read_station_inventory, - read_event_and_picks) + from .input import read_traces st = read_traces() trace1 = st[0] if len(st) else None st.sort() - inventory = read_station_inventory() + from .input import read_station_metadata + inventory = read_station_metadata() + from .input import read_event_and_picks ssp_event, picks = read_event_and_picks(trace1) # Now that we have an evid, we can rename the outdir and the log file diff --git a/sourcespec2/ssp_read_event_metadata.py b/sourcespec2/ssp_read_event_metadata.py deleted file mode 100644 index 5b365bf9..00000000 --- a/sourcespec2/ssp_read_event_metadata.py +++ /dev/null @@ -1,720 +0,0 @@ -# -*- coding: utf-8 -*- -# SPDX-License-Identifier: CECILL-2.1 -""" -Read event metadata in QuakeML, SourceSpec Event File, HYPO71 or -HYPOINVERSE format. - -:copyright: - 2012-2025 Claudio Satriano -:license: - CeCILL Free Software License Agreement v2.1 - (http://www.cecill.info/licences.en.html) -""" -import os -import warnings -import contextlib -import logging -import re -import xml.etree.ElementTree as ET -from datetime import datetime -import yaml -from obspy import UTCDateTime -from obspy import read_events -from .setup import config, ssp_exit -from .ssp_event import SSPEvent -from .ssp_pick import SSPPick -logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) - - -def parse_qml(): - """ - Parse event metadata and picks from a QuakeML file. - - :return: a tuple of (SSPEvent, picks) - :rtype: tuple - """ - ssp_event = None - picks = [] - qml_file = config.options.qml_file - event_id = config.options.evid - # No need to parse event name from QuakeML if event name is given - # in the command line - if config.options.evname is None: - qml_event_description = config.qml_event_description - qml_event_description_regex = config.qml_event_description_regex - else: - qml_event_description = False - qml_event_description_regex = None - if qml_file is None: - return ssp_event, picks - try: - qml_event = _get_event_from_qml(qml_file, event_id) - ssp_event, qml_origin = _parse_qml_event( - qml_event, - parse_event_name_from_description=qml_event_description, - event_description_regex=qml_event_description_regex) - picks = _parse_picks_from_qml_event(qml_event, qml_origin) - except Exception as err: - logger.error(err) - ssp_exit(1) - log_messages = [] - with contextlib.suppress(Exception): - _parse_moment_tensor_from_qml_event(qml_event, ssp_event) - # Compute focal mechanism, scalar moment and magnitude. - # They will be overwritten later on, if they are found in the - # QuakeML file. - ssp_event.focal_mechanism.from_moment_tensor(ssp_event.moment_tensor) - ssp_event.scalar_moment.from_moment_tensor(ssp_event.moment_tensor) - ssp_event.magnitude.from_scalar_moment(ssp_event.scalar_moment) - with contextlib.suppress(Exception): - _parse_scalar_moment_from_qml_event(qml_event, ssp_event) - # Compute magnitude from scalar moment. It will be overwritten later - # on, if it is found in the QuakeML file. - ssp_event.magnitude.from_scalar_moment(ssp_event.scalar_moment) - with contextlib.suppress(Exception): - _parse_magnitude_from_qml_event(qml_event, ssp_event) - with contextlib.suppress(Exception): - _parse_focal_mechanism_from_qml_event(qml_event, ssp_event) - for msg in log_messages: - logger.info(msg) - return ssp_event, picks - - -def _get_event_from_qml(qml_file, event_id=None): - """ - Get an event from a QuakeML file. - - :param qml_file: QuakeML file - :type qml_file: str - :param event_id: event id - :type event_id: str - - :return: QuakeML event - :rtype: obspy.core.event.Event - """ - cat = read_events(qml_file) - if event_id is not None: - _qml_events = [ev for ev in cat if event_id in str(ev.resource_id)] - try: - qml_event = _qml_events[0] - except IndexError as e: - raise ValueError( - f'Event {event_id} not found in {qml_file}') from e - else: - qml_event = cat[0] - if len(cat) > 1: - logger.warning( - f'Found {len(cat)} events in {qml_file}. ' - 'Using the first one.') - return qml_event - - -def _get_evid_from_resource_id(resource_id): - """ - Get evid from resource_id. - - :param resource_id: resource_id string - :type resource_id: str - - :returns: evid string - :rtype: str - """ - evid = resource_id - if '/' in evid: - evid = resource_id.split('/')[-1] - if '?' in evid: - evid = resource_id.split('?')[-1] - if '&' in evid: - evid = evid.split('&')[0] - if '=' in evid: - evid = evid.split('=')[-1] - return evid - - -def _parse_qml_event( - qml_event, - parse_event_name_from_description=False, event_description_regex=None): - """ - Parse event metadata from a QuakeML event. - - :param qml_event: QuakeML event - :type qml_event: obspy.core.event.Event - :param parse_event_name_from_description: parse event name from description - :type parse_event_name_from_description: bool - :param event_description_regex: regex to extract event name - :type event_description_regex: str - - :return: SSPEvent object - :rtype: SSPEvent - """ - ssp_event = SSPEvent() - ssp_event.event_id = _get_evid_from_resource_id( - str(qml_event.resource_id.id)) - if parse_event_name_from_description: - try: - ssp_event.name = str(qml_event.event_descriptions[0].text) - except IndexError: - logger.warning( - 'QuakeML file does not contain an event description.') - if ssp_event.name and event_description_regex: - pattern = re.compile(event_description_regex) - match = pattern.search(ssp_event.name) - if match: - name = match.group() - # capitalize first letter - name = name[0].upper() + name[1:] - ssp_event.name = name - # See if there is a preferred origin... - origin = qml_event.preferred_origin() - # ...or just use the first one - if origin is None: - origin = qml_event.origins[0] - ssp_event.hypocenter.longitude.value_in_deg = origin.longitude - ssp_event.hypocenter.latitude.value_in_deg = origin.latitude - ssp_event.hypocenter.depth.value = origin.depth - ssp_event.hypocenter.depth.units = 'm' - ssp_event.hypocenter.origin_time = origin.time - return ssp_event, origin - - -def _parse_magnitude_from_qml_event(qml_event, ssp_event): - """ - Parse magnitude from a QuakeML event. - - :param qml_event: QuakeML event - :type qml_event: obspy.core.event.Event - :param ssp_event: SSPEvent object - :type ssp_event: SSPEvent - """ - mag = qml_event.preferred_magnitude() or qml_event.magnitudes[0] - ssp_event.magnitude.value = mag.mag - ssp_event.magnitude.mag_type = mag.magnitude_type - - -def _parse_scalar_moment_from_qml_event(qml_event, ssp_event): - """ - Parse scalar moment from a QuakeML event. - - :param qml_event: QuakeML event - :type qml_event: obspy.core.event.Event - :param ssp_event: SSPEvent object - :type ssp_event: SSPEvent - """ - fm = qml_event.preferred_focal_mechanism() or qml_event.focal_mechanisms[0] - ssp_event.scalar_moment.value = fm.moment_tensor.scalar_moment - ssp_event.scalar_moment.units = 'N-m' - - -def _parse_moment_tensor_from_qml_event(qml_event, ssp_event): - """ - Parse moment tensor from a QuakeML event. - - :param qml_event: QuakeML event - :type qml_event: obspy.core.event.Event - :param ssp_event: SSPEvent object - :type ssp_event: SSPEvent - """ - fm = qml_event.preferred_focal_mechanism() or qml_event.focal_mechanisms[0] - mt = fm.moment_tensor.tensor - ssp_event.moment_tensor.m_rr = mt.m_rr - ssp_event.moment_tensor.m_tt = mt.m_tt - ssp_event.moment_tensor.m_pp = mt.m_pp - ssp_event.moment_tensor.m_rt = mt.m_rt - ssp_event.moment_tensor.m_rp = mt.m_rp - ssp_event.moment_tensor.m_tp = mt.m_tp - ssp_event.moment_tensor.units = 'N-m' - - -def _parse_focal_mechanism_from_qml_event(qml_event, ssp_event): - """ - Parse focal mechanism from a QuakeML event. - - :param qml_event: QuakeML event - :type qml_event: obspy.core.event.Event - :param ssp_event: SSPEvent object - :type ssp_event: SSPEvent - """ - fm = qml_event.focal_mechanisms[0] - nodal_plane = fm.nodal_planes.nodal_plane_1 - ssp_event.focal_mechanism.strike = nodal_plane.strike - ssp_event.focal_mechanism.dip = nodal_plane.dip - ssp_event.focal_mechanism.rake = nodal_plane.rake - - -def _parse_picks_from_qml_event(ev, origin): - """ - Parse picks from a QuakeML event. - - :param ev: QuakeML event - :type ev: obspy.core.event.Event - :param origin: QuakeML origin object - :type origin: obspy.core.event.Origin - - :return: list of SSPPick objects - :rtype: list - """ - picks = [] - for pck in ev.picks: - pick = SSPPick() - pick.station = pck.waveform_id.station_code - pick.network = pck.waveform_id.network_code - pick.channel = pck.waveform_id.channel_code - if pck.waveform_id.location_code is not None: - pick.location = pck.waveform_id.location_code - else: - pick.location = '' - if pck.onset == 'emergent': - pick.flag = 'E' - elif pck.onset == 'impulsive': - pick.flag = 'I' - pick.phase = None - if pck.phase_hint: - pick.phase = pck.phase_hint[:1] - else: - # try to get the phase from the arrival object that uses this pick - pick_id = pck.resource_id.id - arrivals = [ - arr for arr in origin.arrivals if arr.pick_id.id == pick_id] - if arrivals: - pick.phase = arrivals[0].phase[:1] - if pick.phase is None: - # ignore picks with no phase hint - continue - if pck.polarity == 'negative': - pick.polarity = 'D' - elif pck.polarity == 'positive': - pick.polarity = 'U' - pick.time = pck.time - picks.append(pick) - return picks - - -def _parse_hypo71_hypocenter(hypo_file, _): - """ - Parse a hypo71 hypocenter file. - - :param hypo_file: path to hypo71 hypocenter file - :type hypo_file: str - :param _: unused (for consistency with other parsers) - :type _: None - - :return: a tuple of (SSPEvent, picks) - :rtype: tuple - """ - with open(hypo_file, encoding='ascii') as fp: - line = fp.readline() - # Skip the first line if it contain characters in the first 10 digits: - if any(c.isalpha() for c in line[:10]): - line = fp.readline() - timestr = line[:17] - # There are two possible formats for the timestring. We try both of them - try: - dt = datetime.strptime(timestr, '%y%m%d %H %M%S.%f') - except Exception: - try: - dt = datetime.strptime(timestr, '%y%m%d %H%M %S.%f') - except Exception as e: - raise ValueError('Cannot read origin time on first line.') from e - ssp_event = SSPEvent() - hypo = ssp_event.hypocenter - hypo.origin_time = UTCDateTime(dt) - lat = float(line[17:20]) - lat_deg = float(line[21:26]) - hypo.latitude.value_in_deg = lat + lat_deg / 60 - lon = float(line[26:30]) - lon_deg = float(line[31:36]) - hypo.longitude.value_in_deg = lon + lon_deg / 60 - hypo.depth.value = float(line[36:42]) - hypo.depth.units = 'km' - evid = os.path.basename(hypo_file) - evid = evid.replace('.phs', '').replace('.h', '').replace('.hyp', '') - ssp_event.event_id = evid - # empty picks list, for consistency with other parsers - picks = [] - return ssp_event, picks - - -def _parse_hypo2000_hypo_line(line): - """ - Parse a line from a hypo2000 hypocenter file. - - :param line: line from hypo2000 hypocenter file - :type line: str - - :return: SSPEvent object - :rtype: SSPEvent - """ - word = line.split() - ssp_event = SSPEvent() - hypo = ssp_event.hypocenter - timestr = ' '.join(word[:3]) - hypo.origin_time = UTCDateTime(timestr) - n = 3 - if word[n].isnumeric(): - # Check if word is integer - # In this case the format should be: degrees and minutes - latitude = float(word[n]) + float(word[n + 1]) / 60. - n += 2 - elif 'N' in word[n] or 'S' in word[n]: - # Check if there is N or S in the string - # In this case the format should be: degrees and minutes - _word = word[n].replace('N', ' ').replace('S', ' ').split() - latitude = float(_word[0]) + float(_word[1]) / 60. - n += 1 - else: - # Otherwise latitude should be in float format - try: - latitude = float(word[n]) - except Exception as e: - raise ValueError(f'cannot read latitude: {word[n]}') from e - n += 1 - hypo.latitude.value_in_deg = latitude - if word[n].isnumeric(): - # Check if word is integer - # In this case the format should be: degrees and minutes - longitude = float(word[n]) + float(word[n + 1]) / 60. - n += 2 - elif 'E' in word[n] or 'W' in word[n]: - # Check if there is E or W in the string - # In this case the format should be: degrees and minutes - _word = word[n].replace('E', ' ').replace('W', ' ').split() - longitude = float(_word[0]) + float(_word[1]) / 60. - n += 1 - else: - # Otherwise longitude should be in float format - try: - longitude = float(word[n]) - except Exception as e: - raise ValueError(f'cannot read longitude: {word[n]}') from e - n += 1 - hypo.longitude.value_in_deg = longitude - hypo.depth.value = float(word[n]) - # depth is in km, according to the hypo2000 manual - hypo.depth.units = 'km' - return ssp_event - - -def _parse_hypo2000_station_line(line, oldpick, origin_time): - """ - Parse a line from a hypo2000 station file. - - :param line: line from hypo2000 station file - :type line: str - :param oldpick: previous pick - :type oldpick: SSPPick - :param origin_time: origin time - :type origin_time: UTCDateTime - - :return: SSPPick object - :rtype: SSPPick - """ - if oldpick is not None: - oldstation = oldpick.station - oldnetwork = oldpick.network - oldchannel = oldpick.channel - else: - oldstation = '' - oldnetwork = '' - oldchannel = '' - pick = SSPPick() - station = line[1:5].strip() - pick.station = station or oldstation - oldstation = pick.station - network = line[6:8].strip() - pick.network = network or oldnetwork - oldnetwork = pick.network - channel = line[9:12].strip() - pick.channel = channel or oldchannel - oldchannel = pick.channel - # pick.flag = line[4:5] - pick.phase = line[31:34].strip() - if not pick.phase: - raise ValueError('Cannot read pick phase') - seconds = float(line[37:43]) - time = origin_time.replace(second=0, microsecond=0) - pick.time = time + seconds - return pick - - -def _parse_hypo2000_file(hypo_file, _): - """ - Parse a hypo2000 hypocenter file. - - :param hypo_file: path to hypo2000 hypocenter file - :type hypo_file: str - :param _: unused (for consistency with other parsers) - :type _: None - - :return: a tuple of (SSPEvent, picks) - :rtype: tuple - """ - ssp_event = None - picks = [] - hypo_line = False - station_line = False - oldpick = None - with open(hypo_file, encoding='ascii') as fp: - for n, line in enumerate(fp, start=1): - word = line.split() - if not word: - continue - # skip short lines, which are probably comments - if len(line) < 50: - continue - if hypo_line: - ssp_event = _parse_hypo2000_hypo_line(line) - evid = os.path.basename(hypo_file) - evid = evid.replace('.txt', '') - ssp_event.event_id = evid - if station_line and not ssp_event: - raise TypeError('Could not find hypocenter data.') - if station_line: - try: - pick = _parse_hypo2000_station_line( - line, oldpick, ssp_event.hypocenter.origin_time) - oldpick = pick - picks.append(pick) - except Exception as err: - logger.warning( - f'Error parsing line {n} in {hypo_file}: {err}') - continue - if word[0] == 'YEAR': - hypo_line = True - continue - hypo_line = False - if word[0] == 'STA': - station_line = True - if not ssp_event: - raise TypeError('Could not find hypocenter data.') - return ssp_event, picks - - -def _parse_source_spec_event_file(event_file, event_id=None): - """ - Parse a SourceSpec Event File, which is a YAML file. - - :param event_file: path to SourceSpec event file - :type event_file: str - :param evid: event id - :type evid: str - - :return: SSPEvent object - :rtype: SSPEvent - """ - try: - with open(event_file, encoding='utf-8') as fp: - events = yaml.safe_load(fp) - except Exception as e: - raise TypeError('Not a valid YAML file.') from e - # XML is valid YAML, but obviously not a SourceSpec Event File - try: - root = ET.fromstring(events) - except Exception: - root = None - if root: - raise TypeError( - 'The file is an XML file, not a YAML file. ' - 'Try reading it with the "-q" option (QuakeML format).') - # raise TypeError if events is not a list - if not isinstance(events, list): - raise TypeError( - 'This is a valid YAML file, but it does not contain the key: ' - '"- event_id:", preceded by a dash (-).') - # make sure that all the event_id are a string - for ev in events: - ev['event_id'] = str(ev['event_id']) - if event_id is not None: - _events = [ev for ev in events if ev.get('event_id') == event_id] - try: - event = _events[0] - except IndexError as e: - raise ValueError( - f'Event {event_id} not found in {event_file}') from e - else: - event = events[0] - if len(events) > 1: - logger.warning( - f'Found {len(events)} events in {event_file}. ' - 'Using the first one.') - try: - with warnings.catch_warnings(record=True) as w: - ssp_event = SSPEvent(event) - if len(w) > 0: - logger.warning(f'Warnings while parsing {event_file}:') - for warning in w: - logger.warning(warning.message) - except Exception as e: - raise TypeError( - 'This is a valid YAML file, but the following error occurred: ' - f'{e}.' - ) from e - # empty picks list, for consistency with other parsers - picks = [] - return ssp_event, picks - - -# pylint: disable=inconsistent-return-statements -def parse_hypo_file(hypo_file, event_id=None): - """ - Parse a SourceSpec Event File, hypo71 or hypo2000 hypocenter file. - - :param hypo_file: Path to the hypocenter file. - :type hypo_file: str - :param event_id: Event ID. - :type event_id: str - - :return: A tuple of (SSPEvent, picks, format). - :rtype: tuple - """ - if not os.path.exists(hypo_file): - logger.error(f'{hypo_file}: No such file or directory') - ssp_exit(1) - err_msgs = [] - parsers = { - 'ssp_event_file': _parse_source_spec_event_file, - 'hypo71': _parse_hypo71_hypocenter, - 'hypo2000': _parse_hypo2000_file, - } - format_strings = { - 'ssp_event_file': 'SourceSpec Event File', - 'hypo71': 'hypo71 hypocenter file', - 'hypo2000': 'hypo2000 hypocenter file', - } - for file_format, parser in parsers.items(): - try: - ssp_event, picks = parser(hypo_file, event_id) - return ssp_event, picks, file_format - except Exception as err: - format_str = format_strings[file_format] - msg = f'{hypo_file}: Not a {format_str}' - err_msgs.append(msg) - msg = f'Parsing error: {err}' - err_msgs.append(msg) - # If we arrive here, the file was not recognized as valid - for msg in err_msgs: - logger.error(msg) - ssp_exit(1) - - -def _is_hypo71_picks(pick_file): - """ - Check if a file is a hypo71 phase file. - - :param pick_file: path to hypo71 phase file - :type pick_file: str - - :raises TypeError: if the file is not a hypo71 phase file - """ - with open(pick_file, encoding='ascii') as fp: - for line in fp: - # remove newline - line = line.replace('\n', '') - # skip separator and empty lines - stripped_line = line.strip() - if stripped_line in ['10', '']: - continue - # Check if it is a pick line - # 6th character should be alpha (phase name: P or S) - # other character should be digits (date/time) - if not (line[5].isalpha() and - line[9].isdigit() and - line[20].isdigit()): - raise TypeError(f'{pick_file}: Not a hypo71 phase file') - - -def _correct_station_name(station): - """ - Correct station name, based on a traceid map. - - :param station: station name - :type station: str - - :return: corrected station name - :rtype: str - """ - if config.TRACEID_MAP is None: - return station - # get all the keys containing station name in it - keys = [key for key in config.TRACEID_MAP if station == key.split('.')[1]] - # then take just the first one - try: - key = keys[0] - except IndexError: - return station - traceid = config.TRACEID_MAP[key] - return traceid.split('.')[1] - - -def parse_hypo71_picks(): - """ - Parse hypo71 picks file - - :return: list of SSPPick objects - :rtype: list - """ - picks = [] - pick_file = config.options.pick_file - if pick_file is None: - return picks - try: - _is_hypo71_picks(pick_file) - except Exception as err: - logger.error(err) - ssp_exit(1) - with open(pick_file, encoding='ascii') as fp: - for line in fp: - # remove newline - line = line.replace('\n', '') - # skip separator and empty lines - stripped_line = line.strip() - if stripped_line in ['10', '']: - continue - # Check if it is a pick line - # 6th character should be alpha (phase name: P or S) - # other character should be digits (date/time) - if not (line[5].isalpha() and - line[9].isdigit() and - line[20].isdigit()): - continue - pick = SSPPick() - pick.station = line[:4].strip() - pick.station = _correct_station_name(pick.station) - pick.flag = line[4:5] - pick.phase = line[5:6] - pick.polarity = line[6:7] - try: - pick.quality = int(line[7:8]) - except ValueError: - # If we cannot read pick quality, - # we give the pick the lowest quality - pick.quality = 4 - timestr = line[9:24] - dt = datetime.strptime(timestr, '%y%m%d%H%M%S.%f') - pick.time = UTCDateTime(dt) - picks.append(pick) - try: - stime = line[31:36] - except Exception: - continue - if stime.strip() == '': - continue - pick2 = SSPPick() - pick2.station = pick.station - pick2.flag = line[36:37] - pick2.phase = line[37:38] - pick2.polarity = line[38:39] - try: - pick2.quality = int(line[39:40]) - except ValueError: - # If we cannot read pick quality, - # we give the pick the lowest quality - pick2.quality = 4 - # pick2.time has the same date, hour and minutes - # than pick.time - # We therefore make a copy of pick.time, - # and set seconds and microseconds to 0 - pick2.time = pick.time.replace(second=0, microsecond=0) - # finally we add stime - pick2.time += float(stime) - picks.append(pick2) - return picks From 7a7e63c819a50b981329ca4a1e9642cb1b71a7aa Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 16 Jul 2024 18:22:43 +0200 Subject: [PATCH 39/73] Split event parsers into separate files, depending on format --- sourcespec2/input/event_and_picks.py | 49 ++- sourcespec2/input/event_parsers/__init__.py | 17 + sourcespec2/input/event_parsers/asdf.py | 47 +++ sourcespec2/input/event_parsers/hypo2000.py | 172 ++++++++++ sourcespec2/input/event_parsers/hypo71.py | 191 +++++++++++ .../input/event_parsers/obspy_catalog.py | 306 ++++++++++++++++++ sourcespec2/input/event_parsers/quakeml.py | 36 +++ .../input/event_parsers/source_spec_event.py | 84 +++++ 8 files changed, 899 insertions(+), 3 deletions(-) create mode 100644 sourcespec2/input/event_parsers/__init__.py create mode 100644 sourcespec2/input/event_parsers/asdf.py create mode 100644 sourcespec2/input/event_parsers/hypo2000.py create mode 100644 sourcespec2/input/event_parsers/hypo71.py create mode 100644 sourcespec2/input/event_parsers/obspy_catalog.py create mode 100644 sourcespec2/input/event_parsers/quakeml.py create mode 100644 sourcespec2/input/event_parsers/source_spec_event.py diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index d29c18a6..6c073716 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -18,10 +18,53 @@ import sys import logging from ..setup import config, ssp_exit -from .event_metadata import parse_qml, parse_hypo_file, parse_hypo71_picks +from .event_parsers import ( + parse_hypo71_hypocenter, parse_hypo71_picks, parse_hypo2000_file, + parse_qml_file, parse_source_spec_event_file +) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) +# pylint: disable=inconsistent-return-statements +def _parse_hypo_file(hypo_file, event_id=None): + """ + Parse a SourceSpec Event File, hypo71 or hypo2000 hypocenter file. + + :param hypo_file: Path to the hypocenter file. + :type hypo_file: str + :param event_id: Event ID. + :type event_id: str + + :return: A tuple of (SSPEvent, picks, format). + :rtype: tuple + """ + err_msgs = [] + parsers = { + 'ssp_event_file': parse_source_spec_event_file, + 'hypo71': parse_hypo71_hypocenter, + 'hypo2000': parse_hypo2000_file, + } + format_strings = { + 'ssp_event_file': 'SourceSpec Event File', + 'hypo71': 'hypo71 hypocenter file', + 'hypo2000': 'hypo2000 hypocenter file', + } + for file_format, parser in parsers.items(): + try: + ssp_event, picks = parser(hypo_file, event_id) + return ssp_event, picks, file_format + except Exception as err: + format_str = format_strings[file_format] + msg = f'{hypo_file}: Not a {format_str}' + err_msgs.append(msg) + msg = f'Parsing error: {err}' + err_msgs.append(msg) + # If we arrive here, the file was not recognized as valid + for msg in err_msgs: + logger.error(msg) + ssp_exit(1) + + def _log_event_info(ssp_event): """ Log event information. @@ -50,7 +93,7 @@ def read_event_and_picks(trace1=None): ssp_event = None # parse hypocenter file if config.options.hypo_file is not None: - ssp_event, picks, file_format = parse_hypo_file( + ssp_event, picks, file_format = _parse_hypo_file( config.options.hypo_file, config.options.evid) config.hypo_file_format = file_format # parse pick file @@ -58,7 +101,7 @@ def read_event_and_picks(trace1=None): picks = parse_hypo71_picks() # parse QML file if config.options.qml_file is not None: - ssp_event, picks = parse_qml() + ssp_event, picks = parse_qml_file() if ssp_event is not None: _log_event_info(ssp_event) diff --git a/sourcespec2/input/event_parsers/__init__.py b/sourcespec2/input/event_parsers/__init__.py new file mode 100644 index 00000000..c50957e4 --- /dev/null +++ b/sourcespec2/input/event_parsers/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Event and phase picks parsers for SourceSpec. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +from .asdf import parse_asdf_event_picks # noqa +from .hypo71 import parse_hypo71_hypocenter, parse_hypo71_picks # noqa +from .hypo2000 import parse_hypo2000_file # noqa +from .obspy_catalog import parse_obspy_catalog # noqa +from .quakeml import parse_qml_file # noqa +from .source_spec_event import parse_source_spec_event_file # noqa diff --git a/sourcespec2/input/event_parsers/asdf.py b/sourcespec2/input/event_parsers/asdf.py new file mode 100644 index 00000000..34cec752 --- /dev/null +++ b/sourcespec2/input/event_parsers/asdf.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Parse event metadata and picks from ASDF file. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import logging +from ...setup import ssp_exit +from .obspy_catalog import parse_obspy_catalog +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def parse_asdf_event_picks(asdf_file, event_id=None): + """ + Parse event metadata and picks from ASDF file + + :param asdf_file: full path to ASDF file + :type asdf_file: str + :param event_id: event id + :type event_id: str + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + """ + try: + # pylint: disable=import-outside-toplevel + # pyasdf is not a hard dependency, so we import it here + # and check for ImportError + import pyasdf + except ImportError: + logger.error( + 'Error importing pyasdf. ' + 'See https://seismicdata.github.io/pyasdf/ for installation ' + 'instructions.' + ) + ssp_exit(1) + try: + obspy_catalog = pyasdf.ASDFDataSet(asdf_file, mode='r').events + return parse_obspy_catalog(obspy_catalog, event_id, asdf_file) + except Exception as err: + logger.error(err) + ssp_exit(1) diff --git a/sourcespec2/input/event_parsers/hypo2000.py b/sourcespec2/input/event_parsers/hypo2000.py new file mode 100644 index 00000000..bb7cdc88 --- /dev/null +++ b/sourcespec2/input/event_parsers/hypo2000.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read event and phase picks from an hypo2000 file. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import logging +from obspy import UTCDateTime +from ...ssp_event import SSPEvent +from ...ssp_pick import SSPPick +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _parse_hypo2000_hypo_line(line): + """ + Parse a line from a hypo2000 hypocenter file. + + :param line: line from hypo2000 hypocenter file + :type line: str + + :return: SSPEvent object + :rtype: SSPEvent + """ + word = line.split() + ssp_event = SSPEvent() + hypo = ssp_event.hypocenter + timestr = ' '.join(word[:3]) + hypo.origin_time = UTCDateTime(timestr) + n = 3 + if word[n].isnumeric(): + # Check if word is integer + # In this case the format should be: degrees and minutes + latitude = float(word[n]) + float(word[n + 1]) / 60. + n += 2 + elif 'N' in word[n] or 'S' in word[n]: + # Check if there is N or S in the string + # In this case the format should be: degrees and minutes + _word = word[n].replace('N', ' ').replace('S', ' ').split() + latitude = float(_word[0]) + float(_word[1]) / 60. + n += 1 + else: + # Otherwise latitude should be in float format + try: + latitude = float(word[n]) + except Exception as e: + raise ValueError(f'cannot read latitude: {word[n]}') from e + n += 1 + hypo.latitude.value_in_deg = latitude + if word[n].isnumeric(): + # Check if word is integer + # In this case the format should be: degrees and minutes + longitude = float(word[n]) + float(word[n + 1]) / 60. + n += 2 + elif 'E' in word[n] or 'W' in word[n]: + # Check if there is E or W in the string + # In this case the format should be: degrees and minutes + _word = word[n].replace('E', ' ').replace('W', ' ').split() + longitude = float(_word[0]) + float(_word[1]) / 60. + n += 1 + else: + # Otherwise longitude should be in float format + try: + longitude = float(word[n]) + except Exception as e: + raise ValueError(f'cannot read longitude: {word[n]}') from e + n += 1 + hypo.longitude.value_in_deg = longitude + hypo.depth.value = float(word[n]) + # depth is in km, according to the hypo2000 manual + hypo.depth.units = 'km' + return ssp_event + + +def _parse_hypo2000_station_line(line, oldpick, origin_time): + """ + Parse a line from a hypo2000 station file. + + :param line: line from hypo2000 station file + :type line: str + :param oldpick: previous pick + :type oldpick: SSPPick + :param origin_time: origin time + :type origin_time: UTCDateTime + + :return: SSPPick object + :rtype: SSPPick + """ + if oldpick is not None: + oldstation = oldpick.station + oldnetwork = oldpick.network + oldchannel = oldpick.channel + else: + oldstation = '' + oldnetwork = '' + oldchannel = '' + pick = SSPPick() + station = line[1:5].strip() + pick.station = station or oldstation + oldstation = pick.station + network = line[6:8].strip() + pick.network = network or oldnetwork + oldnetwork = pick.network + channel = line[9:12].strip() + pick.channel = channel or oldchannel + oldchannel = pick.channel + # pick.flag = line[4:5] + pick.phase = line[31:34].strip() + if not pick.phase: + raise ValueError('Cannot read pick phase') + seconds = float(line[37:43]) + time = origin_time.replace(second=0, microsecond=0) + pick.time = time + seconds + return pick + + +def parse_hypo2000_file(hypo_file, _): + """ + Parse a hypo2000 hypocenter file. + + :param hypo_file: path to hypo2000 hypocenter file + :type hypo_file: str + :param _: unused (for consistency with other parsers) + :type _: None + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + """ + ssp_event = None + picks = [] + hypo_line = False + station_line = False + oldpick = None + with open(hypo_file, encoding='ascii') as fp: + for n, line in enumerate(fp, start=1): + word = line.split() + if not word: + continue + # skip short lines, which are probably comments + if len(line) < 50: + continue + if hypo_line: + ssp_event = _parse_hypo2000_hypo_line(line) + evid = os.path.basename(hypo_file) + evid = evid.replace('.txt', '') + ssp_event.event_id = evid + if station_line and not ssp_event: + raise TypeError('Could not find hypocenter data.') + if station_line: + try: + pick = _parse_hypo2000_station_line( + line, oldpick, ssp_event.hypocenter.origin_time) + oldpick = pick + picks.append(pick) + except Exception as err: + logger.warning( + f'Error parsing line {n} in {hypo_file}: {err}') + continue + if word[0] == 'YEAR': + hypo_line = True + continue + hypo_line = False + if word[0] == 'STA': + station_line = True + if not ssp_event: + raise TypeError('Could not find hypocenter data.') + return ssp_event, picks diff --git a/sourcespec2/input/event_parsers/hypo71.py b/sourcespec2/input/event_parsers/hypo71.py new file mode 100644 index 00000000..e023cadf --- /dev/null +++ b/sourcespec2/input/event_parsers/hypo71.py @@ -0,0 +1,191 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Parse event metadata and picks in hypo71 format. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import logging +from datetime import datetime +from obspy import UTCDateTime +from ...setup import config, ssp_exit +from ...ssp_event import SSPEvent +from ...ssp_pick import SSPPick +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _is_hypo71_picks(pick_file): + """ + Check if a file is a hypo71 phase file. + + :param pick_file: path to hypo71 phase file + :type pick_file: str + + :raises TypeError: if the file is not a hypo71 phase file + """ + with open(pick_file, encoding='ascii') as fp: + for line in fp: + # remove newline + line = line.replace('\n', '') + # skip separator and empty lines + stripped_line = line.strip() + if stripped_line in ['10', '']: + continue + # Check if it is a pick line + # 6th character should be alpha (phase name: P or S) + # other character should be digits (date/time) + if not (line[5].isalpha() and + line[9].isdigit() and + line[20].isdigit()): + raise TypeError(f'{pick_file}: Not a hypo71 phase file') + + +def _correct_station_name(station): + """ + Correct station name, based on a traceid map. + + :param station: station name + :type station: str + + :return: corrected station name + :rtype: str + """ + if config.TRACEID_MAP is None: + return station + # get all the keys containing station name in it + keys = [key for key in config.TRACEID_MAP if station == key.split('.')[1]] + # then take just the first one + try: + key = keys[0] + except IndexError: + return station + traceid = config.TRACEID_MAP[key] + return traceid.split('.')[1] + + +def parse_hypo71_picks(): + """ + Parse hypo71 picks file + + :return: list of SSPPick objects + :rtype: list + """ + picks = [] + pick_file = config.options.pick_file + if pick_file is None: + return picks + try: + _is_hypo71_picks(pick_file) + except Exception as err: + logger.error(err) + ssp_exit(1) + with open(pick_file, encoding='ascii') as fp: + for line in fp: + # remove newline + line = line.replace('\n', '') + # skip separator and empty lines + stripped_line = line.strip() + if stripped_line in ['10', '']: + continue + # Check if it is a pick line + # 6th character should be alpha (phase name: P or S) + # other character should be digits (date/time) + if not (line[5].isalpha() and + line[9].isdigit() and + line[20].isdigit()): + continue + pick = SSPPick() + pick.station = line[:4].strip() + pick.station = _correct_station_name(pick.station) + pick.flag = line[4:5] + pick.phase = line[5:6] + pick.polarity = line[6:7] + try: + pick.quality = int(line[7:8]) + except ValueError: + # If we cannot read pick quality, + # we give the pick the lowest quality + pick.quality = 4 + timestr = line[9:24] + dt = datetime.strptime(timestr, '%y%m%d%H%M%S.%f') + pick.time = UTCDateTime(dt) + picks.append(pick) + try: + stime = line[31:36] + except Exception: + continue + if stime.strip() == '': + continue + pick2 = SSPPick() + pick2.station = pick.station + pick2.flag = line[36:37] + pick2.phase = line[37:38] + pick2.polarity = line[38:39] + try: + pick2.quality = int(line[39:40]) + except ValueError: + # If we cannot read pick quality, + # we give the pick the lowest quality + pick2.quality = 4 + # pick2.time has the same date, hour and minutes + # than pick.time + # We therefore make a copy of pick.time, + # and set seconds and microseconds to 0 + pick2.time = pick.time.replace(second=0, microsecond=0) + # finally we add stime + pick2.time += float(stime) + picks.append(pick2) + return picks + + +def parse_hypo71_hypocenter(hypo_file, _): + """ + Parse a hypo71 hypocenter file. + + :param hypo_file: path to hypo71 hypocenter file + :type hypo_file: str + :param _: unused (for consistency with other parsers) + :type _: None + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + + .. note:: + The returned picks list is empty, for consistency with other parsers. + """ + with open(hypo_file, encoding='ascii') as fp: + line = fp.readline() + # Skip the first line if it contain characters in the first 10 digits: + if any(c.isalpha() for c in line[:10]): + line = fp.readline() + timestr = line[:17] + # There are two possible formats for the timestring. We try both of them + try: + dt = datetime.strptime(timestr, '%y%m%d %H %M%S.%f') + except Exception: + try: + dt = datetime.strptime(timestr, '%y%m%d %H%M %S.%f') + except Exception as e: + raise ValueError('Cannot read origin time on first line.') from e + ssp_event = SSPEvent() + hypo = ssp_event.hypocenter + hypo.origin_time = UTCDateTime(dt) + lat = float(line[17:20]) + lat_deg = float(line[21:26]) + hypo.latitude.value_in_deg = lat + lat_deg / 60 + lon = float(line[26:30]) + lon_deg = float(line[31:36]) + hypo.longitude.value_in_deg = lon + lon_deg / 60 + hypo.depth.value = float(line[36:42]) + hypo.depth.units = 'km' + evid = os.path.basename(hypo_file) + evid = evid.replace('.phs', '').replace('.h', '').replace('.hyp', '') + ssp_event.event_id = evid + # empty picks list, for consistency with other parsers + picks = [] + return ssp_event, picks diff --git a/sourcespec2/input/event_parsers/obspy_catalog.py b/sourcespec2/input/event_parsers/obspy_catalog.py new file mode 100644 index 00000000..b8b7358f --- /dev/null +++ b/sourcespec2/input/event_parsers/obspy_catalog.py @@ -0,0 +1,306 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Parse event metadata and picks from an ObsPy catalog object. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import contextlib +import logging +import re +from ...setup import config, ssp_exit +from ...ssp_event import SSPEvent +from ...ssp_pick import SSPPick +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _get_evid_from_resource_id(resource_id): + """ + Get evid from resource_id. + + :param resource_id: resource_id string + :type resource_id: str + + :returns: evid string + :rtype: str + """ + evid = resource_id + if '/' in evid: + evid = resource_id.split('/')[-1] + if '?' in evid: + evid = resource_id.split('?')[-1] + if '&' in evid: + evid = evid.split('&')[0] + if '=' in evid: + evid = evid.split('=')[-1] + return evid + + +def _parse_event_metadata(obspy_event): + """ + Parse event metadata from an ObsPy event object. + + :param obspy_event: ObsPy event object + :type obspy_event: obspy.core.event.Event + + :return: a tuple of SSPEvent object and ObsPy origin object + :rtype: (SSPEvent, obspy.core.event.Origin) + """ + # No need to parse event name from the ObsPy event if event name is given + # in the command line + if config.options.evname is None: + parse_event_name_from_description = config.qml_event_description + event_description_regex = config.qml_event_description_regex + else: + parse_event_name_from_description = False + event_description_regex = None + ssp_event = SSPEvent() + ssp_event.event_id = _get_evid_from_resource_id( + str(obspy_event.resource_id.id)) + if parse_event_name_from_description: + try: + ssp_event.name = str(obspy_event.event_descriptions[0].text) + except IndexError: + logger.warning( + 'The event does not contain a description. Cannot parse ' + 'event name from description.' + ) + if ssp_event.name and event_description_regex: + pattern = re.compile(event_description_regex) + match = pattern.search(ssp_event.name) + if match: + name = match.group() + # capitalize first letter + name = name[0].upper() + name[1:] + ssp_event.name = name + # See if there is a preferred origin... + obspy_origin = obspy_event.preferred_origin() + # ...or just use the first one + if obspy_origin is None: + obspy_origin = obspy_event.origins[0] + ssp_event.hypocenter.longitude.value_in_deg = obspy_origin.longitude + ssp_event.hypocenter.latitude.value_in_deg = obspy_origin.latitude + ssp_event.hypocenter.depth.value = obspy_origin.depth + ssp_event.hypocenter.depth.units = 'm' + ssp_event.hypocenter.origin_time = obspy_origin.time + return ssp_event, obspy_origin + + +def _parse_magnitude_from_obspy_event(obspy_event, ssp_event): + """ + Parse magnitude from an ObsPy event. + + :param obspy_event: ObsPy event + :type obspy_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ + mag = obspy_event.preferred_magnitude() or obspy_event.magnitudes[0] + ssp_event.magnitude.value = mag.mag + ssp_event.magnitude.mag_type = mag.magnitude_type + + +def _parse_scalar_moment_from_obspy_event(obspy_event, ssp_event): + """ + Parse scalar moment from an ObsPy event. + + :param obspy_event: ObsPy event + :type obspy_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ + fm = obspy_event.preferred_focal_mechanism()\ + or obspy_event.focal_mechanisms[0] + ssp_event.scalar_moment.value = fm.moment_tensor.scalar_moment + ssp_event.scalar_moment.units = 'N-m' + + +def _parse_moment_tensor_from_obspy_event(obspy_event, ssp_event): + """ + Parse moment tensor from an ObsPy event. + + :param obspy_event: ObsPy event + :type obspy_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ + fm = obspy_event.preferred_focal_mechanism()\ + or obspy_event.focal_mechanisms[0] + mt = fm.moment_tensor.tensor + ssp_event.moment_tensor.m_rr = mt.m_rr + ssp_event.moment_tensor.m_tt = mt.m_tt + ssp_event.moment_tensor.m_pp = mt.m_pp + ssp_event.moment_tensor.m_rt = mt.m_rt + ssp_event.moment_tensor.m_rp = mt.m_rp + ssp_event.moment_tensor.m_tp = mt.m_tp + ssp_event.moment_tensor.units = 'N-m' + + +def _parse_focal_mechanism_from_obspy_event(obspy_event, ssp_event): + """ + Parse focal mechanism from an ObsPy event. + + :param obspy_event: ObsPy event + :type obspy_event: obspy.core.event.Event + :param ssp_event: SSPEvent object + :type ssp_event: SSPEvent + """ + fm = obspy_event.focal_mechanisms[0] + nodal_plane = fm.nodal_planes.nodal_plane_1 + ssp_event.focal_mechanism.strike = nodal_plane.strike + ssp_event.focal_mechanism.dip = nodal_plane.dip + ssp_event.focal_mechanism.rake = nodal_plane.rake + + +def _parse_picks_from_obspy_event(obspy_event, obspy_origin): + """ + Parse picks from an ObsPy event. + + :param obspy_event: ObsPy event + :type obspy_event: obspy.core.event.Event + :param obspy_origin: ObsPy origin object + :type obspy_origin: obspy.core.event.Origin + + :return: list of SSPPick objects + :rtype: list + """ + picks = [] + for pck in obspy_event.picks: + pick = SSPPick() + pick.station = pck.waveform_id.station_code + pick.network = pck.waveform_id.network_code + pick.channel = pck.waveform_id.channel_code + if pck.waveform_id.location_code is not None: + pick.location = pck.waveform_id.location_code + else: + pick.location = '' + if pck.onset == 'emergent': + pick.flag = 'E' + elif pck.onset == 'impulsive': + pick.flag = 'I' + pick.phase = None + if pck.phase_hint: + pick.phase = pck.phase_hint[:1] + else: + # try to get the phase from the arrival object that uses this pick + pick_id = pck.resource_id.id + arrivals = [ + arr for arr in obspy_origin.arrivals + if arr.pick_id.id == pick_id + ] + if arrivals: + pick.phase = arrivals[0].phase[:1] + if pick.phase is None: + # ignore picks with no phase hint + continue + if pck.polarity == 'negative': + pick.polarity = 'D' + elif pck.polarity == 'positive': + pick.polarity = 'U' + pick.time = pck.time + picks.append(pick) + return picks + + +def _get_event_from_obspy_catalog( + obspy_catalog, event_id=None, file_name=''): + """ + Get an event from an ObsPy catalog object. + + :param obspy_catalog: ObsPy catalog object + :type obspy_catalog: instance of :class:`obspy.core.event.Catalog` + :param event_id: event id + :type event_id: str + :param file_name: name of the file containing the catalog + :type file_name: str + + :return: ObsPy event object + :rtype: obspy.core.event.Event + """ + if event_id is not None: + _obspy_events = [ + ev for ev in obspy_catalog if event_id in str(ev.resource_id) + ] + try: + obspy_event = _obspy_events[0] + except IndexError as e: + raise ValueError( + f'Event {event_id} not found in {file_name}') from e + else: + obspy_event = obspy_catalog[0] + if len(obspy_catalog) > 1: + logger.warning( + f'Found {len(obspy_catalog)} events in {file_name}. ' + 'Using the first one.') + return obspy_event + + +def _parse_obspy_event(obspy_event): + """ + Parse event metadata and picks from an ObsPy event object. + + :param obspy_event: ObsPy event object to parse + :type obspy_event: instance of :class:`obspy.core.event.Event` + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + """ + try: + ssp_event, obspy_origin = _parse_event_metadata(obspy_event) + picks = _parse_picks_from_obspy_event(obspy_event, obspy_origin) + except Exception as err: + logger.error(err) + ssp_exit(1) + log_messages = [] + with contextlib.suppress(Exception): + _parse_moment_tensor_from_obspy_event(obspy_event, ssp_event) + # Compute focal mechanism, scalar moment and magnitude. + # They will be overwritten later on, if they are found in the + # ObsPy event. + ssp_event.focal_mechanism.from_moment_tensor(ssp_event.moment_tensor) + ssp_event.scalar_moment.from_moment_tensor(ssp_event.moment_tensor) + ssp_event.magnitude.from_scalar_moment(ssp_event.scalar_moment) + with contextlib.suppress(Exception): + _parse_scalar_moment_from_obspy_event(obspy_event, ssp_event) + # Compute magnitude from scalar moment. It will be overwritten later + # on, if it is found in the ObsPy event. + ssp_event.magnitude.from_scalar_moment(ssp_event.scalar_moment) + with contextlib.suppress(Exception): + _parse_magnitude_from_obspy_event(obspy_event, ssp_event) + with contextlib.suppress(Exception): + _parse_focal_mechanism_from_obspy_event(obspy_event, ssp_event) + for msg in log_messages: + logger.info(msg) + return ssp_event, picks + + +def parse_obspy_catalog(obspy_catalog, event_id=None, file_name=''): + """ + Parse event metadata and picks from an ObsPy catalog object. + + :param obspy_catalog: ObsPy catalog object to parse + :type obspy_catalog: instance of :class:`obspy.core.event.Catalog` + :param event_id: event id to extract from the catalog. If None, the value + of config.options.evid is used. If this is also None, + the first event in the catalog is used. + :type event_id: str + :param file_name: name of the file containing the catalog + :type file_name: str + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + """ + event_id = event_id or config.options.evid + try: + obspy_event = _get_event_from_obspy_catalog( + obspy_catalog, event_id, file_name) + except Exception as err: + logger.error(err) + ssp_exit(1) + else: + ssp_event, picks = _parse_obspy_event(obspy_event) + return ssp_event, picks diff --git a/sourcespec2/input/event_parsers/quakeml.py b/sourcespec2/input/event_parsers/quakeml.py new file mode 100644 index 00000000..d9244d57 --- /dev/null +++ b/sourcespec2/input/event_parsers/quakeml.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Parse event metadata and picks from a QuakeML file. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import logging +from obspy import read_events +from ...setup import config, ssp_exit +from .obspy_catalog import parse_obspy_catalog +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def parse_qml_file(): + """ + Parse event metadata and picks from a QuakeML file. + + :return: a tuple of (SSPEvent, picks) + :rtype: tuple + """ + qml_file = config.options.qml_file + if qml_file is None: + ssp_event = None + picks = [] + return ssp_event, picks + try: + obspy_catalog = read_events(qml_file) + return parse_obspy_catalog(obspy_catalog, file_name=qml_file) + except Exception as err: + logger.error(err) + ssp_exit(1) diff --git a/sourcespec2/input/event_parsers/source_spec_event.py b/sourcespec2/input/event_parsers/source_spec_event.py new file mode 100644 index 00000000..3a730f2c --- /dev/null +++ b/sourcespec2/input/event_parsers/source_spec_event.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Parse event metadata from a SourceSpec event file. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import warnings +import logging +import xml.etree.ElementTree as ET +import yaml +from ...ssp_event import SSPEvent +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def parse_source_spec_event_file(event_file, event_id=None): + """ + Parse a SourceSpec Event File, which is a YAML file. + + :param event_file: path to SourceSpec event file + :type event_file: str + :param evid: event id + :type evid: str + + :return: SSPEvent object + :rtype: SSPEvent + + .. note:: + The returned picks list is empty, for consistency with other parsers. + """ + try: + with open(event_file, encoding='utf-8') as fp: + events = yaml.safe_load(fp) + except Exception as e: + raise TypeError('Not a valid YAML file.') from e + # XML is valid YAML, but obviously not a SourceSpec Event File + try: + root = ET.fromstring(events) + except Exception: + root = None + if root: + raise TypeError( + 'The file is an XML file, not a YAML file. ' + 'Try reading it with the "-q" option (QuakeML format).') + # raise TypeError if events is not a list + if not isinstance(events, list): + raise TypeError( + 'This is a valid YAML file, but it does not contain the key: ' + '"- event_id:", preceded by a dash (-).') + # make sure that all the event_id are a string + for ev in events: + ev['event_id'] = str(ev['event_id']) + if event_id is not None: + _events = [ev for ev in events if ev.get('event_id') == event_id] + try: + event = _events[0] + except IndexError as e: + raise ValueError( + f'Event {event_id} not found in {event_file}') from e + else: + event = events[0] + if len(events) > 1: + logger.warning( + f'Found {len(events)} events in {event_file}. ' + 'Using the first one.') + try: + with warnings.catch_warnings(record=True) as w: + ssp_event = SSPEvent(event) + if len(w) > 0: + logger.warning(f'Warnings while parsing {event_file}:') + for warning in w: + logger.warning(warning.message) + except Exception as e: + raise TypeError( + 'This is a valid YAML file, but the following error occurred: ' + f'{e}.' + ) from e + # empty picks list, for consistency with other parsers + picks = [] + return ssp_event, picks From f89f677804db39ea468031acb1444f35bf433b87 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 17 Jul 2024 10:31:01 +0200 Subject: [PATCH 40/73] Fix regression on reading station metadata from SAC headers --- sourcespec2/input/augment_traces.py | 4 +++- sourcespec2/input/station_metadata.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index 0a573a94..e5480fb2 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -393,7 +393,9 @@ def augment_traces(st, inventory, ssp_event, picks): :param st: Traces to be augmented :type st: :class:`obspy.core.stream.Stream` - :param inventory: Station metadata + :param inventory: Station metadata. If it is None or an empty Inventory + object, the code will try to read the station metadata from the + trace headers (only SAC format is supported). :type inventory: :class:`obspy.core.inventory.Inventory` :param ssp_event: Event information :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` diff --git a/sourcespec2/input/station_metadata.py b/sourcespec2/input/station_metadata.py index bd9bf442..7375f237 100644 --- a/sourcespec2/input/station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -188,8 +188,16 @@ def read_station_metadata(): :return: inventory :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + + .. note:: + - Station metadata can be in StationXML, dataless SEED, SEED RESP, + PAZ (SAC polezero format) format. + - An empty inventory is returned if the station metadata path is + not set in the configuration or if no valid files are found """ inventory = Inventory() + if not config.station_metadata: + return inventory logger.info('Reading station metadata...') metadata_path = config.station_metadata if os.path.isdir(metadata_path): From 9a5ee164a4bc871935ea9fea68e3527d7b2fc751 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 17 Jul 2024 10:31:22 +0200 Subject: [PATCH 41/73] Fix regression in reading event information from SAC header --- sourcespec2/input/augment_traces.py | 12 +++--------- sourcespec2/input/sac_header.py | 14 ++++++++++---- sourcespec2/input/traces.py | 25 ++++++++++++++++++++++--- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index e5480fb2..a8e3263f 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -24,7 +24,7 @@ from .sac_header import ( compute_sensitivity_from_SAC, get_instrument_from_SAC, get_station_coordinates_from_SAC, - get_event_from_SAC, get_picks_from_SAC) + get_picks_from_SAC) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -293,21 +293,15 @@ def _add_coords(trace): _add_coords.skipped = [] # noqa -def _add_event(trace, ssp_event=None): +def _add_event(trace, ssp_event): """ Add ssp_event object to trace. :param trace: ObsPy Trace object :type trace: :class:`obspy.core.trace.Trace` - :param ssp_event: SSPEvent object (default: None) + :param ssp_event: SSPEvent object :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` """ - if ssp_event is None: - # Try to get hypocenter information from the SAC header - try: - ssp_event = get_event_from_SAC(trace) - except Exception: - return trace.stats.event = ssp_event diff --git a/sourcespec2/input/sac_header.py b/sourcespec2/input/sac_header.py index ec26c4ad..ef170643 100644 --- a/sourcespec2/input/sac_header.py +++ b/sourcespec2/input/sac_header.py @@ -126,10 +126,16 @@ def get_event_from_SAC(trace): raise RuntimeError( f'{trace.id}: not a SAC trace: cannot get hypocenter from header' ) from e - evla = sac_hdr['evla'] - evlo = sac_hdr['evlo'] - evdp = sac_hdr['evdp'] - begin = sac_hdr['b'] + try: + evla = sac_hdr['evla'] + evlo = sac_hdr['evlo'] + evdp = sac_hdr['evdp'] + begin = sac_hdr['b'] + except KeyError as e: + raise RuntimeError( + f'{trace.id}: cannot find hypocenter information in SAC header: ' + f'{e}' + ) from e starttime = trace.stats.starttime try: tori = sac_hdr['o'] diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index ec4638b1..1d36bdf6 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -24,6 +24,7 @@ from obspy import read from obspy.core import Stream from ..setup import config, ssp_exit +from .sac_header import get_event_from_SAC logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -108,6 +109,23 @@ def _read_trace_files(): return st +def _read_event_from_traces(stream): + """ + Read event information from trace headers. + The event information is stored in the trace.stats.event attribute. + + Currently supports only the SAC header. + + :param stream: ObsPy Stream object + :type stream: :class:`obspy.core.stream.Stream` + """ + for trace in stream: + try: + trace.stats.event = get_event_from_SAC(trace) + except RuntimeError: + continue + + def read_traces(): """ Read trace files @@ -116,10 +134,11 @@ def read_traces(): :rtype: :class:`obspy.core.stream.Stream` """ logger.info('Reading traces...') - st = _read_trace_files() + stream = _read_trace_files() + _read_event_from_traces(stream) logger.info('Reading traces: done') logger.info('---------------------------------------------------') - if len(st) == 0: + if len(stream) == 0: logger.error('No trace loaded') ssp_exit(1) - return st + return stream From 68ba73978a36b49b98af4577dae0088dafb3fb43 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 17 Jul 2024 15:06:19 +0200 Subject: [PATCH 42/73] Added parse_asdf_inventory function. Choose between parse_asdf_inventory and read_inventory depending on extension in read_station_metadata function. --- sourcespec2/input/station_metadata.py | 44 ++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/sourcespec2/input/station_metadata.py b/sourcespec2/input/station_metadata.py index 7375f237..a300ef38 100644 --- a/sourcespec2/input/station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -182,6 +182,45 @@ def _read_paz_file(file): return paz.to_inventory() +def parse_asdf_inventory(asdf_file): + """ + Read station metadata from ASDF file + + :param asdf_file: full path to ASDF file + :type asdf_file: str + + :return: inventory + :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + """ + inventory = Inventory() + + try: + # pylint: disable=import-outside-toplevel + # pyasdf is not a hard dependency, so we import it here + # and check for ImportError + import pyasdf + except ImportError: + logger.error( + 'Error importing pyasdf. ' + 'See https://seismicdata.github.io/pyasdf/ for installation ' + 'instructions.' + ) + ssp_exit(1) + try: + ds = pyasdf.ASDFDataSet(asdf_file, mode='r') + except Exception as err: + logger.error(err) + ssp_exit(1) + else: + for nw_stat_code in ds.waveforms.list(): + if 'StationXML' in ds.waveforms[nw_stat_code]: + station_inv = ds.waveforms[nw_stat_code].StationXML + inventory += station_inv + ds._close() + + return inventory + + def read_station_metadata(): """ Read station metadata into an ObsPy ``Inventory`` object. @@ -213,7 +252,10 @@ def read_station_metadata(): continue logger.info(f'Reading station metadata from file: {file}') try: - inventory += read_inventory(file) + if os.path.splitext(file)[-1].lower() in ('.asdf', '.h5'): + inventory += parse_asdf_inventory(file) + else: + inventory += read_inventory(file) except Exception: msg1 = f'Unable to parse file "{file}" as Inventory' try: From 1b3f7d440ee43f31ea07c9bf91386e33a73eca7c Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 17 Jul 2024 15:10:48 +0200 Subject: [PATCH 43/73] Added parse_asdf_traces function. Choose between obspy.read and parse_asdf_traces in _read_trace_files function depending on file extension. --- sourcespec2/input/traces.py | 119 +++++++++++++++++++++++++++++++++++- 1 file changed, 118 insertions(+), 1 deletion(-) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 1d36bdf6..56f9410b 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -21,6 +21,7 @@ import tarfile import zipfile import tempfile +import json from obspy import read from obspy.core import Stream from ..setup import config, ssp_exit @@ -69,6 +70,119 @@ def _build_filelist(path, filelist, tmpdir): filelist.append(path) +def _parse_asdf_trace_headers(ds, st, nw_stat_codes, tag): + """ + Parse ASDF trace headers. + + :param ds: ASDF dataset + :type ds: :class:`pyasdf.ASDFDataSet` + :param st: ObsPy Stream object + :type st: :class:`obspy.core.stream.Stream` + :param nw_stat_codes: Network and station codes + :type nw_stat_codes: list + :param tag: waveform tag in ASDF file + :type tag: str + """ + # pylint: disable=import-outside-toplevel + from pyasdf.utils import AuxiliaryDataContainer + for nw_stat_code, tr in zip(nw_stat_codes, st.traces): + if nw_stat_code in ds.auxiliary_data['TraceHeaders']: + auxiliary_root = ds.auxiliary_data['TraceHeaders'][nw_stat_code] + else: + # ESM + _nw_stat_code = nw_stat_code.replace('.', '_') + auxiliary_root = ds.auxiliary_data['TraceHeaders'][_nw_stat_code] + if not tag or tag not in auxiliary_root: + continue + header = None + tr_id = tr.id.replace('.', '_') + if isinstance(auxiliary_root[tag], AuxiliaryDataContainer): + # ESM + header = auxiliary_root[tag].parameters + elif tr_id in auxiliary_root[tag]: + header = auxiliary_root[tag][tr_id].parameters + if not header: + continue + for key, val in header.items(): + try: + val = json.loads(val) + except json.JSONDecodeError: + if isinstance(val, type(b'')): + # ESM + val = val.decode('ascii') + if key == 'processing': + val = [val] + # Try preserving original _format + if ( + key == '_format' + and val != 'ASDF' + and 'original_format' not in header.items() + ): + key = 'original_format' + # Write to trace header + if key not in tr.stats: + tr.stats[key] = val + + +def parse_asdf_traces(asdf_file, tag=None, read_headers=False): + """ + Read all traces from ASDF file with given waveform tag + If tag is not specified, the first available one will be taken + + :param asdf_file: full path to ASDF file + :type asdf_file: str + :param tag: waveform tag in ASDF file + :type tag: str + :param read_headers: flag to control reading of (non-standard) + trace headers + :type read_headers: bool + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` + """ + try: + # pylint: disable=import-outside-toplevel + # pyasdf is not a hard dependency, so we import it here + # and check for ImportError + import pyasdf + except ImportError: + logger.error( + 'Error importing pyasdf. ' + 'See https://seismicdata.github.io/pyasdf/ for installation ' + 'instructions.' + ) + ssp_exit(1) + try: + ds = pyasdf.ASDFDataSet(asdf_file, mode='r') + except Exception as err: + logger.error(err) + ssp_exit(1) + # Read waveform data + st = Stream() + nw_stat_codes = [] + for nw_stat_code in ds.waveforms.list(): + wf_tags = ds.waveforms[nw_stat_code].get_waveform_tags() + # If tag is not specified, take first available tag + if not tag: + # Maybe this should be logged + tag = wf_tags[0] + if tag in wf_tags: + station_st = ds.waveforms[nw_stat_code][tag] + st.extend(station_st) + nw_stat_codes.extend([nw_stat_code] * len(station_st)) + # Try reading trace headers if present + if 'TraceHeaders' in ds.auxiliary_data: + header_key = 'TraceHeaders' + elif 'Headers' in ds.auxiliary_data: + header_key = 'Headers' + else: + header_key = None + if read_headers and header_key: + _parse_asdf_trace_headers(ds, st, nw_stat_codes, tag) + ds._close() + return st + + def _read_trace_files(): """ Read trace files from the path specified in the configuration file. @@ -93,7 +207,10 @@ def _read_trace_files(): st = Stream() for filename in sorted(filelist): try: - tmpst = read(filename, fsize=False) + if os.path.splitext(filename)[-1].lower() in ('.asdf', '.h5'): + tmpst = parse_asdf_traces(filename) + else: + tmpst = read(filename, fsize=False) except Exception: logger.warning( f'{filename}: Unable to read file as a trace: skipping') From 0c44c06e7d714046fa6b214232e29be926d968f0 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 12:25:35 +0200 Subject: [PATCH 44/73] Command line option to parse ASDF file for event and picks information Note: the option will allow also reading traces and metadata from the ASDF file (to be implemented in the next commits). Also, renamed asdf.py to asdf_event.py --- sourcespec2/input/event_and_picks.py | 6 +++++- sourcespec2/input/event_parsers/__init__.py | 2 +- sourcespec2/input/event_parsers/{asdf.py => asdf_event.py} | 0 sourcespec2/ssp_parse_arguments.py | 7 +++++++ 4 files changed, 13 insertions(+), 2 deletions(-) rename sourcespec2/input/event_parsers/{asdf.py => asdf_event.py} (100%) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 6c073716..c32d20ea 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -19,8 +19,9 @@ import logging from ..setup import config, ssp_exit from .event_parsers import ( + parse_source_spec_event_file, parse_hypo71_hypocenter, parse_hypo71_picks, parse_hypo2000_file, - parse_qml_file, parse_source_spec_event_file + parse_qml_file, parse_asdf_event_picks ) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -102,6 +103,9 @@ def read_event_and_picks(trace1=None): # parse QML file if config.options.qml_file is not None: ssp_event, picks = parse_qml_file() + # parse ASDF file + if config.options.asdf_file is not None: + ssp_event, picks = parse_asdf_event_picks(config.options.asdf_file) if ssp_event is not None: _log_event_info(ssp_event) diff --git a/sourcespec2/input/event_parsers/__init__.py b/sourcespec2/input/event_parsers/__init__.py index c50957e4..a3e3b588 100644 --- a/sourcespec2/input/event_parsers/__init__.py +++ b/sourcespec2/input/event_parsers/__init__.py @@ -9,7 +9,7 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -from .asdf import parse_asdf_event_picks # noqa +from .asdf_event import parse_asdf_event_picks # noqa from .hypo71 import parse_hypo71_hypocenter, parse_hypo71_picks # noqa from .hypo2000 import parse_hypo2000_file # noqa from .obspy_catalog import parse_obspy_catalog # noqa diff --git a/sourcespec2/input/event_parsers/asdf.py b/sourcespec2/input/event_parsers/asdf_event.py similarity index 100% rename from sourcespec2/input/event_parsers/asdf.py rename to sourcespec2/input/event_parsers/asdf_event.py diff --git a/sourcespec2/ssp_parse_arguments.py b/sourcespec2/ssp_parse_arguments.py index 9190a6a4..b77ac09e 100644 --- a/sourcespec2/ssp_parse_arguments.py +++ b/sourcespec2/ssp_parse_arguments.py @@ -118,6 +118,13 @@ def _init_parser(description, epilog, nargs): help='get picks and hypocenter information from QuakeML FILE', metavar='FILE' ) + parser.add_argument( + '-a', '--asdffile', dest='asdf_file', + action='store', default=None, + help='get picks, hypocenter information, traces and metadata from\n' + 'ASDF FILE', + metavar='FILE' + ) parser.add_argument( '-H', '--hypocenter', dest='hypo_file', action='store', default=None, From b1cf6e143a87a4689fccedb507ba4d86f414057b Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 12:30:45 +0200 Subject: [PATCH 45/73] Make parse_qml_event_picks() behave like parse_asdf_event_picks() The function was previously named parse_qml_file() --- sourcespec2/input/event_and_picks.py | 4 ++-- sourcespec2/input/event_parsers/__init__.py | 2 +- sourcespec2/input/event_parsers/quakeml.py | 12 ++++++++---- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index c32d20ea..7c1434a7 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -21,7 +21,7 @@ from .event_parsers import ( parse_source_spec_event_file, parse_hypo71_hypocenter, parse_hypo71_picks, parse_hypo2000_file, - parse_qml_file, parse_asdf_event_picks + parse_qml_event_picks, parse_asdf_event_picks ) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -102,7 +102,7 @@ def read_event_and_picks(trace1=None): picks = parse_hypo71_picks() # parse QML file if config.options.qml_file is not None: - ssp_event, picks = parse_qml_file() + ssp_event, picks = parse_qml_event_picks(config.options.qml_file) # parse ASDF file if config.options.asdf_file is not None: ssp_event, picks = parse_asdf_event_picks(config.options.asdf_file) diff --git a/sourcespec2/input/event_parsers/__init__.py b/sourcespec2/input/event_parsers/__init__.py index a3e3b588..e3769c37 100644 --- a/sourcespec2/input/event_parsers/__init__.py +++ b/sourcespec2/input/event_parsers/__init__.py @@ -13,5 +13,5 @@ from .hypo71 import parse_hypo71_hypocenter, parse_hypo71_picks # noqa from .hypo2000 import parse_hypo2000_file # noqa from .obspy_catalog import parse_obspy_catalog # noqa -from .quakeml import parse_qml_file # noqa +from .quakeml import parse_qml_event_picks # noqa from .source_spec_event import parse_source_spec_event_file # noqa diff --git a/sourcespec2/input/event_parsers/quakeml.py b/sourcespec2/input/event_parsers/quakeml.py index d9244d57..624b7ee9 100644 --- a/sourcespec2/input/event_parsers/quakeml.py +++ b/sourcespec2/input/event_parsers/quakeml.py @@ -11,26 +11,30 @@ """ import logging from obspy import read_events -from ...setup import config, ssp_exit +from ...setup import ssp_exit from .obspy_catalog import parse_obspy_catalog logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def parse_qml_file(): +def parse_qml_event_picks(qml_file, event_id=None): """ Parse event metadata and picks from a QuakeML file. + :param qml_file: Path to the QuakeML file. + :type qml_file: str + :param event_id: event id + :type event_id: str + :return: a tuple of (SSPEvent, picks) :rtype: tuple """ - qml_file = config.options.qml_file if qml_file is None: ssp_event = None picks = [] return ssp_event, picks try: obspy_catalog = read_events(qml_file) - return parse_obspy_catalog(obspy_catalog, file_name=qml_file) + return parse_obspy_catalog(obspy_catalog, event_id, qml_file) except Exception as err: logger.error(err) ssp_exit(1) From d7d8d46eba1f4f635997f61473ee080fecc5792b Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 15:16:44 +0200 Subject: [PATCH 46/73] Move asdf trace parsing to a separate module --- sourcespec2/input/trace_parsers/__init__.py | 12 ++ .../input/trace_parsers/asdf_traces.py | 130 ++++++++++++++++ sourcespec2/input/traces.py | 144 ++++-------------- sourcespec2/ssp_parse_arguments.py | 2 +- 4 files changed, 170 insertions(+), 118 deletions(-) create mode 100644 sourcespec2/input/trace_parsers/__init__.py create mode 100644 sourcespec2/input/trace_parsers/asdf_traces.py diff --git a/sourcespec2/input/trace_parsers/__init__.py b/sourcespec2/input/trace_parsers/__init__.py new file mode 100644 index 00000000..5536d152 --- /dev/null +++ b/sourcespec2/input/trace_parsers/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Trace parsers for SourceSpec. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +from .asdf_traces import parse_asdf_traces # noqa diff --git a/sourcespec2/input/trace_parsers/asdf_traces.py b/sourcespec2/input/trace_parsers/asdf_traces.py new file mode 100644 index 00000000..30297d47 --- /dev/null +++ b/sourcespec2/input/trace_parsers/asdf_traces.py @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read traces from ASDF files. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import logging +import json +from obspy.core import Stream +from ...setup import ssp_exit +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def _parse_asdf_trace_headers(ds, stream, nw_stat_codes, tag): + """ + Parse ASDF trace headers. + + :param ds: ASDF dataset + :type ds: :class:`pyasdf.ASDFDataSet` + :param stream: ObsPy Stream object + :type stream: :class:`obspy.core.stream.Stream` + :param nw_stat_codes: Network and station codes + :type nw_stat_codes: list + :param tag: waveform tag in ASDF file + :type tag: str + """ + # pylint: disable=import-outside-toplevel + from pyasdf.utils import AuxiliaryDataContainer + for nw_stat_code, tr in zip(nw_stat_codes, stream.traces): + if nw_stat_code in ds.auxiliary_data['TraceHeaders']: + auxiliary_root = ds.auxiliary_data['TraceHeaders'][nw_stat_code] + else: + # ESM + _nw_stat_code = nw_stat_code.replace('.', '_') + auxiliary_root = ds.auxiliary_data['TraceHeaders'][_nw_stat_code] + if not tag or tag not in auxiliary_root: + continue + header = None + tr_id = tr.id.replace('.', '_') + if isinstance(auxiliary_root[tag], AuxiliaryDataContainer): + # ESM + header = auxiliary_root[tag].parameters + elif tr_id in auxiliary_root[tag]: + header = auxiliary_root[tag][tr_id].parameters + if not header: + continue + for key, val in header.items(): + try: + val = json.loads(val) + except json.JSONDecodeError: + if isinstance(val, type(b'')): + # ESM + val = val.decode('ascii') + if key == 'processing': + val = [val] + # Try preserving original _format + if ( + key == '_format' + and val != 'ASDF' + and 'original_format' not in header.items() + ): + key = 'original_format' + # Write to trace header + if key not in tr.stats: + tr.stats[key] = val + + +def parse_asdf_traces(asdf_file, tag=None, read_headers=False): + """ + Read all traces from ASDF file with given waveform tag + If tag is not specified, the first available one will be taken + + :param asdf_file: full path to ASDF file + :type asdf_file: str + :param tag: waveform tag in ASDF file + :type tag: str + :param read_headers: flag to control reading of (non-standard) + trace headers + :type read_headers: bool + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` + """ + try: + # pylint: disable=import-outside-toplevel + # pyasdf is not a hard dependency, so we import it here + # and check for ImportError + import pyasdf + except ImportError: + logger.error( + 'Error importing pyasdf. ' + 'See https://seismicdata.github.io/pyasdf/ for installation ' + 'instructions.' + ) + ssp_exit(1) + stream = Stream() + try: + ds = pyasdf.ASDFDataSet(asdf_file, mode='r') + except OSError: + logger.warning(f'Unable to read ASDF file: {asdf_file}') + return stream + if not ds.waveforms: + return stream + nw_stat_codes = [] + for nw_stat_code in ds.waveforms.list(): + wf_tags = ds.waveforms[nw_stat_code].get_waveform_tags() + # If tag is not specified, take first available tag + if not tag: + # Maybe this should be logged + tag = wf_tags[0] + if tag in wf_tags: + station_st = ds.waveforms[nw_stat_code][tag] + stream.extend(station_st) + nw_stat_codes.extend([nw_stat_code] * len(station_st)) + # Try reading trace headers if present + if 'TraceHeaders' in ds.auxiliary_data: + header_key = 'TraceHeaders' + elif 'Headers' in ds.auxiliary_data: + header_key = 'Headers' + else: + header_key = None + if read_headers and header_key: + _parse_asdf_trace_headers(ds, stream, nw_stat_codes, tag) + ds._close() + return stream diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 56f9410b..53b66fe0 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -21,11 +21,11 @@ import tarfile import zipfile import tempfile -import json from obspy import read from obspy.core import Stream from ..setup import config, ssp_exit from .sac_header import get_event_from_SAC +from .trace_parsers import parse_asdf_traces logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -70,126 +70,46 @@ def _build_filelist(path, filelist, tmpdir): filelist.append(path) -def _parse_asdf_trace_headers(ds, st, nw_stat_codes, tag): +def _filter_by_station(input_stream): """ - Parse ASDF trace headers. + Select traces for a given station, if specified in the configuration. - :param ds: ASDF dataset - :type ds: :class:`pyasdf.ASDFDataSet` - :param st: ObsPy Stream object - :type st: :class:`obspy.core.stream.Stream` - :param nw_stat_codes: Network and station codes - :type nw_stat_codes: list - :param tag: waveform tag in ASDF file - :type tag: str + :param input_stream: Input stream to filter + :type tmpst: :class:`obspy.core.stream.Stream` + + :return: Stream object + :rtype: :class:`obspy.core.stream.Stream` """ - # pylint: disable=import-outside-toplevel - from pyasdf.utils import AuxiliaryDataContainer - for nw_stat_code, tr in zip(nw_stat_codes, st.traces): - if nw_stat_code in ds.auxiliary_data['TraceHeaders']: - auxiliary_root = ds.auxiliary_data['TraceHeaders'][nw_stat_code] - else: - # ESM - _nw_stat_code = nw_stat_code.replace('.', '_') - auxiliary_root = ds.auxiliary_data['TraceHeaders'][_nw_stat_code] - if not tag or tag not in auxiliary_root: - continue - header = None - tr_id = tr.id.replace('.', '_') - if isinstance(auxiliary_root[tag], AuxiliaryDataContainer): - # ESM - header = auxiliary_root[tag].parameters - elif tr_id in auxiliary_root[tag]: - header = auxiliary_root[tag][tr_id].parameters - if not header: - continue - for key, val in header.items(): - try: - val = json.loads(val) - except json.JSONDecodeError: - if isinstance(val, type(b'')): - # ESM - val = val.decode('ascii') - if key == 'processing': - val = [val] - # Try preserving original _format - if ( - key == '_format' - and val != 'ASDF' - and 'original_format' not in header.items() - ): - key = 'original_format' - # Write to trace header - if key not in tr.stats: - tr.stats[key] = val + if config.options.station is None: + return input_stream + return Stream([ + trace for trace in input_stream.traces + if trace.stats.station == config.options.station + ]) -def parse_asdf_traces(asdf_file, tag=None, read_headers=False): +def _read_asdf_traces(): """ - Read all traces from ASDF file with given waveform tag - If tag is not specified, the first available one will be taken - - :param asdf_file: full path to ASDF file - :type asdf_file: str - :param tag: waveform tag in ASDF file - :type tag: str - :param read_headers: flag to control reading of (non-standard) - trace headers - :type read_headers: bool + Read traces from ASDF file specified in the configuration. :return: ObsPy Stream object :rtype: :class:`obspy.core.stream.Stream` """ - try: - # pylint: disable=import-outside-toplevel - # pyasdf is not a hard dependency, so we import it here - # and check for ImportError - import pyasdf - except ImportError: - logger.error( - 'Error importing pyasdf. ' - 'See https://seismicdata.github.io/pyasdf/ for installation ' - 'instructions.' - ) - ssp_exit(1) - try: - ds = pyasdf.ASDFDataSet(asdf_file, mode='r') - except Exception as err: - logger.error(err) - ssp_exit(1) - # Read waveform data - st = Stream() - nw_stat_codes = [] - for nw_stat_code in ds.waveforms.list(): - wf_tags = ds.waveforms[nw_stat_code].get_waveform_tags() - # If tag is not specified, take first available tag - if not tag: - # Maybe this should be logged - tag = wf_tags[0] - if tag in wf_tags: - station_st = ds.waveforms[nw_stat_code][tag] - st.extend(station_st) - nw_stat_codes.extend([nw_stat_code] * len(station_st)) - # Try reading trace headers if present - if 'TraceHeaders' in ds.auxiliary_data: - header_key = 'TraceHeaders' - elif 'Headers' in ds.auxiliary_data: - header_key = 'Headers' - else: - header_key = None - if read_headers and header_key: - _parse_asdf_trace_headers(ds, st, nw_stat_codes, tag) - ds._close() - return st + asdf_file = config.options.asdf_file + if not asdf_file: + return Stream() + return _filter_by_station(parse_asdf_traces(asdf_file)) def _read_trace_files(): """ - Read trace files from the path specified in the configuration file. + Read trace files from the path specified in the configuration. :return: ObsPy Stream object :rtype: :class:`obspy.core.stream.Stream` """ + if config.options.trace_path is None: + return Stream() # phase 1: build a file list # ph 1.1: create a temporary dir and run '_build_filelist()' # to move files to it and extract all tar archives @@ -207,21 +127,11 @@ def _read_trace_files(): st = Stream() for filename in sorted(filelist): try: - if os.path.splitext(filename)[-1].lower() in ('.asdf', '.h5'): - tmpst = parse_asdf_traces(filename) - else: - tmpst = read(filename, fsize=False) - except Exception: + st += _filter_by_station(read(filename, fsize=False)) + except (TypeError, FileNotFoundError): logger.warning( f'{filename}: Unable to read file as a trace: skipping') continue - for trace in tmpst.traces: - # only use the station specified by the command line option - # "--station", if any - if (config.options.station is not None and - trace.stats.station != config.options.station): - continue - st.append(trace) shutil.rmtree(tmpdir) return st @@ -245,13 +155,13 @@ def _read_event_from_traces(stream): def read_traces(): """ - Read trace files + Read traces from the files or paths specified in the configuration. :return: Traces :rtype: :class:`obspy.core.stream.Stream` """ logger.info('Reading traces...') - stream = _read_trace_files() + stream = _read_asdf_traces() + _read_trace_files() _read_event_from_traces(stream) logger.info('Reading traces: done') logger.info('---------------------------------------------------') diff --git a/sourcespec2/ssp_parse_arguments.py b/sourcespec2/ssp_parse_arguments.py index b77ac09e..2b963706 100644 --- a/sourcespec2/ssp_parse_arguments.py +++ b/sourcespec2/ssp_parse_arguments.py @@ -118,7 +118,7 @@ def _init_parser(description, epilog, nargs): help='get picks and hypocenter information from QuakeML FILE', metavar='FILE' ) - parser.add_argument( + group.add_argument( '-a', '--asdffile', dest='asdf_file', action='store', default=None, help='get picks, hypocenter information, traces and metadata from\n' From b39917c221b33f1bb1add0cb39b26a76ffd982cc Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 15:22:24 +0200 Subject: [PATCH 47/73] Command line argument to specify tag when reading traces from ASDF file --- sourcespec2/input/traces.py | 5 ++++- sourcespec2/ssp_parse_arguments.py | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 53b66fe0..c69d93bf 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -98,7 +98,10 @@ def _read_asdf_traces(): asdf_file = config.options.asdf_file if not asdf_file: return Stream() - return _filter_by_station(parse_asdf_traces(asdf_file)) + asdf_tag = config.options.asdf_tag + return _filter_by_station( + parse_asdf_traces(asdf_file, tag=asdf_tag, read_headers=True) + ) def _read_trace_files(): diff --git a/sourcespec2/ssp_parse_arguments.py b/sourcespec2/ssp_parse_arguments.py index 2b963706..ef86b310 100644 --- a/sourcespec2/ssp_parse_arguments.py +++ b/sourcespec2/ssp_parse_arguments.py @@ -125,6 +125,12 @@ def _init_parser(description, epilog, nargs): 'ASDF FILE', metavar='FILE' ) + parser.add_argument( + '-g', '--tag', dest='asdf_tag', + action='store', default=None, + help='tag to use when reading traces from ASDF file', + metavar='TAG' + ) parser.add_argument( '-H', '--hypocenter', dest='hypo_file', action='store', default=None, From 0ac4b15219866c4502d48bdb06423a225c8cc69f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 15:24:48 +0200 Subject: [PATCH 48/73] Log number of traces loaded and whether traces are read from ASDF file --- sourcespec2/input/traces.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index c69d93bf..75296933 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -99,6 +99,7 @@ def _read_asdf_traces(): if not asdf_file: return Stream() asdf_tag = config.options.asdf_tag + logger.info(f'Reading traces from ASDF file: {asdf_file}') return _filter_by_station( parse_asdf_traces(asdf_file, tag=asdf_tag, read_headers=True) ) @@ -166,9 +167,10 @@ def read_traces(): logger.info('Reading traces...') stream = _read_asdf_traces() + _read_trace_files() _read_event_from_traces(stream) - logger.info('Reading traces: done') + ntraces = len(stream) + logger.info(f'Reading traces: {ntraces} traces loaded') logger.info('---------------------------------------------------') - if len(stream) == 0: + if not ntraces: logger.error('No trace loaded') ssp_exit(1) return stream From c8e816059b31c82cb60d1ed4669a9970ec02ef6f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 15:54:20 +0200 Subject: [PATCH 49/73] Move asdf inventory parsing to a separate module --- sourcespec2/input/station_metadata.py | 69 ++++++++----------- .../station_metadata_parsers/__init__.py | 12 ++++ .../asdf_inventory.py | 51 ++++++++++++++ 3 files changed, 90 insertions(+), 42 deletions(-) create mode 100644 sourcespec2/input/station_metadata_parsers/__init__.py create mode 100644 sourcespec2/input/station_metadata_parsers/asdf_inventory.py diff --git a/sourcespec2/input/station_metadata.py b/sourcespec2/input/station_metadata.py index a300ef38..c588bb2b 100644 --- a/sourcespec2/input/station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -2,7 +2,7 @@ # SPDX-License-Identifier: CECILL-2.1 """ Read station metadata in StationXML, dataless SEED, SEED RESP, -PAZ (SAC polezero format). +PAZ (SAC polezero format), ASDF :copyright: 2012-2025 Claudio Satriano @@ -16,6 +16,7 @@ from obspy import read_inventory from obspy.core.inventory import Inventory, Network, Station, Channel, Response from ..setup import config +from .station_metadata_parsers import parse_asdf_inventory logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -182,46 +183,21 @@ def _read_paz_file(file): return paz.to_inventory() -def parse_asdf_inventory(asdf_file): +def _read_asdf_inventory(): """ - Read station metadata from ASDF file - - :param asdf_file: full path to ASDF file - :type asdf_file: str + Read station metadata from ASDF file specified in the configuration. :return: inventory :rtype: :class:`~obspy.core.inventory.inventory.Inventory` """ - inventory = Inventory() - - try: - # pylint: disable=import-outside-toplevel - # pyasdf is not a hard dependency, so we import it here - # and check for ImportError - import pyasdf - except ImportError: - logger.error( - 'Error importing pyasdf. ' - 'See https://seismicdata.github.io/pyasdf/ for installation ' - 'instructions.' - ) - ssp_exit(1) - try: - ds = pyasdf.ASDFDataSet(asdf_file, mode='r') - except Exception as err: - logger.error(err) - ssp_exit(1) - else: - for nw_stat_code in ds.waveforms.list(): - if 'StationXML' in ds.waveforms[nw_stat_code]: - station_inv = ds.waveforms[nw_stat_code].StationXML - inventory += station_inv - ds._close() - - return inventory + asdf_file = config.options.asdf_file + if not asdf_file: + return Inventory() + logger.info(f'Reading station metadata from ASDF file: {asdf_file}') + return parse_asdf_inventory(asdf_file) -def read_station_metadata(): +def _read_station_metadata_from_files(): """ Read station metadata into an ObsPy ``Inventory`` object. @@ -235,10 +211,9 @@ def read_station_metadata(): not set in the configuration or if no valid files are found """ inventory = Inventory() - if not config.station_metadata: - return inventory - logger.info('Reading station metadata...') metadata_path = config.station_metadata + if not metadata_path: + return inventory if os.path.isdir(metadata_path): filelist = [ os.path.join(metadata_path, file) @@ -252,10 +227,7 @@ def read_station_metadata(): continue logger.info(f'Reading station metadata from file: {file}') try: - if os.path.splitext(file)[-1].lower() in ('.asdf', '.h5'): - inventory += parse_asdf_inventory(file) - else: - inventory += read_inventory(file) + inventory += read_inventory(file) except Exception: msg1 = f'Unable to parse file "{file}" as Inventory' try: @@ -264,6 +236,19 @@ def read_station_metadata(): logger.warning(msg1) logger.warning(msg2) continue - logger.info('Reading station metadata: done') + return inventory + + +def read_station_metadata(): + """ + Read station metadata. + + :return: inventory + :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + """ + logger.info('Reading station metadata...') + inventory = _read_station_metadata_from_files() + _read_asdf_inventory() + nstations = len(inventory.get_contents()['stations']) + logger.info(f'Reading station metadata: {nstations} stations read') logger.info('---------------------------------------------------') return inventory diff --git a/sourcespec2/input/station_metadata_parsers/__init__.py b/sourcespec2/input/station_metadata_parsers/__init__.py new file mode 100644 index 00000000..4b498095 --- /dev/null +++ b/sourcespec2/input/station_metadata_parsers/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Station metadata parsers for SourceSpec. + +:copyright: + 2013-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +from .asdf_inventory import parse_asdf_inventory # noqa \ No newline at end of file diff --git a/sourcespec2/input/station_metadata_parsers/asdf_inventory.py b/sourcespec2/input/station_metadata_parsers/asdf_inventory.py new file mode 100644 index 00000000..916e681a --- /dev/null +++ b/sourcespec2/input/station_metadata_parsers/asdf_inventory.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read station metadata from ASDF files. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import logging +from obspy.core.inventory import Inventory +from ...setup import ssp_exit +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def parse_asdf_inventory(asdf_file): + """ + Read station metadata from ASDF file + + :param asdf_file: full path to ASDF file + :type asdf_file: str + + :return: inventory + :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + """ + try: + # pylint: disable=import-outside-toplevel + # pyasdf is not a hard dependency, so we import it here + # and check for ImportError + import pyasdf + except ImportError: + logger.error( + 'Error importing pyasdf. ' + 'See https://seismicdata.github.io/pyasdf/ for installation ' + 'instructions.' + ) + ssp_exit(1) + inventory = Inventory() + try: + ds = pyasdf.ASDFDataSet(asdf_file, mode='r') + except OSError: + logger.warning(f'Unable to read ASDF file: {asdf_file}') + return inventory + for nw_stat_code in ds.waveforms.list(): + if 'StationXML' in ds.waveforms[nw_stat_code]: + station_inv = ds.waveforms[nw_stat_code].StationXML + inventory += station_inv + ds._close() + return inventory From f7ca629f08fe192c24f24c567cfb675cb34536c9 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Fri, 19 Jul 2024 16:01:24 +0200 Subject: [PATCH 50/73] Move PAZ reader to its own module --- sourcespec2/input/augment_traces.py | 2 +- sourcespec2/input/station_metadata.py | 170 +---------------- .../station_metadata_parsers/__init__.py | 3 +- .../input/station_metadata_parsers/paz.py | 180 ++++++++++++++++++ 4 files changed, 186 insertions(+), 169 deletions(-) create mode 100644 sourcespec2/input/station_metadata_parsers/paz.py diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index a8e3263f..6d594915 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -20,7 +20,7 @@ import contextlib from obspy.core.util import AttribDict from ..setup import config -from .station_metadata import PAZ +from .station_metadata_parsers import PAZ from .sac_header import ( compute_sensitivity_from_SAC, get_instrument_from_SAC, get_station_coordinates_from_SAC, diff --git a/sourcespec2/input/station_metadata.py b/sourcespec2/input/station_metadata.py index c588bb2b..5127dd56 100644 --- a/sourcespec2/input/station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -11,178 +11,14 @@ (http://www.cecill.info/licences.en.html) """ import os -import re import logging from obspy import read_inventory -from obspy.core.inventory import Inventory, Network, Station, Channel, Response +from obspy.core.inventory import Inventory from ..setup import config -from .station_metadata_parsers import parse_asdf_inventory +from .station_metadata_parsers import read_paz_file, parse_asdf_inventory logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -class PAZ(): - """Instrument response defined through poles and zeros.""" - zeros = [] - poles = [] - sensitivity = 1. - _seedID = None - network = None - station = None - location = None - channel = None - input_units = None - linenum = None - - def __init__(self, file=None): - """ - Init PAZ object. - - :param file: path to the PAZ file - :type file: str - """ - if file is not None: - self._read(file) - - def __str__(self): - return ( - f'PAZ: {self.seedID}' - f' zeros: {self.zeros}' - f' poles: {self.poles}' - f' sensitivity: {self.sensitivity}' - f' input_units: {self.input_units}' - ) - - @property - def seedID(self): - """Return the seedID.""" - return self._seedID - - @seedID.setter - def seedID(self, seedID): - try: - self.network, self.station, self.location, self.channel =\ - seedID.split('.') - except ValueError as e: - raise ValueError( - f'Invalid seedID "{seedID}". ' - 'SeedID must be in the form NET.STA.LOC.CHAN' - ) from e - self._seedID = seedID - self._guess_input_units() - - def _guess_input_units(self): - """ - Guess the input units from the seedID. - """ - if len(self.channel) < 3: - return - instr_code = self.channel[1] - self.input_units = None - if instr_code in config.INSTR_CODES_VEL: - band_code = self.channel[0] - # SEED standard band codes for velocity channels - # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming - if band_code in ['B', 'C', 'D', 'E', 'F', 'G', 'H', 'S']: - self.input_units = 'M/S' - elif instr_code in config.INSTR_CODES_ACC: - self.input_units = 'M/S**2' - - def _read(self, file): - """Read a PAZ file.""" - # We cannot use the "with" statement here because we need to keep - # self.lines alive as an iterator - # pylint: disable=consider-using-with - fp = open(file, 'r', encoding='ascii') # sourcery skip - # pylint: enable=consider-using-with - self.lines = enumerate(fp, start=1) - while True: - try: - self._parse_paz_file_lines() - except StopIteration: - break - except Exception as e: - raise TypeError( - f'Unable to parse file "{file}" as PAZ file. ' - f'Error at line {self.linenum}: {e}' - ) from e - fp.close() - - def _parse_paz_file_lines(self): - """Parse a line or a set of lines of a PAZ file.""" - self.linenum, line = next(self.lines) - word = line.split() - if not word: - return - what = word[0].lower() - if what in ['poles', 'zeros']: - nvalues = int(word[1]) - poles_zeros = [] - for _ in range(nvalues): - self.linenum, line = next(self.lines) - value = complex(*map(float, line.split())) - poles_zeros.append(value) - setattr(self, what, poles_zeros) - elif what == 'constant': - self.sensitivity = float(word[1]) - - def to_inventory(self): - """ - Convert PAZ object to an Inventory object. - """ - resp = Response().from_paz( - self.zeros, self.poles, stage_gain=self.sensitivity, - input_units=self.input_units, output_units='COUNTS') - resp.instrument_sensitivity.value = self.sensitivity - channel = Channel( - code=self.channel, location_code=self.location, response=resp, - latitude=0, longitude=0, elevation=123456, depth=123456) - station = Station( - code=self.station, channels=[channel, ], - latitude=0, longitude=0, elevation=123456) - network = Network( - code=self.network, stations=[station, ]) - return Inventory(networks=[network, ]) - - -def _read_paz_file(file): - """ - Read a paz file into an ``Inventory`` object. - - :note: - - paz file must have ".pz" or ".paz" suffix (or no suffix) - - paz file name (without prefix and suffix) can have - the trace_id (NET.STA.LOC.CHAN) of the corresponding trace in the last - part of his name (e.g., 20110208_1600.NOW.IV.CRAC.00.EHZ.paz), - otherwise it will be treaten as a generic paz. - - :param file: path to the PAZ file - :type file: str - - :return: inventory - :rtype: :class:`~obspy.core.inventory.inventory.Inventory` - """ - bname = os.path.basename(file) - # strip .pz suffix, if there - bname = re.sub('.pz$', '', bname) - # strip .paz suffix, if there - bname = re.sub('.paz$', '', bname) - # we assume that the last four fields of bname - # (separated by '.') are the trace_id - trace_id = '.'.join(bname.split('.')[-4:]) - paz = PAZ(file) - try: - paz.seedID = trace_id - except ValueError: - paz.seedID = 'XX.GENERIC.XX.XXX' - logger.info(f'Using generic trace ID for PAZ file {file}') - if paz.input_units is None: - paz.input_units = 'M/S' - logger.warning( - f'Cannot find input units for ID "{paz.seedID}". ' - 'Defaulting to M/S') - return paz.to_inventory() - - def _read_asdf_inventory(): """ Read station metadata from ASDF file specified in the configuration. @@ -231,7 +67,7 @@ def _read_station_metadata_from_files(): except Exception: msg1 = f'Unable to parse file "{file}" as Inventory' try: - inventory += _read_paz_file(file) + inventory += read_paz_file(file) except Exception as msg2: logger.warning(msg1) logger.warning(msg2) diff --git a/sourcespec2/input/station_metadata_parsers/__init__.py b/sourcespec2/input/station_metadata_parsers/__init__.py index 4b498095..48ad1be4 100644 --- a/sourcespec2/input/station_metadata_parsers/__init__.py +++ b/sourcespec2/input/station_metadata_parsers/__init__.py @@ -9,4 +9,5 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -from .asdf_inventory import parse_asdf_inventory # noqa \ No newline at end of file +from .asdf_inventory import parse_asdf_inventory # noqa +from .paz import read_paz_file, PAZ # noqa diff --git a/sourcespec2/input/station_metadata_parsers/paz.py b/sourcespec2/input/station_metadata_parsers/paz.py new file mode 100644 index 00000000..e51c685a --- /dev/null +++ b/sourcespec2/input/station_metadata_parsers/paz.py @@ -0,0 +1,180 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read station metadata from PAZ files. + +:copyright: + 2012-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import os +import re +import logging +from obspy.core.inventory import Inventory, Network, Station, Channel, Response +from ...setup import config +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +class PAZ(): + """Instrument response defined through poles and zeros.""" + zeros = [] + poles = [] + sensitivity = 1. + _seedID = None + network = None + station = None + location = None + channel = None + input_units = None + linenum = None + + def __init__(self, file=None): + """ + Init PAZ object. + + :param file: path to the PAZ file + :type file: str + """ + if file is not None: + self._read(file) + + def __str__(self): + return ( + f'PAZ: {self.seedID}' + f' zeros: {self.zeros}' + f' poles: {self.poles}' + f' sensitivity: {self.sensitivity}' + f' input_units: {self.input_units}' + ) + + @property + def seedID(self): + """Return the seedID.""" + return self._seedID + + @seedID.setter + def seedID(self, seedID): + try: + self.network, self.station, self.location, self.channel =\ + seedID.split('.') + except ValueError as e: + raise ValueError( + f'Invalid seedID "{seedID}". ' + 'SeedID must be in the form NET.STA.LOC.CHAN' + ) from e + self._seedID = seedID + self._guess_input_units() + + def _guess_input_units(self): + """ + Guess the input units from the seedID. + """ + if len(self.channel) < 3: + return + instr_code = self.channel[1] + self.input_units = None + if instr_code in config.INSTR_CODES_VEL: + band_code = self.channel[0] + # SEED standard band codes for velocity channels + # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming + if band_code in ['B', 'C', 'D', 'E', 'F', 'G', 'H', 'S']: + self.input_units = 'M/S' + elif instr_code in config.INSTR_CODES_ACC: + self.input_units = 'M/S**2' + + def _read(self, file): + """Read a PAZ file.""" + # We cannot use the "with" statement here because we need to keep + # self.lines alive as an iterator + # pylint: disable=consider-using-with + fp = open(file, 'r', encoding='ascii') # sourcery skip + # pylint: enable=consider-using-with + self.lines = enumerate(fp, start=1) + while True: + try: + self._parse_paz_file_lines() + except StopIteration: + break + except Exception as e: + raise TypeError( + f'Unable to parse file "{file}" as PAZ file. ' + f'Error at line {self.linenum}: {e}' + ) from e + fp.close() + + def _parse_paz_file_lines(self): + """Parse a line or a set of lines of a PAZ file.""" + self.linenum, line = next(self.lines) + word = line.split() + if not word: + return + what = word[0].lower() + if what in ['poles', 'zeros']: + nvalues = int(word[1]) + poles_zeros = [] + for _ in range(nvalues): + self.linenum, line = next(self.lines) + value = complex(*map(float, line.split())) + poles_zeros.append(value) + setattr(self, what, poles_zeros) + elif what == 'constant': + self.sensitivity = float(word[1]) + + def to_inventory(self): + """ + Convert PAZ object to an Inventory object. + """ + resp = Response().from_paz( + self.zeros, self.poles, stage_gain=self.sensitivity, + input_units=self.input_units, output_units='COUNTS') + resp.instrument_sensitivity.value = self.sensitivity + channel = Channel( + code=self.channel, location_code=self.location, response=resp, + latitude=0, longitude=0, elevation=123456, depth=123456) + station = Station( + code=self.station, channels=[channel, ], + latitude=0, longitude=0, elevation=123456) + network = Network( + code=self.network, stations=[station, ]) + return Inventory(networks=[network, ]) + + +def read_paz_file(file): + """ + Read a paz file into an ``Inventory`` object. + + :note: + - paz file must have ".pz" or ".paz" suffix (or no suffix) + - paz file name (without prefix and suffix) can have + the trace_id (NET.STA.LOC.CHAN) of the corresponding trace in the last + part of his name (e.g., 20110208_1600.NOW.IV.CRAC.00.EHZ.paz), + otherwise it will be treaten as a generic paz. + + :param file: path to the PAZ file + :type file: str + + :return: inventory + :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + """ + bname = os.path.basename(file) + # strip .pz suffix, if there + bname = re.sub('.pz$', '', bname) + # strip .paz suffix, if there + bname = re.sub('.paz$', '', bname) + # we assume that the last four fields of bname + # (separated by '.') are the trace_id + trace_id = '.'.join(bname.split('.')[-4:]) + paz = PAZ(file) + try: + paz.seedID = trace_id + except ValueError: + paz.seedID = 'XX.GENERIC.XX.XXX' + logger.info(f'Using generic trace ID for PAZ file {file}') + if paz.input_units is None: + paz.input_units = 'M/S' + logger.warning( + f'Cannot find input units for ID "{paz.seedID}". ' + 'Defaulting to M/S') + return paz.to_inventory() From 058d557975225bb029ee8d872ccbd5e024797203 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 09:14:12 +0200 Subject: [PATCH 51/73] Fix `_make_symlinks` to handle the case where `config.options.trace_path` is None Also, add `config.options.asdf_file` to the list of files to symlink. --- sourcespec2/ssp_output.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/sourcespec2/ssp_output.py b/sourcespec2/ssp_output.py index c3622aa3..f7a59983 100644 --- a/sourcespec2/ssp_output.py +++ b/sourcespec2/ssp_output.py @@ -409,14 +409,17 @@ def _make_symlinks(): out_data_dir = os.path.join(outdir, 'input_files') rel_path = os.path.relpath(config.workdir, out_data_dir) os.makedirs(out_data_dir, exist_ok=True) - filelist =\ - list(config.options.trace_path) +\ - [ - config.options.station_metadata, - config.options.hypo_file, - config.options.pick_file, - config.options.qml_file, - ] + try: + filelist = list(config.options.trace_path) + except TypeError: + filelist = [] + filelist += [ + config.options.station_metadata, + config.options.hypo_file, + config.options.pick_file, + config.options.qml_file, + config.options.asdf_file, + ] for filename in filelist: if filename is None or not os.path.exists(filename): continue From 925caf05810b3360b875853bce2bb48bbe3b127f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 09:32:53 +0200 Subject: [PATCH 52/73] Use `getattr()` when dealing with `config.options` --- sourcespec2/input/event_and_picks.py | 8 ++++---- sourcespec2/input/event_parsers/obspy_catalog.py | 10 +++++----- sourcespec2/input/traces.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 7c1434a7..6e319629 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -93,18 +93,18 @@ def read_event_and_picks(trace1=None): picks = [] ssp_event = None # parse hypocenter file - if config.options.hypo_file is not None: + if getattr(config.options, 'hypo_file', None): ssp_event, picks, file_format = _parse_hypo_file( config.options.hypo_file, config.options.evid) config.hypo_file_format = file_format # parse pick file - if config.options.pick_file is not None: + if getattr(config.options, 'pick_file', None): picks = parse_hypo71_picks() # parse QML file - if config.options.qml_file is not None: + if getattr(config.options, 'qml_file', None): ssp_event, picks = parse_qml_event_picks(config.options.qml_file) # parse ASDF file - if config.options.asdf_file is not None: + if getattr(config.options, 'asdf_file', None): ssp_event, picks = parse_asdf_event_picks(config.options.asdf_file) if ssp_event is not None: _log_event_info(ssp_event) diff --git a/sourcespec2/input/event_parsers/obspy_catalog.py b/sourcespec2/input/event_parsers/obspy_catalog.py index b8b7358f..6177ce87 100644 --- a/sourcespec2/input/event_parsers/obspy_catalog.py +++ b/sourcespec2/input/event_parsers/obspy_catalog.py @@ -52,12 +52,12 @@ def _parse_event_metadata(obspy_event): """ # No need to parse event name from the ObsPy event if event name is given # in the command line - if config.options.evname is None: - parse_event_name_from_description = config.qml_event_description - event_description_regex = config.qml_event_description_regex - else: + if getattr(config.options, 'evname', None): parse_event_name_from_description = False event_description_regex = None + else: + parse_event_name_from_description = config.qml_event_description + event_description_regex = config.qml_event_description_regex ssp_event = SSPEvent() ssp_event.event_id = _get_evid_from_resource_id( str(obspy_event.resource_id.id)) @@ -294,7 +294,7 @@ def parse_obspy_catalog(obspy_catalog, event_id=None, file_name=''): :return: a tuple of (SSPEvent, picks) :rtype: tuple """ - event_id = event_id or config.options.evid + event_id = event_id or getattr(config.options, 'evid', None) try: obspy_event = _get_event_from_obspy_catalog( obspy_catalog, event_id, file_name) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 75296933..35b465ec 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -80,7 +80,7 @@ def _filter_by_station(input_stream): :return: Stream object :rtype: :class:`obspy.core.stream.Stream` """ - if config.options.station is None: + if getattr(config.options, 'station', None) is None: return input_stream return Stream([ trace for trace in input_stream.traces @@ -112,7 +112,7 @@ def _read_trace_files(): :return: ObsPy Stream object :rtype: :class:`obspy.core.stream.Stream` """ - if config.options.trace_path is None: + if getattr(config.options, 'trace_path', None) is None: return Stream() # phase 1: build a file list # ph 1.1: create a temporary dir and run '_build_filelist()' From 005b06cd25aae5d3da14266422c2408544314b28 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 14:48:50 +0200 Subject: [PATCH 53/73] Warn the user about prioritary sources when reading event and picks --- sourcespec2/input/event_and_picks.py | 119 +++++++++++++++++++--- sourcespec2/input/event_parsers/hypo71.py | 8 +- 2 files changed, 108 insertions(+), 19 deletions(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 6e319629..42e95b8c 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -62,8 +62,63 @@ def _parse_hypo_file(hypo_file, event_id=None): err_msgs.append(msg) # If we arrive here, the file was not recognized as valid for msg in err_msgs: - logger.error(msg) - ssp_exit(1) + logger.warning(msg) + picks = [] + ssp_event = None + file_format = None + return ssp_event, picks, file_format + + +def _replace_event(old_event, new_event, old_source, new_source): + """ + Replace old event with new event. + + :param old_event: Old SSPEvent object + :type old_event: :class:`sourcespec.ssp_event.SSPEvent` + :param new_event: New SSPEvent object + :type new_event: :class:`sourcespec.ssp_event.SSPEvent` + :param old_source: Old source + :type old_source: str + :param new_source: New source + :type new_source: str + + :return: New SSPEvent object, new source + :rtype: tuple + """ + if new_event is None: + return old_event, old_source + if old_event is not None: + logger.warning( + f'Replacing event information found in {old_source} ' + f'with information from {new_source}' + ) + return new_event, new_source + + +def _replace_picks(old_picks, new_picks, old_source, new_source): + """ + Replace old picks with new picks. + + :param old_picks: Old picks + :type old_picks: list of :class:`sourcespec.ssp_event.Pick` + :param new_picks: New picks + :type new_picks: list of :class:`sourcespec.ssp_event.Pick` + :param old_source: Old source + :type old_source: str + :param new_source: New source + :type new_source: str + + :return: New picks, new source + :rtype: tuple + """ + if not new_picks: + return old_picks, old_source + if old_picks: + logger.warning( + f'Replacing picks found in {old_source} ' + f'with picks from {new_source}' + ) + return new_picks, new_source def _log_event_info(ssp_event): @@ -89,23 +144,57 @@ def read_event_and_picks(trace1=None): :rtype: tuple of :class:`sourcespec.ssp_event.SSPEvent`, list of :class:`sourcespec.ssp_event.Pick` + + .. note:: + The function reads event and phase picks from the following sources, + from the less prioritary to the most prioritary: + - ASDF file + - QML file + - Hypocenter file + - Pick file + + If trace1 is provided and contains event information, then this + information is used if no other source is found. """ picks = [] ssp_event = None - # parse hypocenter file - if getattr(config.options, 'hypo_file', None): - ssp_event, picks, file_format = _parse_hypo_file( - config.options.hypo_file, config.options.evid) + asdf_file = getattr(config.options, 'asdf_file', None) + hypo_file = getattr(config.options, 'hypo_file', None) + pick_file = getattr(config.options, 'pick_file', None) + qml_file = getattr(config.options, 'qml_file', None) + _event_source = None + _picks_source = None + # parse ASDF file, lowest priority + if asdf_file is not None: + ssp_event, picks = parse_asdf_event_picks(asdf_file) + _event_source = _picks_source = asdf_file + # parse QML file, possibly replacing event and picks + if qml_file is not None: + _new_ssp_event, _new_picks = parse_qml_event_picks(qml_file) + ssp_event, _event_source = _replace_event( + ssp_event, _new_ssp_event, _event_source, qml_file + ) + picks, _picks_source = _replace_picks( + picks, _new_picks, _picks_source, qml_file + ) + # parse hypocenter file, possibly replacing event and picks + if hypo_file is not None: + _new_ssp_event, _new_picks, file_format = _parse_hypo_file( + hypo_file, config.options.evid) + # this is needed when writing the output file in hypo71 format config.hypo_file_format = file_format - # parse pick file - if getattr(config.options, 'pick_file', None): - picks = parse_hypo71_picks() - # parse QML file - if getattr(config.options, 'qml_file', None): - ssp_event, picks = parse_qml_event_picks(config.options.qml_file) - # parse ASDF file - if getattr(config.options, 'asdf_file', None): - ssp_event, picks = parse_asdf_event_picks(config.options.asdf_file) + ssp_event, _event_source = _replace_event( + ssp_event, _new_ssp_event, _event_source, hypo_file + ) + picks, _picks_source = _replace_picks( + picks, _new_picks, _picks_source, hypo_file + ) + # parse pick file, possibly replacing picks + if pick_file is not None: + _new_picks = parse_hypo71_picks() + picks, _picks_source = _replace_picks( + picks, _new_picks, _picks_source, pick_file + ) if ssp_event is not None: _log_event_info(ssp_event) diff --git a/sourcespec2/input/event_parsers/hypo71.py b/sourcespec2/input/event_parsers/hypo71.py index e023cadf..76adcc38 100644 --- a/sourcespec2/input/event_parsers/hypo71.py +++ b/sourcespec2/input/event_parsers/hypo71.py @@ -13,7 +13,7 @@ import logging from datetime import datetime from obspy import UTCDateTime -from ...setup import config, ssp_exit +from ...setup import config from ...ssp_event import SSPEvent from ...ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -81,9 +81,9 @@ def parse_hypo71_picks(): return picks try: _is_hypo71_picks(pick_file) - except Exception as err: - logger.error(err) - ssp_exit(1) + except TypeError as err: + logger.warning(err) + return picks with open(pick_file, encoding='ascii') as fp: for line in fp: # remove newline From 5ea1f8a95adcf3ace0d7ba6375721ee14ebc1cb3 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 14:56:37 +0200 Subject: [PATCH 54/73] asdf_event, quakeml: return empty event and picks if an exception is raised --- sourcespec2/input/event_parsers/asdf_event.py | 6 ++++-- sourcespec2/input/event_parsers/quakeml.py | 9 ++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sourcespec2/input/event_parsers/asdf_event.py b/sourcespec2/input/event_parsers/asdf_event.py index 34cec752..04dbe96f 100644 --- a/sourcespec2/input/event_parsers/asdf_event.py +++ b/sourcespec2/input/event_parsers/asdf_event.py @@ -43,5 +43,7 @@ def parse_asdf_event_picks(asdf_file, event_id=None): obspy_catalog = pyasdf.ASDFDataSet(asdf_file, mode='r').events return parse_obspy_catalog(obspy_catalog, event_id, asdf_file) except Exception as err: - logger.error(err) - ssp_exit(1) + logger.warning(err) + ssp_event = None + picks = [] + return ssp_event, picks diff --git a/sourcespec2/input/event_parsers/quakeml.py b/sourcespec2/input/event_parsers/quakeml.py index 624b7ee9..66898585 100644 --- a/sourcespec2/input/event_parsers/quakeml.py +++ b/sourcespec2/input/event_parsers/quakeml.py @@ -11,7 +11,6 @@ """ import logging from obspy import read_events -from ...setup import ssp_exit from .obspy_catalog import parse_obspy_catalog logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -28,13 +27,13 @@ def parse_qml_event_picks(qml_file, event_id=None): :return: a tuple of (SSPEvent, picks) :rtype: tuple """ + ssp_event = None + picks = [] if qml_file is None: - ssp_event = None - picks = [] return ssp_event, picks try: obspy_catalog = read_events(qml_file) return parse_obspy_catalog(obspy_catalog, event_id, qml_file) except Exception as err: - logger.error(err) - ssp_exit(1) + logger.warning(err) + return ssp_event, picks From dd5a2263643a02f2605a0b1ef1333f4dc60f0493 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 15:30:18 +0200 Subject: [PATCH 55/73] Make reading event and picks from SAC trace headers the least prioritary source --- sourcespec2/input/augment_traces.py | 10 +- sourcespec2/input/event_and_picks.py | 92 +++++++++----- sourcespec2/input/event_parsers/__init__.py | 1 + sourcespec2/input/event_parsers/sac_event.py | 127 +++++++++++++++++++ sourcespec2/input/sac_header.py | 113 ----------------- sourcespec2/input/traces.py | 19 --- sourcespec2/source_spec.py | 3 +- 7 files changed, 195 insertions(+), 170 deletions(-) create mode 100644 sourcespec2/input/event_parsers/sac_event.py diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index 6d594915..a6b01361 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -24,7 +24,7 @@ from .sac_header import ( compute_sensitivity_from_SAC, get_instrument_from_SAC, get_station_coordinates_from_SAC, - get_picks_from_SAC) +) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -316,12 +316,8 @@ def _add_picks(trace, picks=None): """ if picks is None: picks = [] - trace_picks = [] - with contextlib.suppress(Exception): - trace_picks = get_picks_from_SAC(trace) - for pick in picks: - if pick.station == trace.stats.station: - trace_picks.append(pick) + trace_picks = [ + pick for pick in picks if pick.station == trace.stats.station] trace.stats.picks = trace_picks # Create empty dicts for arrivals, travel_times and takeoff angles. # They will be used later. diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 42e95b8c..65fca3b3 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -21,11 +21,42 @@ from .event_parsers import ( parse_source_spec_event_file, parse_hypo71_hypocenter, parse_hypo71_picks, parse_hypo2000_file, - parse_qml_event_picks, parse_asdf_event_picks + parse_qml_event_picks, parse_asdf_event_picks, + read_event_from_SAC, read_picks_from_SAC ) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) +def _read_event_and_picks_from_stream(stream): + """ + Read event and phase picks from a stream. + + :param stream: ObsPy Stream object containing event info and phase picks + :type stream: :class:`obspy.core.stream.Stream` + + :return: (ssp_event, picks) + :rtype: tuple of + :class:`sourcespec.ssp_event.SSPEvent`, + list of :class:`sourcespec.ssp_event.Pick` + + .. note:: + Currently only SAC files are supported. + """ + ssp_event = None + picks = [] + for trace in stream: + if ssp_event is None: + try: + ssp_event = read_event_from_SAC(trace) + except RuntimeError as err: + logger.warning(err) + try: + picks += read_picks_from_SAC(trace) + except RuntimeError as err: + logger.warning(err) + return ssp_event, picks + + # pylint: disable=inconsistent-return-statements def _parse_hypo_file(hypo_file, event_id=None): """ @@ -133,12 +164,13 @@ def _log_event_info(ssp_event): logger.info('---------------------------------------------------') -def read_event_and_picks(trace1=None): +def read_event_and_picks(stream=None): """ Read event and phase picks - :param trace1: ObsPy Trace object containing event info (optional) - :type trace1: :class:`obspy.core.stream.Stream` + :param stream: ObsPy Stream object containing event info and phase picks + (optional) + :type stream: :class:`obspy.core.stream.Stream` :return: (ssp_event, picks) :rtype: tuple of @@ -148,26 +180,34 @@ def read_event_and_picks(trace1=None): .. note:: The function reads event and phase picks from the following sources, from the less prioritary to the most prioritary: + - SAC trace headers - ASDF file - QML file - Hypocenter file - Pick file - - If trace1 is provided and contains event information, then this - information is used if no other source is found. """ picks = [] ssp_event = None + _event_source = None + _picks_source = None + # first, try to read event and picks from stream (SAC trace headers) + if stream is not None: + ssp_event, picks = _read_event_and_picks_from_stream(stream) + _event_source = 'traces' + _picks_source = 'traces' asdf_file = getattr(config.options, 'asdf_file', None) hypo_file = getattr(config.options, 'hypo_file', None) pick_file = getattr(config.options, 'pick_file', None) qml_file = getattr(config.options, 'qml_file', None) - _event_source = None - _picks_source = None - # parse ASDF file, lowest priority + # parse ASDF file, possibly replacing event and picks if asdf_file is not None: - ssp_event, picks = parse_asdf_event_picks(asdf_file) - _event_source = _picks_source = asdf_file + _new_ssp_event, _new_picks = parse_asdf_event_picks(asdf_file) + ssp_event, _event_source = _replace_event( + ssp_event, _new_ssp_event, _event_source, asdf_file + ) + picks, _picks_source = _replace_picks( + picks, _new_picks, _picks_source, asdf_file + ) # parse QML file, possibly replacing event and picks if qml_file is not None: _new_ssp_event, _new_picks = parse_qml_event_picks(qml_file) @@ -198,20 +238,14 @@ def read_event_and_picks(trace1=None): if ssp_event is not None: _log_event_info(ssp_event) - # if ssp_event is still None, get it from first trace - if ssp_event is None and trace1 is not None: - try: - ssp_event = trace1.stats.event - _log_event_info(ssp_event) - except AttributeError: - logger.error('No hypocenter information found.') - sys.stderr.write( - '\n' - 'Use "-q" or "-H" options to provide hypocenter information\n' - 'or add hypocenter information to the SAC file header\n' - '(if you use the SAC format).\n' - ) - ssp_exit(1) - # TODO: log also if trace1 is None? - - return (ssp_event, picks) + if ssp_event is None: + logger.error('No hypocenter information found.') + sys.stderr.write( + '\n' + 'Use "-q" or "-H" options to provide hypocenter information\n' + 'or add hypocenter information to the SAC file header\n' + '(if you use the SAC format).\n' + ) + ssp_exit(1) + + return ssp_event, picks diff --git a/sourcespec2/input/event_parsers/__init__.py b/sourcespec2/input/event_parsers/__init__.py index e3769c37..8e90af73 100644 --- a/sourcespec2/input/event_parsers/__init__.py +++ b/sourcespec2/input/event_parsers/__init__.py @@ -15,3 +15,4 @@ from .obspy_catalog import parse_obspy_catalog # noqa from .quakeml import parse_qml_event_picks # noqa from .source_spec_event import parse_source_spec_event_file # noqa +from .sac_event import read_event_from_SAC, read_picks_from_SAC # noqa diff --git a/sourcespec2/input/event_parsers/sac_event.py b/sourcespec2/input/event_parsers/sac_event.py new file mode 100644 index 00000000..0b889331 --- /dev/null +++ b/sourcespec2/input/event_parsers/sac_event.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Read metadata from SAC file headers. + +:copyright: + 2023-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +import logging +import contextlib +from ...ssp_event import SSPEvent +from ...ssp_pick import SSPPick +logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) + + +def read_event_from_SAC(trace): + """ + Read event information from SAC header. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core + + :return: Event information + :rtype: :class:`ssp_event.SSPEvent` + """ + try: + sac_hdr = trace.stats.sac + except AttributeError as e: + raise RuntimeError( + f'{trace.id}: not a SAC trace: cannot get hypocenter from header' + ) from e + try: + evla = sac_hdr['evla'] + evlo = sac_hdr['evlo'] + evdp = sac_hdr['evdp'] + begin = sac_hdr['b'] + except KeyError as e: + raise RuntimeError( + f'{trace.id}: cannot find hypocenter information in SAC header: ' + f'{e}' + ) from e + starttime = trace.stats.starttime + try: + tori = sac_hdr['o'] + origin_time = starttime + tori - begin + # make a copy of origin_time and round it to the nearest second + if origin_time.microsecond >= 500000: + evid_time = (origin_time + 1).replace(microsecond=0) + else: + evid_time = origin_time.replace(microsecond=0) + except Exception: + origin_time = None + # make a copy of starttime and round it to the nearest minute + if starttime.second >= 30: + evid_time = (starttime + 60).replace(second=0, microsecond=0) + else: + evid_time = starttime.replace(second=0, microsecond=0) + # Check if kevnm is not empty and does not contain spaces + # (if it has spaces, then kevnm is probably not an evid) + kevnm = sac_hdr.get('kevnm', '').strip() + if kevnm and ' ' not in kevnm: + evid = kevnm + else: + # create evid from origin time + evid = evid_time.strftime('%Y%m%d_%H%M%S') + ssp_event = SSPEvent() + ssp_event.event_id = evid + ssp_event.hypocenter.latitude.value_in_deg = evla + ssp_event.hypocenter.longitude.value_in_deg = evlo + ssp_event.hypocenter.depth.value = evdp + ssp_event.hypocenter.depth.units = 'km' + ssp_event.hypocenter.origin_time = origin_time + return ssp_event + + +def read_picks_from_SAC(trace): + """ + Read picks from SAC header. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core + + :return: List of picks + :rtype: list of :class:`ssp_pick.SSPPick` + """ + try: + sac_hdr = trace.stats.sac + except AttributeError as e: + raise RuntimeError( + f'{trace.id}: not a SAC trace: cannot get picks from header' + ) from e + trace_picks = [] + pick_fields = ( + 'a', 't0', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9') + for field in pick_fields: + try: + time = sac_hdr[field] + begin = sac_hdr['b'] + except KeyError: + continue + pick = SSPPick() + pick.station = trace.stats.station + pick.time = trace.stats.starttime + time - begin + # we will try to get the phase from the label later + pick.phase = 'X' + # now look at labels (ka, kt0, ...) + label = '' + with contextlib.suppress(Exception): + label = sac_hdr[f'k{field}'].strip() + if len(label) == 4: + # label is something like 'IPU0' or 'ES 2' + pick.flag = label[0] + pick.phase = label[1] + pick.polarity = label[2] + pick.quality = label[3] + if pick.phase.upper() not in 'PS' and len(label) > 0: + # we assume that label starts with 'P' or 'S' + pick.phase = label[0] + else: + # no label, use default phase for field + default_phases = {'a': 'P', 't0': 'S'} + pick.phase = default_phases.get(field, 'X') + trace_picks.append(pick) + return trace_picks diff --git a/sourcespec2/input/sac_header.py b/sourcespec2/input/sac_header.py index ef170643..7f6cc2e2 100644 --- a/sourcespec2/input/sac_header.py +++ b/sourcespec2/input/sac_header.py @@ -14,8 +14,6 @@ import contextlib from obspy.core.util import AttribDict from ..setup import config, ssp_exit -from ..ssp_event import SSPEvent -from ..ssp_pick import SSPPick logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -108,114 +106,3 @@ def get_station_coordinates_from_SAC(trace): f'{trace.id}: station coordinates read from SAC header') return coords return None - - -def get_event_from_SAC(trace): - """ - Get event information from SAC header. - - :param trace: ObsPy trace object - :type trace: :class:`obspy.core - - :return: Event information - :rtype: :class:`ssp_event.SSPEvent` - """ - try: - sac_hdr = trace.stats.sac - except AttributeError as e: - raise RuntimeError( - f'{trace.id}: not a SAC trace: cannot get hypocenter from header' - ) from e - try: - evla = sac_hdr['evla'] - evlo = sac_hdr['evlo'] - evdp = sac_hdr['evdp'] - begin = sac_hdr['b'] - except KeyError as e: - raise RuntimeError( - f'{trace.id}: cannot find hypocenter information in SAC header: ' - f'{e}' - ) from e - starttime = trace.stats.starttime - try: - tori = sac_hdr['o'] - origin_time = starttime + tori - begin - # make a copy of origin_time and round it to the nearest second - if origin_time.microsecond >= 500000: - evid_time = (origin_time + 1).replace(microsecond=0) - else: - evid_time = origin_time.replace(microsecond=0) - except Exception: - origin_time = None - # make a copy of starttime and round it to the nearest minute - if starttime.second >= 30: - evid_time = (starttime + 60).replace(second=0, microsecond=0) - else: - evid_time = starttime.replace(second=0, microsecond=0) - # Check if kevnm is not empty and does not contain spaces - # (if it has spaces, then kevnm is probably not an evid) - kevnm = sac_hdr.get('kevnm', '').strip() - if kevnm and ' ' not in kevnm: - evid = kevnm - else: - # create evid from origin time - evid = evid_time.strftime('%Y%m%d_%H%M%S') - ssp_event = SSPEvent() - ssp_event.event_id = evid - ssp_event.hypocenter.latitude.value_in_deg = evla - ssp_event.hypocenter.longitude.value_in_deg = evlo - ssp_event.hypocenter.depth.value = evdp - ssp_event.hypocenter.depth.units = 'km' - ssp_event.hypocenter.origin_time = origin_time - return ssp_event - - -def get_picks_from_SAC(trace): - """ - Get picks from SAC header. - - :param trace: ObsPy trace object - :type trace: :class:`obspy.core - - :return: List of picks - :rtype: list of :class:`ssp_pick.SSPPick` - """ - try: - sac_hdr = trace.stats.sac - except AttributeError as e: - raise RuntimeError( - f'{trace.id}: not a SAC trace: cannot get picks from header' - ) from e - trace_picks = [] - pick_fields = ( - 'a', 't0', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9') - for field in pick_fields: - try: - time = sac_hdr[field] - begin = sac_hdr['b'] - except KeyError: - continue - pick = SSPPick() - pick.station = trace.stats.station - pick.time = trace.stats.starttime + time - begin - # we will try to get the phase from the label later - pick.phase = 'X' - # now look at labels (ka, kt0, ...) - label = '' - with contextlib.suppress(Exception): - label = sac_hdr[f'k{field}'].strip() - if len(label) == 4: - # label is something like 'IPU0' or 'ES 2' - pick.flag = label[0] - pick.phase = label[1] - pick.polarity = label[2] - pick.quality = label[3] - if pick.phase.upper() not in 'PS' and len(label) > 0: - # we assume that label starts with 'P' or 'S' - pick.phase = label[0] - else: - # no label, use default phase for field - default_phases = {'a': 'P', 't0': 'S'} - pick.phase = default_phases.get(field, 'X') - trace_picks.append(pick) - return trace_picks diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 35b465ec..214b0956 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -24,7 +24,6 @@ from obspy import read from obspy.core import Stream from ..setup import config, ssp_exit -from .sac_header import get_event_from_SAC from .trace_parsers import parse_asdf_traces logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -140,23 +139,6 @@ def _read_trace_files(): return st -def _read_event_from_traces(stream): - """ - Read event information from trace headers. - The event information is stored in the trace.stats.event attribute. - - Currently supports only the SAC header. - - :param stream: ObsPy Stream object - :type stream: :class:`obspy.core.stream.Stream` - """ - for trace in stream: - try: - trace.stats.event = get_event_from_SAC(trace) - except RuntimeError: - continue - - def read_traces(): """ Read traces from the files or paths specified in the configuration. @@ -166,7 +148,6 @@ def read_traces(): """ logger.info('Reading traces...') stream = _read_asdf_traces() + _read_trace_files() - _read_event_from_traces(stream) ntraces = len(stream) logger.info(f'Reading traces: {ntraces} traces loaded') logger.info('---------------------------------------------------') diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 2580e62e..9e2bf32f 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -167,12 +167,11 @@ def main(): # Read all required information from disk from .input import read_traces st = read_traces() - trace1 = st[0] if len(st) else None st.sort() from .input import read_station_metadata inventory = read_station_metadata() from .input import read_event_and_picks - ssp_event, picks = read_event_and_picks(trace1) + ssp_event, picks = read_event_and_picks(st) # Now that we have an evid, we can rename the outdir and the log file from .setup import move_outdir, remove_old_outdir From 2091b996739295c759202567c4e0d69294fe4cb0 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 16:37:56 +0200 Subject: [PATCH 56/73] Issue SAC header warnings only if no event or pick information was found --- sourcespec2/input/event_and_picks.py | 33 ++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 65fca3b3..41fcfd3d 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -49,12 +49,14 @@ def _read_event_and_picks_from_stream(stream): try: ssp_event = read_event_from_SAC(trace) except RuntimeError as err: - logger.warning(err) + _read_event_and_picks_from_stream.event_warnings.append(err) try: picks += read_picks_from_SAC(trace) except RuntimeError as err: - logger.warning(err) + _read_event_and_picks_from_stream.picks_warnings.append(err) return ssp_event, picks +_read_event_and_picks_from_stream.event_warnings = [] # noqa +_read_event_and_picks_from_stream.picks_warnings = [] # noqa # pylint: disable=inconsistent-return-statements @@ -152,15 +154,33 @@ def _replace_picks(old_picks, new_picks, old_source, new_source): return new_picks, new_source -def _log_event_info(ssp_event): +def _log_event_and_pick_info(ssp_event, picks, event_source, picks_source): """ Log event information. :param ssp_event: SSPEvent object :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` + :param picks: List of Pick objects + :type picks: list of :class:`sourcespec.ssp_event.Pick` + :param event_source: Event source + :type event_source: str + :param picks_source: Picks source """ - for line in str(ssp_event).splitlines(): - logger.info(line) + if ssp_event is None: + # only log warnings if no event information was found + for warning in _read_event_and_picks_from_stream.event_warnings: + logger.warning(warning) + else: + logger.info(f'Event information read from: {event_source}') + for line in str(ssp_event).splitlines(): + logger.info(line) + if not picks: + # only log warnings if no pick information was found + for warning in _read_event_and_picks_from_stream.picks_warnings: + logger.warning(warning) + else: + logger.info(f'Pick information read from: {picks_source}') + logger.info(f'{len(picks)} picks read') logger.info('---------------------------------------------------') @@ -235,8 +255,7 @@ def read_event_and_picks(stream=None): picks, _picks_source = _replace_picks( picks, _new_picks, _picks_source, pick_file ) - if ssp_event is not None: - _log_event_info(ssp_event) + _log_event_and_pick_info(ssp_event, picks, _event_source, _picks_source) if ssp_event is None: logger.error('No hypocenter information found.') From 6cfe38a378f907f50b2a5c1d4059a267732ae9ae Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 16:39:29 +0200 Subject: [PATCH 57/73] Make it possible to read event, picks, traces and metadata from multiple ASDF files --- sourcespec2/input/event_and_picks.py | 70 ++++++++++++++++++--------- sourcespec2/input/station_metadata.py | 13 +++-- sourcespec2/input/traces.py | 19 +++++--- sourcespec2/setup/configure_cli.py | 4 ++ sourcespec2/ssp_output.py | 3 +- sourcespec2/ssp_parse_arguments.py | 6 +-- 6 files changed, 76 insertions(+), 39 deletions(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 41fcfd3d..70a95801 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -59,6 +59,31 @@ def _read_event_and_picks_from_stream(stream): _read_event_and_picks_from_stream.picks_warnings = [] # noqa +def _read_event_and_picks_from_ASDF_file_list(filelist): + """ + Read event and phase picks from a list of ASDF files. + + :param filelist: List of ASDF files + :type filelist: list of str + + :return: (ssp_event, picks, event_source) + :rtype: tuple of + :class:`sourcespec.ssp_event.SSPEvent`, + list of :class:`sourcespec.ssp_event.Pick`, + str + """ + ssp_event = None + picks = [] + event_source = None + for asdf_file in filelist: + _ssp_event, _picks = parse_asdf_event_picks(asdf_file) + if ssp_event is None: + ssp_event = _ssp_event + event_source = asdf_file + picks += _picks + return ssp_event, picks, event_source + + # pylint: disable=inconsistent-return-statements def _parse_hypo_file(hypo_file, event_id=None): """ @@ -208,34 +233,35 @@ def read_event_and_picks(stream=None): """ picks = [] ssp_event = None - _event_source = None - _picks_source = None + event_source = None + picks_source = None # first, try to read event and picks from stream (SAC trace headers) if stream is not None: ssp_event, picks = _read_event_and_picks_from_stream(stream) - _event_source = 'traces' - _picks_source = 'traces' - asdf_file = getattr(config.options, 'asdf_file', None) + event_source = 'traces' + picks_source = 'traces' + asdf_path = getattr(config.options, 'asdf_path', None) hypo_file = getattr(config.options, 'hypo_file', None) pick_file = getattr(config.options, 'pick_file', None) qml_file = getattr(config.options, 'qml_file', None) # parse ASDF file, possibly replacing event and picks - if asdf_file is not None: - _new_ssp_event, _new_picks = parse_asdf_event_picks(asdf_file) - ssp_event, _event_source = _replace_event( - ssp_event, _new_ssp_event, _event_source, asdf_file + if asdf_path is not None: + _new_ssp_event, _new_picks, _new_event_source =\ + _read_event_and_picks_from_ASDF_file_list(asdf_path) + ssp_event, event_source = _replace_event( + ssp_event, _new_ssp_event, event_source, _new_event_source ) - picks, _picks_source = _replace_picks( - picks, _new_picks, _picks_source, asdf_file + picks, picks_source = _replace_picks( + picks, _new_picks, picks_source, 'ASDF files' ) # parse QML file, possibly replacing event and picks if qml_file is not None: _new_ssp_event, _new_picks = parse_qml_event_picks(qml_file) - ssp_event, _event_source = _replace_event( - ssp_event, _new_ssp_event, _event_source, qml_file + ssp_event, event_source = _replace_event( + ssp_event, _new_ssp_event, event_source, qml_file ) - picks, _picks_source = _replace_picks( - picks, _new_picks, _picks_source, qml_file + picks, picks_source = _replace_picks( + picks, _new_picks, picks_source, qml_file ) # parse hypocenter file, possibly replacing event and picks if hypo_file is not None: @@ -243,19 +269,19 @@ def read_event_and_picks(stream=None): hypo_file, config.options.evid) # this is needed when writing the output file in hypo71 format config.hypo_file_format = file_format - ssp_event, _event_source = _replace_event( - ssp_event, _new_ssp_event, _event_source, hypo_file + ssp_event, event_source = _replace_event( + ssp_event, _new_ssp_event, event_source, hypo_file ) - picks, _picks_source = _replace_picks( - picks, _new_picks, _picks_source, hypo_file + picks, picks_source = _replace_picks( + picks, _new_picks, picks_source, hypo_file ) # parse pick file, possibly replacing picks if pick_file is not None: _new_picks = parse_hypo71_picks() - picks, _picks_source = _replace_picks( - picks, _new_picks, _picks_source, pick_file + picks, picks_source = _replace_picks( + picks, _new_picks, picks_source, pick_file ) - _log_event_and_pick_info(ssp_event, picks, _event_source, _picks_source) + _log_event_and_pick_info(ssp_event, picks, event_source, picks_source) if ssp_event is None: logger.error('No hypocenter information found.') diff --git a/sourcespec2/input/station_metadata.py b/sourcespec2/input/station_metadata.py index 5127dd56..ce533185 100644 --- a/sourcespec2/input/station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -26,11 +26,14 @@ def _read_asdf_inventory(): :return: inventory :rtype: :class:`~obspy.core.inventory.inventory.Inventory` """ - asdf_file = config.options.asdf_file - if not asdf_file: - return Inventory() - logger.info(f'Reading station metadata from ASDF file: {asdf_file}') - return parse_asdf_inventory(asdf_file) + inventory = Inventory() + asdf_path = getattr(config.options, 'asdf_path', None) + if not asdf_path: + return inventory + for asdf_file in asdf_path: + logger.info(f'Reading station metadata from ASDF file: {asdf_file}') + inventory += parse_asdf_inventory(asdf_file) + return inventory def _read_station_metadata_from_files(): diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 214b0956..c04ccbb3 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -94,14 +94,17 @@ def _read_asdf_traces(): :return: ObsPy Stream object :rtype: :class:`obspy.core.stream.Stream` """ - asdf_file = config.options.asdf_file - if not asdf_file: - return Stream() - asdf_tag = config.options.asdf_tag - logger.info(f'Reading traces from ASDF file: {asdf_file}') - return _filter_by_station( - parse_asdf_traces(asdf_file, tag=asdf_tag, read_headers=True) - ) + stream = Stream() + asdf_path = getattr(config.options, 'asdf_path', None) + if not asdf_path: + return stream + asdf_tag = getattr(config.options, 'asdf_tag', None) + for asdf_file in asdf_path: + logger.info(f'Reading traces from ASDF file: {asdf_file}') + stream += _filter_by_station( + parse_asdf_traces(asdf_file, tag=asdf_tag, read_headers=True) + ) + return stream def _read_trace_files(): diff --git a/sourcespec2/setup/configure_cli.py b/sourcespec2/setup/configure_cli.py index c6850c5d..0540a81d 100644 --- a/sourcespec2/setup/configure_cli.py +++ b/sourcespec2/setup/configure_cli.py @@ -277,6 +277,10 @@ def configure_cli(options=None, progname='source_spec', config_overrides=None): # trace_path is a list options.trace_path = [ _fix_and_expand_path(path) for path in options.trace_path] + if getattr(options, 'asdf_path', None): + # asdf_path is a list + options.asdf_path = [ + _fix_and_expand_path(path) for path in options.asdf_path] if getattr(options, 'qml_file', None): options.qml_file = _fix_and_expand_path(options.qml_file) if getattr(options, 'hypo_file', None): diff --git a/sourcespec2/ssp_output.py b/sourcespec2/ssp_output.py index f7a59983..83a2e018 100644 --- a/sourcespec2/ssp_output.py +++ b/sourcespec2/ssp_output.py @@ -418,8 +418,9 @@ def _make_symlinks(): config.options.hypo_file, config.options.pick_file, config.options.qml_file, - config.options.asdf_file, ] + with contextlib.suppress(TypeError): + filelist += list(config.options.asdf_path) for filename in filelist: if filename is None or not os.path.exists(filename): continue diff --git a/sourcespec2/ssp_parse_arguments.py b/sourcespec2/ssp_parse_arguments.py index ef86b310..7fae2b81 100644 --- a/sourcespec2/ssp_parse_arguments.py +++ b/sourcespec2/ssp_parse_arguments.py @@ -119,11 +119,11 @@ def _init_parser(description, epilog, nargs): metavar='FILE' ) group.add_argument( - '-a', '--asdffile', dest='asdf_file', + '-a', '--asdf_path', nargs=nargs, action='store', default=None, help='get picks, hypocenter information, traces and metadata from\n' - 'ASDF FILE', - metavar='FILE' + 'one or more ASDF files', + metavar='ASDF_FILE' ) parser.add_argument( '-g', '--tag', dest='asdf_tag', From f90cc3f4ff82171c1ea259ccc27a74df552d0d12 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 16:43:44 +0200 Subject: [PATCH 58/73] `_Config()` class: use the term "paramter" instead of "option" --- sourcespec2/setup/config.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/sourcespec2/setup/config.py b/sourcespec2/setup/config.py index 58a7a4df..45580e91 100644 --- a/sourcespec2/setup/config.py +++ b/sourcespec2/setup/config.py @@ -125,7 +125,7 @@ def update(self, other): :param other: The dictionary with the new values :type other: dict - :raises ValueError: If an error occurs while parsing the options + :raises ValueError: If an error occurs while parsing the parameters """ for key, value in other.items(): self[key] = value @@ -182,7 +182,7 @@ def validate(self): """ Validate the configuration. - :raises ValueError: If an error occurs while validating the options + :raises ValueError: If an error occurs while validating the parameters """ config_obj = ConfigObj(self, configspec=parse_configspec()) val = Validator() @@ -214,7 +214,7 @@ def _check_deprecated_config_params(self): """ Check the deprecated configuration parameters. - :raises ValueError: If an error occurs while parsing the options + :raises ValueError: If an error occurs while parsing the parameters """ deprecation_msgs = [] for param, msgs in deprecated_config_params.items(): @@ -243,7 +243,7 @@ def _check_mandatory_config_params(self): """ Check the mandatory configuration parameters. - :raises ValueError: If an error occurs while parsing the options + :raises ValueError: If an error occurs while parsing the parameters """ messages = [] for par in mandatory_config_params: @@ -256,9 +256,9 @@ def _check_mandatory_config_params(self): def _check_force_list(self): """ - Check the force_list options and convert them to lists of floats. + Check the force_list parameters and convert them to lists of floats. - :raises ValueError: If an error occurs while parsing the options + :raises ValueError: If an error occurs while parsing the parameters """ try: for param in [ @@ -274,7 +274,7 @@ def _check_list_lengths(self): """ Check that the lists describing the source model have the same length. - :raises ValueError: If an error occurs while parsing the options + :raises ValueError: If an error occurs while parsing the parameters """ n_vp_source = _none_lenght(self['vp_source']) n_vs_source = _none_lenght(self['vs_source']) @@ -291,9 +291,9 @@ def _check_list_lengths(self): def _check_Er_freq_range(self): """ - Check the Er_freq_range option. + Check the Er_freq_range parameter. - :raises ValueError: If an error occurs while parsing the options + :raises ValueError: If an error occurs while parsing the parameters """ if self['Er_freq_range'] is None: self['Er_freq_range'] = [None, None] @@ -309,17 +309,17 @@ def _check_Er_freq_range(self): def _check_html_report(self): """ - Check the html_report option. + Check the html_report parameter. """ if self['html_report']: if not self['plot_save']: self['warnings'].append( - 'The "html_report" option is selected but "plot_save" ' + 'The "html_report" parameter is selected but "plot_save" ' 'is "False". HTML report will have no plots.' ) if self['plot_save_format'] not in ['png', 'svg']: self['warnings'].append( - 'The "html_report" option is selected but ' + 'The "html_report" parameter is selected but ' '"plot_save_format" is not "png" or "svg". ' 'HTML report will have no plots.' ) From 13cea573409cd4e97c8eeaf2e06692dda2d73ba1 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Mon, 22 Jul 2024 18:32:03 +0200 Subject: [PATCH 59/73] Custom `_Options` class, with builtin checks for API users. --- sourcespec2/setup/config.py | 71 ++++++++++++++++++++++++++++++++++--- 1 file changed, 67 insertions(+), 4 deletions(-) diff --git a/sourcespec2/setup/config.py b/sourcespec2/setup/config.py index 45580e91..2e01a48b 100644 --- a/sourcespec2/setup/config.py +++ b/sourcespec2/setup/config.py @@ -10,8 +10,8 @@ (http://www.cecill.info/licences.en.html) """ import os -import types import contextlib +import warnings from collections import defaultdict from .configobj_helpers import parse_configspec, get_default_config_obj from .mandatory_deprecated import ( @@ -65,6 +65,69 @@ def _none_lenght(input_list): # ---- End Helper functions ---- +class _Options(): + """ + Options class for sourcespec, with builtin checks for API users. + """ + def __init__(self): + self._trace_path = None + self._asdf_path = None + + def __getitem__(self, key): + """Make attributes accessible as dictionary keys.""" + return getattr(self, key) + + def __setitem__(self, key, value): + """Make attributes accessible as dictionary keys.""" + setattr(self, key, value) + + def __str__(self): + """Return a dict-like representation of the object.""" + return str({ + k: getattr(self, k) for k in dir(self) if not k.startswith('_')} + ) + + @property + def trace_path(self): + """Path to the trace files. Must be a list.""" + return self._trace_path + + @trace_path.setter + def trace_path(self, value): + if value is None or isinstance(value, (list, tuple)): + self._trace_path = value + else: + warnings.warn( + '"trace_path" must be a list. Converting to a list with one ' + 'element.' + ) + self._trace_path = [value] + + @trace_path.deleter + def trace_path(self): + self._trace_path = None + + @property + def asdf_path(self): + """Path to the ASDF files. Must be a list.""" + return self._asdf_path + + @asdf_path.setter + def asdf_path(self, value): + if value is None or isinstance(value, (list, tuple)): + self._asdf_path = value + else: + warnings.warn( + '"asdf_path" must be a list. Converting to a list with one ' + 'element.' + ) + self._asdf_path = [value] + + @asdf_path.deleter + def asdf_path(self): + self._asdf_path = None + + class _Config(dict): """ Config class for sourcespec. @@ -81,14 +144,14 @@ class _Config(dict): """ def __init__(self): # Additional config values that must exist for the code to run without - # errors. Tey must be defined using the dict syntax. + # errors. They must be defined using the dict syntax. self['running_from_command_line'] = False self['vertical_channel_codes'] = ['Z'] self['horizontal_channel_codes_1'] = ['N', 'R'] self['horizontal_channel_codes_2'] = ['E', 'T'] self['TRACEID_MAP'] = None - # Empty options object, for compatibility with the command line version - self['options'] = types.SimpleNamespace() + # options object with some built-in checks for API users + self['options'] = _Options() # A list of warnings to be issued when logger is set up self['warnings'] = [] # Create a dict to store figure paths From dd78cd696ffc18070ae302bee105c3bb2c6b64c3 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 23 Jul 2024 09:02:37 +0200 Subject: [PATCH 60/73] Move `sac_header.py` to `station_metadata_parsers/sac_station_metadata.py --- sourcespec2/input/augment_traces.py | 7 ++++--- sourcespec2/input/station_metadata_parsers/__init__.py | 4 ++++ .../sac_station_metadata.py} | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) rename sourcespec2/input/{sac_header.py => station_metadata_parsers/sac_station_metadata.py} (95%) diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index a6b01361..1e57a8d0 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -20,10 +20,11 @@ import contextlib from obspy.core.util import AttribDict from ..setup import config -from .station_metadata_parsers import PAZ -from .sac_header import ( +from .station_metadata_parsers import ( + PAZ, compute_sensitivity_from_SAC, - get_instrument_from_SAC, get_station_coordinates_from_SAC, + get_instrument_from_SAC, + get_station_coordinates_from_SAC, ) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) diff --git a/sourcespec2/input/station_metadata_parsers/__init__.py b/sourcespec2/input/station_metadata_parsers/__init__.py index 48ad1be4..49442e72 100644 --- a/sourcespec2/input/station_metadata_parsers/__init__.py +++ b/sourcespec2/input/station_metadata_parsers/__init__.py @@ -11,3 +11,7 @@ """ from .asdf_inventory import parse_asdf_inventory # noqa from .paz import read_paz_file, PAZ # noqa +from .sac_station_metadata import ( # noqa + compute_sensitivity_from_SAC, get_instrument_from_SAC, + get_station_coordinates_from_SAC, +) diff --git a/sourcespec2/input/sac_header.py b/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py similarity index 95% rename from sourcespec2/input/sac_header.py rename to sourcespec2/input/station_metadata_parsers/sac_station_metadata.py index 7f6cc2e2..1112c8ed 100644 --- a/sourcespec2/input/sac_header.py +++ b/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # SPDX-License-Identifier: CECILL-2.1 """ -Read metadata from SAC file headers. +Read station metadata from SAC file headers. :copyright: 2023-2025 Claudio Satriano @@ -13,7 +13,7 @@ import logging import contextlib from obspy.core.util import AttribDict -from ..setup import config, ssp_exit +from ...setup import config, ssp_exit logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -49,7 +49,7 @@ def compute_sensitivity_from_SAC(trace): # handpicked list of instruments, instrtypes and band/instr codes, # mainly for ISNet compatibility -instruments = { +_instruments = { 'CMG-5T': {'instrtype': 'acc', 'band_code': 'H', 'instr_code': 'N'}, 'CMG-40T': {'instrtype': 'broadb', 'band_code': 'H', 'instr_code': 'H'}, 'TRILLIUM': {'instrtype': 'broadb', 'band_code': 'H', 'instr_code': 'H'}, @@ -69,7 +69,7 @@ def get_instrument_from_SAC(trace): :rtype: tuple """ try: - codes = instruments[trace.stats.sac.kinst] + codes = _instruments[trace.stats.sac.kinst] instrtype = codes['instrtype'] band_code = codes['band_code'] instr_code = codes['instr_code'] From c820df2fc054f44a38fa269fb703453370983d76 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 23 Jul 2024 12:43:44 +0200 Subject: [PATCH 61/73] Move instrument metadata parsing from SAC header to `read_traces` and `read_station_metadata`. --- sourcespec2/input/augment_traces.py | 96 ++--------- sourcespec2/input/instrument_type.py | 42 +++++ sourcespec2/input/station_metadata.py | 156 +++++++++++++++++- .../sac_station_metadata.py | 7 +- sourcespec2/input/traces.py | 59 +++++++ sourcespec2/source_spec.py | 2 +- 6 files changed, 271 insertions(+), 91 deletions(-) create mode 100644 sourcespec2/input/instrument_type.py diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index 1e57a8d0..67ec9539 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -20,12 +20,7 @@ import contextlib from obspy.core.util import AttribDict from ..setup import config -from .station_metadata_parsers import ( - PAZ, - compute_sensitivity_from_SAC, - get_instrument_from_SAC, - get_station_coordinates_from_SAC, -) +from .instrument_type import get_instrument_type logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -104,24 +99,6 @@ def _select_components(st): st.traces[:] = traces_to_keep[:] -def _correct_traceid(trace): - """ - Correct traceid from config.TRACEID_MAP, if available. - - :param trace: ObsPy Trace object - :type trace: :class:`obspy.core.trace.Trace` - """ - if config.TRACEID_MAP is None: - return - with contextlib.suppress(KeyError): - traceid = config.TRACEID_MAP[trace.get_id()] - net, sta, loc, chan = traceid.split('.') - trace.stats.network = net - trace.stats.station = sta - trace.stats.location = loc - trace.stats.channel = chan - - def _add_instrtype(trace): """ Add instrtype to trace. @@ -129,30 +106,7 @@ def _add_instrtype(trace): :param trace: ObsPy Trace object :type trace: :class:`obspy.core.trace.Trace` """ - instrtype = None - band_code = None - instr_code = None - trace.stats.instrtype = None - # First, try to get the instrtype from channel name - chan = trace.stats.channel - if len(chan) > 2: - band_code = chan[0] - instr_code = chan[1] - if instr_code in config.INSTR_CODES_VEL: - # SEED standard band codes from higher to lower sampling rate - # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ - if band_code in ['G', 'D', 'E', 'S']: - instrtype = 'shortp' - if band_code in ['F', 'C', 'H', 'B']: - instrtype = 'broadb' - if instr_code in config.INSTR_CODES_ACC: - instrtype = 'acc' - if instrtype is None: - # Let's see if there is an instrument name in SAC header (ISNet format) - # In this case, we define band and instrument codes a posteriori - instrtype, band_code, instr_code = get_instrument_from_SAC(trace) - orientation = trace.stats.channel[-1] - trace.stats.channel = ''.join((band_code, instr_code, orientation)) + instrtype = get_instrument_type(trace) trace.stats.instrtype = instrtype trace.stats.info = f'{trace.id} {trace.stats.instrtype}' @@ -179,30 +133,6 @@ def _add_inventory(trace, inventory): inv.networks[0].stations[0].code = sta inv.networks[0].stations[0].channels[0].code = chan inv.networks[0].stations[0].channels[0].location_code = loc - # If a "sensitivity" config option is provided, override the Inventory - # object with a new one constructed from the sensitivity value - if config.sensitivity is not None: - # save coordinates from the inventory, if available - coords = None - with contextlib.suppress(Exception): - coords = inv.get_coordinates(trace.id, trace.stats.starttime) - paz = PAZ() - paz.seedID = trace.id - paz.sensitivity = compute_sensitivity_from_SAC(trace) - paz.poles = [] - paz.zeros = [] - if inv: - logger.warning( - f'Overriding response for {trace.id} with constant ' - f'sensitivity {paz.sensitivity}') - inv = paz.to_inventory() - # restore coordinates, if available - if coords is not None: - chan = inv.networks[0].stations[0].channels[0] - chan.latitude = coords['latitude'] - chan.longitude = coords['longitude'] - chan.elevation = coords['elevation'] - chan.depth = coords['local_depth'] trace.stats.inventory = inv @@ -266,24 +196,19 @@ def _add_coords(trace): # if coordinates are not found coords = trace.stats.inventory.get_coordinates( trace.id, trace.stats.starttime) - if coords is not None: - # Build an AttribDict and make sure that coordinates are floats - coords = AttribDict({ - key: float(value) for key, value in coords.items()}) - coords = ( - None if ( - coords.longitude == 0 and coords.latitude == 0 and - coords.local_depth == 123456 and coords.elevation == 123456) - else coords) - if coords is None: - # If we still don't have trace coordinates, - # we try to get them from SAC header - coords = get_station_coordinates_from_SAC(trace) if coords is None: # Give up! _add_coords.skipped.append(trace.id) raise RuntimeError( f'{trace.id}: could not find coords for trace: skipping trace') + # Build an AttribDict and make sure that coordinates are floats + coords = AttribDict({ + key: float(value) for key, value in coords.items()}) + coords = ( + None if ( + coords.longitude == 0 and coords.latitude == 0 and + coords.local_depth == 123456 and coords.elevation == 123456) + else coords) if coords.latitude == coords.longitude == 0: logger.warning( f'{trace.id}: trace has latitude and longitude equal to zero!') @@ -398,7 +323,6 @@ def augment_traces(st, inventory, ssp_event, picks): # Then, augment the traces and remove the problematic ones traces_to_keep = [] for trace in st: - _correct_traceid(trace) try: _augment_trace(trace, inventory, ssp_event, picks) except Exception as err: diff --git a/sourcespec2/input/instrument_type.py b/sourcespec2/input/instrument_type.py new file mode 100644 index 00000000..634b177e --- /dev/null +++ b/sourcespec2/input/instrument_type.py @@ -0,0 +1,42 @@ +# -*- coding: utf-8 -*- +# SPDX-License-Identifier: CECILL-2.1 +""" +Get the instrument type from the channel name of a trace. + +:copyright: + 2023-2024 Claudio Satriano +:license: + CeCILL Free Software License Agreement v2.1 + (http://www.cecill.info/licences.en.html) +""" +from ..setup import config + + +def get_instrument_type(trace): + """ + Get the instrument type from the channel name of the trace. + + :param trace: ObsPy Trace object + :type trace: :class:`obspy.core.trace.Trace` + + :return: Instrument type + :rtype: str + """ + instrtype = None + _band_code = None + _instr_code = None + # First, try to get the instrtype from channel name + chan = trace.stats.channel + if len(chan) > 2: + _band_code = chan[0] + _instr_code = chan[1] + if _instr_code in config.INSTR_CODES_VEL: + # SEED standard band codes from higher to lower sampling rate + # https://ds.iris.edu/ds/nodes/dmc/data/formats/seed-channel-naming/ + if _band_code in ['G', 'D', 'E', 'S']: + instrtype = 'shortp' + if _band_code in ['F', 'C', 'H', 'B']: + instrtype = 'broadb' + if _instr_code in config.INSTR_CODES_ACC: + instrtype = 'acc' + return instrtype diff --git a/sourcespec2/input/station_metadata.py b/sourcespec2/input/station_metadata.py index ce533185..fe1f916e 100644 --- a/sourcespec2/input/station_metadata.py +++ b/sourcespec2/input/station_metadata.py @@ -12,13 +12,160 @@ """ import os import logging +import contextlib from obspy import read_inventory from obspy.core.inventory import Inventory from ..setup import config -from .station_metadata_parsers import read_paz_file, parse_asdf_inventory +from .station_metadata_parsers import ( + PAZ, + read_paz_file, parse_asdf_inventory, + compute_sensitivity_from_SAC, + get_station_coordinates_from_SAC +) logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) +def _update_coords(obj, coords): + """ + Update coordinates the given object. + + :param obj: ObsPy Station or Channel object + :type obj: :class:`obspy.core.inventory.station.Station` + or :class:`obspy.core.inventory.channel.Channel` + :param coords: Dictionary containing coordinates + :type coords: dict + """ + obj.latitude = coords['latitude'] + obj.longitude = coords['longitude'] + obj.elevation = coords['elevation'] + with contextlib.suppress(KeyError): + obj.depth = coords['local_depth'] + + +def _update_inventory_with_trace_coords(trace, inventory): + """ + Update inventory with station coordinates from a trace. + + The inventory is updated in place. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + :param inventory: ObsPy Inventory object to update + :type inventory: :class:`obspy.core + + .. note:: + Currently only SAC files are supported. + """ + if not hasattr(trace.stats, 'sac'): + return + # get coordinates from SAC headers + sac_coords = get_station_coordinates_from_SAC(trace) + if sac_coords is None: + return + for net in inventory: + if net.code != trace.stats.network: + continue + for sta in net: + if sta.code != trace.stats.station: + continue + for chan in sta: + if chan.location_code != trace.stats.location: + continue + if chan.code != trace.stats.channel: + continue + _update_coords(chan, sac_coords) + _update_coords(sta, sac_coords) + + +def _update_inventory_with_trace_sensitivity(trace, inventory): + """ + Update inventory with sensitivity from a trace. + + :param trace: ObsPy trace object + :type trace: :class:`obspy.core.trace.Trace` + :param inventory: ObsPy Inventory object to update + :type inventory: :class:`obspy.core.inventory.inventory.Inventory` + + :return: inventory + :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + + .. note:: + Currently only SAC files are supported. + """ + if not hasattr(trace.stats, 'sac'): + return inventory + update_inventory = inventory.copy() + trace_inv = update_inventory.select( + network=trace.stats.network, + station=trace.stats.station, + location=trace.stats.location, + channel=trace.stats.channel, + starttime=trace.stats.starttime + ) + # save coordinates from the inventory, if available + coords = None + with contextlib.suppress(Exception): + coords = trace_inv.get_coordinates( + trace.id, trace.stats.starttime) + paz = PAZ() + paz.seedID = trace.id + paz.sensitivity = compute_sensitivity_from_SAC(trace) + paz.poles = [] + paz.zeros = [] + if trace_inv: + logger.warning( + f'Overriding response for {trace.id} with constant ' + f'sensitivity {paz.sensitivity}') + # override the trace_inv object with a new one constructed from the + # sensitivity value + trace_inv = paz.to_inventory() + # restore coordinates, if available + if coords is not None: + for sta in trace_inv[0].stations: + _update_coords(sta, coords) + for chan in sta.channels: + _update_coords(chan, coords) + else: + _update_inventory_with_trace_coords(trace, trace_inv) + # update inventory + update_inventory = update_inventory.remove( + network=trace.stats.network, + station=trace.stats.station, + location=trace.stats.location, + channel=trace.stats.channel + ) + trace_inv + return update_inventory + + +def _update_inventory_from_stream(stream, inventory): + """ + Update inventory with station metadata from a stream. + + :param stream: ObsPy Stream object containing station metadata + :type stream: :class:`obspy.core.stream.Stream` + :param inventory: ObsPy Inventory object to update + :type inventory: :class:`obspy.core.inventory.inventory.Inventory` + + :return: inventory + :rtype: :class:`~obspy.core.inventory.inventory.Inventory` + + .. note:: + Currently only SAC files are supported. + """ + update_inventory = inventory.copy() + for trace in stream: + # ignore traces that are not in SAC format + if not hasattr(trace.stats, 'sac'): + continue + _update_inventory_with_trace_coords(trace, update_inventory) + # If a "sensitivity" config option is provided, override the Inventory + # object with a new one constructed from the sensitivity value + if config.sensitivity is not None: + update_inventory = _update_inventory_with_trace_sensitivity( + trace, update_inventory) + return update_inventory + + def _read_asdf_inventory(): """ Read station metadata from ASDF file specified in the configuration. @@ -78,15 +225,20 @@ def _read_station_metadata_from_files(): return inventory -def read_station_metadata(): +def read_station_metadata(stream=None): """ Read station metadata. + :param stream: ObsPy Stream object containing station metadata (optional) + :type stream: :class:`obspy.core.stream.Stream` + :return: inventory :rtype: :class:`~obspy.core.inventory.inventory.Inventory` """ logger.info('Reading station metadata...') inventory = _read_station_metadata_from_files() + _read_asdf_inventory() + if stream is not None: + inventory = _update_inventory_from_stream(stream, inventory) nstations = len(inventory.get_contents()['stations']) logger.info(f'Reading station metadata: {nstations} stations read') logger.info('---------------------------------------------------') diff --git a/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py b/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py index 1112c8ed..9d986c37 100644 --- a/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py +++ b/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py @@ -102,7 +102,10 @@ def get_station_coordinates_from_SAC(trace): stlo = trace.stats.sac.stlo stel = trace.stats.sac.get('stel', 0.) coords = AttribDict(latitude=stla, longitude=stlo, elevation=stel) - logger.info( - f'{trace.id}: station coordinates read from SAC header') + msg = f'{trace.id}: station coordinates read from SAC header' + if msg not in get_station_coordinates_from_SAC.msgs: + logger.info(msg) + get_station_coordinates_from_SAC.msgs.append(msg) return coords return None +get_station_coordinates_from_SAC.msgs = [] # noqa diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index c04ccbb3..0a92452d 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -17,6 +17,7 @@ """ import os import logging +import contextlib import shutil import tarfile import zipfile @@ -25,6 +26,8 @@ from obspy.core import Stream from ..setup import config, ssp_exit from .trace_parsers import parse_asdf_traces +from .station_metadata_parsers import get_instrument_from_SAC +from .instrument_type import get_instrument_type logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) @@ -142,6 +145,60 @@ def _read_trace_files(): return st +def _correct_traceids(stream): + """ + Correct traceids from config.TRACEID_MAP, if available. + + :param stream: ObsPy Stream object + :type stream: :class:`obspy.core.stream.Stream` + """ + if config.TRACEID_MAP is None: + return + for trace in stream: + with contextlib.suppress(KeyError): + traceid = config.TRACEID_MAP[trace.get_id()] + net, sta, loc, chan = traceid.split('.') + trace.stats.network = net + trace.stats.station = sta + trace.stats.location = loc + trace.stats.channel = chan + + +def _update_non_standard_trace_ids(stream): + """ + Update non-standard trace IDs with a standard SEED ID obtained from the + instrument type. + + :param stream: Stream object + :type stream: :class:`obspy.core + + .. note:: + Currently only SAC files are supported. + """ + traces_to_skip = [] + for trace in stream: + if not hasattr(trace.stats, 'sac'): + continue + instrtype = get_instrument_type(trace) + if instrtype is not None: + continue + try: + instrtype, band_code, instr_code = get_instrument_from_SAC(trace) + except RuntimeError as e: + logger.warning(e) + traces_to_skip.append(trace) + continue + old_id = trace.id + orientation = trace.stats.channel[-1] + trace.stats.channel = ''.join((band_code, instr_code, orientation)) + msg = f'{old_id}: non-standard trace ID updated to {trace.id}' + if msg not in _update_non_standard_trace_ids.msgs: + logger.info(msg) + _update_non_standard_trace_ids.msgs.append(msg) + stream.traces = [trace for trace in stream if trace not in traces_to_skip] +_update_non_standard_trace_ids.msgs = [] # noqa + + def read_traces(): """ Read traces from the files or paths specified in the configuration. @@ -151,6 +208,8 @@ def read_traces(): """ logger.info('Reading traces...') stream = _read_asdf_traces() + _read_trace_files() + _correct_traceids(stream) + _update_non_standard_trace_ids(stream) ntraces = len(stream) logger.info(f'Reading traces: {ntraces} traces loaded') logger.info('---------------------------------------------------') diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 9e2bf32f..ac7b1dd7 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -169,7 +169,7 @@ def main(): st = read_traces() st.sort() from .input import read_station_metadata - inventory = read_station_metadata() + inventory = read_station_metadata(st) from .input import read_event_and_picks ssp_event, picks = read_event_and_picks(st) From b773bcfdcea62f3b7de7e3046e3bc89fd0fb0749 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 23 Jul 2024 16:49:55 +0200 Subject: [PATCH 62/73] Make `_Options` a subclass of `dict`. Clear `_Options` when clearing `_Config`. --- sourcespec2/setup/config.py | 80 +++++++++++++------------------------ 1 file changed, 27 insertions(+), 53 deletions(-) diff --git a/sourcespec2/setup/config.py b/sourcespec2/setup/config.py index 2e01a48b..fab2e322 100644 --- a/sourcespec2/setup/config.py +++ b/sourcespec2/setup/config.py @@ -65,67 +65,36 @@ def _none_lenght(input_list): # ---- End Helper functions ---- -class _Options(): +class _Options(dict): """ Options class for sourcespec, with builtin checks for API users. """ - def __init__(self): - self._trace_path = None - self._asdf_path = None - - def __getitem__(self, key): - """Make attributes accessible as dictionary keys.""" - return getattr(self, key) - def __setitem__(self, key, value): - """Make attributes accessible as dictionary keys.""" - setattr(self, key, value) - - def __str__(self): - """Return a dict-like representation of the object.""" - return str({ - k: getattr(self, k) for k in dir(self) if not k.startswith('_')} - ) - - @property - def trace_path(self): - """Path to the trace files. Must be a list.""" - return self._trace_path - - @trace_path.setter - def trace_path(self, value): - if value is None or isinstance(value, (list, tuple)): - self._trace_path = value - else: - warnings.warn( - '"trace_path" must be a list. Converting to a list with one ' - 'element.' - ) - self._trace_path = [value] - - @trace_path.deleter - def trace_path(self): - self._trace_path = None - - @property - def asdf_path(self): - """Path to the ASDF files. Must be a list.""" - return self._asdf_path - - @asdf_path.setter - def asdf_path(self, value): - if value is None or isinstance(value, (list, tuple)): - self._asdf_path = value - else: + """ + Make Config keys accessible as attributes. + + Perform specific checks for some parameters. + """ + if ( + key in ['trace_path', 'asdf_path'] and + (value is not None and not isinstance(value, (list, tuple))) + ): warnings.warn( - '"asdf_path" must be a list. Converting to a list with one ' + f'"{key}" must be a list. Converting to a list with one ' 'element.' ) - self._asdf_path = [value] + value = [value] + super().__setattr__(key, value) + super().__setitem__(key, value) + + def __getattr__(self, key): + """Make Config keys accessible as attributes.""" + try: + return self.__getitem__(key) + except KeyError as err: + raise AttributeError(err) from err - @asdf_path.deleter - def asdf_path(self): - self._asdf_path = None + __setattr__ = __setitem__ class _Config(dict): @@ -181,6 +150,11 @@ def __getattr__(self, key): __setattr__ = __setitem__ + def clear(self): + """Clear the configuration and the options.""" + self['options'].clear() + super().clear() + def update(self, other): """ Update the configuration with the values from another dictionary. From d53f32371a0ad29f94ccfc2e0612a8e477e5acd7 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 23 Jul 2024 17:49:22 +0200 Subject: [PATCH 63/73] Move trace selection from config to traces.py --- sourcespec2/input/augment_traces.py | 99 +++-------------------------- sourcespec2/input/traces.py | 71 +++++++++++++++++++++ 2 files changed, 79 insertions(+), 91 deletions(-) diff --git a/sourcespec2/input/augment_traces.py b/sourcespec2/input/augment_traces.py index 67ec9539..d9f7a445 100644 --- a/sourcespec2/input/augment_traces.py +++ b/sourcespec2/input/augment_traces.py @@ -15,90 +15,13 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -import re import logging import contextlib from obspy.core.util import AttribDict -from ..setup import config from .instrument_type import get_instrument_type logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def _skip_traces_from_config(traceid): - """ - Skip traces with unknown channel orientation or ignored from config file. - - :param traceid: Trace ID. - :type traceid: str - - :raises: RuntimeError if traceid is ignored from config file. - """ - network, station, location, channel = traceid.split('.') - orientation_codes = config.vertical_channel_codes +\ - config.horizontal_channel_codes_1 +\ - config.horizontal_channel_codes_2 - orientation = channel[-1] - if orientation not in orientation_codes: - raise RuntimeError( - f'{traceid}: Unknown channel orientation: ' - f'"{orientation}": skipping trace' - ) - # build a list of all possible ids, from station only - # to full net.sta.loc.chan - ss = [ - station, - '.'.join((network, station)), - '.'.join((network, station, location)), - '.'.join((network, station, location, channel)), - ] - if config.use_traceids is not None: - # - combine all use_traceids in a single regex - # - escape the dots, otherwise they are interpreted as any character - # - add a dot before the first asterisk, to avoid a pattern error - combined = ( - "(" + ")|(".join(config.use_traceids) + ")" - ).replace('.', r'\.').replace('(*', '(.*') - if not any(re.match(combined, s) for s in ss): - raise RuntimeError(f'{traceid}: ignored from config file') - if config.ignore_traceids is not None: - # - combine all ignore_traceids in a single regex - # - escape the dots, otherwise they are interpreted as any character - # - add a dot before the first asterisk, to avoid a pattern error - combined = ( - "(" + ")|(".join(config.ignore_traceids) + ")" - ).replace('.', r'\.').replace('(*', '(.*') - if any(re.match(combined, s) for s in ss): - raise RuntimeError(f'{traceid}: ignored from config file') - - -def _select_components(st): - """ - Select requested components from stream - - :param st: ObsPy Stream object - :type st: :class:`obspy.core.stream.Stream` - - :return: ObsPy Stream object - :rtype: :class:`obspy.core.stream.Stream` - """ - traces_to_keep = [] - for trace in st: - try: - _skip_traces_from_config(trace.id) - except RuntimeError as e: - logger.warning(str(e)) - continue - # TODO: should we also filter by station here? - # only use the station specified by the command line option - # "--station", if any - #if (config.options.station is not None and - # trace.stats.station != config.options.station): - # continue - traces_to_keep.append(trace) - # in-place update of st - st.traces[:] = traces_to_keep[:] - - def _add_instrtype(trace): """ Add instrtype to trace. @@ -300,29 +223,23 @@ def _augment_trace(trace, inventory, ssp_event, picks): trace.stats.ignore = False -def augment_traces(st, inventory, ssp_event, picks): +def augment_traces(stream, inventory, ssp_event, picks): """ Add all required information to trace headers. - Only the traces that satisfy the conditions in the config file are kept. - Problematic traces are also removed. + Trace with no or incomplete metadata are skipped. - :param st: Traces to be augmented - :type st: :class:`obspy.core.stream.Stream` - :param inventory: Station metadata. If it is None or an empty Inventory - object, the code will try to read the station metadata from the - trace headers (only SAC format is supported). + :param stream: Traces to be augmented + :type stream: :class:`obspy.core.stream.Stream` + :param inventory: Station metadata :type inventory: :class:`obspy.core.inventory.Inventory` :param ssp_event: Event information :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` :param picks: list of picks :type picks: list of :class:`sourcespec.ssp_event.Pick` """ - # First, select the components based on the config options - _select_components(st) - # Then, augment the traces and remove the problematic ones traces_to_keep = [] - for trace in st: + for trace in stream: try: _augment_trace(trace, inventory, ssp_event, picks) except Exception as err: @@ -331,5 +248,5 @@ def augment_traces(st, inventory, ssp_event, picks): continue traces_to_keep.append(trace) # in-place update of st - st.traces[:] = traces_to_keep[:] - _complete_picks(st) + stream.traces[:] = traces_to_keep[:] + _complete_picks(stream) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 0a92452d..40b015f9 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -16,6 +16,7 @@ (http://www.cecill.info/licences.en.html) """ import os +import re import logging import contextlib import shutil @@ -199,6 +200,75 @@ def _update_non_standard_trace_ids(stream): _update_non_standard_trace_ids.msgs = [] # noqa +def _should_keep_trace(traceid): + """ + Check if trace should be kept. + + :param traceid: Trace ID. + :type traceid: str + + :raises: RuntimeError if traceid is to be skipped. + """ + network, station, location, channel = traceid.split('.') + orientation_codes = config.vertical_channel_codes +\ + config.horizontal_channel_codes_1 +\ + config.horizontal_channel_codes_2 + orientation = channel[-1] + if orientation not in orientation_codes: + raise RuntimeError( + f'{traceid}: Unknown channel orientation: ' + f'"{orientation}": skipping trace' + ) + # build a list of all possible ids, from station only + # to full net.sta.loc.chan + ss = [ + station, + '.'.join((network, station)), + '.'.join((network, station, location)), + '.'.join((network, station, location, channel)), + ] + if config.use_traceids is not None: + # - combine all use_traceids in a single regex + # - escape the dots, otherwise they are interpreted as any character + # - add a dot before the first asterisk, to avoid a pattern error + combined = ( + "(" + ")|(".join(config.use_traceids) + ")" + ).replace('.', r'\.').replace('(*', '(.*') + if not any(re.match(combined, s) for s in ss): + raise RuntimeError(f'{traceid}: ignored from config file') + if config.ignore_traceids is not None: + # - combine all ignore_traceids in a single regex + # - escape the dots, otherwise they are interpreted as any character + # - add a dot before the first asterisk, to avoid a pattern error + combined = ( + "(" + ")|(".join(config.ignore_traceids) + ")" + ).replace('.', r'\.').replace('(*', '(.*') + if any(re.match(combined, s) for s in ss): + raise RuntimeError(f'{traceid}: ignored from config file') + + +def _select_requested_components(stream): + """ + Select requested components from stream + + :param stream: ObsPy Stream object + :type stream: :class:`obspy.core.stream.Stream` + + :return: ObsPy Stream object + :rtype: :class:`obspy.core.stream.Stream` + """ + traces_to_keep = [] + for trace in stream: + try: + _should_keep_trace(trace.id) + except RuntimeError as e: + logger.warning(str(e)) + continue + traces_to_keep.append(trace) + # in-place update of st + stream.traces[:] = traces_to_keep[:] + + def read_traces(): """ Read traces from the files or paths specified in the configuration. @@ -210,6 +280,7 @@ def read_traces(): stream = _read_asdf_traces() + _read_trace_files() _correct_traceids(stream) _update_non_standard_trace_ids(stream) + _select_requested_components(stream) ntraces = len(stream) logger.info(f'Reading traces: {ntraces} traces loaded') logger.info('---------------------------------------------------') From c3302befdba61e2b825de87cddeecbcc5414d176 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Tue, 23 Jul 2024 18:46:29 +0200 Subject: [PATCH 64/73] ssp_exit: error messages and raise exception This commit changes the ssp_exit function to accept a string as an error message. If the status is a string, it is used as an error message and the exit status is set to 1 (failure). The function now also accepts an extra_message parameter, which is printed to stderr before exiting. If the program is not running from the command line and the status corresponds to an error, the function raises a RuntimeError exception. --- sourcespec2/input/augment_event.py | 5 ++-- sourcespec2/input/event_and_picks.py | 15 +++++----- sourcespec2/input/event_parsers/asdf_event.py | 3 +- .../input/event_parsers/obspy_catalog.py | 6 ++-- .../asdf_inventory.py | 3 +- .../sac_station_metadata.py | 3 +- .../input/trace_parsers/asdf_traces.py | 3 +- sourcespec2/input/traces.py | 3 +- sourcespec2/setup/exit.py | 30 +++++++++++++++++-- sourcespec2/ssp_build_spectra.py | 3 +- sourcespec2/ssp_correction.py | 3 +- sourcespec2/ssp_process_traces.py | 3 +- sourcespec2/ssp_sqlite_output.py | 29 +++++++----------- 13 files changed, 59 insertions(+), 50 deletions(-) diff --git a/sourcespec2/input/augment_event.py b/sourcespec2/input/augment_event.py index efa3d7df..23f356cf 100644 --- a/sourcespec2/input/augment_event.py +++ b/sourcespec2/input/augment_event.py @@ -48,16 +48,15 @@ def augment_event(ssp_event): The augmented event is stored in config.event - :param ssp_event: Evento to be augmented + :param ssp_event: Event to be augmented :type ssp_event: :class:`sourcespec.ssp_event.SSPEvent` """ # add velocity info to hypocenter try: _hypo_vel(ssp_event.hypocenter) except Exception as e: - logger.error( + ssp_exit( f'Unable to compute velocity at hypocenter: {e}\n') - ssp_exit(1) if config.options.evname is not None: # add evname from command line, if any, overriding the one in ssp_event ssp_event.name = config.options.evname diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 70a95801..f83f3eff 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -284,13 +284,14 @@ def read_event_and_picks(stream=None): _log_event_and_pick_info(ssp_event, picks, event_source, picks_source) if ssp_event is None: - logger.error('No hypocenter information found.') - sys.stderr.write( - '\n' - 'Use "-q" or "-H" options to provide hypocenter information\n' - 'or add hypocenter information to the SAC file header\n' - '(if you use the SAC format).\n' + ssp_exit( + 'No hypocenter information found.', + extra_message=( + '\n' + 'Use "-q" or "-H" options to provide hypocenter information\n' + 'or add hypocenter information to the SAC file header\n' + '(if you use the SAC format).\n' + ) ) - ssp_exit(1) return ssp_event, picks diff --git a/sourcespec2/input/event_parsers/asdf_event.py b/sourcespec2/input/event_parsers/asdf_event.py index 04dbe96f..8e0b1134 100644 --- a/sourcespec2/input/event_parsers/asdf_event.py +++ b/sourcespec2/input/event_parsers/asdf_event.py @@ -33,12 +33,11 @@ def parse_asdf_event_picks(asdf_file, event_id=None): # and check for ImportError import pyasdf except ImportError: - logger.error( + ssp_exit( 'Error importing pyasdf. ' 'See https://seismicdata.github.io/pyasdf/ for installation ' 'instructions.' ) - ssp_exit(1) try: obspy_catalog = pyasdf.ASDFDataSet(asdf_file, mode='r').events return parse_obspy_catalog(obspy_catalog, event_id, asdf_file) diff --git a/sourcespec2/input/event_parsers/obspy_catalog.py b/sourcespec2/input/event_parsers/obspy_catalog.py index 6177ce87..e231560b 100644 --- a/sourcespec2/input/event_parsers/obspy_catalog.py +++ b/sourcespec2/input/event_parsers/obspy_catalog.py @@ -253,8 +253,7 @@ def _parse_obspy_event(obspy_event): ssp_event, obspy_origin = _parse_event_metadata(obspy_event) picks = _parse_picks_from_obspy_event(obspy_event, obspy_origin) except Exception as err: - logger.error(err) - ssp_exit(1) + ssp_exit(err) log_messages = [] with contextlib.suppress(Exception): _parse_moment_tensor_from_obspy_event(obspy_event, ssp_event) @@ -299,8 +298,7 @@ def parse_obspy_catalog(obspy_catalog, event_id=None, file_name=''): obspy_event = _get_event_from_obspy_catalog( obspy_catalog, event_id, file_name) except Exception as err: - logger.error(err) - ssp_exit(1) + ssp_exit(err) else: ssp_event, picks = _parse_obspy_event(obspy_event) return ssp_event, picks diff --git a/sourcespec2/input/station_metadata_parsers/asdf_inventory.py b/sourcespec2/input/station_metadata_parsers/asdf_inventory.py index 916e681a..463deaaf 100644 --- a/sourcespec2/input/station_metadata_parsers/asdf_inventory.py +++ b/sourcespec2/input/station_metadata_parsers/asdf_inventory.py @@ -31,12 +31,11 @@ def parse_asdf_inventory(asdf_file): # and check for ImportError import pyasdf except ImportError: - logger.error( + ssp_exit( 'Error importing pyasdf. ' 'See https://seismicdata.github.io/pyasdf/ for installation ' 'instructions.' ) - ssp_exit(1) inventory = Inventory() try: ds = pyasdf.ASDFDataSet(asdf_file, mode='r') diff --git a/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py b/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py index 9d986c37..3b78f744 100644 --- a/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py +++ b/sourcespec2/input/station_metadata_parsers/sac_station_metadata.py @@ -42,8 +42,7 @@ def compute_sensitivity_from_SAC(trace): sensitivity = eval(inp, {}, namespace) # pylint: disable=eval-used except NameError as msg: hdr_field = str(msg).split()[1] - logger.error(f'SAC header field {hdr_field} does not exist') - ssp_exit(1) + ssp_exit(f'SAC header field {hdr_field} does not exist') return sensitivity diff --git a/sourcespec2/input/trace_parsers/asdf_traces.py b/sourcespec2/input/trace_parsers/asdf_traces.py index 30297d47..187c36f4 100644 --- a/sourcespec2/input/trace_parsers/asdf_traces.py +++ b/sourcespec2/input/trace_parsers/asdf_traces.py @@ -92,12 +92,11 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): # and check for ImportError import pyasdf except ImportError: - logger.error( + ssp_exit( 'Error importing pyasdf. ' 'See https://seismicdata.github.io/pyasdf/ for installation ' 'instructions.' ) - ssp_exit(1) stream = Stream() try: ds = pyasdf.ASDFDataSet(asdf_file, mode='r') diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 40b015f9..89039524 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -285,6 +285,5 @@ def read_traces(): logger.info(f'Reading traces: {ntraces} traces loaded') logger.info('---------------------------------------------------') if not ntraces: - logger.error('No trace loaded') - ssp_exit(1) + ssp_exit('No traces loaded') return stream diff --git a/sourcespec2/setup/exit.py b/sourcespec2/setup/exit.py index 2be038ee..dabde32c 100644 --- a/sourcespec2/setup/exit.py +++ b/sourcespec2/setup/exit.py @@ -18,15 +18,37 @@ import sys import logging import signal +from . import config logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def ssp_exit(retval=0, abort=False): - """Exit the program.""" +def ssp_exit(status=0, abort=False, extra_message=None): + """ + Exit the program. + + If the status is an integer, it is used as the exit status. + If the status is a string, it is used as an error message and the exit + status is set to 1 (failure). + + :param status: Exit status + :type status: int or str + :param abort: If True, print 'Aborting.' before exiting + :type abort: bool + :param extra_message: Extra message to print before exiting. + It will be printed to stderr. + :type extra_message: str + """ # ssp_exit might have already been called if multiprocessing if ssp_exit.SSP_EXIT_CALLED: return ssp_exit.SSP_EXIT_CALLED = True + if isinstance(status, str): + err_message = status + logger.error(err_message) + retval = 1 + else: + err_message = None + retval = status if abort: print('\nAborting.') if logger is not None: @@ -34,6 +56,10 @@ def ssp_exit(retval=0, abort=False): elif logger is not None: logger.debug('source_spec END') logging.shutdown() + if not config.running_from_command_line and retval: + raise RuntimeError(err_message) + if extra_message: + sys.stderr.write(extra_message) sys.exit(retval) diff --git a/sourcespec2/ssp_build_spectra.py b/sourcespec2/ssp_build_spectra.py index 7605c044..63e52d2b 100644 --- a/sourcespec2/ssp_build_spectra.py +++ b/sourcespec2/ssp_build_spectra.py @@ -962,8 +962,7 @@ def _build_signal_and_noise_spectral_streams(signal_st, noise_st, original_st): spec_st.append(spec) specnoise_st.append(specnoise) if not spec_st: - logger.error('No spectra left! Exiting.') - ssp_exit() + ssp_exit('No spectra left! Exiting.') # build H component _build_H( spec_st, specnoise_st, config.vertical_channel_codes, config.wave_type) diff --git a/sourcespec2/ssp_correction.py b/sourcespec2/ssp_correction.py index 40e091c3..3ace2082 100644 --- a/sourcespec2/ssp_correction.py +++ b/sourcespec2/ssp_correction.py @@ -38,8 +38,7 @@ def station_correction(spec_st): try: residual = read_spectra(res_filepath) except Exception as msg: - logger.error(msg) - ssp_exit(1) + ssp_exit(msg) H_specs = [spec for spec in spec_st if spec.stats.channel[-1] == 'H'] for spec in H_specs: diff --git a/sourcespec2/ssp_process_traces.py b/sourcespec2/ssp_process_traces.py index 693b1b24..3c3decd5 100644 --- a/sourcespec2/ssp_process_traces.py +++ b/sourcespec2/ssp_process_traces.py @@ -604,8 +604,7 @@ def process_traces(st): continue if len(out_st) == 0: - logger.error('No traces left! Exiting.') - ssp_exit() + ssp_exit('No traces left! Exiting.') # Rotate traces, if SH or SV is requested if config.wave_type in ['SH', 'SV']: diff --git a/sourcespec2/ssp_sqlite_output.py b/sourcespec2/ssp_sqlite_output.py index b2355a9d..4c0f1651 100644 --- a/sourcespec2/ssp_sqlite_output.py +++ b/sourcespec2/ssp_sqlite_output.py @@ -45,10 +45,10 @@ def _open_sqlite_db(db_file): try: conn = sqlite3.connect(db_file, timeout=60) except Exception as msg: - logger.error(msg) - logger.info( - f'Please check whether "{db_file}" is a valid SQLite file.') - ssp_exit(1) + ssp_exit( + f'{msg}\n' + f'Please check whether "{db_file}" is a valid SQLite file.' + ) return conn, conn.cursor() @@ -65,21 +65,17 @@ def _check_db_version(cursor, db_file): if db_version == DB_VERSION: return if db_version > DB_VERSION: - logger.error( + ssp_exit( f'"{db_file}" has a newer database version: ' f'"{db_version}" Current supported version is "{DB_VERSION}".' ) - ssp_exit(1) - logger.error( + ssp_exit( f'"{db_file}" has an old database version: ' f'"{db_version}" Current supported version is "{DB_VERSION}".' - ) - logger.info( - 'Use the following command to update your database ' + '\nUse the following command to update your database ' '(the current database will be backed up):\n\n' f' source_spec --updatedb {db_file}\n' ) - ssp_exit(1) def _set_db_version(cursor): @@ -101,14 +97,13 @@ def _log_db_write_error(db_err, db_file): :param db_file: SQLite database file :type db_file: str """ - logger.error(f'Unable to insert values: {db_err}') - logger.info('Maybe your sqlite database has an old format.') - logger.info( - 'Use the following command to update your database ' + ssp_exit( + f'Unable to insert values: {db_err}' + '\nMaybe your sqlite database has an old format.' + '\nUse the following command to update your database ' '(the current database will be backed up):\n\n' f' source_spec --updatedb {db_file}\n' ) - ssp_exit(1) def _create_stations_table(cursor, db_file): @@ -195,7 +190,6 @@ def _write_stations_table(cursor, db_file, sspec_output): cursor.execute(sql_insert_into_stations, t) except Exception as msg: _log_db_write_error(msg, db_file) - ssp_exit(1) return nobs @@ -393,7 +387,6 @@ def _write_events_table(cursor, db_file, sspec_output, nobs): cursor.execute(sql_insert_into_events, t) except Exception as msg: _log_db_write_error(msg, db_file) - ssp_exit(1) def write_sqlite(sspec_output): From 1a8c9917f1dbc652f974744262eaf43afb05d4c4 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 24 Jul 2024 10:00:06 +0200 Subject: [PATCH 65/73] Manage more cases of missing options in config --- sourcespec2/input/augment_event.py | 2 +- sourcespec2/source_spec.py | 2 +- sourcespec2/ssp_output.py | 29 +++++++++++++++-------------- sourcespec2/ssp_qml_output.py | 2 +- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/sourcespec2/input/augment_event.py b/sourcespec2/input/augment_event.py index 23f356cf..fc52e66d 100644 --- a/sourcespec2/input/augment_event.py +++ b/sourcespec2/input/augment_event.py @@ -57,7 +57,7 @@ def augment_event(ssp_event): except Exception as e: ssp_exit( f'Unable to compute velocity at hypocenter: {e}\n') - if config.options.evname is not None: + if getattr(config.options, 'evname', None) is not None: # add evname from command line, if any, overriding the one in ssp_event ssp_event.name = config.options.evname else: diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index ac7b1dd7..5c5782cc 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -52,7 +52,7 @@ def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): # Create output folder if required, save config and setup logging from .setup import get_outdir_path, save_config, setup_logging - if config.options.outdir: + if getattr(config.options, 'outdir', None): outdir = get_outdir_path(ssp_event.event_id) if not os.path.exists(outdir): os.makedirs(outdir) diff --git a/sourcespec2/ssp_output.py b/sourcespec2/ssp_output.py index 83a2e018..4241ea62 100644 --- a/sourcespec2/ssp_output.py +++ b/sourcespec2/ssp_output.py @@ -285,7 +285,7 @@ def _write_parfile(sspec_output): parfile.write( f'\n*** Run completed on: {config.end_of_run} ' f'{config.end_of_run_tz}') - if config.options.run_id: + if getattr(config.options, 'run_id', None): parfile.write(f'\n*** Run ID: {config.options.run_id}') _write_author_and_agency_to_parfile(parfile) @@ -371,7 +371,7 @@ def _write_hypo71(sspec_output): :param sspec_output: Output of the spectral inversion. :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` """ - if not config.options.hypo_file: + if not getattr(config.options, 'hypo_file', None): return if config.hypo_file_format != 'hypo71': return @@ -409,17 +409,18 @@ def _make_symlinks(): out_data_dir = os.path.join(outdir, 'input_files') rel_path = os.path.relpath(config.workdir, out_data_dir) os.makedirs(out_data_dir, exist_ok=True) - try: - filelist = list(config.options.trace_path) - except TypeError: - filelist = [] - filelist += [ - config.options.station_metadata, - config.options.hypo_file, - config.options.pick_file, - config.options.qml_file, - ] - with contextlib.suppress(TypeError): + filelist = [] + with contextlib.suppress(AttributeError, TypeError): + filelist += list(config.options.trace_path) + with contextlib.suppress(AttributeError): + filelist += [config.options.station_metadata] + with contextlib.suppress(AttributeError): + filelist += [config.options.hypo_file] + with contextlib.suppress(AttributeError): + filelist += [config.options.pick_file] + with contextlib.suppress(AttributeError): + filelist += [config.options.qml_file] + with contextlib.suppress(AttributeError, TypeError): filelist += list(config.options.asdf_path) for filename in filelist: if filename is None or not os.path.exists(filename): @@ -448,7 +449,7 @@ def write_output(sspec_output): tz = get_localzone() config.end_of_run_tz = tz.tzname(config.end_of_run) run_info.run_completed = f'{config.end_of_run} {config.end_of_run_tz}' - if config.options.run_id: + if getattr(config.options, 'run_id', None): run_info.run_id = config.options.run_id run_info.author_name = config.author_name run_info.author_email = config.author_email diff --git a/sourcespec2/ssp_qml_output.py b/sourcespec2/ssp_qml_output.py index b810660a..1d14fc01 100644 --- a/sourcespec2/ssp_qml_output.py +++ b/sourcespec2/ssp_qml_output.py @@ -71,7 +71,7 @@ def write_qml(sspec_output): :param sspec_output: Output from spectral inversion. :type sspec_output: :class:`~sourcespec.ssp_data_types.SourceSpecOutput` """ - if not config.options.qml_file: + if not getattr(config.options, 'qml_file', None): config.qml_file_out = None return qml_file = config.options.qml_file From 600af3ed45d47d50d54b768751c67eca465498a4 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 24 Jul 2024 11:38:48 +0200 Subject: [PATCH 66/73] Allowed 'tag' argument of parse_asdf_traces function to contain wildcards. Added 'header_key' argument to _parse_asdf_headers function, and allowed 'tag' argument to be list. Improved error catching when trying to deserialize header values in _parse_asdf_headers function. --- .../input/trace_parsers/asdf_traces.py | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/sourcespec2/input/trace_parsers/asdf_traces.py b/sourcespec2/input/trace_parsers/asdf_traces.py index 187c36f4..602e3a9e 100644 --- a/sourcespec2/input/trace_parsers/asdf_traces.py +++ b/sourcespec2/input/trace_parsers/asdf_traces.py @@ -11,12 +11,13 @@ """ import logging import json +import fnmatch from obspy.core import Stream from ...setup import ssp_exit logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def _parse_asdf_trace_headers(ds, stream, nw_stat_codes, tag): +def _parse_asdf_trace_headers(ds, stream, header_key, nw_stat_codes, tag): """ Parse ASDF trace headers. @@ -24,20 +25,28 @@ def _parse_asdf_trace_headers(ds, stream, nw_stat_codes, tag): :type ds: :class:`pyasdf.ASDFDataSet` :param stream: ObsPy Stream object :type stream: :class:`obspy.core.stream.Stream` + :param header_key: key for trace headers in auxiliary data + :type header_key: str :param nw_stat_codes: Network and station codes :type nw_stat_codes: list - :param tag: waveform tag in ASDF file - :type tag: str + :param tag: waveform tag(s) in ASDF file + :type tag: str or list of str """ # pylint: disable=import-outside-toplevel from pyasdf.utils import AuxiliaryDataContainer - for nw_stat_code, tr in zip(nw_stat_codes, stream.traces): - if nw_stat_code in ds.auxiliary_data['TraceHeaders']: - auxiliary_root = ds.auxiliary_data['TraceHeaders'][nw_stat_code] + if not isinstance(tag, (list, tuple)): + trace_tags = [tag] + else: + trace_tags = tag + if len(trace_tags) == 1 and len(stream) > 1: + trace_tags *= len(stream) + for nw_stat_code, tr, tag in zip(nw_stat_codes, stream.traces, trace_tags): + if nw_stat_code in ds.auxiliary_data[header_key]: + auxiliary_root = ds.auxiliary_data[header_key][nw_stat_code] else: # ESM _nw_stat_code = nw_stat_code.replace('.', '_') - auxiliary_root = ds.auxiliary_data['TraceHeaders'][_nw_stat_code] + auxiliary_root = ds.auxiliary_data[header_key][_nw_stat_code] if not tag or tag not in auxiliary_root: continue header = None @@ -52,7 +61,7 @@ def _parse_asdf_trace_headers(ds, stream, nw_stat_codes, tag): for key, val in header.items(): try: val = json.loads(val) - except json.JSONDecodeError: + except (json.JSONDecodeError, TypeError): if isinstance(val, type(b'')): # ESM val = val.decode('ascii') @@ -77,7 +86,7 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): :param asdf_file: full path to ASDF file :type asdf_file: str - :param tag: waveform tag in ASDF file + :param tag: waveform tag in ASDF file (may contain wildcards) :type tag: str :param read_headers: flag to control reading of (non-standard) trace headers @@ -106,6 +115,7 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): if not ds.waveforms: return stream nw_stat_codes = [] + trace_tags = [] for nw_stat_code in ds.waveforms.list(): wf_tags = ds.waveforms[nw_stat_code].get_waveform_tags() # If tag is not specified, take first available tag @@ -116,6 +126,15 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): station_st = ds.waveforms[nw_stat_code][tag] stream.extend(station_st) nw_stat_codes.extend([nw_stat_code] * len(station_st)) + trace_tags.extend([tag] * len(station_st)) + elif '?' in tag or '*' in tag: + for _tag in fnmatch.filter(wf_tags, tag): + _st = ds.waveforms[nw_stat_code][_tag] + stream.extend(_st) + nw_stat_codes.extend([nw_stat_code] * len(_st)) + trace_tags.extend([_tag] * len(_st)) + else: + read_headers = False # Try reading trace headers if present if 'TraceHeaders' in ds.auxiliary_data: header_key = 'TraceHeaders' @@ -124,6 +143,7 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): else: header_key = None if read_headers and header_key: - _parse_asdf_trace_headers(ds, stream, nw_stat_codes, tag) + _parse_asdf_trace_headers(ds, stream, header_key, + nw_stat_codes, trace_tags) ds._close() return stream From 9bff212b9482decaf1740526e3139e4e7742c687 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Wed, 24 Jul 2024 12:27:25 +0200 Subject: [PATCH 67/73] Added support for separate tags for each ASDF file in _read_asdf_traces function. --- sourcespec2/input/traces.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 89039524..d59aab7f 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -102,8 +102,11 @@ def _read_asdf_traces(): asdf_path = getattr(config.options, 'asdf_path', None) if not asdf_path: return stream - asdf_tag = getattr(config.options, 'asdf_tag', None) - for asdf_file in asdf_path: + asdf_tags = getattr(config.options, 'asdf_tag', None) + # Allow 1 tag for all ASDF files or 1 tag for each ASDF file + if isinstance(asdf_tags, type('')): + asdf_tags = [asdf_tags] * len(asdf_path) + for asdf_file, asdf_tag in zip(asdf_path, asdf_tags): logger.info(f'Reading traces from ASDF file: {asdf_file}') stream += _filter_by_station( parse_asdf_traces(asdf_file, tag=asdf_tag, read_headers=True) From 73050300288f2f05fb6db66cfbf44860b000f4fb Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 24 Jul 2024 13:06:13 +0200 Subject: [PATCH 68/73] Small cosmetic changes in asdf parsing Most important change is log message when no ASDF tag is specified. --- .../input/trace_parsers/asdf_traces.py | 21 +++++++++---------- sourcespec2/input/traces.py | 2 +- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/sourcespec2/input/trace_parsers/asdf_traces.py b/sourcespec2/input/trace_parsers/asdf_traces.py index 602e3a9e..0076e61b 100644 --- a/sourcespec2/input/trace_parsers/asdf_traces.py +++ b/sourcespec2/input/trace_parsers/asdf_traces.py @@ -17,7 +17,8 @@ logger = logging.getLogger(__name__.rsplit('.', maxsplit=1)[-1]) -def _parse_asdf_trace_headers(ds, stream, header_key, nw_stat_codes, tag): +def _parse_asdf_trace_headers( + ds, stream, header_key, nw_stat_codes, trace_tags): """ Parse ASDF trace headers. @@ -29,15 +30,13 @@ def _parse_asdf_trace_headers(ds, stream, header_key, nw_stat_codes, tag): :type header_key: str :param nw_stat_codes: Network and station codes :type nw_stat_codes: list - :param tag: waveform tag(s) in ASDF file - :type tag: str or list of str + :param trace_tags: waveform tag(s) in ASDF file + :type trace_tags: str or list of str """ # pylint: disable=import-outside-toplevel from pyasdf.utils import AuxiliaryDataContainer - if not isinstance(tag, (list, tuple)): - trace_tags = [tag] - else: - trace_tags = tag + if not isinstance(trace_tags, (list, tuple)): + trace_tags = [trace_tags] if len(trace_tags) == 1 and len(stream) > 1: trace_tags *= len(stream) for nw_stat_code, tr, tag in zip(nw_stat_codes, stream.traces, trace_tags): @@ -118,10 +117,10 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): trace_tags = [] for nw_stat_code in ds.waveforms.list(): wf_tags = ds.waveforms[nw_stat_code].get_waveform_tags() - # If tag is not specified, take first available tag if not tag: - # Maybe this should be logged tag = wf_tags[0] + logger.info( + f'No ASDF tag specified, taking first available tag: {tag}') if tag in wf_tags: station_st = ds.waveforms[nw_stat_code][tag] stream.extend(station_st) @@ -143,7 +142,7 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): else: header_key = None if read_headers and header_key: - _parse_asdf_trace_headers(ds, stream, header_key, - nw_stat_codes, trace_tags) + _parse_asdf_trace_headers( + ds, stream, header_key, nw_stat_codes, trace_tags) ds._close() return stream diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index d59aab7f..6e2294a7 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -104,7 +104,7 @@ def _read_asdf_traces(): return stream asdf_tags = getattr(config.options, 'asdf_tag', None) # Allow 1 tag for all ASDF files or 1 tag for each ASDF file - if isinstance(asdf_tags, type('')): + if isinstance(asdf_tags, str): asdf_tags = [asdf_tags] * len(asdf_path) for asdf_file, asdf_tag in zip(asdf_path, asdf_tags): logger.info(f'Reading traces from ASDF file: {asdf_file}') From 203c86a7e4d957d4030955496a8d29ec3c4bb7a9 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 24 Jul 2024 13:18:08 +0200 Subject: [PATCH 69/73] Update docstrings in asdf_traces.py for consistency --- sourcespec2/input/trace_parsers/asdf_traces.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sourcespec2/input/trace_parsers/asdf_traces.py b/sourcespec2/input/trace_parsers/asdf_traces.py index 0076e61b..d49b9c86 100644 --- a/sourcespec2/input/trace_parsers/asdf_traces.py +++ b/sourcespec2/input/trace_parsers/asdf_traces.py @@ -30,7 +30,7 @@ def _parse_asdf_trace_headers( :type header_key: str :param nw_stat_codes: Network and station codes :type nw_stat_codes: list - :param trace_tags: waveform tag(s) in ASDF file + :param trace_tags: trace tag(s) in ASDF file :type trace_tags: str or list of str """ # pylint: disable=import-outside-toplevel @@ -85,7 +85,7 @@ def parse_asdf_traces(asdf_file, tag=None, read_headers=False): :param asdf_file: full path to ASDF file :type asdf_file: str - :param tag: waveform tag in ASDF file (may contain wildcards) + :param tag: trace tag in ASDF file (may contain wildcards) :type tag: str :param read_headers: flag to control reading of (non-standard) trace headers From 714e088be42c681814ab96b675f36e561021afd4 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 23 Oct 2024 11:53:59 +0200 Subject: [PATCH 70/73] Remove unused import in event_and_picks.py --- sourcespec2/input/event_and_picks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index f83f3eff..10af8858 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -15,7 +15,6 @@ CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) """ -import sys import logging from ..setup import config, ssp_exit from .event_parsers import ( From cd834117c5fcfa4d58799245e802383e2a6a4e4f Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 23 Oct 2024 12:02:43 +0200 Subject: [PATCH 71/73] Better logging for when no event or pick information is found --- sourcespec2/input/event_and_picks.py | 9 +++++++++ sourcespec2/input/event_parsers/sac_event.py | 14 ++++++++------ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 10af8858..9c70fb32 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -49,10 +49,16 @@ def _read_event_and_picks_from_stream(stream): ssp_event = read_event_from_SAC(trace) except RuntimeError as err: _read_event_and_picks_from_stream.event_warnings.append(err) + except ValueError: + # ValueError is raised when trace is not in SAC format + continue try: picks += read_picks_from_SAC(trace) except RuntimeError as err: _read_event_and_picks_from_stream.picks_warnings.append(err) + except ValueError: + # ValueError is raised when trace is not in SAC format + continue return ssp_event, picks _read_event_and_picks_from_stream.event_warnings = [] # noqa _read_event_and_picks_from_stream.picks_warnings = [] # noqa @@ -191,6 +197,7 @@ def _log_event_and_pick_info(ssp_event, picks, event_source, picks_source): :param picks_source: Picks source """ if ssp_event is None: + logger.warning('No event information found') # only log warnings if no event information was found for warning in _read_event_and_picks_from_stream.event_warnings: logger.warning(warning) @@ -198,7 +205,9 @@ def _log_event_and_pick_info(ssp_event, picks, event_source, picks_source): logger.info(f'Event information read from: {event_source}') for line in str(ssp_event).splitlines(): logger.info(line) + logger.info('---------------------------------------------------') if not picks: + logger.warning('No pick information found') # only log warnings if no pick information was found for warning in _read_event_and_picks_from_stream.picks_warnings: logger.warning(warning) diff --git a/sourcespec2/input/event_parsers/sac_event.py b/sourcespec2/input/event_parsers/sac_event.py index 0b889331..2416e7b8 100644 --- a/sourcespec2/input/event_parsers/sac_event.py +++ b/sourcespec2/input/event_parsers/sac_event.py @@ -25,13 +25,14 @@ def read_event_from_SAC(trace): :return: Event information :rtype: :class:`ssp_event.SSPEvent` + + :raises: ValueError if trace is not a SAC trace + :raises: RuntimeError if hypocenter information is not found in header """ try: sac_hdr = trace.stats.sac except AttributeError as e: - raise RuntimeError( - f'{trace.id}: not a SAC trace: cannot get hypocenter from header' - ) from e + raise ValueError(f'{trace.id}: not a SAC trace') from e try: evla = sac_hdr['evla'] evlo = sac_hdr['evlo'] @@ -85,13 +86,14 @@ def read_picks_from_SAC(trace): :return: List of picks :rtype: list of :class:`ssp_pick.SSPPick` + + :raises: ValueError if trace is not a SAC trace + :raises: RuntimeError if pick information is not found in header """ try: sac_hdr = trace.stats.sac except AttributeError as e: - raise RuntimeError( - f'{trace.id}: not a SAC trace: cannot get picks from header' - ) from e + raise ValueError(f'{trace.id}: not a SAC trace') from e trace_picks = [] pick_fields = ( 'a', 't0', 't1', 't2', 't3', 't4', 't5', 't6', 't7', 't8', 't9') From 029158b58ce7a6ef728aaaaaab079708151bacd3 Mon Sep 17 00:00:00 2001 From: Claudio Satriano Date: Wed, 8 Jan 2025 12:01:08 +0100 Subject: [PATCH 72/73] Update copyright year to 2025 --- sourcespec2/input/__init__.py | 2 +- sourcespec2/input/augment_event.py | 2 +- sourcespec2/input/event_and_picks.py | 2 +- sourcespec2/input/event_parsers/__init__.py | 2 +- sourcespec2/input/event_parsers/asdf_event.py | 2 +- sourcespec2/input/event_parsers/hypo2000.py | 2 +- sourcespec2/input/event_parsers/hypo71.py | 2 +- sourcespec2/input/event_parsers/obspy_catalog.py | 2 +- sourcespec2/input/event_parsers/quakeml.py | 2 +- sourcespec2/input/event_parsers/sac_event.py | 2 +- sourcespec2/input/event_parsers/source_spec_event.py | 2 +- sourcespec2/input/instrument_type.py | 2 +- sourcespec2/input/station_metadata_parsers/__init__.py | 2 +- sourcespec2/input/station_metadata_parsers/asdf_inventory.py | 2 +- sourcespec2/input/station_metadata_parsers/paz.py | 2 +- sourcespec2/input/trace_parsers/__init__.py | 2 +- sourcespec2/input/trace_parsers/asdf_traces.py | 2 +- sourcespec2/input/traces.py | 2 +- sourcespec2/setup/__init__.py | 2 +- sourcespec2/setup/configobj_helpers.py | 2 +- sourcespec2/setup/configure_cli.py | 2 +- sourcespec2/setup/exit.py | 2 +- sourcespec2/setup/library_versions.py | 2 +- sourcespec2/setup/mandatory_deprecated.py | 2 +- sourcespec2/setup/outdir.py | 2 +- 25 files changed, 25 insertions(+), 25 deletions(-) diff --git a/sourcespec2/input/__init__.py b/sourcespec2/input/__init__.py index 244f8be8..8f40af01 100644 --- a/sourcespec2/input/__init__.py +++ b/sourcespec2/input/__init__.py @@ -4,7 +4,7 @@ Input functions for SourceSpec. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/augment_event.py b/sourcespec2/input/augment_event.py index fc52e66d..16f9bd9e 100644 --- a/sourcespec2/input/augment_event.py +++ b/sourcespec2/input/augment_event.py @@ -9,7 +9,7 @@ 2013-2014 Claudio Satriano , Emanuela Matrullo - 2015-2024 Claudio Satriano , + 2015-2025 Claudio Satriano , Sophie Lambotte :license: CeCILL Free Software License Agreement v2.1 diff --git a/sourcespec2/input/event_and_picks.py b/sourcespec2/input/event_and_picks.py index 9c70fb32..c1765260 100644 --- a/sourcespec2/input/event_and_picks.py +++ b/sourcespec2/input/event_and_picks.py @@ -9,7 +9,7 @@ 2013-2014 Claudio Satriano , Emanuela Matrullo - 2015-2024 Claudio Satriano , + 2015-2025 Claudio Satriano , Sophie Lambotte :license: CeCILL Free Software License Agreement v2.1 diff --git a/sourcespec2/input/event_parsers/__init__.py b/sourcespec2/input/event_parsers/__init__.py index 8e90af73..38e9a088 100644 --- a/sourcespec2/input/event_parsers/__init__.py +++ b/sourcespec2/input/event_parsers/__init__.py @@ -4,7 +4,7 @@ Event and phase picks parsers for SourceSpec. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/asdf_event.py b/sourcespec2/input/event_parsers/asdf_event.py index 8e0b1134..03b5fcc2 100644 --- a/sourcespec2/input/event_parsers/asdf_event.py +++ b/sourcespec2/input/event_parsers/asdf_event.py @@ -4,7 +4,7 @@ Parse event metadata and picks from ASDF file. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/hypo2000.py b/sourcespec2/input/event_parsers/hypo2000.py index bb7cdc88..231474fe 100644 --- a/sourcespec2/input/event_parsers/hypo2000.py +++ b/sourcespec2/input/event_parsers/hypo2000.py @@ -4,7 +4,7 @@ Read event and phase picks from an hypo2000 file. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/hypo71.py b/sourcespec2/input/event_parsers/hypo71.py index 76adcc38..e8b66a8a 100644 --- a/sourcespec2/input/event_parsers/hypo71.py +++ b/sourcespec2/input/event_parsers/hypo71.py @@ -4,7 +4,7 @@ Parse event metadata and picks in hypo71 format. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/obspy_catalog.py b/sourcespec2/input/event_parsers/obspy_catalog.py index e231560b..9d5d9d9b 100644 --- a/sourcespec2/input/event_parsers/obspy_catalog.py +++ b/sourcespec2/input/event_parsers/obspy_catalog.py @@ -4,7 +4,7 @@ Parse event metadata and picks from an ObsPy catalog object. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/quakeml.py b/sourcespec2/input/event_parsers/quakeml.py index 66898585..7227cd82 100644 --- a/sourcespec2/input/event_parsers/quakeml.py +++ b/sourcespec2/input/event_parsers/quakeml.py @@ -4,7 +4,7 @@ Parse event metadata and picks from a QuakeML file. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/sac_event.py b/sourcespec2/input/event_parsers/sac_event.py index 2416e7b8..22024d31 100644 --- a/sourcespec2/input/event_parsers/sac_event.py +++ b/sourcespec2/input/event_parsers/sac_event.py @@ -4,7 +4,7 @@ Read metadata from SAC file headers. :copyright: - 2023-2024 Claudio Satriano + 2023-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/event_parsers/source_spec_event.py b/sourcespec2/input/event_parsers/source_spec_event.py index 3a730f2c..1be38fb7 100644 --- a/sourcespec2/input/event_parsers/source_spec_event.py +++ b/sourcespec2/input/event_parsers/source_spec_event.py @@ -4,7 +4,7 @@ Parse event metadata from a SourceSpec event file. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/instrument_type.py b/sourcespec2/input/instrument_type.py index 634b177e..ed3f6e12 100644 --- a/sourcespec2/input/instrument_type.py +++ b/sourcespec2/input/instrument_type.py @@ -4,7 +4,7 @@ Get the instrument type from the channel name of a trace. :copyright: - 2023-2024 Claudio Satriano + 2023-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/station_metadata_parsers/__init__.py b/sourcespec2/input/station_metadata_parsers/__init__.py index 49442e72..b45d8608 100644 --- a/sourcespec2/input/station_metadata_parsers/__init__.py +++ b/sourcespec2/input/station_metadata_parsers/__init__.py @@ -4,7 +4,7 @@ Station metadata parsers for SourceSpec. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/station_metadata_parsers/asdf_inventory.py b/sourcespec2/input/station_metadata_parsers/asdf_inventory.py index 463deaaf..e73942d2 100644 --- a/sourcespec2/input/station_metadata_parsers/asdf_inventory.py +++ b/sourcespec2/input/station_metadata_parsers/asdf_inventory.py @@ -4,7 +4,7 @@ Read station metadata from ASDF files. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/station_metadata_parsers/paz.py b/sourcespec2/input/station_metadata_parsers/paz.py index e51c685a..35c478fa 100644 --- a/sourcespec2/input/station_metadata_parsers/paz.py +++ b/sourcespec2/input/station_metadata_parsers/paz.py @@ -4,7 +4,7 @@ Read station metadata from PAZ files. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/trace_parsers/__init__.py b/sourcespec2/input/trace_parsers/__init__.py index 5536d152..46fc82cd 100644 --- a/sourcespec2/input/trace_parsers/__init__.py +++ b/sourcespec2/input/trace_parsers/__init__.py @@ -4,7 +4,7 @@ Trace parsers for SourceSpec. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/trace_parsers/asdf_traces.py b/sourcespec2/input/trace_parsers/asdf_traces.py index d49b9c86..73051065 100644 --- a/sourcespec2/input/trace_parsers/asdf_traces.py +++ b/sourcespec2/input/trace_parsers/asdf_traces.py @@ -4,7 +4,7 @@ Read traces from ASDF files. :copyright: - 2012-2024 Claudio Satriano + 2012-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/input/traces.py b/sourcespec2/input/traces.py index 6e2294a7..a1a80d34 100644 --- a/sourcespec2/input/traces.py +++ b/sourcespec2/input/traces.py @@ -9,7 +9,7 @@ 2013-2014 Claudio Satriano , Emanuela Matrullo - 2015-2024 Claudio Satriano , + 2015-2025 Claudio Satriano , Sophie Lambotte :license: CeCILL Free Software License Agreement v2.1 diff --git a/sourcespec2/setup/__init__.py b/sourcespec2/setup/__init__.py index 3a29dc59..d5c077a1 100644 --- a/sourcespec2/setup/__init__.py +++ b/sourcespec2/setup/__init__.py @@ -4,7 +4,7 @@ Configuration classes and functions for SourceSpec :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/setup/configobj_helpers.py b/sourcespec2/setup/configobj_helpers.py index c8d8cba4..db2271f9 100644 --- a/sourcespec2/setup/configobj_helpers.py +++ b/sourcespec2/setup/configobj_helpers.py @@ -4,7 +4,7 @@ Helper functions for using ConfigObj in SourceSpec. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/setup/configure_cli.py b/sourcespec2/setup/configure_cli.py index 0540a81d..71cac21e 100644 --- a/sourcespec2/setup/configure_cli.py +++ b/sourcespec2/setup/configure_cli.py @@ -4,7 +4,7 @@ Configure SourceSpec from command line arguments and config file. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/setup/exit.py b/sourcespec2/setup/exit.py index dabde32c..573a3444 100644 --- a/sourcespec2/setup/exit.py +++ b/sourcespec2/setup/exit.py @@ -10,7 +10,7 @@ Emanuela Matrullo , Agnes Chounet - 2015-2024 Claudio Satriano + 2015-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/setup/library_versions.py b/sourcespec2/setup/library_versions.py index 6531ecd8..70799238 100644 --- a/sourcespec2/setup/library_versions.py +++ b/sourcespec2/setup/library_versions.py @@ -10,7 +10,7 @@ Emanuela Matrullo , Agnes Chounet - 2015-2024 Claudio Satriano + 2015-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/setup/mandatory_deprecated.py b/sourcespec2/setup/mandatory_deprecated.py index eb665041..335a3a3f 100644 --- a/sourcespec2/setup/mandatory_deprecated.py +++ b/sourcespec2/setup/mandatory_deprecated.py @@ -4,7 +4,7 @@ Mandatory and deprecated config parameters for sourcespec. :copyright: - 2013-2024 Claudio Satriano + 2013-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) diff --git a/sourcespec2/setup/outdir.py b/sourcespec2/setup/outdir.py index 012f795b..76c9c1c1 100644 --- a/sourcespec2/setup/outdir.py +++ b/sourcespec2/setup/outdir.py @@ -10,7 +10,7 @@ Emanuela Matrullo , Agnes Chounet - 2015-2024 Claudio Satriano + 2015-2025 Claudio Satriano :license: CeCILL Free Software License Agreement v2.1 (http://www.cecill.info/licences.en.html) From c6e92188ca324f3d7d7219e1f16b5858d16ff1e1 Mon Sep 17 00:00:00 2001 From: Kris Vanneste Date: Fri, 10 Jan 2025 13:32:46 +0100 Subject: [PATCH 73/73] Added ssp_clear_state function. Clear state from possible previous run at beginning of ssp_run function. --- sourcespec2/source_spec.py | 42 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/sourcespec2/source_spec.py b/sourcespec2/source_spec.py index 5c5782cc..fa77a375 100644 --- a/sourcespec2/source_spec.py +++ b/sourcespec2/source_spec.py @@ -17,6 +17,45 @@ """ +def ssp_clear_state(clear_config=False, clear_options=True): + """ + Clear state from a previous run. + Some submodules contain global variables caching information + about the current run, which need to be cleared before + launching a new run. + + :param clear_config: whether to clear global config as well + :type clear_config: bool + :param clear_options: whether to clear options in global config + :type clear_options: bool + """ + from .setup import logging + logging.OLDLOGFILE = None + # Not sure if we should reset LOGGER? + #logging.LOGGER = None + + from . import ssp_wave_arrival + ssp_wave_arrival.add_arrival_to_trace.pick_cache = dict() + ssp_wave_arrival.add_arrival_to_trace.travel_time_cache = dict() + ssp_wave_arrival.add_arrival_to_trace.angle_cache = dict() + + from . import ssp_plot_traces + ssp_plot_traces.SAVED_FIGURE_CODES = [] + ssp_plot_traces.BBOX = None + + from . import ssp_plot_spectra + ssp_plot_spectra.SAVED_FIGURE_CODES = [] + ssp_plot_spectra.BBOX = None + + from .setup import config + if clear_config: + # This clears the entire config, which is not what we want + #config.clear() + config.__init__() + elif clear_options: + config.options.clear() + + def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): """ Run source_spec as function with collected traces, station inventory, @@ -50,6 +89,9 @@ def ssp_run(st, inventory, ssp_event, picks, allow_exit=False): if not allow_exit: ssp_exit.SSP_EXIT_CALLED = True + # Clear state from possible previous run + ssp_clear_state(clear_config=False, clear_options=False) + # Create output folder if required, save config and setup logging from .setup import get_outdir_path, save_config, setup_logging if getattr(config.options, 'outdir', None):