diff --git a/.github/workflows/pyxrf_requirements.txt b/.github/workflows/pyxrf_requirements.txt index 3208689d9..e31436c51 100644 --- a/.github/workflows/pyxrf_requirements.txt +++ b/.github/workflows/pyxrf_requirements.txt @@ -7,3 +7,7 @@ scikit-beam tomopy # These are required for installing the pyxrf-utils environment hatchling +# Pandas needs to be pinned to prevent a bug in xrf-tomo, until +# it gets fixed and a new release comes out. +# See: https://github.com/NSLS-II-SRX/xrf-tomo/pull/10 +pandas<3.0 diff --git a/tomviz/CMakeLists.txt b/tomviz/CMakeLists.txt index 8c86439d8..0ecd815d4 100644 --- a/tomviz/CMakeLists.txt +++ b/tomviz/CMakeLists.txt @@ -626,6 +626,7 @@ set(tomviz_python_modules __init__.py _internal.py dataset.py + executor.py external_dataset.py fix_pdb.py operators.py @@ -635,7 +636,6 @@ set(tomviz_python_modules utils.py web.py modules.py - views.py ) file(MAKE_DIRECTORY "${tomviz_python_binary_dir}/tomviz") diff --git a/tomviz/DataPropertiesPanel.cxx b/tomviz/DataPropertiesPanel.cxx index 68e39c526..1bb91690e 100644 --- a/tomviz/DataPropertiesPanel.cxx +++ b/tomviz/DataPropertiesPanel.cxx @@ -499,7 +499,7 @@ void DataPropertiesPanel::updateComponentsCombo() auto blocked = QSignalBlocker(combo); // Only make this editor visible if there is more than one component - bool visible = dsource->scalars()->GetNumberOfComponents() > 1; + bool visible = dsource->scalars() && dsource->scalars()->GetNumberOfComponents() > 1; label->setVisible(visible); combo->setVisible(visible); diff --git a/tomviz/DataSource.h b/tomviz/DataSource.h index 9a5b8478b..fa426ac05 100644 --- a/tomviz/DataSource.h +++ b/tomviz/DataSource.h @@ -171,6 +171,10 @@ class DataSource : public QObject /// Set/get the dark/white data, used in Data Exchange currently vtkImageData* whiteData() const; + // Used to track if a volume visualization module was added + bool volumeModuleAutoAdded() const; + void setVolumeModuleAutoAdded(bool b); + /// Check to see if the data was subsampled while reading bool wasSubsampled() const; @@ -458,6 +462,7 @@ protected slots: MetadataType m_metadata; bool m_changingTimeStep = false; + bool m_volumeModuleAutoAdded = false; }; inline void DataSource::clearTiltAngles() @@ -495,6 +500,16 @@ inline void DataSource::setSubsampleVolumeBounds(int bs[6]) setSubsampleVolumeBounds(dataObject(), bs); } +inline bool DataSource::volumeModuleAutoAdded() const +{ + return m_volumeModuleAutoAdded; +} + +inline void DataSource::setVolumeModuleAutoAdded(bool b) +{ + m_volumeModuleAutoAdded = b; +} + } // namespace tomviz #endif diff --git a/tomviz/ExternalPythonExecutor.cxx b/tomviz/ExternalPythonExecutor.cxx index 6753bda9b..1a71516ef 100644 --- a/tomviz/ExternalPythonExecutor.cxx +++ b/tomviz/ExternalPythonExecutor.cxx @@ -84,7 +84,7 @@ Pipeline::Future* ExternalPythonExecutor::execute(vtkDataObject* data, } else if (exitCode != 0) { displayError( "External Python Error", - QString("The external python return a non-zero exist code: %1\n\n " + QString("The external python returned a non-zero exit code: %1\n\n " "command: %2 \n\n stderr:\n%3 \n\n stdout:\n%4 \n") .arg(exitCode) .arg(commandLine(this->m_process.data())) diff --git a/tomviz/Pipeline.cxx b/tomviz/Pipeline.cxx index 345c9b035..141aa3e05 100644 --- a/tomviz/Pipeline.cxx +++ b/tomviz/Pipeline.cxx @@ -361,6 +361,11 @@ void Pipeline::branchFinished() // doesn't already have an explicit child data source i.e. // hasChildDataSource is true. auto lastOp = start->operators().last(); + if (!lastOp->isCompleted()) { + // Cannot continue + return; + } + if (!lastOp->hasChildDataSource()) { DataSource* newChildDataSource = nullptr; if (lastOp->childDataSource() == nullptr) { diff --git a/tomviz/PtychoDialog.cxx b/tomviz/PtychoDialog.cxx index 438424a6c..add48be46 100644 --- a/tomviz/PtychoDialog.cxx +++ b/tomviz/PtychoDialog.cxx @@ -15,8 +15,8 @@ #include #include #include +#include #include -#include #include #include #include @@ -83,6 +83,9 @@ class PtychoDialog::Internal : public QObject connect(ui.filterSIDsString, &QLineEdit::editingFinished, this, &Internal::updateFilteredSidList); + connect(ui.loadSidsFromTxt, &QPushButton::clicked, this, + &Internal::onLoadSidsFromTxtClicked); + connect(ui.selectOutputDirectory, &QPushButton::clicked, this, &Internal::selectOutputDirectory); @@ -535,6 +538,14 @@ class PtychoDialog::Internal : public QObject return; } + // If "recon_result" exists underneath the selected directory, + // it means the parent directory was selected. + // We should automatically select the child one. + auto possibleChildPath = QDir(file).filePath("recon_result"); + if (QFile::exists(possibleChildPath)) { + file = possibleChildPath; + } + setPtychoDirectory(file); ptychoDirEdited(); } @@ -677,6 +688,50 @@ class PtychoDialog::Internal : public QObject updateTable(); } + void onLoadSidsFromTxtClicked() + { + QString caption = "Select txt file"; + QString filter = "*.txt"; + auto startPath = ptychoDirectory(); + auto filePath = + QFileDialog::getOpenFileName(parent.data(), caption, startPath, filter); + + if (filePath.isEmpty()) { + return; + } + + QFile file(filePath); + if (!file.exists()) { + qCritical() << QString("Txt file does not exist: %1").arg(filePath); + return; + } + + if (!file.open(QIODevice::ReadOnly)) { + qCritical() + << QString("Failed to open file \"%1\" with error: ").arg(filePath) + << file.errorString(); + return; + } + + QTextStream reader(&file); + + // Now load the SIDs + QStringList sids; + while (!reader.atEnd()) { + auto line = reader.readLine().trimmed(); + if (line.isEmpty() || line.startsWith('#')) { + // Skip over it + continue; + } + + sids.append(line.split(' ')[0]); + } + + ui.filterSIDsString->setText(sids.join(", ")); + + updateFilteredSidList(); + } + void selectLoadFromCSV() { QString caption = "Select CSV file to load Use and Version settings"; diff --git a/tomviz/PtychoDialog.ui b/tomviz/PtychoDialog.ui index bfe3bd838..c85354685 100644 --- a/tomviz/PtychoDialog.ui +++ b/tomviz/PtychoDialog.ui @@ -26,47 +26,26 @@ 6 - - + + - <html><head/><body><p>The directory where the output files will be written. If one is not specified, the Ptycho directory will be used.</p></body></html> + <html><head/><body><p>Select which SIDs to use for each version.</p><p><br/></p><p>The list of SIDs to use can be specified via numpy slice syntax (for example, '157391:157637'), a comma-delimited list (for example, '157391, 157394'), or loaded from a txt/csv file.</p><p><br/></p><p>Any SIDs which are missing data or angles will be automatically ignored. A list of the ones ignored may be seen from the console and messages output.</p></body></html> - - - - - - <html><head/><body><p>The orientation of the tilt series does match what Tomviz expects internally. Rotating the data ensures that the later Tomviz operators will be able to run properly.</p></body></html> + + QAbstractItemView::NoEditTriggers - - Rotate datasets to Tomviz convention? + + QAbstractItemView::NoSelection - + true - - - - - - - <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> - - - Filter SIDs: - - - - - - - <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> - + - - + + - <html><head/><body><p>The Ptycho directory should contain data organized as created from the Ptycho GUI output. It will be used to determine the contents of the table below.<br/></p><p>The Ptycho directory is expected to have, at its top level, a list of scan directories all starting with 'S'. For example, 'S157391' is the directory for scan number 157391.<br/></p><p>Within the scan directory are version directories. For example, 't1'.<br/></p><p>Within the version directories are 'recon_data' and 'recon_pic'. The 'recon_data' should contain all the necessary datasets from the ptychography reconstruction.</p></body></html> + <html><head/><body><p>Select a pyxrf-style CSV file from which the &quot;Use&quot; and &quot;Version&quot; settings will be loaded.<br/></p><p>This expects columns of &quot;Scan ID&quot;, &quot;Use&quot;, and &quot;Version&quot; to be within the file. If it indeed has these columns (except for &quot;Version&quot;, which is optional), all current SID &quot;Use&quot; settings will be checked off, and only SIDs marked for &quot;Use&quot; within the CSV file will be checked back on.<br/></p><p>Additionally, if &quot;Version&quot; is present as well, the version will be automatically selected for each SID, assuming that the version is indeed a viable option within the ptycho directory for that SID.<br/></p><p>The user may still modify settings afterwards.</p></body></html> @@ -94,7 +73,44 @@ - + + + + <html><head/><body><p>The Ptycho directory should contain data organized as created from the Ptycho GUI output. It will be used to determine the contents of the table below.<br/></p><p>The Ptycho directory is expected to have, at its top level, a list of scan directories all starting with 'S'. For example, 'S157391' is the directory for scan number 157391.<br/></p><p>Within the scan directory are version directories. For example, 't1'.<br/></p><p>Within the version directories are 'recon_data' and 'recon_pic'. The 'recon_data' should contain all the necessary datasets from the ptychography reconstruction.</p></body></html> + + + + + + + <html><head/><body><p>Select a pyxrf-style CSV file from which the &quot;Use&quot; and &quot;Version&quot; settings will be loaded.<br/></p><p>This expects columns of &quot;Scan ID&quot;, &quot;Use&quot;, and &quot;Version&quot; to be within the file. If it indeed has these columns (except for &quot;Version&quot;, which is optional), all current SID &quot;Use&quot; settings will be checked off, and only SIDs marked for &quot;Use&quot; within the CSV file will be checked back on.<br/></p><p>Additionally, if &quot;Version&quot; is present as well, the version will be automatically selected for each SID, assuming that the version is indeed a viable option within the ptycho directory for that SID.<br/></p><p>The user may still modify settings afterwards.</p></body></html> + + + Load settings from CSV: + + + + + + + <html><head/><body><p>The Ptycho directory should contain data organized as created from the Ptycho GUI output. It will be used to determine the contents of the table below.<br/></p><p>The Ptycho directory is expected to have, at its top level, a list of scan directories all starting with 'S'. For example, 'S157391' is the directory for scan number 157391.<br/></p><p>Within the scan directory are version directories. For example, 't1'.<br/></p><p>Within the version directories are 'recon_data' and 'recon_pic'. The 'recon_data' should contain all the necessary datasets from the ptychography reconstruction.</p></body></html> + + + Ptycho Directory: + + + + + + + <html><head/><body><p>The directory where the output files will be written. If one is not specified, the Ptycho directory will be used.</p></body></html> + + + Select + + + + Qt::Horizontal @@ -114,7 +130,7 @@ - + <html><head/><body><p>The directory where the output files will be written. If one is not specified, the Ptycho directory will be used.</p></body></html> @@ -137,68 +153,66 @@ - - + + - <html><head/><body><p>The Ptycho directory should contain data organized as created from the Ptycho GUI output. It will be used to determine the contents of the table below.<br/></p><p>The Ptycho directory is expected to have, at its top level, a list of scan directories all starting with 'S'. For example, 'S157391' is the directory for scan number 157391.<br/></p><p>Within the scan directory are version directories. For example, 't1'.<br/></p><p>Within the version directories are 'recon_data' and 'recon_pic'. The 'recon_data' should contain all the necessary datasets from the ptychography reconstruction.</p></body></html> + <html><head/><body><p>The orientation of the tilt series does match what Tomviz expects internally. Rotating the data ensures that the later Tomviz operators will be able to run properly.</p></body></html> - Ptycho Directory: - - - - - - - <html><head/><body><p>Select which SIDs to use for each version.</p><p><br/></p><p>The list of SIDs to use can be specified via numpy slice syntax (for example, '157391:157637'), a comma-delimited list (for example, '157391, 157394'), or loaded from a txt/csv file.</p><p><br/></p><p>Any SIDs which are missing data or angles will be automatically ignored. A list of the ones ignored may be seen from the console and messages output.</p></body></html> - - - QAbstractItemView::NoEditTriggers - - - QAbstractItemView::NoSelection + Rotate datasets to Tomviz convention? - + true - - - - - - - <html><head/><body><p>The directory where the output files will be written. If one is not specified, the Ptycho directory will be used.</p></body></html> - - - Select - - + + <html><head/><body><p>Select a pyxrf-style CSV file from which the &quot;Use&quot; and &quot;Version&quot; settings will be loaded.<br/></p><p>This expects columns of &quot;Scan ID&quot;, &quot;Use&quot;, and &quot;Version&quot; to be within the file. If it indeed has these columns (except for &quot;Version&quot;, which is optional), all current SID &quot;Use&quot; settings will be checked off, and only SIDs marked for &quot;Use&quot; within the CSV file will be checked back on.<br/></p><p>Additionally, if &quot;Version&quot; is present as well, the version will be automatically selected for each SID, assuming that the version is indeed a viable option within the ptycho directory for that SID.<br/></p><p>The user may still modify settings afterwards.</p></body></html> - Load settings from CSV: + Select - - + + - <html><head/><body><p>Select a pyxrf-style CSV file from which the &quot;Use&quot; and &quot;Version&quot; settings will be loaded.<br/></p><p>This expects columns of &quot;Scan ID&quot;, &quot;Use&quot;, and &quot;Version&quot; to be within the file. If it indeed has these columns (except for &quot;Version&quot;, which is optional), all current SID &quot;Use&quot; settings will be checked off, and only SIDs marked for &quot;Use&quot; within the CSV file will be checked back on.<br/></p><p>Additionally, if &quot;Version&quot; is present as well, the version will be automatically selected for each SID, assuming that the version is indeed a viable option within the ptycho directory for that SID.<br/></p><p>The user may still modify settings afterwards.</p></body></html> + <html><head/><body><p>The directory where the output files will be written. If one is not specified, the Ptycho directory will be used.</p></body></html> - - - - <html><head/><body><p>Select a pyxrf-style CSV file from which the &quot;Use&quot; and &quot;Version&quot; settings will be loaded.<br/></p><p>This expects columns of &quot;Scan ID&quot;, &quot;Use&quot;, and &quot;Version&quot; to be within the file. If it indeed has these columns (except for &quot;Version&quot;, which is optional), all current SID &quot;Use&quot; settings will be checked off, and only SIDs marked for &quot;Use&quot; within the CSV file will be checked back on.<br/></p><p>Additionally, if &quot;Version&quot; is present as well, the version will be automatically selected for each SID, assuming that the version is indeed a viable option within the ptycho directory for that SID.<br/></p><p>The user may still modify settings afterwards.</p></body></html> - - - Select - - + + + + + + <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> + + + Filter SIDs: + + + + + + + <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> + + + + + + + <html><head/><body><p>Load a list of SIDs from a txt file. The txt file should be formatted so that on each row, the first number that occurs is the SID, like so:<br/></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383563 -90.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383565 -88.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383567 -86.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383569 -84.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383571 -82.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383573 -80.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383575 -78.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383577 -76.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383579 -74.000</span><br/></p><p>The second column of angles is optional (the angles, if present, will be ignored). It is only necessary that the first column be the SID numbers.</p></body></html> + + + Load from txt + + + + @@ -210,6 +224,7 @@ loadFromCSVFile selectLoadFromCSVFile filterSIDsString + loadSidsFromTxt table outputDirectory selectOutputDirectory diff --git a/tomviz/PyXRFMakeHDF5Dialog.cxx b/tomviz/PyXRFMakeHDF5Dialog.cxx index f65c9c73e..245597d2b 100644 --- a/tomviz/PyXRFMakeHDF5Dialog.cxx +++ b/tomviz/PyXRFMakeHDF5Dialog.cxx @@ -42,6 +42,8 @@ class PyXRFMakeHDF5Dialog::Internal : public QObject &Internal::updateEnableStates); connect(ui.selectWorkingDirectory, &QPushButton::clicked, this, &Internal::selectWorkingDirectory); + connect(ui.selectLogFile, &QPushButton::clicked, this, + &Internal::selectLogFile); connect(ui.buttonBox, &QDialogButtonBox::accepted, this, &Internal::accepted); @@ -115,6 +117,47 @@ class PyXRFMakeHDF5Dialog::Internal : public QObject setWorkingDirectory(directory); } + QString logFile() const + { + return ui.logFile->text().trimmed(); + } + + QString logFileOrDefault() const + { + if (!writingNewLogFile()) { + // Return an empty log file + return ""; + } + + auto name = logFile(); + return name.isEmpty() ? defaultLogFile() : name; + } + + void setLogFile(QString s) { ui.logFile->setText(s); } + + QString defaultLogFile() const + { + return QDir(workingDirectory()).filePath("tomo_info.csv"); + } + + void selectLogFile() + { + QString caption = "Select Output CSV File"; + auto fileName = QFileDialog::getSaveFileName(parent.data(), caption, + logFileOrDefault(), + "CSV Files (*.csv)"); + if (fileName.isEmpty()) { + return; + } + + setLogFile(fileName); + } + + bool writingNewLogFile() const + { + return method() == "New" || remakeCsvFile(); + } + void accepted() { QString reason; @@ -162,6 +205,19 @@ class PyXRFMakeHDF5Dialog::Internal : public QObject return false; } + auto logFile = logFileOrDefault(); + if (writingNewLogFile() && QFile(logFile).exists()) { + QString title = "CSV Log File Exists"; + auto text = QString("CSV log file \"%1\" already exists. It " + "will be overwritten. Proceed?") + .arg(logFile); + + if (QMessageBox::question(parent, title, text) == QMessageBox::No) { + reason = "Not overwriting log file: " + logFile; + return false; + } + } + if (scanStart() > scanStop()) { reason = QString("Scan start, %1, cannot be greater than scan stop, %2") .arg(scanStart()) @@ -174,8 +230,11 @@ class PyXRFMakeHDF5Dialog::Internal : public QObject void updateEnableStates() { - bool enable = method() == "New" || remakeCsvFile(); - ui.scanNumbersGroup->setEnabled(enable); + auto b = writingNewLogFile(); + ui.scanNumbersGroup->setEnabled(b); + ui.logFileLabel->setEnabled(b); + ui.logFile->setEnabled(b); + ui.selectLogFile->setEnabled(b); } void readSettings() @@ -196,6 +255,7 @@ class PyXRFMakeHDF5Dialog::Internal : public QObject setSuccessfulScansOnly( settings->value("successfulScansOnly", true).toBool()); setRemakeCsvFile(settings->value("remakeCsvFile", false).toBool()); + setLogFile(settings->value("logFile", "").toString()); settings->endGroup(); settings->endGroup(); @@ -217,6 +277,7 @@ class PyXRFMakeHDF5Dialog::Internal : public QObject settings->setValue("scanStop", scanStop()); settings->setValue("successfulScansOnly", successfulScansOnly()); settings->setValue("remakeCsvFile", remakeCsvFile()); + settings->setValue("logFile", logFile()); settings->endGroup(); settings->endGroup(); @@ -271,4 +332,9 @@ bool PyXRFMakeHDF5Dialog::remakeCsvFile() const return m_internal->remakeCsvFile(); } +QString PyXRFMakeHDF5Dialog::logFile() const +{ + return m_internal->logFileOrDefault(); +} + } // namespace tomviz diff --git a/tomviz/PyXRFMakeHDF5Dialog.h b/tomviz/PyXRFMakeHDF5Dialog.h index 9e689def7..6db79a20d 100644 --- a/tomviz/PyXRFMakeHDF5Dialog.h +++ b/tomviz/PyXRFMakeHDF5Dialog.h @@ -26,6 +26,7 @@ class PyXRFMakeHDF5Dialog : public QDialog int scanStop() const; bool successfulScansOnly() const; bool remakeCsvFile() const; + QString logFile() const; private: class Internal; diff --git a/tomviz/PyXRFMakeHDF5Dialog.ui b/tomviz/PyXRFMakeHDF5Dialog.ui index 6ed3f9360..6c9e38bf4 100644 --- a/tomviz/PyXRFMakeHDF5Dialog.ui +++ b/tomviz/PyXRFMakeHDF5Dialog.ui @@ -7,7 +7,7 @@ 0 0 595 - 341 + 369 @@ -26,22 +26,28 @@ 6 - - - - Qt::Horizontal + + + + <html><head/><body><p>Command for running pyxrf-utils, which will be used to run the make-hdf5 command.</p></body></html> - - QDialogButtonBox::Close|QDialogButtonBox::Help|QDialogButtonBox::Ok + + pyxrf-utils - - - - Select + + + + Qt::Vertical - + + + 20 + 40 + + + @@ -92,62 +98,26 @@ - - - - - - - Method: - - - method - - - - - + + - <html><head/><body><p>Command for running pyxrf-utils, which will be used to run the make-hdf5 command.</p></body></html> - - - pyxrf-utils + <html><head/><body><p>The directory where the data files will be written if the &quot;New&quot; method is selected, or the directory already containing the data files if the &quot;Already Existing&quot; method is selected.</p></body></html> - - - - Qt::Vertical - - - - 20 - 40 - - - - - + + + <html><head/><body><p>The directory where the data files will be written if the &quot;New&quot; method is selected, or the directory already containing the data files if the &quot;Already Existing&quot; method is selected.</p></body></html> + - Working directory: + Data directory: workingDirectory - - - - <html><head/><body><p>Command for running pyxrf-utils, which will be used to run the make-hdf5 command.</p></body></html> - - - PyXRF Utils Command: - - - @@ -191,15 +161,83 @@ + + + + <html><head/><body><p>The directory where the data files will be written if the &quot;New&quot; method is selected, or the directory already containing the data files if the &quot;Already Existing&quot; method is selected.</p></body></html> + + + Select + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Close|QDialogButtonBox::Help|QDialogButtonBox::Ok + + + + + + + Method: + + + method + + + + + + + <html><head/><body><p>Command for running pyxrf-utils, which will be used to run the make-hdf5 command.</p></body></html> + + + PyXRF Utils Command: + + + + + + + <html><head/><body><p>Specify the filepath for the new CSV log file. If not specified, `tomo_info.csv` will be written to the data directory.</p></body></html> + + + Output CSV File: + + + + + + + <html><head/><body><p>Specify the filepath for the new CSV log file. If not specified, `tomo_info.csv` will be written to the data directory.</p></body></html> + + + + + + + <html><head/><body><p>Specify the filepath for the new CSV log file. If not specified, `tomo_info.csv` will be written to the data directory.</p></body></html> + + + Select + + + command method methodWidget + successfulScansOnly + remakeCsvFile scanStart scanStop - successfulScansOnly workingDirectory selectWorkingDirectory diff --git a/tomviz/PyXRFProcessDialog.cxx b/tomviz/PyXRFProcessDialog.cxx index c8d1696c8..a02cc8c34 100644 --- a/tomviz/PyXRFProcessDialog.cxx +++ b/tomviz/PyXRFProcessDialog.cxx @@ -47,21 +47,17 @@ class PyXRFProcessDialog::Internal : public QObject Python::Module pyxrfModule; - Internal(QString workingDir, PyXRFProcessDialog* p) + Internal(QString workingDir, QString logFile, PyXRFProcessDialog* p) : parent(p), workingDirectory(workingDir) { ui.setupUi(p); setParent(p); + setLogFile(logFile); + setupTableColumns(); setupComboBoxes(); setupConnections(); - - if (QDir(workingDirectory).exists("tomo_info.csv")) { - // Set the csv file automatically - auto path = QDir(workingDirectory).filePath("tomo_info.csv"); - setLogFile(path); - } } void setupConnections() @@ -71,6 +67,8 @@ class PyXRFProcessDialog::Internal : public QObject connect(ui.logFile, &QLineEdit::textChanged, this, &Internal::updateTable); connect(ui.filterSidsString, &QLineEdit::editingFinished, this, &Internal::onFilterSidsStringChanged); + connect(ui.loadSidsFromTxt, &QPushButton::clicked, this, + &Internal::onLoadSidsFromTxtClicked); connect(ui.selectLogFile, &QPushButton::clicked, this, &Internal::selectLogFile); @@ -490,6 +488,50 @@ class PyXRFProcessDialog::Internal : public QObject updateTable(); } + void onLoadSidsFromTxtClicked() + { + QString caption = "Select txt file"; + QString filter = "*.txt"; + auto startPath = workingDirectory; + auto filePath = + QFileDialog::getOpenFileName(parent.data(), caption, startPath, filter); + + if (filePath.isEmpty()) { + return; + } + + QFile file(filePath); + if (!file.exists()) { + qCritical() << QString("Txt file does not exist: %1").arg(filePath); + return; + } + + if (!file.open(QIODevice::ReadOnly)) { + qCritical() + << QString("Failed to open file \"%1\" with error: ").arg(filePath) + << file.errorString(); + return; + } + + QTextStream reader(&file); + + // Now load the SIDs + QStringList sids; + while (!reader.atEnd()) { + auto line = reader.readLine().trimmed(); + if (line.isEmpty() || line.startsWith('#')) { + // Skip over it + continue; + } + + sids.append(line.split(' ')[0]); + } + + ui.filterSidsString->setText(sids.join(", ")); + + onFilterSidsStringChanged(); + } + void updateFilteredSidList() { auto filterString = filterSidsString().trimmed(); @@ -748,8 +790,9 @@ class PyXRFProcessDialog::Internal : public QObject }; PyXRFProcessDialog::PyXRFProcessDialog(QString workingDirectory, + QString logFile, QWidget* parent) - : QDialog(parent), m_internal(new Internal(workingDirectory, this)) + : QDialog(parent), m_internal(new Internal(workingDirectory, logFile, this)) { } diff --git a/tomviz/PyXRFProcessDialog.h b/tomviz/PyXRFProcessDialog.h index 281921236..ddb70cee3 100644 --- a/tomviz/PyXRFProcessDialog.h +++ b/tomviz/PyXRFProcessDialog.h @@ -14,7 +14,8 @@ class PyXRFProcessDialog : public QDialog Q_OBJECT public: - explicit PyXRFProcessDialog(QString workingDirectory, QWidget* parent); + explicit PyXRFProcessDialog(QString workingDirectory, QString logFile, + QWidget* parent); ~PyXRFProcessDialog() override; virtual void show(); diff --git a/tomviz/PyXRFProcessDialog.ui b/tomviz/PyXRFProcessDialog.ui index e4990aaab..537e0970e 100644 --- a/tomviz/PyXRFProcessDialog.ui +++ b/tomviz/PyXRFProcessDialog.ui @@ -7,7 +7,7 @@ 0 0 595 - 494 + 497 @@ -26,73 +26,40 @@ 6 - + <html><head/><body><p>The directory where the output will be written.</p></body></html> - - - - <html><head/><body><p>The directory where the output will be written.</p></body></html> - - - Output Directory: - - - outputDirectory - - - - - - - Qt::Horizontal - - - QDialogButtonBox::Close|QDialogButtonBox::Help|QDialogButtonBox::Ok - - - - - + + - <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> + <html><head/><body><p>Select the parameters file that was generated using the PyXRF GUI on one of the datasets.</p></body></html> - Filter SIDs: + Select - - - - <html><head/><body><p>xrf-tomo will store the output from processed scans within the input HDF5 files. If this is checked, any HDF5 files that have already been processed will be skipped over, and their old output re-used.</p><p><br/></p><p>This should be unchecked if you change the parameters file or the normalization channel - otherwise, previously processed files will not be processed with the updated values.</p></body></html> - + + - Skip already processed scans? - - - true + PyXRF GUI Command: - - + + - <html><head/><body><p>Select the parameters file that was generated using the PyXRF GUI on one of the datasets.</p></body></html> + <html><head/><body><p>The directory where the output will be written.</p></body></html> - Select + Output Directory: - - - - - - PyXRF Utils Command: + + outputDirectory @@ -137,14 +104,20 @@ - - - - PyXRF GUI Command: + + + + QAbstractItemView::NoEditTriggers + + + QAbstractItemView::NoSelection + + true + - + <html><head/><body><p>Name of the scalar for normalization of fluorescence data.</p></body></html> @@ -157,41 +130,34 @@ - - - - <html><head/><body><p>Use the PyXRF GUI to generate a parameters file with one of the datasets. Then, select the parameters file below.</p></body></html> + + + + Qt::Horizontal - - Start PyXRF GUI + + QDialogButtonBox::Close|QDialogButtonBox::Help|QDialogButtonBox::Ok - - + + - <html><head/><body><p>Name of the scalar for normalization of fluorescence data.</p></body></html> + <html><head/><body><p>The directory where the output will be written.</p></body></html> - - - - - pyxrf + Select - - - - <html><head/><body><p>The directory where the output will be written.</p></body></html> - + + - Select + pyxrf - + <html><head/><body><p>Select the parameters file that was generated using the PyXRF GUI on one of the datasets.</p></body></html> @@ -204,7 +170,21 @@ - + + + + <html><head/><body><p>Select the parameters file that was generated using the PyXRF GUI on one of the datasets.</p></body></html> + + + + + + + <html><head/><body><p>Name of the scalar for normalization of fluorescence data.</p></body></html> + + + + <html><head/><body><p>The orientation of the tilt series from PyXRF does match what Tomviz expects internally. Rotating the data ensures that the later Tomviz operators will be able to run properly.</p></body></html> @@ -217,33 +197,67 @@ - - - - <html><head/><body><p>Select the parameters file that was generated using the PyXRF GUI on one of the datasets.</p></body></html> + + + + PyXRF Utils Command: - - - - QAbstractItemView::NoEditTriggers + + + + <html><head/><body><p>Use the PyXRF GUI to generate a parameters file with one of the datasets. Then, select the parameters file below.</p></body></html> - - QAbstractItemView::NoSelection + + Start PyXRF GUI - - true - - - + + - <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> + <html><head/><body><p>xrf-tomo will store the output from processed scans within the input HDF5 files. If this is checked, any HDF5 files that have already been processed will be skipped over, and their old output re-used.</p><p><br/></p><p>This should be unchecked if you change the parameters file or the normalization channel - otherwise, previously processed files will not be processed with the updated values.</p></body></html> + + + Skip already processed scans? + + + true + + + + + + <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> + + + Filter SIDs: + + + + + + + <html><head/><body><p>Filter out SIDs using numpy-like slice syntax. For example: <br/></p><p>157394:157413:3<br/></p><p>This would only allow every third number between 157394 (inclusive) and 157413 (exclusive) to be shown.<br/></p><p>The filtered out SIDs are hidden and won't be used in the next step.<br/></p><p>This also supports separate sets of comma-delimited slices, for example:<br/></p><p>157394:157413:3, 157420:157500:2</p></body></html> + + + + + + + <html><head/><body><p>Load a list of SIDs from a txt file. The txt file should be formatted so that on each row, the first number that occurs is the SID, like so:<br/></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383563 -90.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383565 -88.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383567 -86.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383569 -84.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383571 -82.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383573 -80.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383575 -78.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383577 -76.000</span></p><p><span style=" font-family:'Courier New','Courier','monospace','arial','sans-serif'; font-size:14px; color:#000000; background-color:#ffffff;">383579 -74.000</span><br/></p><p>The second column of angles is optional (the angles, if present, will be ignored). It is only necessary that the first column be the SID numbers.</p></body></html> + + + Load from txt + + + + + @@ -251,6 +265,7 @@ logFile selectLogFile filterSidsString + loadSidsFromTxt logFileTable pyxrfGUICommand startPyXRFGUI diff --git a/tomviz/PyXRFRunner.cxx b/tomviz/PyXRFRunner.cxx index 914817844..37f1ccf3e 100644 --- a/tomviz/PyXRFRunner.cxx +++ b/tomviz/PyXRFRunner.cxx @@ -3,6 +3,7 @@ #include "PyXRFRunner.h" +#include "CameraReaction.h" #include "DataSource.h" #include "EmdFormat.h" #include "LoadDataReaction.h" @@ -73,11 +74,10 @@ class PyXRFRunner::Internal : public QObject int scanStop = 0; bool successfulScansOnly = true; bool remakeCsvFile = false; - QString defaultLogFileName = "tomo_info.csv"; + QString logFile; // Process projection options QString parametersFile; - QString logFile; QString icName; QString outputDirectory; bool skipProcessed = true; @@ -240,6 +240,7 @@ class PyXRFRunner::Internal : public QObject scanStop = makeHDF5Dialog->scanStop(); successfulScansOnly = makeHDF5Dialog->successfulScansOnly(); remakeCsvFile = makeHDF5Dialog->remakeCsvFile(); + logFile = makeHDF5Dialog->logFile(); auto useAlreadyExistingData = makeHDF5Dialog->useAlreadyExistingData(); if (useAlreadyExistingData) { @@ -266,7 +267,7 @@ class PyXRFRunner::Internal : public QObject args << "make-hdf5" << workingDirectory << "-s" << QString::number(scanStart) << "-e" << QString::number(scanStop) - << "-l" << defaultLogFileName; + << "-l" << logFile; if (successfulScansOnly) { args.append("-b"); @@ -322,7 +323,7 @@ class PyXRFRunner::Internal : public QObject args << "make-csv" << "-i" << "-w" << workingDirectory << "-s" << rangeString - << defaultLogFileName; + << logFile; qInfo() << "Running:" << program + " " + args.join(" "); remakeCsvFileProcess.start(program, args); @@ -372,7 +373,8 @@ class PyXRFRunner::Internal : public QObject clearWidget(processDialog); - processDialog = new PyXRFProcessDialog(workingDirectory, parentWidget); + processDialog = new PyXRFProcessDialog(workingDirectory, logFile, + parentWidget); connect(processDialog.data(), &QDialog::accepted, this, &Internal::processDialogAccepted); // If the user rejects the process dialog, go back to @@ -687,6 +689,10 @@ class PyXRFRunner::Internal : public QObject QString saveFile = QFileInfo(sortedList[0]).dir().absoluteFilePath("extracted_elements.emd"); EmdFormat::write(saveFile.toStdString(), dataSource); dataSource->setFileName(saveFile); + + // Automatically update camera to BNL convention + CameraReaction::resetPositiveZ(); + CameraReaction::rotateCamera(-90); } }; diff --git a/tomviz/operators/Operator.h b/tomviz/operators/Operator.h index 243982862..5deac7cd7 100644 --- a/tomviz/operators/Operator.h +++ b/tomviz/operators/Operator.h @@ -259,6 +259,7 @@ public slots: /// Distinction between this and isFinished is necessary to prevent cascading /// errors bool isCompleted() { return m_state == OperatorState::Complete; } + bool failed() { return m_state == OperatorState::Error; } bool isFinished() { return m_state == OperatorState::Complete || diff --git a/tomviz/operators/OperatorPython.cxx b/tomviz/operators/OperatorPython.cxx index a2af5ed3a..970727821 100644 --- a/tomviz/operators/OperatorPython.cxx +++ b/tomviz/operators/OperatorPython.cxx @@ -217,6 +217,7 @@ class OperatorPython::OPInternals Python::Function IsCancelableFunction; Python::Function IsCompletableFunction; Python::Function DeleteModuleFunction; + Python::Function TransformMethodWrapper; }; OperatorPython::OperatorPython(DataSource* parentObject) @@ -245,7 +246,6 @@ OperatorPython::OperatorPython(DataSource* parentObject) d->IsCompletableFunction = d->InternalModule.findFunction("is_completable"); if (!d->IsCompletableFunction.isValid()) { qCritical() << "Unable to locate is_completable."; - return; } d->FindTransformFunction = @@ -258,6 +258,11 @@ OperatorPython::OperatorPython(DataSource* parentObject) if (!d->DeleteModuleFunction.isValid()) { qCritical() << "Unable to locate delete_module."; } + + d->TransformMethodWrapper = d->InternalModule.findFunction("transform_method_wrapper"); + if (!d->TransformMethodWrapper.isValid()) { + qCritical() << "Unable to locate transform_method_wrapper."; + } } auto connectionType = Qt::BlockingQueuedConnection; @@ -520,7 +525,8 @@ bool OperatorPython::applyTransform(vtkDataObject* data) if (m_script.isEmpty()) { return false; } - if (!d->OperatorModule.isValid() || !d->TransformMethod.isValid()) { + if (!d->OperatorModule.isValid() || !d->TransformMethod.isValid() || + !d->TransformMethodWrapper.isValid()) { return false; } @@ -532,18 +538,25 @@ bool OperatorPython::applyTransform(vtkDataObject* data) { Python python; - Python::Tuple args(1); + Python::Tuple args(3); + + // Serialize the operator, so that the transform wrapper can + // decide whether to execute this one internally or externally + auto operatorSerialized = Python::Object(QString::fromUtf8(QJsonDocument(serialize()).toJson())); + + args.set(0, d->TransformMethod); + args.set(1, operatorSerialized); // Get the name of the function auto transformMethod = d->TransformMethod.getAttr("__name__").toString(); if (transformMethod == "transform_scalars") { // Use the arguments for transform_scalars() Python::Object pydata = Python::VTK::GetObjectFromPointer(data); - args.set(0, pydata); + args.set(2, pydata); } else if (transformMethod == "transform") { // Use the arguments for transform() Python::Object pydata = Python::createDataset(data, *dataSource()); - args.set(0, pydata); + args.set(2, pydata); } else { qDebug() << "Unknown TransformMethod name: " << transformMethod; return false; @@ -572,7 +585,7 @@ bool OperatorPython::applyTransform(vtkDataObject* data) kwargs.set(key, v); } - result = d->TransformMethod.call(args, kwargs); + result = d->TransformMethodWrapper.call(args, kwargs); if (!result.isValid()) { qCritical("Failed to execute the script."); return false; @@ -812,6 +825,17 @@ void OperatorPython::updateChildDataSource(vtkSmartPointer data) auto dataSource = childDataSource(); Q_ASSERT(dataSource); + if (!dataSource->volumeModuleAutoAdded()) { + // Automatically add a volume so that users can see live updates + // This also fixes some strange issue where the application will + // crash after this function if no visualization module was ever + // created for this child data source, before the new data was + // copied over. + ModuleManager::instance().createAndAddModule("Volume", dataSource, + ActiveObjects::instance().activeView()); + dataSource->setVolumeModuleAutoAdded(true); + } + // Now deep copy the new data to the child source data if needed dataSource->copyData(data); emit dataSource->dataChanged(); diff --git a/tomviz/operators/TranslateAlignOperator.cxx b/tomviz/operators/TranslateAlignOperator.cxx index cbab231ec..9bcb4754a 100644 --- a/tomviz/operators/TranslateAlignOperator.cxx +++ b/tomviz/operators/TranslateAlignOperator.cxx @@ -10,6 +10,7 @@ #include "vtkImageData.h" #include "vtkIntArray.h" #include "vtkNew.h" +#include "vtkPointData.h" #include "vtkTable.h" #include @@ -100,12 +101,22 @@ bool TranslateAlignOperator::applyTransform(vtkDataObject* data) vtkImageData* inImage = vtkImageData::SafeDownCast(data); assert(inImage); outImage->DeepCopy(data); - switch (inImage->GetScalarType()) { - vtkTemplateMacro( - applyImageOffsets(reinterpret_cast(inImage->GetScalarPointer()), - reinterpret_cast(outImage->GetScalarPointer()), - inImage, offsets)); + + auto numArrays = inImage->GetPointData()->GetNumberOfArrays(); + + for (int i = 0; i < numArrays; ++i) { + std::string arrayName = inImage->GetPointData()->GetArrayName(i); + switch (inImage->GetScalarType()) { + vtkTemplateMacro( + applyImageOffsets( + reinterpret_cast( + inImage->GetPointData()->GetScalars(arrayName.c_str())->GetVoidPointer(0)), + reinterpret_cast( + outImage->GetPointData()->GetScalars(arrayName.c_str())->GetVoidPointer(0)), + inImage, offsets)); + } } + offsetsToResult(); data->ShallowCopy(outImage); return true; diff --git a/tomviz/python/AddConstant.py b/tomviz/python/AddConstant.py index 382cb0d7b..80be964b1 100644 --- a/tomviz/python/AddConstant.py +++ b/tomviz/python/AddConstant.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, constant=0.0): """Add a constant to the data set""" diff --git a/tomviz/python/AddPoissonNoise.py b/tomviz/python/AddPoissonNoise.py index 499f70826..d36073f5d 100644 --- a/tomviz/python/AddPoissonNoise.py +++ b/tomviz/python/AddPoissonNoise.py @@ -1,9 +1,11 @@ import numpy as np import tomviz.operators +from tomviz.utils import apply_to_each_array class AddPoissonNoiseOperator(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset, N=25): """Add Poisson noise to tilt images""" self.progress.maximum = 1 diff --git a/tomviz/python/AutoCenterOfMassTiltImageAlignment.py b/tomviz/python/AutoCenterOfMassTiltImageAlignment.py index 723b88518..984ae1f0d 100644 --- a/tomviz/python/AutoCenterOfMassTiltImageAlignment.py +++ b/tomviz/python/AutoCenterOfMassTiltImageAlignment.py @@ -1,7 +1,9 @@ from tomviz import utils -import numpy as np import tomviz.operators +import numpy as np +from scipy import ndimage + class CenterOfMassAlignmentOperator(tomviz.operators.CancelableOperator): @@ -11,6 +13,8 @@ def transform(self, dataset): tiltSeries = dataset.active_scalars.astype(float) + apply_to_all_arrays = True + self.progress.maximum = tiltSeries.shape[2] step = 1 @@ -30,6 +34,10 @@ def transform(self, dataset): self.progress.value = step dataset.active_scalars = tiltSeries + if apply_to_all_arrays: + names = [name for name in dataset.scalars_names + if name != dataset.active_name] + self.apply_offsets_to_scalar_names(dataset, offsets, names) # Create a spreadsheet data set from table data column_names = ["X Offset", "Y Offset"] @@ -39,6 +47,20 @@ def transform(self, dataset): returnValues["alignments"] = offsetsTable return returnValues + def apply_offsets_to_scalar_names(self, dataset, offsets, names): + # Now apply the same offsets to all other arrays + for name in names: + print(f'Applying shifts to {name}...') + array = dataset.scalars(name) + for i in range(len(offsets)): + shifts = offsets[i] + array[:, :, i] = ndimage.shift(array[:, :, i], + shift=shifts, + order=1, + mode='wrap') + + dataset.set_scalars(name, array) + def centerOfMassAlign(image): """Shift image so that the center of mass of is at origin""" diff --git a/tomviz/python/AutoCrossCorrelationTiltImageAlignment.py b/tomviz/python/AutoCrossCorrelationTiltImageAlignment.py index 54fdcb7a6..135b2f908 100644 --- a/tomviz/python/AutoCrossCorrelationTiltImageAlignment.py +++ b/tomviz/python/AutoCrossCorrelationTiltImageAlignment.py @@ -44,13 +44,11 @@ def transform_generate(self, dataset, apply_to_all_arrays, # determine reference image index zeroDegreeTiltImage = None + referenceIndex = tiltSeries.shape[2] // 2 if tiltAngles is not None: zeroDegreeTiltImage = np.where(tiltAngles == 0)[0] - - if zeroDegreeTiltImage: - referenceIndex = zeroDegreeTiltImage[0] - else: - referenceIndex = tiltSeries.shape[2] // 2 + if zeroDegreeTiltImage.size > 0: + referenceIndex = zeroDegreeTiltImage[0] # create Fourier space filter filterCutoff = 4 diff --git a/tomviz/python/AutoTiltAxisRotationAlignment.py b/tomviz/python/AutoTiltAxisRotationAlignment.py index 7253b9b5f..45849ea8e 100644 --- a/tomviz/python/AutoTiltAxisRotationAlignment.py +++ b/tomviz/python/AutoTiltAxisRotationAlignment.py @@ -89,14 +89,17 @@ def transform(self, dataset): self.progress.message = 'Rotating tilt series' axes = ((0, 1)) shape = utils.rotate_shape(tiltSeries, -rot_ang, axes=axes) - result = np.empty(shape, tiltSeries.dtype, order='F') - ndimage.interpolation.rotate( - tiltSeries, -rot_ang, axes=axes, output=result) print("rotate tilt series by %f degrees" % -rot_ang) - # Set the result as the new scalars. - dataset.active_scalars = result + for name in dataset.scalars_names: + array = dataset.scalars(name) + result = np.empty(shape, array.dtype, order='F') + ndimage.interpolation.rotate( + array, -rot_ang, axes=axes, output=result) + + # Set the result as the new scalars. + dataset.set_scalars(name, result) def calculateLineIntensity(Intensity_var, angle_d, N): diff --git a/tomviz/python/AutoTiltAxisShiftAlignment.json b/tomviz/python/AutoTiltAxisShiftAlignment.json index 987253757..3d50d450a 100644 --- a/tomviz/python/AutoTiltAxisShiftAlignment.json +++ b/tomviz/python/AutoTiltAxisShiftAlignment.json @@ -49,7 +49,7 @@ "type" : "int", "default" : 0, "minimum" : 0, - "maximum" : 10000000000000, + "maximum" : 1000000000, "visible_if" : "transform_source == 'generate'" }, { diff --git a/tomviz/python/ClipEdges.py b/tomviz/python/ClipEdges.py index 8b5287a30..ad661c9ca 100644 --- a/tomviz/python/ClipEdges.py +++ b/tomviz/python/ClipEdges.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, clipNum=5): """Set values outside a cirular range to minimum(dataset) to remove reconstruction artifacts""" diff --git a/tomviz/python/DummyMolecule.py b/tomviz/python/DummyMolecule.py index 11dc02e61..e9ef51a47 100644 --- a/tomviz/python/DummyMolecule.py +++ b/tomviz/python/DummyMolecule.py @@ -6,7 +6,7 @@ class DummyMoleculeOperator(tomviz.operators.CancelableOperator): - def transform_scalars(self, dataset): + def transform(self, dataset): """Reconstruct atomic positions""" atomic_numbers = [6, 1, 6, 1, 6, 1, 6, 1, 6, 1, 6, 1] diff --git a/tomviz/python/FFT_AbsLog.py b/tomviz/python/FFT_AbsLog.py index bab146a2f..06d60cef2 100644 --- a/tomviz/python/FFT_AbsLog.py +++ b/tomviz/python/FFT_AbsLog.py @@ -5,7 +5,10 @@ # # WARNING: Be patient! Large datasets may take a while. +from tomviz.utils import apply_to_each_array + +@apply_to_each_array def transform(dataset): import numpy as np diff --git a/tomviz/python/GaussianFilter.py b/tomviz/python/GaussianFilter.py index 6c3210b21..7cf5fea49 100644 --- a/tomviz/python/GaussianFilter.py +++ b/tomviz/python/GaussianFilter.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, sigma=2.0): """Apply a Gaussian filter to volume dataset.""" """Gaussian Filter blurs the image and reduces the noise and details.""" diff --git a/tomviz/python/GaussianFilterTiltSeries.py b/tomviz/python/GaussianFilterTiltSeries.py index 01548a757..328ce537c 100644 --- a/tomviz/python/GaussianFilterTiltSeries.py +++ b/tomviz/python/GaussianFilterTiltSeries.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, sigma=2.0): """Apply a Gaussian filter to tilt images.""" """Gaussian Filter blurs the image and reduces the noise and details.""" diff --git a/tomviz/python/GenerateTiltSeries.py b/tomviz/python/GenerateTiltSeries.py index 2faccafaf..1a5dfcd25 100644 --- a/tomviz/python/GenerateTiltSeries.py +++ b/tomviz/python/GenerateTiltSeries.py @@ -1,10 +1,13 @@ import numpy as np import scipy.ndimage + +from tomviz.utils import apply_to_each_array import tomviz.operators class GenerateTiltSeriesOperator(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset, start_angle=-90.0, angle_increment=3.0, num_tilts=60): """Generate Tilt Series from Volume""" diff --git a/tomviz/python/GradientMagnitude2D_Sobel.py b/tomviz/python/GradientMagnitude2D_Sobel.py index 07c9110c7..9370d2ec5 100644 --- a/tomviz/python/GradientMagnitude2D_Sobel.py +++ b/tomviz/python/GradientMagnitude2D_Sobel.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): """Calculate gradient magnitude of each tilt image using Sobel operator""" diff --git a/tomviz/python/GradientMagnitude_Sobel.py b/tomviz/python/GradientMagnitude_Sobel.py index e3e2f6b7a..886ea742d 100644 --- a/tomviz/python/GradientMagnitude_Sobel.py +++ b/tomviz/python/GradientMagnitude_Sobel.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): """Calculate 3D gradient magnitude using Sobel operator""" diff --git a/tomviz/python/HannWindow3D.py b/tomviz/python/HannWindow3D.py index 9d5ac6e22..19a6a85c0 100644 --- a/tomviz/python/HannWindow3D.py +++ b/tomviz/python/HannWindow3D.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): import numpy as np diff --git a/tomviz/python/InvertData.py b/tomviz/python/InvertData.py index 2cce92ab3..1ae006aaf 100644 --- a/tomviz/python/InvertData.py +++ b/tomviz/python/InvertData.py @@ -1,10 +1,13 @@ +from tomviz.utils import apply_to_each_array import tomviz.operators + NUMBER_OF_CHUNKS = 10 class InvertOperator(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset): import numpy as np self.progress.maximum = NUMBER_OF_CHUNKS diff --git a/tomviz/python/LaplaceFilter.py b/tomviz/python/LaplaceFilter.py index c81e7b109..3d6936767 100644 --- a/tomviz/python/LaplaceFilter.py +++ b/tomviz/python/LaplaceFilter.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): """Apply a Laplace filter to dataset.""" diff --git a/tomviz/python/ManualManipulation.py b/tomviz/python/ManualManipulation.py index ab17bdbea..10867ccf2 100644 --- a/tomviz/python/ManualManipulation.py +++ b/tomviz/python/ManualManipulation.py @@ -1,17 +1,21 @@ +import numpy as np +from scipy.ndimage.interpolation import zoom + +from tomviz import utils +from tomviz.utils import apply_to_each_array + + + def apply_shift(array, shift): if shift is None: return - import numpy as np - array[:] = np.roll(array, shift[0], axis=0) array[:] = np.roll(array, shift[1], axis=1) array[:] = np.roll(array, shift[2], axis=2) def apply_rotation(array, rotation, spacing): - import numpy as np - if rotation is None or all(np.isclose(x, 0) for x in rotation): # No rotation. Nothing to do. return @@ -75,16 +79,10 @@ def apply_resampling(array, spacing, reference_spacing): resampling_factor = [x / y for x, y in zip(spacing, reference_spacing)] - import numpy as np - if np.allclose(resampling_factor, 1): # Nothing to do return array - from tomviz import utils - - from scipy.ndimage.interpolation import zoom - # Transform the dataset. result_shape = utils.zoom_shape(array, resampling_factor) result = np.empty(result_shape, array.dtype, order='F') @@ -101,8 +99,6 @@ def apply_resize(array, reference_shape): # Nothing to do... return array - import numpy as np - padding = [] cropping = [] for x, y in zip(array.shape, reference_shape): @@ -136,13 +132,12 @@ def apply_alignment(array, spacing, reference_spacing, reference_shape): return apply_resize(array, reference_shape) +@apply_to_each_array def transform(dataset, scaling=None, rotation=None, shift=None, align_with_reference=False, reference_spacing=None, reference_shape=None): array = dataset.active_scalars - import numpy as np - convert_to_float = ( not all(np.isclose(x, 0) for x in rotation) and not array.dtype == np.float32 diff --git a/tomviz/python/MedianFilter.py b/tomviz/python/MedianFilter.py index 9490fa1ee..02c754e03 100644 --- a/tomviz/python/MedianFilter.py +++ b/tomviz/python/MedianFilter.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, size=2): """Apply a Median filter to dataset.""" """ Median filter is a nonlinear filter used to reduce noise.""" diff --git a/tomviz/python/NormalizeTiltSeries.py b/tomviz/python/NormalizeTiltSeries.py index 935f0b826..e9babaf70 100644 --- a/tomviz/python/NormalizeTiltSeries.py +++ b/tomviz/python/NormalizeTiltSeries.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): """ Normalize tilt series so that each tilt image has the same total intensity. diff --git a/tomviz/python/PeronaMalikAnisotropicDiffusion.py b/tomviz/python/PeronaMalikAnisotropicDiffusion.py index 4206b9741..91fd9e376 100644 --- a/tomviz/python/PeronaMalikAnisotropicDiffusion.py +++ b/tomviz/python/PeronaMalikAnisotropicDiffusion.py @@ -1,8 +1,10 @@ +from tomviz.utils import apply_to_each_array import tomviz.operators class PeronaMalikAnisotropicDiffusion(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset, conductance=1.0, iterations=100, timestep=0.0625): """This filter performs anisotropic diffusion on an image using diff --git a/tomviz/python/Recon_DFT_constraint.json b/tomviz/python/Recon_DFT_constraint.json index 27c1a57e3..12d4d740b 100644 --- a/tomviz/python/Recon_DFT_constraint.json +++ b/tomviz/python/Recon_DFT_constraint.json @@ -53,7 +53,7 @@ "type" : "int", "default" : 0, "minimum" : 0, - "maximum" : 10000000000000 + "maximum" : 1000000000 } ] } diff --git a/tomviz/python/Recon_WBP.py b/tomviz/python/Recon_WBP.py index f9b259a8f..6bc728763 100644 --- a/tomviz/python/Recon_WBP.py +++ b/tomviz/python/Recon_WBP.py @@ -1,11 +1,13 @@ import numpy as np from scipy.interpolate import interp1d import tomviz.operators +from tomviz.utils import apply_to_each_array import time class ReconWBPOperator(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset, Nrecon=None, filter=None, interp=None, Nupdates=None): """ diff --git a/tomviz/python/RemoveBadPixelsTiltSeries.py b/tomviz/python/RemoveBadPixelsTiltSeries.py index 2e15cb47d..bca96db04 100644 --- a/tomviz/python/RemoveBadPixelsTiltSeries.py +++ b/tomviz/python/RemoveBadPixelsTiltSeries.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, threshold=None): """Remove bad pixels in tilt series.""" diff --git a/tomviz/python/RotationAlign.py b/tomviz/python/RotationAlign.py index e6abf1d5e..ade67c686 100644 --- a/tomviz/python/RotationAlign.py +++ b/tomviz/python/RotationAlign.py @@ -1,8 +1,10 @@ # Perform alignment to the estimated rotation axis # # Developed as part of the tomviz project (www.tomviz.com). +from tomviz.utils import apply_to_each_array +@apply_to_each_array def transform(dataset, SHIFT=None, rotation_angle=90.0, tilt_axis=0): from tomviz import utils from scipy import ndimage diff --git a/tomviz/python/SetNegativeVoxelsToZero.py b/tomviz/python/SetNegativeVoxelsToZero.py index 2db321cdd..be2787c23 100644 --- a/tomviz/python/SetNegativeVoxelsToZero.py +++ b/tomviz/python/SetNegativeVoxelsToZero.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): """Set negative voxels to zero""" diff --git a/tomviz/python/ShiftTiltSeriesRandomly.py b/tomviz/python/ShiftTiltSeriesRandomly.py index 2d1989235..e35cfe654 100644 --- a/tomviz/python/ShiftTiltSeriesRandomly.py +++ b/tomviz/python/ShiftTiltSeriesRandomly.py @@ -12,16 +12,19 @@ def transform(self, dataset, maxShift=1): raise RuntimeError("No scalars found!") self.progress.maximum = tiltSeries.shape[2] + arrays = {k: dataset.scalars(k) for k in dataset.scalars_names} step = 0 for i in range(tiltSeries.shape[2]): if self.canceled: return shifts = (np.random.rand(2) * 2 - 1) * maxShift - tiltSeries[:, :, i] = np.roll( - tiltSeries[:, :, i], int(shifts[0]), axis=0) - tiltSeries[:, :, i] = np.roll( - tiltSeries[:, :, i], int(shifts[1]), axis=1) + for array in arrays.values(): + array[:, :, i] = np.roll( + array[:, :, i], int(shifts[0]), axis=0) + array[:, :, i] = np.roll( + array[:, :, i], int(shifts[1]), axis=1) step += 1 self.progress.value = step - dataset.active_scalars = tiltSeries + for name, array in arrays.items(): + dataset.set_scalars(name, array) diff --git a/tomviz/python/Square_Root_Data.py b/tomviz/python/Square_Root_Data.py index 366a867df..408b4f1b5 100644 --- a/tomviz/python/Square_Root_Data.py +++ b/tomviz/python/Square_Root_Data.py @@ -1,3 +1,4 @@ +from tomviz.utils import apply_to_each_array import tomviz.operators NUMBER_OF_CHUNKS = 10 @@ -5,6 +6,7 @@ class SquareRootOperator(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset): """Define this method for Python operators that transform input scalars""" @@ -17,7 +19,7 @@ def transform(self, dataset): raise RuntimeError("No scalars found!") if scalars.min() < 0: - print("WARNING: Square root of negative values results in NaN!") + raise RuntimeError("Square root of negative values results in NaN!") else: # transform the dataset # Process dataset in chunks so the user gets an opportunity to diff --git a/tomviz/python/Subtract_TiltSer_Background.py b/tomviz/python/Subtract_TiltSer_Background.py index 610ee6864..1e298d7fc 100644 --- a/tomviz/python/Subtract_TiltSer_Background.py +++ b/tomviz/python/Subtract_TiltSer_Background.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, XRANGE=None, YRANGE=None, ZRANGE=None): '''For each tilt image, the method uses average pixel value of selected region as the background level and subtracts it from the image.''' diff --git a/tomviz/python/Subtract_TiltSer_Background_Auto.py b/tomviz/python/Subtract_TiltSer_Background_Auto.py index b3366616a..a96ce3aa6 100644 --- a/tomviz/python/Subtract_TiltSer_Background_Auto.py +++ b/tomviz/python/Subtract_TiltSer_Background_Auto.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset): """ For each tilt image, the method calculates its histogram diff --git a/tomviz/python/TV_Filter.py b/tomviz/python/TV_Filter.py index ddae73098..50d7778b6 100644 --- a/tomviz/python/TV_Filter.py +++ b/tomviz/python/TV_Filter.py @@ -1,10 +1,14 @@ +from tomviz.utils import apply_to_each_array +import tomviz.operators + import numpy as np from numpy.fft import fftn, fftshift, ifftn, ifftshift -import tomviz.operators + class ArtifactsTVOperator(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset, Niter=100, a=0.1, wedgeSize=5, kmin=5, theta=0): """ diff --git a/tomviz/python/UnsharpMask.py b/tomviz/python/UnsharpMask.py index 497fc9842..2f8a06560 100644 --- a/tomviz/python/UnsharpMask.py +++ b/tomviz/python/UnsharpMask.py @@ -1,8 +1,10 @@ +from tomviz.utils import apply_to_each_array import tomviz.operators class UnsharpMask(tomviz.operators.CancelableOperator): + @apply_to_each_array def transform(self, dataset, amount=0.5, threshold=0.0, sigma=1.0): """This filter performs anisotropic diffusion on an image using the classic Perona-Malik, gradient magnitude-based equation. diff --git a/tomviz/python/WienerFilter.py b/tomviz/python/WienerFilter.py index aa8a6d7b8..3ec80686c 100644 --- a/tomviz/python/WienerFilter.py +++ b/tomviz/python/WienerFilter.py @@ -1,3 +1,7 @@ +from tomviz.utils import apply_to_each_array + + +@apply_to_each_array def transform(dataset, SX=0.5, SY=0.5, SZ=0.5, noise=15.0): """Deblur Images with a Weiner Filter.""" diff --git a/tomviz/python/ctf_correct.py b/tomviz/python/ctf_correct.py index 462f1092d..371010f6d 100755 --- a/tomviz/python/ctf_correct.py +++ b/tomviz/python/ctf_correct.py @@ -1,8 +1,11 @@ import numpy as np +from tomviz.utils import apply_to_each_array + # Given an dataset containing one or more 2D images, # apply CTF operations on them. +@apply_to_each_array def transform(dataset, apix=None, df1=None, df2=None, ast=None, ampcon=None, cs=None, kev=None, ctf_method=None, snr=None): diff --git a/tomviz/python/tomviz/_internal.py b/tomviz/python/tomviz/_internal.py index c29816280..2de288d9a 100644 --- a/tomviz/python/tomviz/_internal.py +++ b/tomviz/python/tomviz/_internal.py @@ -4,15 +4,20 @@ # This source file is part of the Tomviz project, https://tomviz.org/. # It is released under the 3-Clause BSD License, see "LICENSE". ############################################################################### -import inspect -import sys -import os import fnmatch import importlib.machinery import importlib.util +import inspect import json +import os +import subprocess +import sys +import tempfile import traceback +from pathlib import Path +from typing import Callable + import tomviz import tomviz.operators @@ -125,6 +130,117 @@ def find_transform_function(transform_module, op=None): return transform_function +def transform_method_wrapper(transform_method: Callable, + operator_serialized: str, *args, **kwargs): + # We take the serialized operator as input because we may need it + # later. If we need to execute this in an external environment, + # we need it serialized. It's easier to have the C++ side do the + # serialization right now. + operator_dict = json.loads(operator_serialized) + tomviz_pipeline_env = None + + operator_description = operator_dict.get('description') + if operator_description: + description_json = json.loads(operator_description) + tomviz_pipeline_env = description_json.get('tomviz_pipeline_env') + + if not tomviz_pipeline_env: + # Execute internally as normal + return transform_method(*args, **kwargs) + + return transform_single_external_operator(transform_method, operator_serialized, *args, **kwargs) + + +def transform_single_external_operator(transform_method: Callable, + operator_serialized: str, *args, **kwargs): + from tomviz.executor import load_dataset, _write_emd + + operator_dict = json.loads(operator_serialized) + description_dict = json.loads(operator_dict['description']) + tomviz_pipeline_env = description_dict['tomviz_pipeline_env'] + + # Find the `tomviz-pipeline` executable + exec_path = Path(tomviz_pipeline_env) / 'bin/tomviz-pipeline' + if not exec_path.exists(): + msg = f'Tomviz pipeline executable does not exist: {exec_path}' + raise RuntimeError(msg) + + # Get the dataset as a Dataset class + dataset = convert_to_dataset(args[0]) + + # Set up the temporary directory for reading/writing + with tempfile.TemporaryDirectory() as tmpdir_path: + tmpdir_path = Path(tmpdir_path) + + # Save the input data to an EMD file + input_path = tmpdir_path / 'original.emd' + _write_emd(input_path, dataset) + + # Build the state file dict + state_dict = { + 'dataSources': [{ + 'reader': { + 'fileNames': [str(input_path)], + }, + 'operators': [operator_dict], + }], + } + + # Write the state file + state_path = tmpdir_path / 'state.tvsm' + with open(state_path, 'w') as wf: + json.dump(state_dict, wf) + + output_path = tmpdir_path / 'output.emd' + + progress_path = tmpdir_path / 'progress' + # FIXME: set up 'socket' progress mode for Linux, and + # 'files' progress mode for Mac/Windows. We need to implement + # a Python reader for each of these. They should just forward + # progress to the main progress. + progress_mode = 'tqdm' + + # Set up the command + cmd = [ + exec_path, + '-s', + state_path, + '-o', + output_path, + '-p', + progress_mode, + '-u', + progress_path, + ] + cmd = [str(x) for x in cmd] + + # Slightly customize the environment + custom_env = os.environ.copy() + custom_env.pop('TOMVIZ_APPLICATION', None) + custom_env.pop('PYTHONHOME', None) + custom_env.pop('PYTHONPATH', None) + custom_env['PYTHONUNBUFFERED'] = 'ON' + + print('Executing operator with command:', ' '.join(cmd)) + + # Run the operator + subprocess.run(cmd, check=True, env=custom_env) + + # Load and return the result + output_dataset = load_dataset(output_path) + + # Now modify the input dataset with any changes made. + # FIXME: put these in a function to copy one dataset to another? + for name in output_dataset.scalars_names: + dataset.set_scalars(name, output_dataset.scalars(name)) + dataset.tilt_angles = output_dataset.tilt_angles + dataset.spacing = output_dataset.spacing + + # FIXME: for functions that normally return a dict with things like + # a child dataset, what should we do? + return None + + def _load_module(operator_dir, python_file): module_name, _ = os.path.splitext(python_file) spec = importlib.machinery.PathFinder.find_spec(module_name, [operator_dir]) diff --git a/tomviz/python/tomviz/executor.py b/tomviz/python/tomviz/executor.py index 0536dde37..6ed22dbb4 100644 --- a/tomviz/python/tomviz/executor.py +++ b/tomviz/python/tomviz/executor.py @@ -393,6 +393,13 @@ def _swap_dims(dims, i, j): def _read_emd(path, options=None): + def bytes_to_str(x): + if isinstance(x, np.bytes_): + return x.decode('utf-8') + + return x + + with h5py.File(path, 'r') as f: tomography = f['data/tomography'] @@ -400,8 +407,8 @@ def _read_emd(path, options=None): for dim in DIMS: dims.append(Dim(dim, tomography[dim][:], - tomography[dim].attrs['name'][0], - tomography[dim].attrs['units'][0])) + bytes_to_str(tomography[dim].attrs['name']), + bytes_to_str(tomography[dim].attrs['units']))) data = tomography['data'] # We default the name to ImageScalars @@ -466,9 +473,12 @@ def _get_arrays_for_writing(dataset): active_array = dataset.active_scalars # Separate out the extra channels/arrays as we store them separately - extra_arrays = {name: array for name, array - in dataset.arrays.items() - if id(array) != id(active_array)} + extra_arrays = {} + for name in dataset.scalars_names: + if name == dataset.active_name: + continue + + extra_arrays[name] = dataset.scalars(name) # If this is a tilt series, swap the X and Z axes if tilt_angles is not None and tilt_axis == 2: @@ -544,15 +554,15 @@ def _write_emd(path, dataset, dims=None): tomography_group = data_group.create_group('tomography') tomography_group.attrs.create('emd_group_type', 1, dtype='uint32') data = tomography_group.create_dataset('data', data=active_array) - data.attrs['name'] = np.bytes_(dataset.active_name) + data.attrs['name'] = dataset.active_name dims = _get_dims_for_writing(dataset, data, dims) # add dimension vectors for dim in dims: d = tomography_group.create_dataset(dim.path, dim.values.shape) - d.attrs['name'] = np.bytes_(dim.name) - d.attrs['units'] = np.bytes_(dim.units) + d.attrs['name'] = dim.name + d.attrs['units'] = dim.units d[:] = dim.values # If we have extra scalars add them under tomviz_scalars diff --git a/tomviz/python/tomviz/modules.py b/tomviz/python/tomviz/modules.py new file mode 100644 index 000000000..e69de29bb diff --git a/tomviz/python/tomviz/ptycho/ptycho.py b/tomviz/python/tomviz/ptycho/ptycho.py index 251c9abd8..9bf5a4fb9 100644 --- a/tomviz/python/tomviz/ptycho/ptycho.py +++ b/tomviz/python/tomviz/ptycho/ptycho.py @@ -178,6 +178,10 @@ def filter_sid_list(sid_list: list[int], filter_string: str) -> list[int]: if this_slice.stop is None: this_slice = slice(this_slice.start, max(sid_list) + 1, this_slice.step) + else: + # Unlike numpy, we want to be inclusive of the last number + this_slice = slice(this_slice.start, this_slice.stop + 1, + this_slice.step) valid_sids += np.r_[this_slice].tolist() else: @@ -275,9 +279,19 @@ def load_stack_ptycho(version_list: list[str], # imt = tf.imread(filelist[0][0]) # load fluor1 and get shape of first array - probes = np.asarray(tempPtyprb) - probes_phase = np.angle(probes) - probes_amp = np.abs(probes) + has_probes = True + try: + probes = np.asarray(tempPtyprb) + probes_phase = np.angle(probes) + probes_amp = np.abs(probes) + except Exception as e: + has_probes = False + msg = ( + f'Failed to stack probes with error message: {e}\n' + 'Skipping over probe data...' + ) + print(msg, file=sys.stderr) + # imt = ptfluor[0] # l, w = imt.shape # factor= 2 @@ -301,9 +315,13 @@ def load_stack_ptycho(version_list: list[str], arrays = { 'Phase': ptychodatanew, 'Amplitude': ampdatanew, - 'Probes Phase': probes_phase, - 'Probes Amplitude': probes_amp, } + if has_probes: + arrays = { + **arrays, + 'Probes Phase': probes_phase, + 'Probes Amplitude': probes_amp, + } # Do all necessary processing of the output arrays. for key, array in arrays.items(): @@ -321,9 +339,11 @@ def load_stack_ptycho(version_list: list[str], datasets = { # Ptycho and Amp have the same shape, so we write them together 'ptycho_object.emd': ['Phase', 'Amplitude'], - # Probe has a different shape - 'ptycho_probe.emd': ['Probes Phase', 'Probes Amplitude'], } + if has_probes: + # Probe has a different shape + datasets['ptycho_probe.emd'] = ['Probes Phase', 'Probes Amplitude'] + for filename, array_names in datasets.items(): dataset = Dataset({key: arrays[key] for key in array_names}) dataset.tilt_angles = np.array([x[2] for x in currentsidlist]) diff --git a/tomviz/python/tomviz/pyxrf/pyxrf-utils/pixi.toml b/tomviz/python/tomviz/pyxrf/pyxrf-utils/pixi.toml index 25f9aa6ed..78b0d8dd6 100644 --- a/tomviz/python/tomviz/pyxrf/pyxrf-utils/pixi.toml +++ b/tomviz/python/tomviz/pyxrf/pyxrf-utils/pixi.toml @@ -9,7 +9,10 @@ platforms = ["linux-64"] python = ">=3.10,<3.14" numpy = "*" h5py = "*" -pandas = "*" +# Pandas needs to be pinned to prevent a bug in xrf-tomo, until +# it gets fixed and a new release comes out. +# See: https://github.com/NSLS-II-SRX/xrf-tomo/pull/10 +pandas = "<3.0" xraylib = "*" tomopy = "*" scikit-beam = "*" diff --git a/tomviz/python/tomviz/pyxrf/sids.py b/tomviz/python/tomviz/pyxrf/sids.py index 3180c7827..26204e068 100644 --- a/tomviz/python/tomviz/pyxrf/sids.py +++ b/tomviz/python/tomviz/pyxrf/sids.py @@ -14,10 +14,14 @@ def filter_sids(all_sids: list[str], filter_string: str) -> list[str]: this_slice = slice( *(int(s) if s else None for s in this_str.split(':')) ) - # If there was no stop specified, go to the end of the sid_list if this_slice.stop is None: + # If there was no stop specified, go to the end of the sid_list this_slice = slice(this_slice.start, max_sid + 1, this_slice.step) + else: + # Unlike numpy, we want to be inclusive of the last number + this_slice = slice(this_slice.start, this_slice.stop + 1, + this_slice.step) valid_sids += np.r_[this_slice].tolist() else: diff --git a/tomviz/python/tomviz/utils.py b/tomviz/python/tomviz/utils.py index e2089e106..84b53c1d8 100644 --- a/tomviz/python/tomviz/utils.py +++ b/tomviz/python/tomviz/utils.py @@ -151,7 +151,12 @@ def wrapper(*args, **kwargs): image_data = orig_do else: image_data = vtkImageData() - image_data.ShallowCopy(orig_do) + # This must be a deep copy for any operators that + # include progress to work properly. Otherwise, there + # is a crash. + # But this deep copy is probably not very expensive + # since the data object is empty... + image_data.DeepCopy(orig_do) this_pd = image_data.GetPointData() this_pd.AddArray(all_arrays[i])