diff --git a/protzilla/disk_operator.py b/protzilla/disk_operator.py index ce7f07ad..e41d5bcb 100644 --- a/protzilla/disk_operator.py +++ b/protzilla/disk_operator.py @@ -93,6 +93,7 @@ class KEYS: STEP_PLOTS = "plots" STEP_INSTANCE_IDENTIFIER = "instance_identifier" STEP_TYPE = "type" + STEP_CALCULATION_STATUS = "calculation_status" DF_MODE = "df_mode" @@ -176,7 +177,7 @@ def check_file_validity(self, file: Path, steps: StepManager) -> bool: if steps.current_step.instance_identifier in file.name: return False return any( - step.instance_identifier in file.name and step.finished + step.instance_identifier in file.name and step.calculation_status!="incomplete" for step in steps.all_steps ) @@ -213,6 +214,7 @@ def _read_step(self, step_data: dict, steps: StepManager) -> Step: step.output = self._read_outputs(step_data.get(KEYS.STEP_OUTPUTS, {})) step.plots = self._read_plots(step_data.get(KEYS.STEP_PLOTS, [])) step.form_inputs = step_data.get(KEYS.STEP_FORM_INPUTS, {}) + step.calculation_status = step_data.get(KEYS.STEP_CALCULATION_STATUS,"incomplete") return step def _write_step(self, step: Step, workflow_mode: bool = False) -> dict: @@ -232,6 +234,7 @@ def _write_step(self, step: Step, workflow_mode: bool = False) -> dict: instance_identifier=step.instance_identifier, output=step.output ) step_data[KEYS.STEP_MESSAGES] = step.messages.messages + step_data[KEYS.STEP_CALCULATION_STATUS] = step.calculation_status return step_data def _read_outputs(self, output: dict) -> Output: diff --git a/protzilla/run.py b/protzilla/run.py index 1f451e89..978ef740 100644 --- a/protzilla/run.py +++ b/protzilla/run.py @@ -144,6 +144,11 @@ def step_remove( def step_calculate(self, inputs: dict | None = None) -> None: self.steps.current_step.calculate(self.steps, inputs) + @error_handling + @auto_save + def update_inputs(self, inputs: dict) -> None: + self.steps.current_step.updateInputs(inputs) + @error_handling @auto_save def step_plot(self, inputs: dict | None = None) -> None: @@ -162,6 +167,10 @@ def step_previous(self) -> None: def step_goto(self, step_index: int, section: str) -> None: self.steps.goto_step(step_index, section) + @error_handling + def step_set_outdated(self, offset: int = 0) -> int: + return self.steps.set_steps_outdated(offset) + @error_handling @auto_save def step_change_method(self, new_method: str) -> None: diff --git a/protzilla/runner.py b/protzilla/runner.py index fa206a19..5c78c57c 100644 --- a/protzilla/runner.py +++ b/protzilla/runner.py @@ -89,7 +89,7 @@ def compute_workflow(self): log_messages(self.run.current_messages) self.run.current_messages.clear() - if not step.finished: + if step.calculation_status!="complete": break self.run.step_next() diff --git a/protzilla/steps.py b/protzilla/steps.py index eecf0d04..f7f99a79 100644 --- a/protzilla/steps.py +++ b/protzilla/steps.py @@ -7,6 +7,7 @@ from enum import Enum from io import BytesIO from pathlib import Path +from typing import Literal import pandas as pd import plotly @@ -29,6 +30,7 @@ class Step: method_description: str = None input_keys: list[str] = [] output_keys: list[str] = [] + calculation_status: Literal["complete", "outdated", "incomplete", "failed"] = "incomplete" def __init__(self, instance_identifier: str | None = None): self.form_inputs: dict = {} @@ -54,7 +56,12 @@ def __eq__(self, other): and self.output == other.output ) - def calculate(self, steps: StepManager, inputs: dict) -> None: + def updateInputs(self, inputs: dict) -> None: + if inputs: + self.inputs = inputs.copy() + self.form_inputs = self.inputs.copy() + + def calculate(self, steps: StepManager, inputs: dict) -> bool: """ Core calculation method for all steps, receives the inputs from the front-end and calculates the output. @@ -62,22 +69,27 @@ def calculate(self, steps: StepManager, inputs: dict) -> None: :param inputs: These inputs will be supplied to the method. Only keys in the input_keys of the method class will actually be supplied to the method :return: None """ - steps._clear_future_steps() - - if inputs: - self.inputs = inputs.copy() - self.form_inputs = self.inputs.copy() - + stepIndex = steps.all_steps.index(self) + previousStep = steps.all_steps[stepIndex-1] + try: + if (previousStep.calculation_status == "outdated" ): + if not previousStep.calculate(steps,inputs): + return False + + if (steps.current_step_index == stepIndex): + self.updateInputs(inputs) self.messages.clear() self.insert_dataframes(steps, self.inputs) self.validate_inputs() - output_dict = self.method(self.inputs) self.handle_outputs(output_dict) - self.handle_messages(output_dict) - + self.handle_messages(output_dict,steps,stepIndex) self.validate_outputs() + self.calculation_status = "complete" + if (steps.failed_step_index == stepIndex): + steps.failed_step_index = -1 + return True except NotImplementedError as e: self.messages.append( dict( @@ -103,16 +115,17 @@ def calculate(self, steps: StepManager, inputs: dict) -> None: ) ) except Exception as e: - self.messages.append( - dict( - level=logging.ERROR, - msg=( - f"An error occurred while calculating this step: {e.__class__.__name__} {e} " - f"Please check your parameters or report a potential programming issue." - ), - trace=format_trace(traceback.format_exception(e)), + self.messages.append( + dict( + level=logging.ERROR, + msg=( + f"An error occurred while calculating this step: {e.__class__.__name__} {e} " + f"Please check your parameters or report a potential programming issue." + ), + trace=format_trace(traceback.format_exception(e)), + ) ) - ) + return False def method(self, **kwargs) -> dict: raise NotImplementedError("This method must be implemented in a subclass.") @@ -135,7 +148,7 @@ def handle_outputs(self, outputs: dict) -> None: raise ValueError("Output of calculation is empty.") self.output = Output(outputs) - def handle_messages(self, outputs: dict) -> None: + def handle_messages(self, outputs: dict, steps: StepManager, stepIndex: int) -> None: """ Handles the messages from the calculation method and creates a Messages object from it. Responsible for clearing and setting the messages attribute of the class. @@ -144,6 +157,11 @@ def handle_messages(self, outputs: dict) -> None: """ messages = outputs.get("messages", []) self.messages.extend(messages) + for message in messages: + if message["level"] == logging.ERROR: + self.calculation_status = "failed" + steps.failed_step_index = stepIndex + raise Exception("Calculation failed") def plot(self, inputs: dict = None) -> None: raise NotImplementedError( @@ -198,17 +216,6 @@ def validate_outputs( return False return True - @property - def finished(self) -> bool: - """ - Return whether the step has valid outputs and is therefore considered finished. - Plot steps without required outputs are considered finished if they have plots. - :return: True if the step is finished, False otherwise - """ - if len(self.output_keys) == 0: - return not self.plots.empty - return self.validate_outputs(soft_check=True) - class Output: def __init__(self, output: dict = None): @@ -324,6 +331,7 @@ def __init__( self.df_mode = df_mode self.disk_operator = disk_operator self.current_step_index = 0 + self.failed_step_index = -1 self.importing = [] self.data_preprocessing = [] self.data_analysis = [] @@ -467,10 +475,22 @@ def all_steps_in_section(self, section: str) -> list[Step]: return self.sections[section] else: raise ValueError(f"Unknown section {section}") + + def set_steps_outdated(self, offset: int) -> None: + count = 0 + for step in self.following_steps[offset:]: + if (step.calculation_status == "complete"): + step.calculation_status = "outdated" + count+=1 + return count @property def previous_steps(self) -> list[Step]: return self.all_steps[: self.current_step_index] + + @property + def following_steps(self) -> list[Step]: + return self.all_steps[self.current_step_index :] @property def current_step(self) -> Step: @@ -515,7 +535,7 @@ def preprocessed_output(self) -> Output: if self.current_section == "data_preprocessing": return ( self.current_step.output - if self.current_step.finished + if self.current_step.calculation_status!="incomplete" else self.previous_steps[-1].output ) return self.data_preprocessing[-1].output @@ -624,10 +644,7 @@ def goto_step(self, step_index: int, section: str) -> None: step = self.all_steps_in_section(section)[step_index] new_step_index = self.all_steps.index(step) - if new_step_index < self.current_step_index: - self.current_step_index = new_step_index - else: - raise ValueError("Cannot go to a step that is after the current step") + self.current_step_index = new_step_index def name_current_step_instance(self, new_instance_identifier: str) -> None: """ diff --git a/tests/protzilla/test_run.py b/tests/protzilla/test_run.py index 7c9a111c..1e469c58 100644 --- a/tests/protzilla/test_run.py +++ b/tests/protzilla/test_run.py @@ -91,7 +91,7 @@ def test_step_previous(self, run_imported): def test_step_goto(self, caplog, run_imported): step = ImputationByMinPerProtein() run_imported.step_add(step) - run_imported.step_goto(0, "data_preprocessing") + run_imported.step_goto(0, "data_preprocessing_wrong") assert any( message["level"] == logging.ERROR and "ValueError" in message["msg"] for message in run_imported.current_messages @@ -105,3 +105,13 @@ def test_step_goto(self, caplog, run_imported): def test_step_change_method(self, run_imported): run_imported.step_change_method("DiannImport") assert run_imported.current_step.__class__.__name__ == "DiannImport" + + def test_set_steps_outdated(self,run_imported,maxquant_data_file): + step = ImputationByMinPerProtein() + run_imported.step_add(step) + run_imported.step_next() + assert run_imported.current_step.calculation_status == "incomplete" + run_imported.step_calculate(inputs={"shrinking_value": 0.5}) + assert run_imported.current_step.calculation_status == "complete" + run_imported.step_set_outdated() + assert run_imported.current_step.calculation_status == "outdated" \ No newline at end of file diff --git a/tests/protzilla/test_runner.py b/tests/protzilla/test_runner.py index 18080f48..a3260cad 100644 --- a/tests/protzilla/test_runner.py +++ b/tests/protzilla/test_runner.py @@ -41,11 +41,7 @@ def mock_current_parameters(*args, **kwargs): mock_perform.methods.append(str(runner.run.current_step)) mock_perform.inputs.append(runner.run.current_step.inputs) - # side effect to mark the step as finished - runner.run.current_step.output = Output( - {key: "mock_output_value" for key in runner.run.current_step.output_keys}) - if len(runner.run.current_step.output_keys) == 0: - runner.run.current_step.plots = Plots(["mock_plot"]) + runner.run.current_step.calculation_status = "complete" mock_perform.side_effect = mock_current_parameters diff --git a/tests/ui/test_views.py b/tests/ui/test_views.py index eb0aece6..a3cf8f62 100644 --- a/tests/ui/test_views.py +++ b/tests/ui/test_views.py @@ -106,7 +106,7 @@ def test_all_button_parameters(): def test_step_finished(run_standard): - assert not run_standard.current_step.finished + assert run_standard.current_step.calculation_status == "incomplete" parameters = { "file_path": f"{PROJECT_PATH}/tests/proteinGroups_small_cut.txt", @@ -116,11 +116,11 @@ def test_step_finished(run_standard): } run_standard.step_calculate(parameters) - assert run_standard.current_step.finished + assert run_standard.current_step.calculation_status == "complete" run_standard.step_next() - assert not run_standard.current_step.finished + assert run_standard.current_step.calculation_status == "incomplete" parameters = { "file_path": f"", @@ -128,7 +128,7 @@ def test_step_finished(run_standard): } run_standard.step_calculate(parameters) - assert not run_standard.current_step.finished + assert run_standard.current_step.calculation_status == "failed" parameters = { "file_path": f"{PROJECT_PATH}/tests/nonexistent_file.txt", @@ -136,7 +136,7 @@ def test_step_finished(run_standard): } run_standard.step_calculate(parameters) - assert not run_standard.current_step.finished + assert run_standard.current_step.calculation_status == "failed" parameters = { "file_path": f"{PROJECT_PATH}/tests/metadata_cut_columns.csv", @@ -144,4 +144,4 @@ def test_step_finished(run_standard): } run_standard.step_calculate(parameters) - assert run_standard.current_step.finished + assert run_standard.current_step.calculation_status == "complete" diff --git a/ui/runs/forms/base.py b/ui/runs/forms/base.py index ba6b9385..270085d4 100644 --- a/ui/runs/forms/base.py +++ b/ui/runs/forms/base.py @@ -91,10 +91,16 @@ def replace_file_fields_with_paths(self, pretty_file_names: bool) -> None: label=field.label, initial=file_name_to_show ) - def submit(self, run: Run) -> None: + def add_missing_fields(self)->None: # add the missing fields to the form for field_name, field in self.initial_fields.items(): if field_name not in self.fields: self.fields[field_name] = field self.cleaned_data[field_name] = None + def submit(self, run: Run) -> None: + self.add_missing_fields() run.step_calculate(self.cleaned_data) + + def update_form(self, run: Run) -> None: + self.add_missing_fields() + run.update_inputs(self.cleaned_data) \ No newline at end of file diff --git a/ui/runs/static/runs/runs.js b/ui/runs/static/runs/runs.js index 67cf344d..bd1f0f82 100644 --- a/ui/runs/static/runs/runs.js +++ b/ui/runs/static/runs/runs.js @@ -32,6 +32,29 @@ $(document).ready(function () { $('#chosen-' + id).text(this.files[0].name); }); + + //save forms on change + $('.calc_form').on( "change", function() { + var triggeredForm = $(this); + var formId = triggeredForm.attr('id'); + var index = Number(formId.split("_").pop()); + + $.ajax({ + url: `/runs/${run_name}/update_form`, // The URL for the Django view + type: 'POST', + headers: { + 'X-CSRFToken': $('[name=csrfmiddlewaretoken]').val() // CSRF token for security + }, + data: $(this).serialize(), + success: function(response) { + for (let i=0; i - + {# TODO 129 Better buttons for analysis and importing #} @@ -124,7 +127,7 @@

{{ display_name }}

{# if there are plot parameters, display method and plot parameters next to each other #} {% if plot_form %}
-
{% csrf_token %}
@@ -141,7 +144,7 @@

{{ display_name }}

-
{% csrf_token %}
@@ -158,7 +161,7 @@

{{ display_name }}

{% else %}
{% if step != "plot" %} - {% csrf_token %}
@@ -191,7 +194,7 @@

{{ display_name }}

{% else %} -
{% csrf_token %} {{ method_dropdown }} diff --git a/ui/runs/templates/runs/sidebar_section.html b/ui/runs/templates/runs/sidebar_section.html index b76528b8..98dcbd30 100644 --- a/ui/runs/templates/runs/sidebar_section.html +++ b/ui/runs/templates/runs/sidebar_section.html @@ -1,4 +1,5 @@ -{# params: run_name section{id, name, finished, selected, possible_steps{id, name, methods{id, name, description}}, steps{id, name, finished, selected, index, method_name}} #} +{% load static %} +{# params: run_name section{id, finished, name, selected, possible_steps{id, name, methods{id, name, description}}, steps{id, name, finished, selected, index, index_global, method_name, calculation_icon_path}} #} {% block js %}