diff --git a/fireatlas/FireConsts.py b/fireatlas/FireConsts.py index b84b6b4..3dfdc90 100644 --- a/fireatlas/FireConsts.py +++ b/fireatlas/FireConsts.py @@ -186,6 +186,10 @@ def check_epsg(cls, epsg: int): ) N_DASK_WORKERS: int = Field(6, description="How many dask workers to use for Run.") + DEPARALLELIZE_POSTPROCESS: bool = Field( + False, description="whether to avoid dask for postprocessing in case of worker memory errors" + ) + DO_NIFC_MATCHING: bool = Field( False, description="If True, reads from the NIFC incident database for current " diff --git a/fireatlas/postprocess.py b/fireatlas/postprocess.py index 9bf9032..213e92d 100644 --- a/fireatlas/postprocess.py +++ b/fireatlas/postprocess.py @@ -411,10 +411,11 @@ def merge_and_save_fire(data, fid): futures = [] processed_gdfs = [] for fid, data in gdf[gdf["fireID"].isin(large_fires)].groupby("fireID"): - if client: - futures.append(client.submit(merge_and_save_fire, data, fid)) - else: + if settings.DEPARALLELIZE_POSTPROCESS == True: processed_gdfs.append(merge_and_save_fire(data, fid)) + else: + futures.append(client.submit(merge_and_save_fire, data, fid)) + if futures: processed_gdfs = client.gather(futures)