diff --git a/ami/jobs/models.py b/ami/jobs/models.py index 89be29312..53504c871 100644 --- a/ami/jobs/models.py +++ b/ami/jobs/models.py @@ -614,7 +614,8 @@ def run(cls, job: "Job"): """ job.progress.add_stage(cls.name) - job.progress.add_stage_param(cls.key, "Total files", "") + job.progress.add_stage_param(cls.key, "Total files", 0) + job.progress.add_stage_param(cls.key, "Failed", 0) job.update_status(JobState.STARTED) job.started_at = datetime.datetime.now() job.finished_at = None diff --git a/ami/main/api/serializers.py b/ami/main/api/serializers.py index 1a7265b1d..6d0d93762 100644 --- a/ami/main/api/serializers.py +++ b/ami/main/api/serializers.py @@ -1106,6 +1106,7 @@ class Meta: "deployment", "event", "url", + "path", # "thumbnail", "timestamp", "width", diff --git a/ami/main/api/views.py b/ami/main/api/views.py index df29946f5..18536e7d5 100644 --- a/ami/main/api/views.py +++ b/ami/main/api/views.py @@ -501,6 +501,7 @@ class SourceImageViewSet(DefaultViewSet, ProjectMixin): "taxa_count", "deployment__name", "event__start", + "path", ] permission_classes = [ObjectPermission] diff --git a/ami/main/models.py b/ami/main/models.py index 400932fc3..0bad68531 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -797,6 +797,7 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J s3_config = deployment.data_source.config total_size = 0 total_files = 0 + failed = 0 source_images = [] django_batch_size = batch_size sql_batch_size = 1000 @@ -814,8 +815,34 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J logger.debug(f"Processing file {file_index}: {obj}") if not obj: continue - source_image = _create_source_image_for_sync(deployment, obj) + try: + source_image = _create_source_image_for_sync(deployment, obj) + except Exception: + failed += 1 + msg = f"Failed to process {obj.get('Key', '?')}" + if job: + job.logger.exception(msg) + else: + logger.exception(msg) + continue + if source_image: + # Skip images with unparseable timestamps — they can't be grouped into events + if source_image.timestamp is None: + failed += 1 + msg = f"No timestamp parsed from filename: {obj['Key']}" + if job: + job.logger.error(msg) + else: + logger.error(msg) + continue + elif source_image.timestamp.year < 2000: + msg = f"Suspicious timestamp ({source_image.timestamp.year}) for: {obj['Key']}" + if job: + job.logger.warning(msg) + else: + logger.warning(msg) + total_files += 1 total_size += obj.get("Size", 0) source_images.append(source_image) @@ -827,7 +854,7 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J source_images = [] if job: job.logger.info(f"Processed {total_files} files") - job.progress.update_stage(job.job_type().key, total_files=total_files) + job.progress.update_stage(job.job_type().key, total_files=total_files, failed=failed) job.update_progress() if source_images: @@ -837,7 +864,7 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J ) if job: job.logger.info(f"Processed {total_files} files") - job.progress.update_stage(job.job_type().key, total_files=total_files) + job.progress.update_stage(job.job_type().key, total_files=total_files, failed=failed) job.update_progress() _compare_totals_for_sync(deployment, total_files) diff --git a/ami/utils/dates.py b/ami/utils/dates.py index a1764172b..02dfc4350 100644 --- a/ami/utils/dates.py +++ b/ami/utils/dates.py @@ -35,6 +35,9 @@ def get_image_timestamp_from_filename(img_path, raise_error=False) -> datetime.d >>> # Snapshot date format from Wingscape camera from Newfoundland >>> get_image_timestamp_from_filename("Project_20230801023001_4393.JPG").strftime(out_fmt) '2023-08-01 02:30:01' + >>> # 2-digit year format (e.g., Farmscape/NSCF cameras) + >>> get_image_timestamp_from_filename("NSCF----_250927194802_0017.JPG").strftime(out_fmt) + '2025-09-27 19:48:02' """ name = pathlib.Path(img_path).stem @@ -47,19 +50,30 @@ def get_image_timestamp_from_filename(img_path, raise_error=False) -> datetime.d two_groups_pattern = r"\d{8}[^\d]+\d{6}" # YYYYMMDD*HHMMSS # Allow single non-digit delimiters within components, and one or more between DD and HH delimited_pattern = r"\d{4}[^\d]\d{2}[^\d]\d{2}[^\d]+\d{2}[^\d]\d{2}[^\d]\d{2}" # YYYY*MM*DD*+HH*MM*SS + # 2-digit year: YYMMDDHHMMSS (12 consecutive digits, bounded by non-digits or string edges) + short_year_pattern = r"(? TableColumn[] = ( ), }, + { + id: 'filename', + name: translate(STRING.FIELD_LABEL_FILENAME), + sortField: 'path', + renderCell: (item: Capture) => , + }, + { + id: 'path', + name: translate(STRING.FIELD_LABEL_PATH), + sortField: 'path', + renderCell: (item: Capture) => , + }, { id: 'occurrences', name: translate(STRING.FIELD_LABEL_OCCURRENCES), diff --git a/ui/src/pages/captures/captures.tsx b/ui/src/pages/captures/captures.tsx index 8d47b0595..474af8e69 100644 --- a/ui/src/pages/captures/captures.tsx +++ b/ui/src/pages/captures/captures.tsx @@ -33,6 +33,8 @@ export const Captures = () => { session: true, size: true, dimensions: true, + filename: false, + path: false, }) const { selectedView, setSelectedView } = useSelectedView('table') const { filters } = useFilters() diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index 87eb0e1df..20d5ad0ea 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -96,6 +96,7 @@ export enum STRING { FIELD_LABEL_EMAIL, FIELD_LABEL_ENDPOINT, FIELD_LABEL_ERRORS, + FIELD_LABEL_FILENAME, FIELD_LABEL_FILE_SIZE, FIELD_LABEL_FINISHED_AT, FIELD_LABEL_FIRST_DATE, @@ -386,6 +387,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.FIELD_LABEL_EMAIL]: 'Email', [STRING.FIELD_LABEL_ENDPOINT]: 'Endpoint URL', [STRING.FIELD_LABEL_ERRORS]: 'Errors', + [STRING.FIELD_LABEL_FILENAME]: 'Filename', [STRING.FIELD_LABEL_FILE_SIZE]: 'File size', [STRING.FIELD_LABEL_FINISHED_AT]: 'Finished at', [STRING.FIELD_LABEL_FIRST_DATE]: 'First date',