fix: 2-digit year timestamp parsing, skip null timestamps, sync error reporting, captures columns

- Add YYMMDDHHMMSS (12-digit) regex pattern for cameras like Farmscape/NSCF
  that produce 2-digit year filenames
- Skip images with unparseable timestamps during sync instead of importing
  them with NULL — they can't be grouped into events anyway
- Wrap per-image sync processing in try/except, track failed count, and
  report it as a "Failed" stage param in job progress UI
- Warn on suspicious pre-2000 timestamps
- Add filename and path columns to captures table (hidden by default)
- Expose path field in SourceImage list serializer and sortable fields

Co-Authored-By: Claude <noreply@anthropic.com>

Author: mihow

ami/jobs/models.py

@@ -606,7 +606,8 @@ def run(cls, job: "Job"):
         """
 
         job.progress.add_stage(cls.name)
-        job.progress.add_stage_param(cls.key, "Total files", "")
+        job.progress.add_stage_param(cls.key, "Total files", 0)
+        job.progress.add_stage_param(cls.key, "Failed", 0)
         job.update_status(JobState.STARTED)
         job.started_at = datetime.datetime.now()
         job.finished_at = None

ami/main/api/serializers.py

@@ -1106,6 +1106,7 @@ class Meta:
             "deployment",
             "event",
             "url",
+            "path",
             # "thumbnail",
             "timestamp",
             "width",

ami/main/api/views.py

@@ -501,6 +501,7 @@ class SourceImageViewSet(DefaultViewSet, ProjectMixin):
         "taxa_count",
         "deployment__name",
         "event__start",
+        "path",
     ]
     permission_classes = [ObjectPermission]
 

ami/main/models.py

@@ -797,6 +797,7 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J
         s3_config = deployment.data_source.config
         total_size = 0
         total_files = 0
+        failed = 0
         source_images = []
         django_batch_size = batch_size
         sql_batch_size = 1000
@@ -814,8 +815,34 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J
             logger.debug(f"Processing file {file_index}: {obj}")
             if not obj:
                 continue
-            source_image = _create_source_image_for_sync(deployment, obj)
+            try:
+                source_image = _create_source_image_for_sync(deployment, obj)
+            except Exception:
+                failed += 1
+                msg = f"Failed to process {obj.get('Key', '?')}"
+                if job:
+                    job.logger.exception(msg)
+                else:
+                    logger.exception(msg)
+                continue
+
             if source_image:
+                # Skip images with unparseable timestamps — they can't be grouped into events
+                if source_image.timestamp is None:
+                    failed += 1
+                    msg = f"No timestamp parsed from filename: {obj['Key']}"
+                    if job:
+                        job.logger.error(msg)
+                    else:
+                        logger.error(msg)
+                    continue
+                elif source_image.timestamp.year < 2000:
+                    msg = f"Suspicious timestamp ({source_image.timestamp.year}) for: {obj['Key']}"
+                    if job:
+                        job.logger.warning(msg)
+                    else:
+                        logger.warning(msg)
+
                 total_files += 1
                 total_size += obj.get("Size", 0)
                 source_images.append(source_image)
@@ -827,7 +854,7 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J
                 source_images = []
                 if job:
                     job.logger.info(f"Processed {total_files} files")
-                    job.progress.update_stage(job.job_type().key, total_files=total_files)
+                    job.progress.update_stage(job.job_type().key, total_files=total_files, failed=failed)
                     job.update_progress()
 
         if source_images:
@@ -837,7 +864,7 @@ def sync_captures(self, batch_size=1000, regroup_events_per_batch=False, job: "J
             )
         if job:
             job.logger.info(f"Processed {total_files} files")
-            job.progress.update_stage(job.job_type().key, total_files=total_files)
+            job.progress.update_stage(job.job_type().key, total_files=total_files, failed=failed)
             job.update_progress()
 
         _compare_totals_for_sync(deployment, total_files)

ami/utils/dates.py

@@ -35,6 +35,9 @@ def get_image_timestamp_from_filename(img_path, raise_error=False) -> datetime.d
     >>> # Snapshot date format from Wingscape camera from Newfoundland
     >>> get_image_timestamp_from_filename("Project_20230801023001_4393.JPG").strftime(out_fmt)
     '2023-08-01 02:30:01'
+    >>> # 2-digit year format (e.g., Farmscape/NSCF cameras)
+    >>> get_image_timestamp_from_filename("NSCF----_250927194802_0017.JPG").strftime(out_fmt)
+    '2025-09-27 19:48:02'
 
     """
     name = pathlib.Path(img_path).stem
@@ -47,19 +50,30 @@ def get_image_timestamp_from_filename(img_path, raise_error=False) -> datetime.d
     two_groups_pattern = r"\d{8}[^\d]+\d{6}"  # YYYYMMDD*HHMMSS
     # Allow single non-digit delimiters within components, and one or more between DD and HH
     delimited_pattern = r"\d{4}[^\d]\d{2}[^\d]\d{2}[^\d]+\d{2}[^\d]\d{2}[^\d]\d{2}"  # YYYY*MM*DD*+HH*MM*SS
+    # 2-digit year: YYMMDDHHMMSS (12 consecutive digits, bounded by non-digits or string edges)
+    short_year_pattern = r"(?<!\d)\d{12}(?!\d)"  # YYMMDDHHMMSS
 
     # Combine patterns with OR '|' but keep them in their own groups
-    pattern = re.compile(f"({consecutive_pattern})|({two_groups_pattern})|({delimited_pattern})")
+    # Order matters: longer/more specific patterns first
+    pattern = re.compile(
+        f"({consecutive_pattern})|({two_groups_pattern})|({delimited_pattern})|({short_year_pattern})"
+    )
 
     match = pattern.search(name)
     if match:
         # Get the full string matched by any of the patterns
         matched_string = match.group(0)
-        # Remove all non-digit characters to create YYYYMMDDHHMMSS
+        # Remove all non-digit characters to create YYYYMMDDHHMMSS or YYMMDDHHMMSS
         consecutive_date_string = re.sub(r"[^\d]", "", matched_string)
 
+        # Determine format based on length (12 digits = 2-digit year, 14 = 4-digit year)
+        if len(consecutive_date_string) == 12:
+            fmt = "%y%m%d%H%M%S"
+        else:
+            fmt = strptime_format
+
         try:
-            date = datetime.datetime.strptime(consecutive_date_string, strptime_format)
+            date = datetime.datetime.strptime(consecutive_date_string, fmt)
         except ValueError:
             pass
 

ami/utils/tests.py

@@ -21,6 +21,9 @@ def test_extract_timestamps(self):
             ("mothbox/2024_01_01 12_00_00.jpg", "2024-01-01 12:00:00"),
             ("other_common/2024-01-01 12:00:00.jpg", "2024-01-01 12:00:00"),
             ("other_common/2024-01-01T12:00:00.jpg", "2024-01-01 12:00:00"),
+            # 2-digit year: YYMMDDHHMMSS (Farmscape/NSCF cameras)
+            ("farmscape/NSCF----_250927194802_0017.JPG", "2025-09-27 19:48:02"),
+            ("farmscape/NSCF----_251004210001_0041.JPG", "2025-10-04 21:00:01"),
         ]
 
         for filename, expected_date in filenames_and_expected_dates:

ui/src/data-services/models/capture.ts

@@ -167,6 +167,15 @@ export class Capture {
     return new Date(this._capture.timestamp)
   }
 
+  get path(): string {
+    return this._capture.path ?? ''
+  }
+
+  get filename(): string {
+    const path = this.path
+    return path ? path.split('/').pop() ?? path : ''
+  }
+
   get width(): number | null {
     return this._capture.width
   }

ui/src/pages/captures/capture-columns.tsx

@@ -134,6 +134,18 @@ export const columns: (projectId: string) => TableColumn<Capture>[] = (
       <BasicTableCell value={item.dimensionsLabel} />
     ),
   },
+  {
+    id: 'filename',
+    name: translate(STRING.FIELD_LABEL_FILENAME),
+    sortField: 'path',
+    renderCell: (item: Capture) => <BasicTableCell value={item.filename} />,
+  },
+  {
+    id: 'path',
+    name: translate(STRING.FIELD_LABEL_PATH),
+    sortField: 'path',
+    renderCell: (item: Capture) => <BasicTableCell value={item.path} />,
+  },
   {
     id: 'occurrences',
     name: translate(STRING.FIELD_LABEL_OCCURRENCES),

ui/src/pages/captures/captures.tsx

@@ -33,6 +33,8 @@ export const Captures = () => {
     session: true,
     size: true,
     dimensions: true,
+    filename: false,
+    path: false,
   })
   const { selectedView, setSelectedView } = useSelectedView('table')
   const { filters } = useFilters()

ui/src/utils/language.ts

@@ -96,6 +96,7 @@ export enum STRING {
   FIELD_LABEL_EMAIL,
   FIELD_LABEL_ENDPOINT,
   FIELD_LABEL_ERRORS,
+  FIELD_LABEL_FILENAME,
   FIELD_LABEL_FILE_SIZE,
   FIELD_LABEL_FINISHED_AT,
   FIELD_LABEL_FIRST_DATE,
@@ -386,6 +387,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = {
   [STRING.FIELD_LABEL_EMAIL]: 'Email',
   [STRING.FIELD_LABEL_ENDPOINT]: 'Endpoint URL',
   [STRING.FIELD_LABEL_ERRORS]: 'Errors',
+  [STRING.FIELD_LABEL_FILENAME]: 'Filename',
   [STRING.FIELD_LABEL_FILE_SIZE]: 'File size',
   [STRING.FIELD_LABEL_FINISHED_AT]: 'Finished at',
   [STRING.FIELD_LABEL_FIRST_DATE]: 'First date',