From 4b48e4a4ee60e3fa66f0f4a3de22fde2b8b5dc00 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:29:51 -0700
Subject: [PATCH 01/10] feat(media): quarantine private resources with durable
scan and cleanup
---
.github/workflows/media-security.yml | 57 ++++
backend/media-service/pom.xml | 29 ++
.../media/MediaServiceApplication.java | 2 +
.../media/api/CourseResourceController.java | 41 ++-
.../media/api/CourseResourceResponse.java | 8 +-
.../media/application/ClamAvScanner.java | 85 ++++++
.../application/CourseResourceRepository.java | 17 --
.../application/CourseResourceService.java | 264 +++++++-----------
.../LocalCourseResourceStorage.java | 13 +-
.../media/application/MalwareScanner.java | 9 +
.../application/PrivateResourceStorage.java | 23 ++
.../media/application/ResourceLifecycle.java | 211 ++++++++++++++
.../media/application/ResourceWorker.java | 123 ++++++++
.../media/application/UploadValidator.java | 183 ++++++++++++
.../chanter/media/domain/CourseResource.java | 14 +-
.../infra/HttpResourceIngestionClient.java | 22 +-
.../infra/JdbcCourseResourceRepository.java | 138 ---------
.../infra/LocalPrivateResourceStorage.java | 58 ++++
.../media/infra/S3PrivateResourceStorage.java | 96 +++++++
.../src/main/resources/application.yml | 20 +-
.../V2__private_resource_lifecycle.sql | 24 ++
.../media/api/CourseResourceSmokeTest.java | 36 ++-
.../media/application/ClamAvScannerTest.java | 99 +++++++
.../LocalPrivateResourceStorageTest.java | 31 ++
.../PrivateStorageIntegrationTest.java | 118 ++++++++
.../application/ResourceLifecycleTest.java | 93 ++++++
.../application/ResourceWorkerSafetyTest.java | 154 ++++++++++
.../application/UploadValidatorTest.java | 59 ++++
.../src/test/resources/application-test.yml | 5 +-
...ecture-review-chanter-private-resources.md | 69 +++++
docs/operations/issue-244-change-log.md | 28 ++
docs/operations/private-course-resources.md | 82 ++++++
infra/media-security/compose.yml | 45 +++
scripts/media/wait-dependencies.py | 28 ++
34 files changed, 1910 insertions(+), 374 deletions(-)
create mode 100644 .github/workflows/media-security.yml
create mode 100644 backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java
delete mode 100644 backend/media-service/src/main/java/com/chanter/media/application/CourseResourceRepository.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/application/MalwareScanner.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/application/PrivateResourceStorage.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/application/UploadValidator.java
delete mode 100644 backend/media-service/src/main/java/com/chanter/media/infra/JdbcCourseResourceRepository.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java
create mode 100644 backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java
create mode 100644 backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/LocalPrivateResourceStorageTest.java
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/ResourceLifecycleTest.java
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/UploadValidatorTest.java
create mode 100644 docs/engineering/records/architecture-review-chanter-private-resources.md
create mode 100644 docs/operations/issue-244-change-log.md
create mode 100644 docs/operations/private-course-resources.md
create mode 100644 infra/media-security/compose.yml
create mode 100644 scripts/media/wait-dependencies.py
diff --git a/.github/workflows/media-security.yml b/.github/workflows/media-security.yml
new file mode 100644
index 00000000..f2b5d7d7
--- /dev/null
+++ b/.github/workflows/media-security.yml
@@ -0,0 +1,57 @@
+name: Private media storage
+
+on:
+ pull_request:
+ paths: ['backend/media-service/**', 'infra/media-security/**', 'scripts/media/**', '.github/workflows/media-security.yml']
+ push:
+ branches: [main]
+ paths: ['backend/media-service/**', 'infra/media-security/**', 'scripts/media/**', '.github/workflows/media-security.yml']
+ workflow_dispatch:
+
+permissions:
+ contents: read
+
+jobs:
+ actual-storage-and-scanner:
+ if: github.event.repository.private == false
+ strategy:
+ fail-fast: false
+ matrix:
+ runner: [ubuntu-24.04, ubuntu-24.04-arm]
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 35
+ env:
+ MEDIA_INTEGRATION: 'true'
+ MEDIA_RESTART_PHASE: 'false'
+ COMPOSE_PROJECT_NAME: media-security-test
+ steps:
+ - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+ with:
+ persist-credentials: false
+ - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961
+ with:
+ distribution: temurin
+ java-version: '21.0.12+8.0.LTS'
+ - name: Start real PostgreSQL, S3 emulator and malware scanner
+ run: |
+ docker compose -f infra/media-security/compose.yml up -d
+ python3 scripts/media/wait-dependencies.py
+ - name: Verify private upload, real malware rejection and metered object operations
+ run: scripts/java21.sh mvn -B -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false
+ - name: Restart database and storage processes while preserving their volumes
+ run: |
+ docker compose -f infra/media-security/compose.yml restart postgres s3 clamav
+ python3 scripts/media/wait-dependencies.py
+ - name: Verify durable metadata, object bytes, idempotency and deletion after restart
+ env:
+ MEDIA_RESTART_PHASE: 'true'
+ run: scripts/java21.sh mvn -B -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false
+ - name: Capture resource limits and actual container use
+ if: always()
+ run: |
+ docker stats --no-stream
+ docker compose -f infra/media-security/compose.yml ps
+ docker compose -f infra/media-security/compose.yml logs --tail 60
+ - name: Remove only the ephemeral test stack
+ if: always()
+ run: docker compose -f infra/media-security/compose.yml down -v
diff --git a/backend/media-service/pom.xml b/backend/media-service/pom.xml
index 6ad8a2a0..25179ac6 100644
--- a/backend/media-service/pom.xml
+++ b/backend/media-service/pom.xml
@@ -13,7 +13,36 @@
media-service
Chanter Media Service
+
+
+
+ software.amazon.awssdk
+ bom
+ 2.54.17
+ pom
+ import
+
+
+
+
+
+ software.amazon.awssdk
+ s3
+
+ software.amazon.awssdkapache-client
+ software.amazon.awssdknetty-nio-client
+
+
+
+ software.amazon.awssdk
+ url-connection-client
+
+
+ org.apache.tika
+ tika-core
+ 4.0.0
+
com.chanter
common
diff --git a/backend/media-service/src/main/java/com/chanter/media/MediaServiceApplication.java b/backend/media-service/src/main/java/com/chanter/media/MediaServiceApplication.java
index c9f56df6..d975ee71 100644
--- a/backend/media-service/src/main/java/com/chanter/media/MediaServiceApplication.java
+++ b/backend/media-service/src/main/java/com/chanter/media/MediaServiceApplication.java
@@ -2,8 +2,10 @@
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
@SpringBootApplication
+@EnableScheduling
public class MediaServiceApplication {
public static void main(String[] args) {
diff --git a/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceController.java b/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceController.java
index 5fdd55f1..4577be15 100644
--- a/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceController.java
+++ b/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceController.java
@@ -12,6 +12,10 @@
import org.springframework.http.HttpHeaders;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.core.io.Resource;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.RequestHeader;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
@@ -21,7 +25,6 @@
import org.springframework.web.bind.annotation.RequestPart;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.web.multipart.MultipartFile;
-import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
@RestController
@RequestMapping(ServiceInfo.API_V1_PREFIX)
@@ -39,21 +42,20 @@ public ResponseEntity uploadCourseResource(
@RequestAttribute(AuthRequestAttributes.USER_ID) UUID uploaderUserId,
@RequestParam(required = false) String title,
@RequestParam boolean aiApproved,
- @RequestPart("file") MultipartFile file
+ @RequestPart("file") MultipartFile file,
+ @RequestHeader(value = "Idempotency-Key", required = false) UUID idempotencyKey,
+ @RequestHeader(value = "X-Content-SHA256", required = false) String checksum
) {
CourseResource courseResource = courseResourceService.uploadCourseResource(
courseId,
uploaderUserId,
title,
aiApproved,
- file
+ file, idempotencyKey, checksum
);
- URI location = ServletUriComponentsBuilder.fromCurrentRequest()
- .path("/{resourceId}")
- .buildAndExpand(courseResource.id())
- .toUri();
+ URI location = URI.create(ServiceInfo.API_V1_PREFIX + "/course-resources/" + courseResource.id());
- return ResponseEntity.created(location).body(CourseResourceResponse.from(courseResource));
+ return ResponseEntity.accepted().location(location).body(CourseResourceResponse.from(courseResource));
}
@GetMapping("/courses/{courseId}/course-resources")
@@ -71,7 +73,7 @@ public CourseResourceListResponse listCourseResources(
}
@GetMapping("/course-resources/{resourceId}/content")
- public ResponseEntity downloadCourseResource(
+ public ResponseEntity downloadCourseResource(
@PathVariable UUID resourceId,
@RequestAttribute(AuthRequestAttributes.USER_ID) UUID viewerUserId
) {
@@ -86,8 +88,27 @@ public ResponseEntity downloadCourseResource(
return ResponseEntity.ok()
.header(HttpHeaders.CONTENT_DISPOSITION, disposition.toString())
+ .header("X-Content-Type-Options", "nosniff")
+ .header(HttpHeaders.CACHE_CONTROL, "no-store")
.contentType(safeContentType(stored.courseResource().contentType()))
- .body(stored.content());
+ .contentLength(stored.courseResource().byteSize())
+ .body(new InputStreamResource(stored.content()));
+ }
+
+ @GetMapping("/course-resources/{resourceId}")
+ public CourseResourceResponse resource(@PathVariable UUID resourceId, @RequestAttribute(AuthRequestAttributes.USER_ID) UUID user) {
+ return CourseResourceResponse.from(courseResourceService.getCourseResource(resourceId, user));
+ }
+
+ @DeleteMapping("/course-resources/{resourceId}")
+ public ResponseEntity delete(@PathVariable UUID resourceId, @RequestAttribute(AuthRequestAttributes.USER_ID) UUID user) {
+ courseResourceService.deleteCourseResource(resourceId, user);
+ return ResponseEntity.noContent().build();
+ }
+
+ @GetMapping("/courses/{courseId}/course-resources/usage")
+ public com.chanter.media.application.ResourceLifecycle.Usage usage(@PathVariable UUID courseId, @RequestAttribute(AuthRequestAttributes.USER_ID) UUID user) {
+ return courseResourceService.usage(courseId, user);
}
/** Parse stored content type for download; fall back if missing/invalid (SEC-17). */
diff --git a/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceResponse.java b/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceResponse.java
index 74d2283e..56ec12ee 100644
--- a/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceResponse.java
+++ b/backend/media-service/src/main/java/com/chanter/media/api/CourseResourceResponse.java
@@ -13,7 +13,9 @@ public record CourseResourceResponse(
long byteSize,
boolean aiApproved,
UUID uploadedByUserId,
- Instant createdAt
+ Instant createdAt,
+ String status,
+ String sha256
) {
public static CourseResourceResponse from(CourseResource courseResource) {
@@ -26,7 +28,9 @@ public static CourseResourceResponse from(CourseResource courseResource) {
courseResource.byteSize(),
courseResource.aiApproved(),
courseResource.uploadedByUserId(),
- courseResource.createdAt()
+ courseResource.createdAt(),
+ courseResource.publicStatus(),
+ courseResource.sha256()
);
}
}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java b/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java
new file mode 100644
index 00000000..f7db10fc
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java
@@ -0,0 +1,85 @@
+package com.chanter.media.application;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ClamAvScanner implements MalwareScanner {
+ private static final java.util.concurrent.ScheduledExecutorService DEADLINES = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(
+ runnable -> { var thread = new Thread(runnable, "clamav-socket-deadlines"); thread.setDaemon(true); return thread; });
+ private final String host;
+ private final int port;
+ private final Duration timeout;
+ private final Duration maximumAge;
+ private final Clock clock;
+ public ClamAvScanner(@Value("${chanter.media.scanner.host:localhost}") String host,
+ @Value("${chanter.media.scanner.port:3310}") int port,
+ @Value("${chanter.media.scanner.timeout:20s}") Duration timeout,
+ @Value("${chanter.media.scanner.maximum-definition-age:72h}") Duration maximumAge, Clock clock) {
+ if (port < 1 || port > 65535 || timeout.toMillis() < 1 || timeout.compareTo(Duration.ofSeconds(30)) > 0
+ || maximumAge.isNegative() || maximumAge.isZero() || maximumAge.compareTo(Duration.ofDays(7)) > 0) throw new IllegalArgumentException("Invalid scanner policy");
+ this.host = host; this.port = port; this.timeout = timeout; this.maximumAge = maximumAge; this.clock = clock;
+ }
+ @Override public Verdict scan(Path file) throws IOException {
+ try (var socket = connect()) {
+ socket.getOutputStream().write("zVERSION\0".getBytes(StandardCharsets.US_ASCII));
+ String[] version = response(socket).split("/", 3);
+ if (version.length != 3) throw new IOException("Scanner definitions are unavailable");
+ try {
+ var updated = LocalDateTime.parse(version[2].strip().replaceAll("\\s+", " "), DateTimeFormatter.ofPattern("EEE MMM d HH:mm:ss yyyy", Locale.US)).toInstant(ZoneOffset.UTC);
+ if (updated.isBefore(clock.instant().minus(maximumAge)) || updated.isAfter(clock.instant().plusSeconds(300))) {
+ throw new IOException("Scanner definitions are stale");
+ }
+ } catch (java.time.format.DateTimeParseException exception) { throw new IOException("Scanner definitions are unavailable"); }
+ }
+ try (var socket = connect(); var source = Files.newInputStream(file)) {
+ socket.getOutputStream().write("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII));
+ var output = new DataOutputStream(socket.getOutputStream());
+ byte[] buffer = new byte[8192]; int count; long size = 0;
+ while ((count = source.read(buffer)) != -1) {
+ if ((size += count) > 10L * 1024 * 1024) throw new IOException("Scanner stream limit exceeded");
+ output.writeInt(count); output.write(buffer, 0, count);
+ }
+ output.writeInt(0); output.flush();
+ String result = response(socket);
+ if (result.equals("stream: OK")) return Verdict.CLEAN;
+ if (result.startsWith("stream: ") && result.endsWith(" FOUND")) return Verdict.INFECTED;
+ throw new IOException("Scanner did not verify the resource");
+ }
+ }
+ private Socket connect() throws IOException {
+ // SO_TIMEOUT covers reads only. Closing the socket also bounds a blocked INSTREAM write.
+ Socket socket = new Socket() {
+ private final java.util.concurrent.ScheduledFuture deadline = DEADLINES.schedule(() -> {
+ try { close(); } catch (IOException ignored) { }
+ }, timeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);
+ @Override public void close() throws IOException { if (deadline != null) deadline.cancel(false); super.close(); }
+ };
+ try {
+ socket.connect(new InetSocketAddress(host, port), Math.min(3000, Math.toIntExact(timeout.toMillis())));
+ socket.setSoTimeout(Math.toIntExact(timeout.toMillis()));
+ return socket;
+ } catch (IOException exception) { socket.close(); throw new IOException("Scanner is unavailable"); }
+ }
+ private static String response(Socket socket) throws IOException {
+ var bytes = new java.io.ByteArrayOutputStream(); int value;
+ while ((value = socket.getInputStream().read()) != 0) {
+ if (value == -1 || bytes.size() >= 4096) throw new IOException("Invalid scanner response");
+ bytes.write(value);
+ }
+ return bytes.toString(StandardCharsets.US_ASCII).strip();
+ }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceRepository.java b/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceRepository.java
deleted file mode 100644
index fb772989..00000000
--- a/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceRepository.java
+++ /dev/null
@@ -1,17 +0,0 @@
-package com.chanter.media.application;
-
-import com.chanter.media.domain.CourseResource;
-import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
-
-public interface CourseResourceRepository {
-
- CourseResource save(CourseResource courseResource);
-
- Optional findById(UUID resourceId);
-
- List findByCourseId(UUID courseId);
-
- void deleteById(UUID resourceId);
-}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceService.java b/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceService.java
index 8c79e45a..9fbd78ae 100644
--- a/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceService.java
+++ b/backend/media-service/src/main/java/com/chanter/media/application/CourseResourceService.java
@@ -1,206 +1,132 @@
package com.chanter.media.application;
import com.chanter.media.domain.CourseResource;
+import java.io.FilterInputStream;
import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
import java.time.Clock;
import java.time.temporal.ChronoUnit;
import java.util.List;
-import java.util.Locale;
-import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.springframework.http.HttpStatus;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.server.ResponseStatusException;
@Service
public class CourseResourceService {
-
- private static final long MAX_FILE_BYTES = 10L * 1024L * 1024L;
-
- /** Allowed upload MIME types (SEC-17). Parameters (e.g. charset) are stripped before compare. */
- static final Set ALLOWED_CONTENT_TYPES = Set.of(
- "text/plain",
- "text/markdown",
- "application/pdf",
- "application/vnd.openxmlformats-officedocument.presentationml.presentation",
- "application/vnd.ms-powerpoint",
- "audio/mpeg",
- "audio/mp4",
- "audio/wav",
- "audio/ogg",
- "audio/webm",
- "video/mp4",
- "video/webm",
- "video/quicktime"
- );
-
- private final CourseResourceRepository repository;
+ private final ResourceLifecycle lifecycle;
private final CourseResourceAccessClient accessClient;
- private final LocalCourseResourceStorage storage;
- private final ResourceIngestionClient resourceIngestionClient;
+ private final PrivateResourceStorage storage;
+ private final UploadValidator validator;
private final Clock clock;
+ private final Semaphore transfers = new Semaphore(2);
- public CourseResourceService(
- CourseResourceRepository repository,
- CourseResourceAccessClient accessClient,
- LocalCourseResourceStorage storage,
- ResourceIngestionClient resourceIngestionClient,
- Clock clock
- ) {
- this.repository = repository;
- this.accessClient = accessClient;
- this.storage = storage;
- this.resourceIngestionClient = resourceIngestionClient;
- this.clock = clock;
+ public CourseResourceService(ResourceLifecycle lifecycle, CourseResourceAccessClient accessClient,
+ PrivateResourceStorage storage, UploadValidator validator, Clock clock) {
+ this.lifecycle = lifecycle; this.accessClient = accessClient; this.storage = storage; this.validator = validator; this.clock = clock;
}
- @Transactional
- public CourseResource uploadCourseResource(
- UUID courseId,
- UUID uploaderUserId,
- String title,
- boolean aiApproved,
- MultipartFile file
- ) {
- CourseResourceAccess access = accessClient.requireAccess(courseId, uploaderUserId);
- if (!access.canUploadCourseResource()) {
- throw new ResponseStatusException(HttpStatus.FORBIDDEN, "Only Course Instructors can upload Course Resources");
- }
-
- if (file == null || file.isEmpty()) {
- throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Course Resource file must not be empty");
- }
- if (file.getSize() > MAX_FILE_BYTES) {
- throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Course Resource file exceeds the 10 MB limit");
- }
-
- String fileName = sanitizeFileName(file.getOriginalFilename());
- if (fileName.isBlank()) {
- throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Course Resource file name must not be blank");
- }
-
- String normalizedTitle = title == null || title.isBlank() ? fileName : title.trim();
- if (normalizedTitle.isEmpty()) {
- throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Course Resource title must not be blank");
- }
-
- UUID resourceId = UUID.randomUUID();
- String contentType = requireAllowedContentType(file.getContentType());
-
- byte[] content;
- try {
- content = file.getBytes();
+ public CourseResource uploadCourseResource(UUID courseId, UUID userId, String title, boolean aiApproved,
+ MultipartFile file, UUID idempotencyKey, String checksum) {
+ requireUpload(courseId, userId);
+ acquire();
+ try (var upload = validator.validate(file, checksum)) {
+ String normalizedTitle = title == null || title.isBlank() ? upload.fileName() : title.strip();
+ if (normalizedTitle.length() > 255 || normalizedTitle.chars().anyMatch(Character::isISOControl)) {
+ throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Course Resource title is invalid");
+ }
+ UUID id = UUID.randomUUID();
+ var candidate = new CourseResource(id, courseId, normalizedTitle, upload.fileName(), upload.contentType(), upload.byteSize(),
+ PrivateResourceStorage.PREFIX + courseId + "/" + id + "/" + UUID.randomUUID(), aiApproved, userId,
+ clock.instant().truncatedTo(ChronoUnit.MICROS), "STAGING", upload.sha256(),
+ idempotencyKey == null ? UUID.randomUUID() : idempotencyKey, storage.backend());
+ CourseResource reserved = lifecycle.reserve(candidate);
+ if (!reserved.id().equals(id)) return reserved;
+ try {
+ storage.put(candidate.storageKey(), upload.path(), upload.sha256());
+ lifecycle.quarantine(id);
+ } catch (Exception unavailable) {
+ // A timed-out put may have succeeded. Keep the reservation until a worker confirms deletion.
+ lifecycle.requestDelete(id);
+ }
+ return lifecycle.find(id).orElseThrow();
} catch (IOException exception) {
- throw new ResponseStatusException(HttpStatus.BAD_REQUEST, "Unable to read uploaded Course Resource");
- }
-
- CourseResource courseResource = new CourseResource(
- resourceId,
- courseId,
- normalizedTitle,
- fileName,
- contentType,
- content.length,
- resourceId.toString(),
- aiApproved,
- uploaderUserId,
- clock.instant().truncatedTo(ChronoUnit.MICROS)
- );
-
- repository.save(courseResource);
-
- try {
- storage.store(resourceId, content);
- } catch (IOException exception) {
- throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Unable to store Course Resource");
- }
-
- if (aiApproved) {
- resourceIngestionClient.ingestAiApprovedResource(courseId, resourceId, fileName, content);
- }
-
- return courseResource;
+ throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Course Resource processing is unavailable");
+ } finally { transfers.release(); }
}
- static String sanitizeFileName(String originalFileName) {
- if (originalFileName == null) {
- return "";
- }
-
- String normalized = originalFileName.replace('\\', '/').trim();
- int lastSlash = normalized.lastIndexOf('/');
- if (lastSlash >= 0) {
- normalized = normalized.substring(lastSlash + 1);
- }
-
- if (normalized.equals(".") || normalized.equals("..")) {
- return "";
- }
-
- return normalized;
+ public List listCourseResources(UUID course, UUID user) {
+ var access = requireView(course, user);
+ return lifecycle.list(course, access.canUploadCourseResource());
}
- static String requireAllowedContentType(String rawContentType) {
- String normalized = normalizeContentType(rawContentType);
- if (normalized == null || !ALLOWED_CONTENT_TYPES.contains(normalized)) {
- throw new ResponseStatusException(
- HttpStatus.BAD_REQUEST,
- "Course Resource content type is not allowed"
- );
- }
- return normalized;
+ public CourseResource getCourseResource(UUID id, UUID user) {
+ var resource = existing(id);
+ var access = requireView(resource.courseId(), user);
+ if (!access.canUploadCourseResource() && !resource.state().equals("AVAILABLE")) throw missing();
+ return resource;
}
- /** Strip parameters and lowercase type/subtype; null/blank → null. */
- static String normalizeContentType(String rawContentType) {
- if (rawContentType == null || rawContentType.isBlank()) {
- return null;
- }
- String withoutParams = rawContentType.split(";", 2)[0].trim().toLowerCase(Locale.ROOT);
- if (withoutParams.isEmpty() || withoutParams.indexOf('/') < 1) {
- return null;
- }
- return withoutParams;
+ public void deleteCourseResource(UUID id, UUID user) {
+ var resource = lifecycle.find(id).orElseThrow(CourseResourceService::missing);
+ requireUpload(resource.courseId(), user);
+ lifecycle.requestDelete(id);
}
- @Transactional(readOnly = true)
- public List listCourseResources(UUID courseId, UUID viewerUserId) {
- CourseResourceAccess access = accessClient.requireAccess(courseId, viewerUserId);
- if (!access.canViewCourseResources()) {
- throw new ResponseStatusException(
- HttpStatus.FORBIDDEN,
- "Course Resource access requires Cohort Enrollment or Instructor role"
- );
- }
-
- return repository.findByCourseId(courseId);
+ public ResourceLifecycle.Usage usage(UUID course, UUID user) {
+ requireUpload(course, user);
+ return lifecycle.courseUsage(course);
}
- @Transactional(readOnly = true)
- public StoredCourseResourceContent downloadCourseResource(UUID resourceId, UUID viewerUserId) {
- CourseResource courseResource = repository.findById(resourceId)
- .orElseThrow(() -> new ResponseStatusException(HttpStatus.NOT_FOUND, "Course Resource not found"));
-
- CourseResourceAccess access = accessClient.requireAccess(courseResource.courseId(), viewerUserId);
- if (!access.canViewCourseResources()) {
- throw new ResponseStatusException(
- HttpStatus.FORBIDDEN,
- "Course Resource access requires Cohort Enrollment or Instructor role"
- );
- }
-
- byte[] content;
+ public StoredCourseResourceContent downloadCourseResource(UUID id, UUID user) {
+ var resource = existing(id);
+ requireView(resource.courseId(), user);
+ if (!resource.state().equals("AVAILABLE")) throw new ResponseStatusException(HttpStatus.CONFLICT, "Course Resource is not available");
+ if (!resource.storageBackend().equals(storage.backend())) throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Course Resource migration is required");
+ acquire();
+ java.nio.file.Path temporary = null;
try {
- content = storage.load(resourceId);
- } catch (IOException exception) {
- throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, "Course Resource content is unavailable");
+ var path = validator.verifiedDownload(storage.open(resource.storageKey()), resource.byteSize(), resource.sha256());
+ temporary = path;
+ // Deletion accepted during a provider read must win before any bytes are exposed.
+ var latest = lifecycle.find(id);
+ if (latest.isEmpty() || !latest.get().state().equals("AVAILABLE")) { Files.deleteIfExists(path); throw missing(); }
+ var closed = new AtomicBoolean();
+ InputStream content = new FilterInputStream(Files.newInputStream(path)) {
+ @Override public void close() throws IOException {
+ if (closed.compareAndSet(false, true)) {
+ try { super.close(); } finally { try { Files.deleteIfExists(path); } finally { transfers.release(); } }
+ }
+ }
+ };
+ return new StoredCourseResourceContent(resource, content);
+ } catch (Exception unavailable) {
+ if (temporary != null) try { Files.deleteIfExists(temporary); } catch (IOException ignored) { }
+ transfers.release();
+ if (unavailable instanceof ResponseStatusException status) throw status;
+ throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Course Resource content is unavailable");
}
-
- return new StoredCourseResourceContent(courseResource, content);
}
- public record StoredCourseResourceContent(CourseResource courseResource, byte[] content) {
+ private CourseResource existing(UUID id) {
+ return lifecycle.find(id).filter(resource -> !List.of("DELETE_PENDING", "DELETED").contains(resource.state()))
+ .orElseThrow(CourseResourceService::missing);
+ }
+ private CourseResourceAccess requireView(UUID course, UUID user) {
+ var access = accessClient.requireAccess(course, user);
+ if (!access.canViewCourseResources()) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "Course Resource access requires enrollment or Instructor role");
+ return access;
+ }
+ private void requireUpload(UUID course, UUID user) {
+ if (!accessClient.requireAccess(course, user).canUploadCourseResource()) throw new ResponseStatusException(HttpStatus.FORBIDDEN, "Only Course Instructors can manage Course Resources");
+ }
+ private void acquire() {
+ if (!transfers.tryAcquire()) throw new ResponseStatusException(HttpStatus.TOO_MANY_REQUESTS, "Course Resource transfers are busy; retry later");
}
+ private static ResponseStatusException missing() { return new ResponseStatusException(HttpStatus.NOT_FOUND, "Course Resource not found"); }
+ public record StoredCourseResourceContent(CourseResource courseResource, InputStream content) { }
}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/LocalCourseResourceStorage.java b/backend/media-service/src/main/java/com/chanter/media/application/LocalCourseResourceStorage.java
index 1aedd247..ed8f498b 100644
--- a/backend/media-service/src/main/java/com/chanter/media/application/LocalCourseResourceStorage.java
+++ b/backend/media-service/src/main/java/com/chanter/media/application/LocalCourseResourceStorage.java
@@ -15,15 +15,14 @@ public class LocalCourseResourceStorage {
public LocalCourseResourceStorage(
@Value("${chanter.media.storage-dir:./data/course-resources}") String storageDir
) throws IOException {
- this.storageRoot = Path.of(storageDir);
+ this.storageRoot = Path.of(storageDir).toAbsolutePath().normalize();
Files.createDirectories(storageRoot);
}
- public void store(UUID resourceId, byte[] content) throws IOException {
- Files.write(storageRoot.resolve(resourceId.toString()), content);
- }
-
- public byte[] load(UUID resourceId) throws IOException {
- return Files.readAllBytes(storageRoot.resolve(resourceId.toString()));
+ public Path legacyPath(UUID resourceId) throws IOException {
+ Path file = storageRoot.resolve(resourceId.toString());
+ if (Files.isSymbolicLink(storageRoot) || !Files.isRegularFile(file, java.nio.file.LinkOption.NOFOLLOW_LINKS)) throw new IOException("Legacy resource is missing");
+ return file;
}
+ public void deleteLegacy(UUID resourceId) throws IOException { Files.deleteIfExists(storageRoot.resolve(resourceId.toString())); }
}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/MalwareScanner.java b/backend/media-service/src/main/java/com/chanter/media/application/MalwareScanner.java
new file mode 100644
index 00000000..0b504a45
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/application/MalwareScanner.java
@@ -0,0 +1,9 @@
+package com.chanter.media.application;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+public interface MalwareScanner {
+ Verdict scan(Path file) throws IOException;
+ enum Verdict { CLEAN, INFECTED }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/PrivateResourceStorage.java b/backend/media-service/src/main/java/com/chanter/media/application/PrivateResourceStorage.java
new file mode 100644
index 00000000..a216ba80
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/application/PrivateResourceStorage.java
@@ -0,0 +1,23 @@
+package com.chanter.media.application;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.util.List;
+
+public interface PrivateResourceStorage {
+ String PREFIX = "resources/v1/";
+ String backend();
+ void put(String key, Path content, String sha256) throws IOException;
+ InputStream open(String key) throws IOException;
+ void delete(String key) throws IOException;
+ Page list(String cursor) throws IOException;
+ record ObjectInfo(String key, Instant modifiedAt) { }
+ record Page(List objects, String nextCursor) { }
+ static void requireKey(String key) {
+ if (key == null || !key.matches("resources/v1/[a-f0-9-]{36}/[a-f0-9-]{36}/[a-f0-9-]{36}")) {
+ throw new IllegalArgumentException("Invalid private resource key");
+ }
+ }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java b/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
new file mode 100644
index 00000000..02ab2241
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
@@ -0,0 +1,211 @@
+package com.chanter.media.application;
+
+import com.chanter.media.domain.CourseResource;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.YearMonth;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.jdbc.core.simple.JdbcClient;
+import org.springframework.stereotype.Repository;
+import org.springframework.transaction.annotation.Propagation;
+import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.server.ResponseStatusException;
+
+/** Owns durable reservations, idempotency, worker leases and deletion accounting. No network calls run in its transactions. */
+@Repository
+public class ResourceLifecycle {
+ private final JdbcClient jdbc;
+ private final Clock clock;
+ private final long byteLimit;
+ private final int requestLimit;
+ private final int cleanupReserve;
+
+ public ResourceLifecycle(JdbcClient jdbc, Clock clock,
+ @Value("${chanter.media.byte-limit:8000000000}") long byteLimit,
+ @Value("${chanter.media.request-limit:40000}") int requestLimit,
+ @Value("${chanter.media.cleanup-request-reserve:4000}") int cleanupReserve) {
+ if (byteLimit < 1 || byteLimit > 8_000_000_000L || requestLimit < 1 || requestLimit > 40_000
+ || cleanupReserve < 1 || cleanupReserve >= requestLimit) throw new IllegalArgumentException("Invalid free storage budget");
+ this.jdbc = jdbc; this.clock = clock; this.byteLimit = byteLimit; this.requestLimit = requestLimit; this.cleanupReserve = cleanupReserve;
+ }
+
+ @Transactional
+ public CourseResource reserve(CourseResource r) {
+ long used = jdbc.sql("SELECT reserved_bytes FROM media_storage_budget WHERE id=1 FOR UPDATE").query(Long.class).single();
+ Optional existing = jdbc.sql("SELECT * FROM course_resources WHERE uploaded_by_user_id=:user AND idempotency_key=:key")
+ .param("user", r.uploadedByUserId()).param("key", r.idempotencyKey()).query(ResourceLifecycle::map).optional();
+ if (existing.isPresent()) {
+ CourseResource old = existing.get();
+ if (!old.courseId().equals(r.courseId()) || !old.title().equals(r.title()) || !old.fileName().equals(r.fileName())
+ || !old.contentType().equals(r.contentType()) || old.aiApproved() != r.aiApproved()
+ || old.byteSize() != r.byteSize() || !old.sha256().equals(r.sha256())) {
+ throw new ResponseStatusException(HttpStatus.CONFLICT, "Idempotency key belongs to a different upload");
+ }
+ return old;
+ }
+ if (r.byteSize() < 1 || r.byteSize() > byteLimit - used) throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE, "Course Resource storage quota reached");
+ jdbc.sql("""
+ INSERT INTO course_resources (id,course_id,title,file_name,content_type,byte_size,storage_key,ai_approved,
+ uploaded_by_user_id,created_at,state,sha256,idempotency_key,storage_backend,updated_at)
+ VALUES (:id,:course,:title,:file,:type,:bytes,:object,:ai,:user,:created,'STAGING',:hash,:key,:backend,:created)
+ """).param("id", r.id()).param("course", r.courseId()).param("title", r.title()).param("file", r.fileName())
+ .param("type", r.contentType()).param("bytes", r.byteSize()).param("object", r.storageKey()).param("ai", r.aiApproved())
+ .param("user", r.uploadedByUserId()).param("created", time(r.createdAt())).param("hash", r.sha256())
+ .param("key", r.idempotencyKey()).param("backend", r.storageBackend()).update();
+ jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=reserved_bytes+:bytes WHERE id=1").param("bytes", r.byteSize()).update();
+ return r;
+ }
+
+ @Transactional(propagation = Propagation.REQUIRES_NEW)
+ public void countRequest(boolean maintenance) {
+ var budget = jdbc.sql("SELECT * FROM media_storage_budget WHERE id=1 FOR UPDATE").query((rs, n) ->
+ new Budget(rs.getString("request_month"), rs.getInt("foreground_requests"), rs.getInt("maintenance_requests"))).single();
+ String month = YearMonth.now(clock).toString();
+ int foreground = budget.month().equals(month) ? budget.foreground() : 0;
+ int background = budget.month().equals(month) ? budget.maintenance() : 0;
+ if (foreground + background >= requestLimit || (!maintenance && foreground >= requestLimit - cleanupReserve)) {
+ throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, "Private storage request budget reached");
+ }
+ jdbc.sql("UPDATE media_storage_budget SET request_month=:month, foreground_requests=:foreground, maintenance_requests=:maintenance WHERE id=1")
+ .param("month", month).param("foreground", foreground + (maintenance ? 0 : 1))
+ .param("maintenance", background + (maintenance ? 1 : 0)).update();
+ }
+
+ public Optional find(UUID id) {
+ return jdbc.sql("SELECT * FROM course_resources WHERE id=:id").param("id", id).query(ResourceLifecycle::map).optional();
+ }
+
+ public List list(UUID course, boolean instructor) {
+ return jdbc.sql("SELECT * FROM course_resources WHERE course_id=:course AND "
+ + (instructor ? "state NOT IN ('DELETE_PENDING','DELETED')" : "state='AVAILABLE'") + " ORDER BY created_at,id")
+ .param("course", course).query(ResourceLifecycle::map).list();
+ }
+
+ @Transactional
+ public void quarantine(UUID id) {
+ jdbc.sql("UPDATE course_resources SET state='QUARANTINED', updated_at=:now WHERE id=:id AND state='STAGING'")
+ .param("now", now()).param("id", id).update();
+ }
+
+ @Transactional
+ public void requestDelete(UUID id) {
+ jdbc.sql("UPDATE course_resources SET state='DELETE_PENDING', updated_at=:now, retry_at=NULL WHERE id=:id AND state<>'DELETED'")
+ .param("now", now()).param("id", id).update();
+ }
+
+ @Transactional
+ public Optional claim(boolean migrateLegacy) {
+ Instant instant = clock.instant();
+ var row = jdbc.sql("""
+ SELECT * FROM course_resources WHERE
+ (lease_until IS NULL OR lease_until<:now) AND (retry_at IS NULL OR retry_at<=:now)
+ AND (storage_backend<>'legacy' OR :migrate=TRUE OR state='DELETE_PENDING') AND (
+ state IN ('QUARANTINED','SCANNING','DELETE_PENDING')
+ OR (state='SCAN_FAILED' AND byte_reservation=TRUE AND (attempts<5 OR (updated_at<:expired AND storage_backend<>'legacy')))
+ OR (state='REJECTED' AND byte_reservation=TRUE)
+ OR (state='STAGING' AND created_at<:abandoned)
+ OR (state='LEGACY' AND :migrate=TRUE))
+ ORDER BY updated_at,id LIMIT 1 FOR UPDATE SKIP LOCKED
+ """).param("now", time(instant)).param("expired", time(instant.minusSeconds(86400)))
+ .param("abandoned", time(instant.minusSeconds(600))).param("migrate", migrateLegacy)
+ .query((rs, n) -> new Candidate(map(rs, n), rs.getInt("attempts"), rs.getString("migration_key"))).optional();
+ if (row.isEmpty()) return Optional.empty();
+ CourseResource r = row.get().resource();
+ boolean delete = List.of("DELETE_PENDING", "REJECTED", "STAGING").contains(r.state())
+ || (r.state().equals("SCAN_FAILED") && row.get().attempts() >= 5);
+ String state = delete ? (r.state().equals("STAGING") ? "DELETE_PENDING" : r.state()) : "SCANNING";
+ UUID lease = UUID.randomUUID();
+ String migrationKey = row.get().migrationKey();
+ if (!delete && r.storageBackend().equals("legacy") && migrationKey == null) migrationKey = PrivateResourceStorage.PREFIX + r.courseId() + "/" + r.id() + "/" + UUID.randomUUID();
+ jdbc.sql("UPDATE course_resources SET state=:state, lease_id=:lease, lease_until=:until, migration_key=:migration, attempts=attempts+1, updated_at=:now WHERE id=:id")
+ .param("state", state).param("lease", lease).param("until", time(instant.plusSeconds(180)))
+ .param("migration", migrationKey).param("now", time(instant)).param("id", r.id()).update();
+ return Optional.of(new Job(r, lease, delete ? "DELETE" : r.storageBackend().equals("legacy") ? "MIGRATE" : "SCAN", migrationKey));
+ }
+
+ @Transactional
+ public void finishMigration(Job job, UploadValidator.ValidatedUpload upload, String backend) {
+ int changed = jdbc.sql("""
+ UPDATE course_resources SET storage_key=:key, storage_backend=:backend, sha256=:hash,
+ file_name=:file,content_type=:type,state='QUARANTINED',lease_id=NULL,lease_until=NULL,
+ attempts=0,retry_at=NULL,updated_at=:now WHERE id=:id AND lease_id=:lease AND state='SCANNING'
+ """).param("key", job.migrationKey()).param("backend", backend).param("hash", upload.sha256())
+ .param("file", upload.fileName()).param("type", upload.contentType()).param("now", now())
+ .param("id", job.resource().id()).param("lease", job.leaseId()).update();
+ if (changed == 0) releaseLease(job.resource().id(), job.leaseId());
+ }
+
+ @Transactional
+ public boolean finishScan(UUID id, UUID lease, String state) {
+ if (!List.of("AVAILABLE", "REJECTED", "SCAN_FAILED").contains(state)) throw new IllegalArgumentException("Invalid scan result");
+ int changed = jdbc.sql("""
+ UPDATE course_resources SET state=:state, lease_id=NULL, lease_until=NULL, updated_at=:now,
+ retry_at=:retry WHERE id=:id AND lease_id=:lease AND state='SCANNING'
+ """).param("state", state).param("now", now())
+ .param("retry", state.equals("SCAN_FAILED") ? time(clock.instant().plusSeconds(60)) : null)
+ .param("id", id).param("lease", lease).update();
+ if (changed == 0) releaseLease(id, lease);
+ return changed == 1;
+ }
+
+ @Transactional
+ public void finishDelete(UUID id, UUID lease) {
+ // All quota mutations take this row first, preventing inversion with upload reservations.
+ jdbc.sql("SELECT reserved_bytes FROM media_storage_budget WHERE id=1 FOR UPDATE").query(Long.class).single();
+ var row = jdbc.sql("SELECT * FROM course_resources WHERE id=:id AND lease_id=:lease FOR UPDATE")
+ .param("id", id).param("lease", lease).query((rs, n) -> new Deleted(rs.getLong("byte_size"), rs.getBoolean("byte_reservation"))).optional();
+ if (row.isEmpty()) return;
+ jdbc.sql("""
+ UPDATE course_resources SET state=CASE WHEN state='DELETE_PENDING' THEN 'DELETED' ELSE state END,
+ byte_reservation=FALSE, lease_id=NULL,lease_until=NULL,retry_at=NULL,updated_at=:now WHERE id=:id
+ """).param("now", now()).param("id", id).update();
+ if (row.get().reserved()) jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=reserved_bytes-:bytes WHERE id=1")
+ .param("bytes", row.get().bytes()).update();
+ }
+
+ @Transactional
+ public void retryJob(UUID id, UUID lease) {
+ jdbc.sql("UPDATE course_resources SET lease_id=NULL,lease_until=NULL,retry_at=:retry WHERE id=:id AND lease_id=:lease")
+ .param("retry", time(clock.instant().plusSeconds(60))).param("id", id).param("lease", lease).update();
+ }
+
+ private void releaseLease(UUID id, UUID lease) {
+ jdbc.sql("UPDATE course_resources SET lease_id=NULL,lease_until=NULL WHERE id=:id AND lease_id=:lease")
+ .param("id", id).param("lease", lease).update();
+ }
+
+ public Usage courseUsage(UUID course) {
+ return jdbc.sql("""
+ SELECT COALESCE(SUM(CASE WHEN byte_reservation THEN byte_size ELSE 0 END),0) AS reserved,
+ COALESCE(SUM(CASE WHEN state='AVAILABLE' THEN byte_size ELSE 0 END),0) AS available
+ FROM course_resources WHERE course_id=:course
+ """).param("course", course).query((rs, n) -> new Usage(rs.getLong("reserved"), rs.getLong("available"))).single();
+ }
+
+ public boolean orphaned(String key) {
+ return jdbc.sql("SELECT COUNT(*) FROM course_resources WHERE (storage_key=:key OR migration_key=:key) AND byte_reservation=TRUE")
+ .param("key", key).query(Integer.class).single() == 0;
+ }
+
+ private OffsetDateTime now() { return time(clock.instant()); }
+ private static OffsetDateTime time(Instant instant) { return instant.atOffset(ZoneOffset.UTC); }
+ private static CourseResource map(ResultSet rs, int row) throws SQLException {
+ return new CourseResource(rs.getObject("id", UUID.class), rs.getObject("course_id", UUID.class), rs.getString("title"),
+ rs.getString("file_name"), rs.getString("content_type"), rs.getLong("byte_size"), rs.getString("storage_key"),
+ rs.getBoolean("ai_approved"), rs.getObject("uploaded_by_user_id", UUID.class), rs.getObject("created_at", OffsetDateTime.class).toInstant(),
+ rs.getString("state"), rs.getString("sha256"), rs.getObject("idempotency_key", UUID.class), rs.getString("storage_backend"));
+ }
+ private record Budget(String month, int foreground, int maintenance) { }
+ private record Candidate(CourseResource resource, int attempts, String migrationKey) { }
+ private record Deleted(long bytes, boolean reserved) { }
+ public record Job(CourseResource resource, UUID leaseId, String operation, String migrationKey) { }
+ public record Usage(long reservedBytes, long availableBytes) { }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java b/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java
new file mode 100644
index 00000000..b9065b73
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java
@@ -0,0 +1,123 @@
+package com.chanter.media.application;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+/** Database leases fence completion; provider calls never hold a database transaction. */
+@Component
+public class ResourceWorker {
+ private static final Logger log = LoggerFactory.getLogger(ResourceWorker.class);
+ private final ResourceLifecycle lifecycle;
+ private final PrivateResourceStorage storage;
+ private final LocalCourseResourceStorage legacy;
+ private final UploadValidator validator;
+ private final MalwareScanner scanner;
+ private final ResourceIngestionClient ingestion;
+ private final Clock clock;
+ private final boolean enabled;
+ private final boolean migrateLegacy;
+ private String cursor;
+
+ public ResourceWorker(ResourceLifecycle lifecycle, PrivateResourceStorage storage,
+ LocalCourseResourceStorage legacy, UploadValidator validator, MalwareScanner scanner,
+ ResourceIngestionClient ingestion, Clock clock,
+ @Value("${chanter.media.worker-enabled:true}") boolean enabled,
+ @Value("${chanter.media.migrate-legacy:false}") boolean migrateLegacy) {
+ this.lifecycle = lifecycle; this.storage = storage; this.legacy = legacy;
+ this.validator = validator; this.scanner = scanner; this.ingestion = ingestion;
+ this.clock = clock; this.enabled = enabled; this.migrateLegacy = migrateLegacy;
+ }
+
+ @Scheduled(fixedDelayString = "${chanter.media.worker-delay-ms:5000}")
+ public void poll() { if (enabled) runOnce(); }
+
+ public void runOnce() {
+ lifecycle.claim(migrateLegacy).ifPresent(job -> {
+ try {
+ switch (job.operation()) {
+ case "DELETE" -> delete(job);
+ case "MIGRATE" -> migrate(job);
+ case "SCAN" -> scan(job);
+ default -> throw new IllegalStateException("Unsupported resource operation");
+ }
+ } catch (Exception failure) {
+ // Provider errors and scanner signatures can contain private information.
+ log.warn("Course Resource work deferred resourceId={} operation={}", job.resource().id(), job.operation());
+ if (job.operation().equals("DELETE")) lifecycle.retryJob(job.resource().id(), job.leaseId());
+ else lifecycle.finishScan(job.resource().id(), job.leaseId(), "SCAN_FAILED");
+ }
+ });
+ }
+
+ private void scan(ResourceLifecycle.Job job) throws IOException {
+ var resource = job.resource();
+ requireBackend(resource.storageBackend());
+ Path verified = validator.verifiedDownload(storage.open(resource.storageKey()), resource.byteSize(), resource.sha256());
+ try {
+ var verdict = scanner.scan(verified);
+ if (verdict == MalwareScanner.Verdict.INFECTED) {
+ lifecycle.finishScan(resource.id(), job.leaseId(), "REJECTED");
+ } else if (verdict == MalwareScanner.Verdict.CLEAN) {
+ if (resource.aiApproved()) ingestion.ingestAiApprovedResource(resource.courseId(), resource.id(), resource.fileName(), Files.readAllBytes(verified));
+ lifecycle.finishScan(resource.id(), job.leaseId(), "AVAILABLE");
+ } else throw new IOException("Scanner did not return a verdict");
+ } finally { Files.deleteIfExists(verified); }
+ }
+
+ private void delete(ResourceLifecycle.Job job) throws IOException {
+ var resource = job.resource();
+ if (resource.storageBackend().equals("legacy")) legacy.deleteLegacy(resource.id());
+ else {
+ requireBackend(resource.storageBackend());
+ storage.delete(resource.storageKey());
+ }
+ if (job.migrationKey() != null && !job.migrationKey().equals(resource.storageKey())) storage.delete(job.migrationKey());
+ ingestion.deleteResourceChunks(resource.id());
+ lifecycle.finishDelete(resource.id(), job.leaseId());
+ }
+
+ private void migrate(ResourceLifecycle.Job job) throws IOException {
+ var resource = job.resource();
+ // Existing vectors predate the quarantine guarantee. Purge them before rebuilding from a clean scan.
+ ingestion.deleteResourceChunks(resource.id());
+ try (var upload = validator.validateExisting(legacy.legacyPath(resource.id()), resource.fileName(), resource.contentType())) {
+ if (upload.byteSize() != resource.byteSize()) throw new IOException("Legacy metadata does not match stored bytes");
+ try { storage.put(job.migrationKey(), upload.path(), upload.sha256()); }
+ catch (IOException uncertainWrite) {
+ // An immutable PUT may have succeeded before its response was interrupted.
+ Path confirmed = validator.verifiedDownload(storage.open(job.migrationKey()), upload.byteSize(), upload.sha256());
+ Files.deleteIfExists(confirmed);
+ }
+ lifecycle.finishMigration(job, upload, storage.backend());
+ }
+ }
+
+ @Scheduled(initialDelayString = "${chanter.media.reconcile-delay-ms:3600000}", fixedDelayString = "${chanter.media.reconcile-delay-ms:3600000}")
+ public void reconcile() {
+ if (!enabled) return;
+ try {
+ validator.cleanup(clock.instant().minusSeconds(3600));
+ // Prefix and age restrict cleanup to abandoned objects owned by this module.
+ for (int pageNumber = 0; pageNumber < 10; pageNumber++) {
+ var page = storage.list(cursor);
+ for (var object : page.objects()) {
+ PrivateResourceStorage.requireKey(object.key());
+ if (object.modifiedAt().isBefore(clock.instant().minusSeconds(86400)) && lifecycle.orphaned(object.key())) storage.delete(object.key());
+ }
+ cursor = page.nextCursor();
+ if (cursor == null) break;
+ }
+ } catch (Exception unavailable) { log.warn("Course Resource reconciliation deferred"); }
+ }
+
+ private void requireBackend(String backend) throws IOException {
+ if (!storage.backend().equals(backend)) throw new IOException("Resource storage migration is required");
+ }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/UploadValidator.java b/backend/media-service/src/main/java/com/chanter/media/application/UploadValidator.java
new file mode 100644
index 00000000..a7778544
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/application/UploadValidator.java
@@ -0,0 +1,183 @@
+package com.chanter.media.application;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.text.Normalizer;
+import java.time.Instant;
+import java.util.HexFormat;
+import java.util.Locale;
+import java.util.Map;
+import java.util.zip.ZipInputStream;
+import org.apache.tika.detect.DefaultDetector;
+import org.apache.tika.io.TikaInputStream;
+import org.apache.tika.metadata.Metadata;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.http.HttpStatus;
+import org.springframework.stereotype.Component;
+import org.springframework.web.multipart.MultipartFile;
+import org.springframework.web.server.ResponseStatusException;
+
+@Component
+public class UploadValidator {
+ private static final Map TYPES = Map.ofEntries(
+ Map.entry("txt", "text/plain"), Map.entry("md", "text/markdown"), Map.entry("markdown", "text/markdown"),
+ Map.entry("pdf", "application/pdf"), Map.entry("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"),
+ Map.entry("mp3", "audio/mpeg"), Map.entry("m4a", "audio/mp4"), Map.entry("wav", "audio/wav"),
+ Map.entry("ogg", "audio/ogg"), Map.entry("mp4", "video/mp4"), Map.entry("webm", "video/webm"),
+ Map.entry("mov", "video/quicktime"));
+ private final Path spool;
+ private final long limit;
+
+ public UploadValidator(@Value("${chanter.media.spool-dir:./data/media-spool}") String directory,
+ @Value("${chanter.media.max-file-bytes:10485760}") long limit) throws IOException {
+ if (limit < 1 || limit > 10L * 1024 * 1024) throw new IllegalArgumentException("Invalid resource size limit");
+ this.spool = Path.of(directory).toAbsolutePath().normalize();
+ Files.createDirectories(spool);
+ if (Files.getFileStore(spool).supportsFileAttributeView("posix")) Files.setPosixFilePermissions(spool, java.nio.file.attribute.PosixFilePermissions.fromString("rwx------"));
+ this.limit = limit;
+ }
+
+ public ValidatedUpload validate(MultipartFile file, String expectedChecksum) {
+ if (file == null || file.isEmpty()) throw bad("Course Resource must not be empty");
+ if (file.getSize() > limit) throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE, "Course Resource exceeds the size limit");
+ String filename = filename(file.getOriginalFilename());
+ String extension = filename.substring(filename.lastIndexOf('.') + 1).toLowerCase(Locale.ROOT);
+ String expected = TYPES.get(extension);
+ String declared = file.getContentType() == null ? "" : file.getContentType().split(";", 2)[0].strip().toLowerCase(Locale.ROOT);
+ if (expected == null || !compatible(expected, declared)) throw bad("Course Resource type is not allowed");
+ if (expectedChecksum != null && !expectedChecksum.matches("[a-fA-F0-9]{64}")) throw bad("Invalid SHA-256 checksum");
+ Path temporary = null;
+ try {
+ temporary = Files.createTempFile(spool, "upload-", ".part");
+ long size = copyBounded(file.getInputStream(), temporary, limit);
+ if (size == 0) throw bad("Course Resource must not be empty");
+ String detected;
+ try (var input = TikaInputStream.get(temporary)) {
+ detected = new DefaultDetector().detect(input, new Metadata(), new org.apache.tika.parser.ParseContext()).toString();
+ }
+ if (expected.equals(TYPES.get("pptx"))) {
+ if (!detected.contains("zip") && !detected.contains("ooxml")) throw bad("File bytes do not match the file type");
+ verifyPresentation(temporary);
+ } else if (expected.startsWith("text/")) {
+ if (!detected.equals("text/plain")) throw bad("File bytes do not match the file type");
+ String text = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT)
+ .decode(ByteBuffer.wrap(Files.readAllBytes(temporary))).toString();
+ if (text.chars().anyMatch(c -> (c < 32 && c != 9 && c != 10 && c != 13) || c == 127)) throw bad("Text contains unsupported control bytes");
+ } else if (!compatible(expected, detected)) throw bad("File bytes do not match the file type");
+ if (extension.equals("pdf")) {
+ byte[] bytes = Files.readAllBytes(temporary);
+ String end = new String(bytes, Math.max(0, bytes.length - 1024), Math.min(1024, bytes.length), StandardCharsets.ISO_8859_1);
+ if (!end.contains("%%EOF")) throw bad("PDF is incomplete");
+ }
+ String checksum = checksum(temporary);
+ if (expectedChecksum != null && !MessageDigest.isEqual(checksum.getBytes(StandardCharsets.US_ASCII),
+ expectedChecksum.toLowerCase(Locale.ROOT).getBytes(StandardCharsets.US_ASCII))) throw bad("SHA-256 checksum mismatch");
+ return new ValidatedUpload(temporary, filename, expected, size, checksum);
+ } catch (ResponseStatusException exception) {
+ delete(temporary); throw exception;
+ } catch (Exception exception) {
+ delete(temporary); throw bad("Unable to validate Course Resource bytes");
+ }
+ }
+
+ public Path verifiedDownload(InputStream source, long size, String checksum) throws IOException {
+ Path temporary = null;
+ try (source) {
+ temporary = Files.createTempFile(spool, "download-", ".part");
+ if (copyBounded(source, temporary, limit) != size || !checksum(temporary).equals(checksum)) throw new IOException("Resource integrity check failed");
+ return temporary;
+ } catch (Exception exception) {
+ if (temporary != null) Files.deleteIfExists(temporary);
+ if (exception instanceof IOException io) throw io;
+ throw new IOException("Resource integrity check failed");
+ }
+ }
+
+ public ValidatedUpload validateExisting(Path file, String filename, String contentType) throws IOException {
+ return validate(new MultipartFile() {
+ public String getName() { return "file"; }
+ public String getOriginalFilename() { return filename; }
+ public String getContentType() { return contentType; }
+ public boolean isEmpty() { return getSize() == 0; }
+ public long getSize() { try { return Files.size(file); } catch (IOException e) { throw new java.io.UncheckedIOException(e); } }
+ public byte[] getBytes() throws IOException { throw new IOException("Unbounded reads are disabled"); }
+ public InputStream getInputStream() throws IOException { return Files.newInputStream(file); }
+ public void transferTo(java.io.File destination) throws IOException { throw new IOException("Unbounded copies are disabled"); }
+ }, null);
+ }
+
+ public void cleanup(Instant cutoff) throws IOException {
+ try (var files = Files.list(spool)) {
+ for (Path file : files.filter(path -> path.getFileName().toString().matches("(upload|download)-.*\\.part")).toList()) {
+ if (Files.getLastModifiedTime(file, java.nio.file.LinkOption.NOFOLLOW_LINKS).toInstant().isBefore(cutoff)) Files.deleteIfExists(file);
+ }
+ }
+ }
+
+ public static long copyBounded(InputStream source, Path target, long limit) throws IOException {
+ try (source; var out = Files.newOutputStream(target)) {
+ byte[] buffer = new byte[8192]; long size = 0; int count;
+ while ((count = source.read(buffer)) != -1) {
+ size += count;
+ if (size > limit) throw new ResponseStatusException(HttpStatus.PAYLOAD_TOO_LARGE, "Course Resource exceeds the size limit");
+ out.write(buffer, 0, count);
+ }
+ return size;
+ }
+ }
+
+ public static String checksum(Path file) throws IOException {
+ try {
+ var digest = MessageDigest.getInstance("SHA-256");
+ try (var input = Files.newInputStream(file)) {
+ byte[] buffer = new byte[8192]; int count;
+ while ((count = input.read(buffer)) != -1) digest.update(buffer, 0, count);
+ }
+ return HexFormat.of().formatHex(digest.digest());
+ } catch (java.security.NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); }
+ }
+
+ private static void verifyPresentation(Path file) throws IOException {
+ boolean types = false, presentation = false; long expanded = 0; int entries = 0;
+ try (var zip = new ZipInputStream(Files.newInputStream(file))) {
+ java.util.zip.ZipEntry entry; byte[] buffer = new byte[8192];
+ while ((entry = zip.getNextEntry()) != null) {
+ String name = entry.getName();
+ if (++entries > 1024 || name.startsWith("/") || name.contains("..") || name.contains("\\")
+ || name.toLowerCase(Locale.ROOT).contains("vbaproject")) throw bad("Presentation archive is not supported");
+ types |= name.equals("[Content_Types].xml"); presentation |= name.equals("ppt/presentation.xml");
+ int count;
+ while ((count = zip.read(buffer)) != -1) if ((expanded += count) > 40L * 1024 * 1024) throw bad("Presentation archive is too large");
+ }
+ }
+ if (!types || !presentation) throw bad("File is not a PowerPoint presentation");
+ }
+
+ public static String filename(String original) {
+ if (original == null) throw bad("Course Resource filename is required");
+ String normalized = Normalizer.normalize(original.replace('\\', '/'), Normalizer.Form.NFC);
+ normalized = normalized.substring(normalized.lastIndexOf('/') + 1).replaceAll("[\\p{Cntrl}\\p{Cf}]", "").strip();
+ if (normalized.isBlank() || normalized.length() > 180 || normalized.startsWith(".") || normalized.endsWith(".")
+ || normalized.indexOf('.') < 1 || normalized.matches(".*[<>:\"|?*].*")) throw bad("Course Resource filename is invalid");
+ return normalized;
+ }
+
+ private static boolean compatible(String expected, String actual) {
+ return expected.equals(actual) || (expected.equals("audio/wav") && actual.equals("audio/vnd.wave"))
+ || (expected.equals("audio/wav") && actual.equals("audio/x-wav"))
+ || (expected.equals("text/markdown") && actual.equals("text/plain"))
+ || (expected.equals("audio/mp4") && actual.equals("video/mp4"))
+ || (expected.equals("audio/ogg") && actual.equals("application/ogg"));
+ }
+ private static ResponseStatusException bad(String message) { return new ResponseStatusException(HttpStatus.BAD_REQUEST, message); }
+ private static void delete(Path file) { if (file != null) try { Files.deleteIfExists(file); } catch (IOException ignored) { } }
+ public record ValidatedUpload(Path path, String fileName, String contentType, long byteSize, String sha256) implements AutoCloseable {
+ @Override public void close() throws IOException { Files.deleteIfExists(path); }
+ }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/domain/CourseResource.java b/backend/media-service/src/main/java/com/chanter/media/domain/CourseResource.java
index a9991a22..dba8d7ff 100644
--- a/backend/media-service/src/main/java/com/chanter/media/domain/CourseResource.java
+++ b/backend/media-service/src/main/java/com/chanter/media/domain/CourseResource.java
@@ -13,6 +13,18 @@ public record CourseResource(
String storageKey,
boolean aiApproved,
UUID uploadedByUserId,
- Instant createdAt
+ Instant createdAt,
+ String state,
+ String sha256,
+ UUID idempotencyKey,
+ String storageBackend
) {
+ public String publicStatus() {
+ return switch (state) {
+ case "AVAILABLE" -> "AVAILABLE";
+ case "REJECTED" -> "REJECTED";
+ case "SCAN_FAILED", "DELETE_PENDING", "DELETED" -> "FAILED";
+ default -> "PROCESSING";
+ };
+ }
}
diff --git a/backend/media-service/src/main/java/com/chanter/media/infra/HttpResourceIngestionClient.java b/backend/media-service/src/main/java/com/chanter/media/infra/HttpResourceIngestionClient.java
index 9b08a8af..9598d831 100644
--- a/backend/media-service/src/main/java/com/chanter/media/infra/HttpResourceIngestionClient.java
+++ b/backend/media-service/src/main/java/com/chanter/media/infra/HttpResourceIngestionClient.java
@@ -10,8 +10,6 @@
import java.util.Locale;
import java.util.Map;
import java.util.UUID;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Profile;
import org.springframework.http.client.JdkClientHttpRequestFactory;
@@ -23,8 +21,6 @@
@Profile("!test")
public class HttpResourceIngestionClient implements ResourceIngestionClient {
- private static final Logger log = LoggerFactory.getLogger(HttpResourceIngestionClient.class);
-
private final RestClient restClient;
private final String serviceToken;
@@ -50,11 +46,6 @@ public HttpResourceIngestionClient(
@Override
public void ingestAiApprovedResource(UUID courseId, UUID resourceId, String fileName, byte[] content) {
if (!isTextResource(fileName)) {
- log.info(
- "Skipping resource ingestion for unsupported file resourceId={} fileName={}",
- resourceId,
- fileName
- );
return;
}
if (content == null) {
@@ -75,12 +66,7 @@ public void ingestAiApprovedResource(UUID courseId, UUID resourceId, String file
.retrieve()
.toBodilessEntity();
} catch (RestClientException exception) {
- log.warn(
- "Failed to ingest AI-approved resource chunks resourceId={} courseId={}: {}",
- resourceId,
- courseId,
- exception.getMessage()
- );
+ throw new IllegalStateException("Resource indexing is unavailable");
}
}
@@ -93,11 +79,7 @@ public void deleteResourceChunks(UUID resourceId) {
.retrieve()
.toBodilessEntity();
} catch (RestClientException exception) {
- log.warn(
- "Failed to delete resource chunks resourceId={}: {}",
- resourceId,
- exception.getMessage()
- );
+ throw new IllegalStateException("Resource index deletion is unavailable");
}
}
diff --git a/backend/media-service/src/main/java/com/chanter/media/infra/JdbcCourseResourceRepository.java b/backend/media-service/src/main/java/com/chanter/media/infra/JdbcCourseResourceRepository.java
deleted file mode 100644
index ca29bf90..00000000
--- a/backend/media-service/src/main/java/com/chanter/media/infra/JdbcCourseResourceRepository.java
+++ /dev/null
@@ -1,138 +0,0 @@
-package com.chanter.media.infra;
-
-import com.chanter.media.application.CourseResourceRepository;
-import com.chanter.media.domain.CourseResource;
-import java.time.OffsetDateTime;
-import java.time.ZoneOffset;
-import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
-import org.springframework.jdbc.core.simple.JdbcClient;
-import org.springframework.stereotype.Repository;
-import org.springframework.transaction.annotation.Transactional;
-
-@Repository
-public class JdbcCourseResourceRepository implements CourseResourceRepository {
-
- private final JdbcClient jdbcClient;
-
- public JdbcCourseResourceRepository(JdbcClient jdbcClient) {
- this.jdbcClient = jdbcClient;
- }
-
- @Override
- @Transactional
- public CourseResource save(CourseResource courseResource) {
- OffsetDateTime createdAt = OffsetDateTime.ofInstant(courseResource.createdAt(), ZoneOffset.UTC);
-
- jdbcClient.sql("""
- INSERT INTO course_resources (
- id,
- course_id,
- title,
- file_name,
- content_type,
- byte_size,
- storage_key,
- ai_approved,
- uploaded_by_user_id,
- created_at
- )
- VALUES (
- :id,
- :courseId,
- :title,
- :fileName,
- :contentType,
- :byteSize,
- :storageKey,
- :aiApproved,
- :uploadedByUserId,
- :createdAt
- )
- """)
- .param("id", courseResource.id())
- .param("courseId", courseResource.courseId())
- .param("title", courseResource.title())
- .param("fileName", courseResource.fileName())
- .param("contentType", courseResource.contentType())
- .param("byteSize", courseResource.byteSize())
- .param("storageKey", courseResource.storageKey())
- .param("aiApproved", courseResource.aiApproved())
- .param("uploadedByUserId", courseResource.uploadedByUserId())
- .param("createdAt", createdAt)
- .update();
-
- return courseResource;
- }
-
- @Override
- @Transactional(readOnly = true)
- public Optional findById(UUID resourceId) {
- return jdbcClient.sql("""
- SELECT
- id,
- course_id,
- title,
- file_name,
- content_type,
- byte_size,
- storage_key,
- ai_approved,
- uploaded_by_user_id,
- created_at
- FROM course_resources
- WHERE id = :resourceId
- """)
- .param("resourceId", resourceId)
- .query(this::mapCourseResource)
- .optional();
- }
-
- @Override
- @Transactional
- public void deleteById(UUID resourceId) {
- jdbcClient.sql("DELETE FROM course_resources WHERE id = :resourceId")
- .param("resourceId", resourceId)
- .update();
- }
-
- @Override
- @Transactional(readOnly = true)
- public List findByCourseId(UUID courseId) {
- return jdbcClient.sql("""
- SELECT
- id,
- course_id,
- title,
- file_name,
- content_type,
- byte_size,
- storage_key,
- ai_approved,
- uploaded_by_user_id,
- created_at
- FROM course_resources
- WHERE course_id = :courseId
- ORDER BY created_at ASC
- """)
- .param("courseId", courseId)
- .query(this::mapCourseResource)
- .list();
- }
-
- private CourseResource mapCourseResource(java.sql.ResultSet rs, int rowNum) throws java.sql.SQLException {
- return new CourseResource(
- rs.getObject("id", UUID.class),
- rs.getObject("course_id", UUID.class),
- rs.getString("title"),
- rs.getString("file_name"),
- rs.getString("content_type"),
- rs.getLong("byte_size"),
- rs.getString("storage_key"),
- rs.getBoolean("ai_approved"),
- rs.getObject("uploaded_by_user_id", UUID.class),
- rs.getObject("created_at", OffsetDateTime.class).toInstant()
- );
- }
-}
diff --git a/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java b/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java
new file mode 100644
index 00000000..db61989d
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java
@@ -0,0 +1,58 @@
+package com.chanter.media.infra;
+
+import com.chanter.media.application.PrivateResourceStorage;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.LinkOption;
+import java.nio.file.Path;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+@Component
+@ConditionalOnProperty(name = "chanter.media.storage-backend", havingValue = "local")
+public class LocalPrivateResourceStorage implements PrivateResourceStorage {
+ private final Path root;
+ public LocalPrivateResourceStorage(@Value("${chanter.media.storage-dir}") String root) throws IOException {
+ this.root = Path.of(root).toAbsolutePath().normalize();
+ Files.createDirectories(this.root);
+ if (Files.getFileStore(this.root).supportsFileAttributeView("posix")) Files.setPosixFilePermissions(this.root, java.nio.file.attribute.PosixFilePermissions.fromString("rwx------"));
+ }
+ @Override public String backend() { return "local"; }
+ private Path path(String key) throws IOException {
+ PrivateResourceStorage.requireKey(key);
+ Path target = root.resolve(key).normalize();
+ for (Path current = target; current != null && current.startsWith(root); current = current.getParent()) {
+ if (Files.isSymbolicLink(current)) throw new IOException("Symbolic link in private storage");
+ }
+ return target;
+ }
+ @Override public void put(String key, Path content, String checksum) throws IOException {
+ Path target = path(key);
+ Files.createDirectories(target.getParent());
+ // CREATE_NEW atomically rejects existing keys; the DB keeps partial writes quarantined until cleanup.
+ try (var out = Files.newOutputStream(target, java.nio.file.StandardOpenOption.CREATE_NEW, java.nio.file.StandardOpenOption.WRITE)) {
+ Files.copy(content, out);
+ }
+ }
+ @Override public InputStream open(String key) throws IOException {
+ Path target = path(key);
+ if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS)) throw new IOException("Private resource is missing");
+ return Files.newInputStream(target);
+ }
+ @Override public void delete(String key) throws IOException { Files.deleteIfExists(path(key)); }
+ @Override public Page list(String cursor) throws IOException {
+ Path prefix = root.resolve(PREFIX);
+ if (!Files.exists(prefix)) return new Page(java.util.List.of(), null);
+ try (var files = Files.walk(prefix)) {
+ var page = files.filter(path -> Files.isRegularFile(path, LinkOption.NOFOLLOW_LINKS))
+ .map(path -> root.relativize(path).toString().replace('\\', '/'))
+ .filter(key -> !key.substring(key.lastIndexOf('/') + 1).startsWith("pending-"))
+ .filter(key -> cursor == null || key.compareTo(cursor) > 0).sorted().limit(1001).toList();
+ var result = new java.util.ArrayList();
+ for (String key : page.stream().limit(1000).toList()) result.add(new ObjectInfo(key, Files.getLastModifiedTime(path(key)).toInstant()));
+ return new Page(result, page.size() > 1000 ? page.get(999) : null);
+ }
+ }
+}
diff --git a/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java b/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java
new file mode 100644
index 00000000..27555c25
--- /dev/null
+++ b/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java
@@ -0,0 +1,96 @@
+package com.chanter.media.infra;
+
+import com.chanter.media.application.PrivateResourceStorage;
+import com.chanter.media.application.ResourceLifecycle;
+import jakarta.annotation.PreDestroy;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.util.Base64;
+import java.util.Map;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.checksums.RequestChecksumCalculation;
+import software.amazon.awssdk.core.checksums.ResponseChecksumValidation;
+import software.amazon.awssdk.core.retry.RetryPolicy;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.*;
+
+@Component
+@ConditionalOnProperty(name = "chanter.media.storage-backend", havingValue = "s3")
+public class S3PrivateResourceStorage implements PrivateResourceStorage {
+ private final S3Client client;
+ private final String bucket;
+ private final ResourceLifecycle lifecycle;
+
+ public S3PrivateResourceStorage(ResourceLifecycle lifecycle,
+ @Value("${chanter.media.s3.endpoint}") String endpoint,
+ @Value("${chanter.media.s3.region}") String region,
+ @Value("${chanter.media.s3.bucket}") String bucket,
+ @Value("${chanter.media.s3.access-key}") String accessKey,
+ @Value("${chanter.media.s3.secret-key}") String secretKey,
+ @Value("${chanter.media.s3.allow-local-http:false}") boolean localHttp) {
+ URI uri;
+ try { uri = URI.create(endpoint); }
+ catch (IllegalArgumentException invalid) { throw new IllegalArgumentException("Invalid private S3 configuration"); }
+ boolean local = localHttp && "http".equals(uri.getScheme()) && java.util.Set.of("127.0.0.1", "localhost").contains(uri.getHost());
+ if ((!"https".equals(uri.getScheme()) && !local) || uri.getHost() == null || uri.getUserInfo() != null
+ || uri.getQuery() != null || uri.getFragment() != null || !java.util.Set.of("", "/").contains(uri.getPath())
+ || !bucket.matches("[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]") || accessKey.isBlank() || secretKey.isBlank() || region.isBlank()) {
+ throw new IllegalArgumentException("Invalid private S3 configuration");
+ }
+ this.lifecycle = lifecycle; this.bucket = bucket;
+ this.client = S3Client.builder().endpointOverride(uri).region(Region.of(region))
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)))
+ .httpClientBuilder(UrlConnectionHttpClient.builder().connectionTimeout(Duration.ofSeconds(3)).socketTimeout(Duration.ofSeconds(15)))
+ .serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).chunkedEncodingEnabled(false).build())
+ .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
+ .responseChecksumValidation(ResponseChecksumValidation.WHEN_REQUIRED)
+ .overrideConfiguration(config -> config.retryPolicy(RetryPolicy.none()).apiCallTimeout(Duration.ofSeconds(30))
+ .apiCallAttemptTimeout(Duration.ofSeconds(20))).build();
+ }
+ @Override public String backend() { return "s3"; }
+ @Override public void put(String key, Path content, String sha256) throws IOException {
+ PrivateResourceStorage.requireKey(key);
+ try {
+ String md5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(Files.readAllBytes(content)));
+ lifecycle.countRequest(false);
+ client.putObject(PutObjectRequest.builder().bucket(bucket).key(key).ifNoneMatch("*").contentMD5(md5)
+ .contentType("application/octet-stream").metadata(Map.of("sha256", sha256)).build(), RequestBody.fromFile(content));
+ } catch (Exception exception) { throw failure(); }
+ }
+ @Override public InputStream open(String key) throws IOException {
+ PrivateResourceStorage.requireKey(key);
+ lifecycle.countRequest(false);
+ try { return client.getObject(GetObjectRequest.builder().bucket(bucket).key(key).build()); }
+ catch (Exception exception) { throw failure(); }
+ }
+ @Override public void delete(String key) throws IOException {
+ PrivateResourceStorage.requireKey(key);
+ lifecycle.countRequest(true);
+ try { client.deleteObject(DeleteObjectRequest.builder().bucket(bucket).key(key).build()); }
+ catch (S3Exception exception) { if (exception.statusCode() != 404) throw failure(); }
+ catch (Exception exception) { throw failure(); }
+ }
+ @Override public Page list(String cursor) throws IOException {
+ lifecycle.countRequest(false);
+ try {
+ var result = client.listObjectsV2(ListObjectsV2Request.builder().bucket(bucket).prefix(PREFIX).maxKeys(1000).continuationToken(cursor).build());
+ return new Page(result.contents().stream().map(object -> new ObjectInfo(object.key(), object.lastModified())).toList(),
+ Boolean.TRUE.equals(result.isTruncated()) ? result.nextContinuationToken() : null);
+ } catch (Exception exception) { throw failure(); }
+ }
+ private static IOException failure() { return new IOException("Private object storage is unavailable"); }
+ @PreDestroy public void close() { client.close(); }
+}
diff --git a/backend/media-service/src/main/resources/application.yml b/backend/media-service/src/main/resources/application.yml
index 01bd2808..2b67efb7 100644
--- a/backend/media-service/src/main/resources/application.yml
+++ b/backend/media-service/src/main/resources/application.yml
@@ -14,7 +14,7 @@ spring:
servlet:
multipart:
max-file-size: 10MB
- max-request-size: 10MB
+ max-request-size: 11MB
chanter:
jwt:
@@ -30,6 +30,24 @@ chanter:
service-token: ${CHANTER_INTERNAL_SERVICE_TOKEN}
media:
storage-dir: ${COURSE_RESOURCE_STORAGE_DIR:./data/course-resources}
+ storage-backend: ${CHANTER_MEDIA_STORAGE_BACKEND:local}
+ spool-dir: ${CHANTER_MEDIA_SPOOL_DIR:./data/media-spool}
+ byte-limit: ${CHANTER_MEDIA_BYTE_LIMIT:8000000000}
+ request-limit: ${CHANTER_MEDIA_REQUEST_LIMIT:40000}
+ cleanup-request-reserve: ${CHANTER_MEDIA_CLEANUP_REQUEST_RESERVE:4000}
+ worker-enabled: ${CHANTER_MEDIA_WORKER_ENABLED:true}
+ migrate-legacy: ${CHANTER_MEDIA_MIGRATE_LEGACY:false}
+ s3:
+ endpoint: ${CHANTER_S3_ENDPOINT:}
+ region: ${CHANTER_S3_REGION:}
+ bucket: ${CHANTER_S3_BUCKET:}
+ access-key: ${CHANTER_S3_ACCESS_KEY:}
+ secret-key: ${CHANTER_S3_SECRET_KEY:}
+ scanner:
+ host: ${CHANTER_CLAMAV_HOST:localhost}
+ port: ${CHANTER_CLAMAV_PORT:3310}
+ timeout: 20s
+ maximum-definition-age: 72h
management:
endpoints:
diff --git a/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql b/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
new file mode 100644
index 00000000..7e0d0027
--- /dev/null
+++ b/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
@@ -0,0 +1,24 @@
+ALTER TABLE course_resources ADD COLUMN state VARCHAR(32) NOT NULL DEFAULT 'LEGACY';
+ALTER TABLE course_resources ADD COLUMN sha256 VARCHAR(64);
+ALTER TABLE course_resources ADD COLUMN idempotency_key UUID;
+ALTER TABLE course_resources ADD COLUMN storage_backend VARCHAR(16) NOT NULL DEFAULT 'legacy';
+ALTER TABLE course_resources ADD COLUMN migration_key VARCHAR(512);
+ALTER TABLE course_resources ADD COLUMN byte_reservation BOOLEAN NOT NULL DEFAULT TRUE;
+ALTER TABLE course_resources ADD COLUMN lease_id UUID;
+ALTER TABLE course_resources ADD COLUMN lease_until TIMESTAMP WITH TIME ZONE;
+ALTER TABLE course_resources ADD COLUMN attempts INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE course_resources ADD COLUMN retry_at TIMESTAMP WITH TIME ZONE;
+ALTER TABLE course_resources ADD COLUMN updated_at TIMESTAMP WITH TIME ZONE;
+UPDATE course_resources SET updated_at = created_at;
+CREATE UNIQUE INDEX uq_resource_upload_idempotency ON course_resources(uploaded_by_user_id, idempotency_key);
+CREATE INDEX idx_resource_processing ON course_resources(state, retry_at, lease_until);
+
+CREATE TABLE media_storage_budget (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ reserved_bytes BIGINT NOT NULL CHECK (reserved_bytes >= 0),
+ request_month VARCHAR(7) NOT NULL,
+ foreground_requests INTEGER NOT NULL,
+ maintenance_requests INTEGER NOT NULL
+);
+INSERT INTO media_storage_budget (id, reserved_bytes, request_month, foreground_requests, maintenance_requests)
+SELECT 1, COALESCE(SUM(byte_size), 0), '1970-01', 0, 0 FROM course_resources;
diff --git a/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java b/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
index b84268f3..4b658e56 100644
--- a/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
@@ -1,6 +1,8 @@
package com.chanter.media.api;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.multipart;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@@ -8,6 +10,8 @@
import com.chanter.common.auth.AuthHeaders;
import com.chanter.media.infra.TestCourseResourceAccessClient;
import com.chanter.media.infra.TestResourceIngestionClient;
+import com.chanter.media.application.MalwareScanner;
+import com.chanter.media.application.ResourceWorker;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.nio.charset.StandardCharsets;
import java.time.temporal.ChronoUnit;
@@ -26,6 +30,10 @@
@AutoConfigureMockMvc
@ActiveProfiles("test")
class CourseResourceSmokeTest {
+ @org.springframework.test.context.bean.override.mockito.MockitoBean
+ MalwareScanner scanner;
+ @Autowired ResourceWorker worker;
+ @Autowired org.springframework.jdbc.core.simple.JdbcClient jdbc;
private static final String INTERNAL_TOKEN = "test-internal-service-token-for-media";
@@ -42,9 +50,12 @@ class CourseResourceSmokeTest {
private TestResourceIngestionClient resourceIngestionClient;
@BeforeEach
- void setUp() {
+ void setUp() throws Exception {
courseResourceAccessClient.clear();
resourceIngestionClient.clear();
+ jdbc.sql("DELETE FROM course_resources").update();
+ jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=0, foreground_requests=0, maintenance_requests=0").update();
+ when(scanner.scan(any())).thenReturn(MalwareScanner.Verdict.CLEAN);
}
@Test
@@ -68,7 +79,7 @@ void instructorCanUploadCourseResourceAndEnrolledLearnerCanListAndDownload() thr
.header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)
.param("title", "Spring Security Guide")
.param("aiApproved", "true"))
- .andExpect(status().isCreated())
+ .andExpect(status().isAccepted())
.andReturn();
CourseResourceResponse uploaded = objectMapper.readValue(
uploadResult.getResponse().getContentAsString(),
@@ -80,6 +91,13 @@ void instructorCanUploadCourseResourceAndEnrolledLearnerCanListAndDownload() thr
assertThat(uploaded.fileName()).isEqualTo("spring-security-guide.md");
assertThat(uploaded.aiApproved()).isTrue();
assertThat(uploaded.uploadedByUserId()).isEqualTo(instructorUserId);
+ assertThat(uploaded.status()).isEqualTo("PROCESSING");
+ assertThat(uploaded.sha256()).hasSize(64);
+ mockMvc.perform(get("/api/v1/course-resources/{id}/content", uploaded.id())
+ .header(AuthHeaders.USER_ID, learnerUserId.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN))
+ .andExpect(status().isConflict());
+ assertThat(resourceIngestionClient.ingestCalls()).isEmpty();
+ worker.runOnce();
MvcResult listResult = mockMvc.perform(get("/api/v1/courses/{courseId}/course-resources", courseId)
.header(AuthHeaders.USER_ID, learnerUserId.toString())
@@ -95,7 +113,7 @@ void instructorCanUploadCourseResourceAndEnrolledLearnerCanListAndDownload() thr
CourseResourceResponse listedResource = listed.courseResources().getFirst();
assertThat(listedResource)
.usingRecursiveComparison()
- .ignoringFields("createdAt")
+ .ignoringFields("createdAt", "status")
.isEqualTo(uploaded);
// Postgres stores timestamps at microsecond precision; upload responses truncate to match.
assertThat(listedResource.createdAt()).isEqualTo(uploaded.createdAt().truncatedTo(ChronoUnit.MICROS));
@@ -109,6 +127,8 @@ void instructorCanUploadCourseResourceAndEnrolledLearnerCanListAndDownload() thr
assertThat(downloadResult.getResponse().getContentAsByteArray()).isEqualTo(fileContent);
assertThat(downloadResult.getResponse().getHeader("Content-Disposition"))
.contains("spring-security-guide.md");
+ assertThat(downloadResult.getResponse().getHeader("X-Content-Type-Options")).isEqualTo("nosniff");
+ assertThat(downloadResult.getResponse().getHeader("Cache-Control")).isEqualTo("no-store");
assertThat(resourceIngestionClient.ingestCalls()).hasSize(1);
TestResourceIngestionClient.IngestCall ingestCall = resourceIngestionClient.ingestCalls().getFirst();
@@ -154,7 +174,7 @@ void uploadAcceptsPdfContentType() throws Exception {
UUID courseId = UUID.randomUUID();
UUID instructorUserId = UUID.randomUUID();
courseResourceAccessClient.grantInstructorUpload(courseId, instructorUserId);
- byte[] pdfBytes = "%PDF-1.4 demo".getBytes(StandardCharsets.UTF_8);
+ byte[] pdfBytes = "%PDF-1.4 demo\n%%EOF".getBytes(StandardCharsets.UTF_8);
MvcResult uploadResult = mockMvc.perform(multipart("/api/v1/courses/{courseId}/course-resources", courseId)
.file(new MockMultipartFile(
@@ -167,7 +187,7 @@ void uploadAcceptsPdfContentType() throws Exception {
.header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)
.param("title", "Syllabus")
.param("aiApproved", "false"))
- .andExpect(status().isCreated())
+ .andExpect(status().isAccepted())
.andReturn();
CourseResourceResponse uploaded = objectMapper.readValue(
@@ -206,7 +226,7 @@ void unauthorizedUserCannotListCourseResources() throws Exception {
.header(AuthHeaders.USER_ID, instructorUserId.toString())
.header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)
.param("aiApproved", "true"))
- .andExpect(status().isCreated());
+ .andExpect(status().isAccepted());
mockMvc.perform(get("/api/v1/courses/{courseId}/course-resources", courseId)
.header(AuthHeaders.USER_ID, strangerUserId.toString())
@@ -251,7 +271,7 @@ void instructorUploadStripsPathFromFileName() throws Exception {
.header(AuthHeaders.USER_ID, instructorUserId.toString())
.header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)
.param("aiApproved", "true"))
- .andExpect(status().isCreated())
+ .andExpect(status().isAccepted())
.andReturn();
CourseResourceResponse uploaded = objectMapper.readValue(
@@ -280,7 +300,7 @@ void unauthorizedUserCannotDownloadCourseResource() throws Exception {
.header(AuthHeaders.USER_ID, instructorUserId.toString())
.header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)
.param("aiApproved", "true"))
- .andExpect(status().isCreated())
+ .andExpect(status().isAccepted())
.andReturn();
CourseResourceResponse uploaded = objectMapper.readValue(
uploadResult.getResponse().getContentAsString(),
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java b/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java
new file mode 100644
index 00000000..fa641c68
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java
@@ -0,0 +1,99 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+
+import java.io.DataInputStream;
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import java.util.concurrent.CompletableFuture;
+import org.junit.jupiter.api.Test;
+
+class ClamAvScannerTest {
+ @org.junit.jupiter.params.ParameterizedTest
+ @org.junit.jupiter.params.provider.ValueSource(booleans = {true, false})
+ void streamsBytesAndDistinguishesMalwareFromClean(boolean infected) throws Exception {
+ try (var server = new ServerSocket(0, 1, java.net.InetAddress.getLoopbackAddress())) {
+ var serving = CompletableFuture.runAsync(() -> {
+ try {
+ try (var socket = server.accept()) {
+ assertThat(readCommand(socket.getInputStream())).isEqualTo("zVERSION");
+ String date = DateTimeFormatter.ofPattern("EEE MMM d HH:mm:ss yyyy", Locale.US).withZone(ZoneOffset.UTC).format(Instant.now());
+ socket.getOutputStream().write(("ClamAV 1.5.2/28000/" + date + "\0").getBytes(StandardCharsets.US_ASCII));
+ }
+ try (var socket = server.accept()) {
+ assertThat(readCommand(socket.getInputStream())).isEqualTo("zINSTREAM");
+ var input = new DataInputStream(socket.getInputStream());
+ int count = input.readInt(); assertThat(input.readNBytes(count)).isEqualTo("scan me".getBytes(StandardCharsets.US_ASCII));
+ assertThat(input.readInt()).isZero();
+ socket.getOutputStream().write((infected ? "stream: Eicar-Test-Signature FOUND\0" : "stream: OK\0").getBytes(StandardCharsets.US_ASCII));
+ }
+ } catch (Exception exception) { throw new RuntimeException(exception); }
+ });
+ Path file = Path.of("target/clam-stream-fixture.txt"); Files.createDirectories(file.getParent()); Files.writeString(file, "scan me");
+ var scanner = new ClamAvScanner("127.0.0.1", server.getLocalPort(), Duration.ofSeconds(2), Duration.ofHours(72), Clock.systemUTC());
+ assertThat(scanner.scan(file)).isEqualTo(infected ? MalwareScanner.Verdict.INFECTED : MalwareScanner.Verdict.CLEAN);
+ serving.get(5, java.util.concurrent.TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ void refusesUnavailableScannerInsteadOfDeclaringTheFileClean() throws Exception {
+ int port; try (var server = new ServerSocket(0)) { port = server.getLocalPort(); }
+ var scanner = new ClamAvScanner("127.0.0.1", port, Duration.ofMillis(100), Duration.ofHours(72), Clock.systemUTC());
+ assertThatThrownBy(() -> scanner.scan(Path.of("target/missing.txt"))).isInstanceOf(java.io.IOException.class);
+ }
+
+ @Test
+ void staleDefinitionsFailClosedBeforeSendingResourceBytes() throws Exception {
+ try (var server = new ServerSocket(0)) {
+ var serving = CompletableFuture.runAsync(() -> {
+ try (var socket = server.accept()) {
+ readCommand(socket.getInputStream());
+ socket.getOutputStream().write("ClamAV 1.5.2/28000/Mon Jan 1 00:00:00 2024\0".getBytes(StandardCharsets.US_ASCII));
+ } catch (Exception exception) { throw new RuntimeException(exception); }
+ });
+ var scanner = new ClamAvScanner("127.0.0.1", server.getLocalPort(), Duration.ofSeconds(1), Duration.ofHours(72), Clock.systemUTC());
+ assertThatThrownBy(() -> scanner.scan(Path.of("target/missing.txt"))).isInstanceOf(java.io.IOException.class).hasMessageContaining("stale");
+ serving.get(2, java.util.concurrent.TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ void blockedScannerWritesHaveABoundedDeadline() throws Exception {
+ try (var server = new ServerSocket(0)) {
+ var serving = CompletableFuture.runAsync(() -> {
+ try {
+ try (var socket = server.accept()) {
+ readCommand(socket.getInputStream());
+ String date = DateTimeFormatter.ofPattern("EEE MMM d HH:mm:ss yyyy", Locale.US).withZone(ZoneOffset.UTC).format(Instant.now());
+ socket.getOutputStream().write(("ClamAV 1.5.2/28000/" + date + "\0").getBytes(StandardCharsets.US_ASCII));
+ }
+ try (var socket = server.accept()) {
+ socket.setReceiveBufferSize(1024);
+ Thread.sleep(2500); // Never consume the file stream.
+ }
+ } catch (Exception exception) { throw new RuntimeException(exception); }
+ });
+ Path file = Path.of("target/clam-blocked-fixture.bin"); Files.createDirectories(file.getParent()); Files.write(file, new byte[10 * 1024 * 1024]);
+ var scanner = new ClamAvScanner("127.0.0.1", server.getLocalPort(), Duration.ofMillis(150), Duration.ofHours(72), Clock.systemUTC());
+ long started = System.nanoTime();
+ assertThatThrownBy(() -> scanner.scan(file)).isInstanceOf(java.io.IOException.class);
+ assertThat(Duration.ofNanos(System.nanoTime() - started)).isLessThan(Duration.ofSeconds(1));
+ serving.get(4, java.util.concurrent.TimeUnit.SECONDS);
+ }
+ }
+
+ private static String readCommand(java.io.InputStream input) throws Exception {
+ var result = new java.io.ByteArrayOutputStream(); int value;
+ while ((value = input.read()) > 0) result.write(value);
+ return result.toString(StandardCharsets.US_ASCII);
+ }
+}
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/LocalPrivateResourceStorageTest.java b/backend/media-service/src/test/java/com/chanter/media/application/LocalPrivateResourceStorageTest.java
new file mode 100644
index 00000000..f2dd76f5
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/LocalPrivateResourceStorageTest.java
@@ -0,0 +1,31 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+import com.chanter.media.infra.LocalPrivateResourceStorage;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class LocalPrivateResourceStorageTest {
+ @TempDir Path directory;
+ @Test void immutableCreationIsAtomicAndBytesSurviveAdapterRestart() throws Exception {
+ var storage = new LocalPrivateResourceStorage(directory.resolve("private").toString());
+ Path source = directory.resolve("source.txt"); Files.writeString(source, "immutable bytes");
+ String key = PrivateResourceStorage.PREFIX + UUID.randomUUID() + "/" + UUID.randomUUID() + "/" + UUID.randomUUID();
+ var successes = new AtomicInteger();
+ var calls = java.util.stream.IntStream.range(0, 8).mapToObj(i -> CompletableFuture.runAsync(() -> {
+ try { storage.put(key, source, "unused by local adapter"); successes.incrementAndGet(); }
+ catch (java.io.IOException expectedConflict) { assertThat(expectedConflict).isInstanceOf(java.nio.file.FileAlreadyExistsException.class); }
+ })).toList(); calls.forEach(CompletableFuture::join);
+ assertThat(successes.get()).isEqualTo(1);
+ var restarted = new LocalPrivateResourceStorage(directory.resolve("private").toString());
+ try (var content = restarted.open(key)) { assertThat(new String(content.readAllBytes(), java.nio.charset.StandardCharsets.UTF_8)).isEqualTo("immutable bytes"); }
+ assertThatThrownBy(() -> storage.open("../../source.txt")).isInstanceOf(IllegalArgumentException.class);
+ storage.delete(key); storage.delete(key);
+ assertThatThrownBy(() -> storage.open(key)).isInstanceOf(java.io.IOException.class);
+ }
+}
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java b/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
new file mode 100644
index 00000000..3a5cb9e0
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
@@ -0,0 +1,118 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+
+import com.chanter.media.infra.TestCourseResourceAccessClient;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+import org.junit.jupiter.api.*;
+import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.jdbc.core.simple.JdbcClient;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.DynamicPropertyRegistry;
+import org.springframework.test.context.DynamicPropertySource;
+
+/** Runs against real PostgreSQL, S3Mock and ClamAV processes; never substitutes a scan verdict. */
+@SpringBootTest
+@ActiveProfiles("test")
+@EnabledIfEnvironmentVariable(named = "MEDIA_INTEGRATION", matches = "true")
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class PrivateStorageIntegrationTest {
+ static final UUID COURSE = UUID.fromString("a0000000-0000-4000-8000-000000000001");
+ static final UUID TEACHER = UUID.fromString("a0000000-0000-4000-8000-000000000002");
+ static final UUID LEARNER = UUID.fromString("a0000000-0000-4000-8000-000000000003");
+ static final UUID RETRY = UUID.fromString("a0000000-0000-4000-8000-000000000004");
+ static final byte[] CLEAN = "Durable private Course Resource".getBytes(StandardCharsets.UTF_8);
+ @Autowired CourseResourceService service;
+ @Autowired ResourceWorker worker;
+ @Autowired ResourceLifecycle lifecycle;
+ @Autowired PrivateResourceStorage storage;
+ @Autowired MalwareScanner scanner;
+ @Autowired TestCourseResourceAccessClient access;
+ @Autowired JdbcClient jdbc;
+
+ @DynamicPropertySource static void properties(DynamicPropertyRegistry registry) {
+ registry.add("spring.datasource.url", () -> "jdbc:postgresql://127.0.0.1:5544/chanter_media");
+ registry.add("spring.datasource.username", () -> "media_test");
+ registry.add("spring.datasource.password", () -> "media-test-only-password");
+ registry.add("chanter.media.storage-backend", () -> "s3");
+ registry.add("chanter.media.s3.endpoint", () -> "http://127.0.0.1:9090");
+ registry.add("chanter.media.s3.region", () -> "us-east-1");
+ registry.add("chanter.media.s3.bucket", () -> "private-media-test");
+ registry.add("chanter.media.s3.access-key", () -> "emulator-only");
+ registry.add("chanter.media.s3.secret-key", () -> "emulator-only");
+ registry.add("chanter.media.s3.allow-local-http", () -> true);
+ registry.add("chanter.media.scanner.host", () -> "127.0.0.1");
+ }
+ @BeforeAll void setup() {
+ access.grantInstructorUpload(COURSE, TEACHER); access.grantLearnerView(COURSE, LEARNER);
+ if (!"true".equals(System.getenv("MEDIA_RESTART_PHASE"))) {
+ jdbc.sql("DELETE FROM course_resources").update();
+ jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=0,foreground_requests=0,maintenance_requests=0").update();
+ }
+ }
+ private com.chanter.media.domain.CourseResource uploadClean() {
+ return service.uploadCourseResource(COURSE, TEACHER, "Durable notes", false,
+ new MockMultipartFile("file", "notes.txt", "text/plain", CLEAN), RETRY, null);
+ }
+ @Test @Order(1) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "false")
+ void actualStorageAndCleanScanGateDownload() throws Exception {
+ var resource = uploadClean(); assertThat(resource.publicStatus()).isEqualTo("PROCESSING");
+ assertThat(service.listCourseResources(COURSE, LEARNER)).isEmpty();
+ worker.runOnce();
+ assertThat(lifecycle.find(resource.id()).orElseThrow().publicStatus()).isEqualTo("AVAILABLE");
+ try (var content = service.downloadCourseResource(resource.id(), LEARNER).content()) { assertThat(content.readAllBytes()).isEqualTo(CLEAN); }
+ }
+ @Test @Order(2) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "false")
+ void realClamAvRejectsEicarInsidePresentationAndDeleteReleasesReservation() throws Exception {
+ // EICAR is the standard harmless antivirus test string. Splitting prevents source scanners mistaking this fixture for an uploaded file.
+ String eicar = "X5O!P%@AP[4\\PZX54(P^)7CC)7}$" + "EICAR-STANDARD-ANTIVIRUS-TEST-FILE!$H+H*";
+ var bytes = new java.io.ByteArrayOutputStream();
+ try (var zip = new ZipOutputStream(bytes)) {
+ for (String name : java.util.List.of("[Content_Types].xml", "ppt/presentation.xml", "fixture.txt")) {
+ zip.putNextEntry(new ZipEntry(name));
+ zip.write((name.endsWith(".txt") ? eicar : "").getBytes(StandardCharsets.UTF_8)); zip.closeEntry();
+ }
+ }
+ long before = service.usage(COURSE, TEACHER).reservedBytes();
+ var resource = service.uploadCourseResource(COURSE, TEACHER, "Scanner fixture", true,
+ new MockMultipartFile("file", "fixture.pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation", bytes.toByteArray()), UUID.randomUUID(), null);
+ worker.runOnce(); assertThat(lifecycle.find(resource.id()).orElseThrow().publicStatus()).isEqualTo("REJECTED");
+ assertThatThrownBy(() -> service.downloadCourseResource(resource.id(), LEARNER)).isInstanceOf(org.springframework.web.server.ResponseStatusException.class);
+ worker.runOnce(); assertThat(service.usage(COURSE, TEACHER).reservedBytes()).isEqualTo(before);
+ assertThatThrownBy(() -> storage.open(resource.storageKey())).isInstanceOf(java.io.IOException.class);
+ }
+ @Test @Order(3) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "false")
+ void conditionalWritesCannotReplaceObjectsAndAllAttemptsAreMetered() throws Exception {
+ String key = PrivateResourceStorage.PREFIX + UUID.randomUUID() + "/" + UUID.randomUUID() + "/" + UUID.randomUUID();
+ Path file = Path.of("target/s3-immutable.txt"); Files.write(file, CLEAN);
+ int normalBefore = jdbc.sql("SELECT foreground_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single();
+ int deletesBefore = jdbc.sql("SELECT maintenance_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single();
+ storage.put(key, file, UploadValidator.checksum(file));
+ Files.writeString(file, "replacement");
+ assertThatThrownBy(() -> storage.put(key, file, UploadValidator.checksum(file))).isInstanceOf(java.io.IOException.class);
+ try (var object = storage.open(key)) { assertThat(object.readAllBytes()).isEqualTo(CLEAN); }
+ assertThat(storage.list(null).objects()).anyMatch(object -> object.key().equals(key));
+ storage.delete(key);
+ assertThatThrownBy(() -> storage.open(key)).isInstanceOf(java.io.IOException.class);
+ assertThat(jdbc.sql("SELECT foreground_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single() - normalBefore).isEqualTo(5);
+ assertThat(jdbc.sql("SELECT maintenance_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single() - deletesBefore).isEqualTo(1);
+ }
+ @Test @Order(4) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "true")
+ void databaseObjectStoreAndApplicationRestartPreserveIdentityAndContent() throws Exception {
+ var resource = uploadClean();
+ assertThat(resource.publicStatus()).isEqualTo("AVAILABLE");
+ try (var content = service.downloadCourseResource(resource.id(), LEARNER).content()) { assertThat(content.readAllBytes()).isEqualTo(CLEAN); }
+ service.deleteCourseResource(resource.id(), TEACHER); worker.runOnce();
+ assertThat(service.usage(COURSE, TEACHER).reservedBytes()).isZero();
+ assertThat(lifecycle.find(resource.id()).orElseThrow().state()).isEqualTo("DELETED");
+ }
+}
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/ResourceLifecycleTest.java b/backend/media-service/src/test/java/com/chanter/media/application/ResourceLifecycleTest.java
new file mode 100644
index 00000000..a4c1f81e
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/ResourceLifecycleTest.java
@@ -0,0 +1,93 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+
+import com.chanter.media.domain.CourseResource;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.UUID;
+import java.util.concurrent.CompletableFuture;
+import java.util.stream.IntStream;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+
+@SpringBootTest
+@ActiveProfiles("test")
+class ResourceLifecycleTest {
+ @Autowired ResourceLifecycle lifecycle;
+ @Autowired org.springframework.jdbc.core.simple.JdbcClient jdbc;
+
+ @org.junit.jupiter.api.BeforeEach
+ void reset() {
+ jdbc.sql("DELETE FROM course_resources").update();
+ jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=0, foreground_requests=0, maintenance_requests=0").update();
+ }
+
+ private CourseResource candidate(UUID key) {
+ UUID id = UUID.randomUUID();
+ return new CourseResource(id, UUID.randomUUID(), "Notes", "notes.txt", "text/plain", 10,
+ "resources/v1/" + UUID.randomUUID() + "/" + id + "/" + UUID.randomUUID(), false,
+ UUID.randomUUID(), Instant.now(), "STAGING", "a".repeat(64), key, "local");
+ }
+
+ @Test
+ void concurrentIdempotentReservationChargesBytesOnlyOnceAndRejectsConflictingPayload() {
+ var resource = candidate(UUID.randomUUID());
+ var results = IntStream.range(0, 8).mapToObj(i -> CompletableFuture.supplyAsync(() -> lifecycle.reserve(resource))).toList();
+ assertThat(results.stream().map(CompletableFuture::join)).allMatch(r -> r.id().equals(resource.id()));
+ assertThat(lifecycle.courseUsage(resource.courseId()).reservedBytes()).isEqualTo(10);
+ var conflict = new CourseResource(UUID.randomUUID(), resource.courseId(), "Different", resource.fileName(), resource.contentType(),
+ resource.byteSize(), resource.storageKey(), false, resource.uploadedByUserId(), resource.createdAt(), "STAGING",
+ resource.sha256(), resource.idempotencyKey(), "local");
+ assertThatThrownBy(() -> lifecycle.reserve(conflict)).isInstanceOf(org.springframework.web.server.ResponseStatusException.class);
+ }
+
+ @Test
+ void deletedResourceCannotBePublishedByAnInFlightScannerAndQuotaReleasesOnlyAfterConfirmedDelete() {
+ var resource = lifecycle.reserve(candidate(UUID.randomUUID()));
+ lifecycle.quarantine(resource.id());
+ var job = lifecycle.claim(false).orElseThrow();
+ lifecycle.requestDelete(resource.id());
+ assertThat(lifecycle.finishScan(resource.id(), job.leaseId(), "AVAILABLE")).isFalse();
+ assertThat(lifecycle.courseUsage(resource.courseId()).reservedBytes()).isEqualTo(10);
+ var deletion = lifecycle.claim(false).orElseThrow();
+ lifecycle.finishDelete(resource.id(), deletion.leaseId());
+ lifecycle.finishDelete(resource.id(), deletion.leaseId());
+ assertThat(lifecycle.courseUsage(resource.courseId()).reservedBytes()).isZero();
+ }
+
+ @Test
+ void concurrentRequestsNeverExceedBudgetAndCleanupKeepsItsReserve() {
+ jdbc.sql("UPDATE media_storage_budget SET request_month=:month,foreground_requests=35999")
+ .param("month", java.time.YearMonth.now(java.time.ZoneOffset.UTC).toString()).update();
+ var accepted = new java.util.concurrent.atomic.AtomicInteger();
+ var calls = IntStream.range(0, 8).mapToObj(i -> CompletableFuture.runAsync(() -> {
+ try { lifecycle.countRequest(false); accepted.incrementAndGet(); }
+ catch (org.springframework.web.server.ResponseStatusException limited) { assertThat(limited.getStatusCode().value()).isEqualTo(503); }
+ })).toList();
+ calls.forEach(CompletableFuture::join);
+ assertThat(accepted.get()).isEqualTo(1);
+ lifecycle.countRequest(true);
+ assertThat(jdbc.sql("SELECT maintenance_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single()).isEqualTo(1);
+ jdbc.sql("UPDATE media_storage_budget SET maintenance_requests=4000").update();
+ assertThatThrownBy(() -> lifecycle.countRequest(true)).isInstanceOf(org.springframework.web.server.ResponseStatusException.class);
+ }
+
+ @Test
+ void staleLeaseCannotPublishAndCleanedFailureIsNotRepeatedlyClaimed() {
+ var resource = lifecycle.reserve(candidate(UUID.randomUUID())); lifecycle.quarantine(resource.id());
+ var old = lifecycle.claim(false).orElseThrow();
+ jdbc.sql("UPDATE course_resources SET lease_until=TIMESTAMP WITH TIME ZONE '2000-01-01 00:00:00Z'").update();
+ var current = lifecycle.claim(false).orElseThrow();
+ assertThat(lifecycle.finishScan(resource.id(), old.leaseId(), "AVAILABLE")).isFalse();
+ lifecycle.finishScan(resource.id(), current.leaseId(), "SCAN_FAILED");
+ jdbc.sql("UPDATE course_resources SET attempts=5,retry_at=NULL,updated_at=TIMESTAMP WITH TIME ZONE '2000-01-01 00:00:00Z'").update();
+ var cleanup = lifecycle.claim(false).orElseThrow(); assertThat(cleanup.operation()).isEqualTo("DELETE");
+ lifecycle.finishDelete(resource.id(), cleanup.leaseId());
+ jdbc.sql("UPDATE course_resources SET updated_at=TIMESTAMP WITH TIME ZONE '2000-01-01 00:00:00Z'").update();
+ assertThat(lifecycle.claim(false)).isEmpty();
+ assertThat(lifecycle.reserve(resource).publicStatus()).isEqualTo("FAILED");
+ }
+}
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java b/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
new file mode 100644
index 00000000..228ca20c
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
@@ -0,0 +1,154 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+
+import com.chanter.media.domain.CourseResource;
+import com.chanter.media.infra.TestCourseResourceAccessClient;
+import com.chanter.media.infra.TestResourceIngestionClient;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.UUID;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.jdbc.core.simple.JdbcClient;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.bean.override.mockito.MockitoBean;
+import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
+import org.springframework.web.server.ResponseStatusException;
+
+@SpringBootTest
+@ActiveProfiles("test")
+class ResourceWorkerSafetyTest {
+ @Autowired CourseResourceService service;
+ @Autowired ResourceWorker worker;
+ @Autowired ResourceLifecycle lifecycle;
+ @Autowired JdbcClient jdbc;
+ @Autowired TestCourseResourceAccessClient access;
+ @Autowired TestResourceIngestionClient ingestion;
+ @Autowired LocalCourseResourceStorage legacy;
+ @Autowired UploadValidator validator;
+ @MockitoBean MalwareScanner scanner;
+ @MockitoSpyBean PrivateResourceStorage storage;
+ UUID course, teacher, learner;
+
+ @BeforeEach void resetData() throws Exception {
+ jdbc.sql("DELETE FROM course_resources").update();
+ jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=0,foreground_requests=0,maintenance_requests=0").update();
+ access.clear(); ingestion.clear();
+ course = UUID.randomUUID(); teacher = UUID.randomUUID(); learner = UUID.randomUUID();
+ access.grantInstructorUpload(course, teacher); access.grantLearnerView(course, learner);
+ doReturn(MalwareScanner.Verdict.CLEAN).when(scanner).scan(any());
+ }
+ private CourseResource upload(UUID key) {
+ return service.uploadCourseResource(course, teacher, "Notes", true,
+ new MockMultipartFile("file", "notes.txt", "text/plain", "bounded notes".getBytes(java.nio.charset.StandardCharsets.UTF_8)), key, null);
+ }
+ private void notAvailable(UUID id, int status) {
+ assertThatThrownBy(() -> service.downloadCourseResource(id, learner)).isInstanceOfSatisfying(ResponseStatusException.class,
+ failure -> assertThat(failure.getStatusCode().value()).isEqualTo(status));
+ }
+
+ @Test void infectedFileNeverReachesLearnersOrAiAndRetryKeepsItsIdentityAfterCleanup() throws Exception {
+ when(scanner.scan(any())).thenReturn(MalwareScanner.Verdict.INFECTED);
+ UUID key = UUID.randomUUID(); var resource = upload(key); worker.runOnce();
+ assertThat(service.getCourseResource(resource.id(), teacher).publicStatus()).isEqualTo("REJECTED");
+ assertThat(service.listCourseResources(course, learner)).isEmpty(); notAvailable(resource.id(), 409);
+ assertThat(ingestion.ingestCalls()).isEmpty();
+ assertThat(upload(key).id()).isEqualTo(resource.id());
+ assertThat(service.usage(course, teacher).reservedBytes()).isEqualTo(resource.byteSize());
+ worker.runOnce();
+ assertThat(service.usage(course, teacher).reservedBytes()).isZero();
+ assertThat(upload(key).publicStatus()).isEqualTo("REJECTED");
+ assertThatThrownBy(() -> storage.open(resource.storageKey())).isInstanceOf(IOException.class);
+ }
+
+ @Test void unavailableScannerFailsClosedThenRetriesWithoutReupload() throws Exception {
+ when(scanner.scan(any())).thenThrow(new IOException("unavailable"));
+ UUID key = UUID.randomUUID(); var resource = upload(key); worker.runOnce();
+ assertThat(upload(key).publicStatus()).isEqualTo("FAILED"); notAvailable(resource.id(), 409);
+ assertThat(ingestion.ingestCalls()).isEmpty(); assertThat(service.listCourseResources(course, learner)).isEmpty();
+ doReturn(MalwareScanner.Verdict.CLEAN).when(scanner).scan(any());
+ jdbc.sql("UPDATE course_resources SET retry_at=NULL").update(); worker.runOnce();
+ assertThat(service.getCourseResource(resource.id(), learner).publicStatus()).isEqualTo("AVAILABLE");
+ verify(storage, times(1)).put(eq(resource.storageKey()), any(), anyString());
+ }
+
+ @Test void uncertainPutAndFailedDeleteKeepQuotaAndNeverExposeBytes() throws Exception {
+ doAnswer(call -> { call.callRealMethod(); throw new IOException("response interrupted after write"); }).when(storage).put(anyString(), any(), anyString());
+ UUID key = UUID.randomUUID(); var resource = upload(key);
+ assertThat(resource.publicStatus()).isEqualTo("FAILED"); notAvailable(resource.id(), 404);
+ assertThat(upload(key).id()).isEqualTo(resource.id());
+ doThrow(new IOException("unavailable")).when(storage).delete(resource.storageKey()); worker.runOnce();
+ assertThat(service.usage(course, teacher).reservedBytes()).isEqualTo(resource.byteSize());
+ doCallRealMethod().when(storage).delete(resource.storageKey());
+ jdbc.sql("UPDATE course_resources SET retry_at=NULL").update(); worker.runOnce();
+ assertThat(service.usage(course, teacher).reservedBytes()).isZero();
+ assertThat(upload(key).id()).isEqualTo(resource.id()); verify(storage, times(1)).put(anyString(), any(), anyString());
+ }
+
+ @Test void corruptDownloadFailsBeforeReturningContentAndAuthorizationDoesNotReadStorage() throws Exception {
+ var resource = upload(UUID.randomUUID()); worker.runOnce(); clearInvocations(storage);
+ assertThatThrownBy(() -> service.downloadCourseResource(resource.id(), UUID.randomUUID())).isInstanceOf(ResponseStatusException.class);
+ verify(storage, never()).open(anyString());
+ doReturn(new java.io.ByteArrayInputStream("tampered bytes".getBytes())).when(storage).open(resource.storageKey());
+ notAvailable(resource.id(), 503);
+ try (var spools = Files.list(Path.of("target/media-spool-test"))) {
+ assertThat(spools.filter(p -> p.getFileName().toString().startsWith("download-")).count()).isZero();
+ }
+ }
+
+ @Test void deletionDuringObjectReadWinsAndRepeatedDeleteIsSafe() throws Exception {
+ var resource = upload(UUID.randomUUID()); worker.runOnce();
+ doAnswer(call -> {
+ var input = (java.io.InputStream) call.callRealMethod();
+ service.deleteCourseResource(resource.id(), teacher);
+ return input;
+ }).when(storage).open(resource.storageKey());
+ notAvailable(resource.id(), 404);
+ service.deleteCourseResource(resource.id(), teacher); worker.runOnce(); service.deleteCourseResource(resource.id(), teacher);
+ assertThat(service.usage(course, teacher).reservedBytes()).isZero();
+ assertThat(ingestion.deleteCalls()).containsExactly(resource.id());
+ }
+
+ @Test void legacyImportPreservesOriginalAndResumesAnUncertainImmutablePutBeforeScanning() throws Exception {
+ UUID id = UUID.randomUUID(); byte[] content = "legacy notes".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ Path original = Path.of("target/media-test", id.toString()); Files.createDirectories(original.getParent()); Files.write(original, content);
+ jdbc.sql("""
+ INSERT INTO course_resources (id,course_id,title,file_name,content_type,byte_size,storage_key,ai_approved,uploaded_by_user_id,created_at,updated_at)
+ VALUES (:id,:course,'Legacy','legacy.txt','text/plain',:size,:key,TRUE,:user,:now,:now)
+ """).param("id", id).param("course", course).param("size", content.length).param("key", id.toString()).param("user", teacher)
+ .param("now", java.time.OffsetDateTime.now(java.time.ZoneOffset.UTC)).update();
+ jdbc.sql("UPDATE media_storage_budget SET reserved_bytes=:size").param("size", content.length).update();
+ notAvailable(id, 409);
+ doAnswer(call -> { call.callRealMethod(); throw new IOException("lost response"); }).when(storage).put(anyString(), any(), anyString());
+ var importer = new ResourceWorker(lifecycle, storage, legacy, validator, scanner, ingestion, Clock.systemUTC(), false, true);
+ importer.runOnce();
+ assertThat(lifecycle.find(id).orElseThrow().state()).isEqualTo("QUARANTINED");
+ assertThat(ingestion.deleteCalls()).containsExactly(id);
+ assertThat(ingestion.ingestCalls()).isEmpty();
+ assertThat(Files.readAllBytes(original)).isEqualTo(content); notAvailable(id, 409);
+ importer.runOnce();
+ try (var download = service.downloadCourseResource(id, learner).content()) { assertThat(download.readAllBytes()).isEqualTo(content); }
+ assertThat(service.usage(course, teacher).reservedBytes()).isEqualTo(content.length);
+ }
+
+ @Test void reconcilerDeletesOnlyOldUnreferencedKeysAndKeepsReservedQuarantine() throws Exception {
+ var resource = upload(UUID.randomUUID());
+ String orphan = PrivateResourceStorage.PREFIX + course + "/" + UUID.randomUUID() + "/" + UUID.randomUUID();
+ doReturn(new PrivateResourceStorage.Page(java.util.List.of(
+ new PrivateResourceStorage.ObjectInfo(resource.storageKey(), Instant.now().minusSeconds(90000)),
+ new PrivateResourceStorage.ObjectInfo(orphan, Instant.now().minusSeconds(90000))), null)).when(storage).list(null);
+ var reconciler = new ResourceWorker(lifecycle, storage, legacy, validator, scanner, ingestion, Clock.systemUTC(), true, false);
+ reconciler.reconcile();
+ verify(storage).delete(orphan); verify(storage, never()).delete(resource.storageKey());
+ assertThat(service.usage(course, teacher).reservedBytes()).isEqualTo(resource.byteSize());
+ }
+}
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/UploadValidatorTest.java b/backend/media-service/src/test/java/com/chanter/media/application/UploadValidatorTest.java
new file mode 100644
index 00000000..c9a014e0
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/UploadValidatorTest.java
@@ -0,0 +1,59 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import org.junit.jupiter.api.Test;
+import org.springframework.mock.web.MockMultipartFile;
+import org.springframework.web.server.ResponseStatusException;
+
+class UploadValidatorTest {
+ @org.junit.jupiter.api.Test
+ void failedDownloadSpoolCreationClosesTheProviderStream() throws Exception {
+ Path directory = Files.createTempDirectory("media-spool-closed-");
+ var validator = new UploadValidator(directory.toString(), 1024);
+ Files.delete(directory);
+ var closed = new java.util.concurrent.atomic.AtomicBoolean();
+ var source = new java.io.ByteArrayInputStream(new byte[] {1}) {
+ @Override public void close() { closed.set(true); }
+ };
+ assertThatThrownBy(() -> validator.verifiedDownload(source, 1, "a".repeat(64))).isInstanceOf(java.io.IOException.class);
+ assertThat(closed.get()).isTrue();
+ }
+ private UploadValidator validator() throws Exception {
+ return new UploadValidator("target/upload-validator-tests", 1024);
+ }
+
+ @Test
+ void validatesBytesNormalizesFilenameAndComputesChecksum() throws Exception {
+ var bytes = "course notes".getBytes(StandardCharsets.UTF_8);
+ try (var upload = validator().validate(new MockMultipartFile("file", "../no\r\ntes.txt", "text/plain", bytes), null)) {
+ assertThat(upload.fileName()).isEqualTo("notes.txt");
+ assertThat(upload.contentType()).isEqualTo("text/plain");
+ assertThat(upload.sha256()).hasSize(64);
+ assertThat(Files.readAllBytes(upload.path())).isEqualTo(bytes);
+ }
+ }
+
+ @Test
+ void rejectsDeclaredPdfWhenBytesAreExecutableAndRejectsChecksumMismatch() throws Exception {
+ assertThatThrownBy(() -> validator().validate(new MockMultipartFile("file", "notes.pdf", "application/pdf",
+ new byte[] {'M', 'Z', 0, 0, 1}), null)).isInstanceOf(ResponseStatusException.class);
+ assertThatThrownBy(() -> validator().validate(new MockMultipartFile("file", "notes.txt", "text/plain",
+ "notes".getBytes(StandardCharsets.UTF_8)), "0".repeat(64))).isInstanceOf(ResponseStatusException.class);
+ }
+
+ @Test
+ void enforcesActualStreamLimitEvenWhenMultipartSizeLies() throws Exception {
+ var file = new MockMultipartFile("file", "notes.txt", "text/plain", new byte[] {1}) {
+ @Override public long getSize() { return 1; }
+ @Override public InputStream getInputStream() { return new ByteArrayInputStream(new byte[2048]); }
+ };
+ assertThatThrownBy(() -> validator().validate(file, null)).isInstanceOf(ResponseStatusException.class)
+ .satisfies(error -> assertThat(((ResponseStatusException) error).getStatusCode().value()).isEqualTo(413));
+ }
+}
diff --git a/backend/media-service/src/test/resources/application-test.yml b/backend/media-service/src/test/resources/application-test.yml
index ec73a43e..93412c8f 100644
--- a/backend/media-service/src/test/resources/application-test.yml
+++ b/backend/media-service/src/test/resources/application-test.yml
@@ -13,4 +13,7 @@ chanter:
access-token-ttl: 15m
internal-service-token: test-internal-service-token-for-media
media:
- storage-dir: ${java.io.tmpdir}/chanter-media-test
+ storage-dir: target/media-test
+ spool-dir: target/media-spool-test
+ storage-backend: local
+ worker-enabled: false
diff --git a/docs/engineering/records/architecture-review-chanter-private-resources.md b/docs/engineering/records/architecture-review-chanter-private-resources.md
new file mode 100644
index 00000000..fb98dbff
--- /dev/null
+++ b/docs/engineering/records/architecture-review-chanter-private-resources.md
@@ -0,0 +1,69 @@
+---
+schemaVersion: 1
+id: architecture-review-chanter-private-resources
+revision: 1
+type: architecture-review
+status: proposed
+title: Private Course Resource lifecycle and free storage boundary
+repository: chanter
+capabilityIds: ["private-course-resources"]
+createdAt: 2026-09-12
+reconstructed: false
+confidence: high
+unknowns: ["Unprovisioned OCI credentials and private bucket policy", "Full 2 OCPU and 12 GB workload capacity with ClamAV", "Provider recovery and browser processing-state acceptance"]
+modules: ["media-service"]
+interfaces: ["course-resource-api", "private-object-storage", "malware-scanner", "resource-chunk-ingestion"]
+seams: ["object-write-and-database-reservation", "scan-and-delete-concurrency", "deployment-schema-epoch"]
+adapters: ["s3-compatible-object-storage", "local-private-files", "clamav-instream", "postgresql"]
+relatedRecords: []
+decisions: []
+incidents: []
+features: []
+capabilities: ["Private resource storage", "Fail-closed malware scanning", "Conservative free-tier accounting"]
+amends: []
+supersedes: []
+learningRefs: []
+sources: [{"label":"Chanter issue #244","url":"https://github.com/Vinosaamaa/chanter/issues/244","kind":"issue"},{"label":"Oracle Always Free limits","url":"https://docs.oracle.com/en-us/iaas/Content/FreeTier/freetier_topic-Always_Free_Resources.htm","kind":"documentation"},{"label":"ClamAV Docker guidance","url":"https://docs.clamav.net/manual/Installing/Docker.html","kind":"documentation"}]
+verification: {"state":"verified","evidenceRefs":["issue:244", "backend/media-service/src/test/java/com/chanter/media/application", "docs/operations/issue-244-change-log.md"]}
+visibility: public-safe
+publicationEligibility: eligible
+issue: 244
+pr: null
+release: null
+run: null
+---
+# Private Course Resource lifecycle and free storage boundary
+
+## Context and decision
+
+The previous upload path wrote local bytes and exposed metadata immediately. There was no durable quarantine, real malware gate, failed-write reconciliation or object budget. Moving the same behavior to an object-store URL would preserve those flaws and bypass course authorization on downloads.
+
+The media module now owns a small private-storage interface and a transactional lifecycle repository. Object bytes stay behind the module. Upload reserves bytes and an immutable key before making a network call; a clean scan is required before publication. Each external operation occurs outside database transactions. Completion compares a durable lease identifier and current state so a deletion or newer worker cannot be overwritten by a stale scanner.
+
+The application has four public statuses, while its internal states distinguish interrupted writes, quarantine, active scan, failure and pending cleanup. This distinction is required for recovery, but object keys and lease states never appear in the API. Idempotency is scoped to uploader plus UUID and binds the complete validated payload. A rejected or deleted retry returns the original identity instead of silently creating a new object.
+
+## Alternatives
+
+- Public or signed object links would move the authorization boundary away from current course permissions. Verified application downloads avoid that change and are practical at the 10 MiB limit.
+- Scanning synchronously inside upload would make scanner downtime an HTTP timeout without durable recovery. A database-backed worker keeps files quarantined and retries safely.
+- A new queue or workflow service would add another durable infrastructure dependency. PostgreSQL leases are enough for this small launch tier.
+- R2 as the mandatory provider would conflict with the owner's no-charge instruction because usage above its allowance is billed. A configurable S3 adapter with an unupgraded Oracle Always Free account is the planned path; no provider is provisioned by this PR.
+- Adding a 4 GiB scanner to the prior 7.625 GiB runtime caps would leave insufficient OS memory on the chosen 12 GB VM. Capacity remains a measured release gate, with no paid fallback.
+
+## Failure and privacy review
+
+Declared length, filename extension and MIME are checked against bounded actual bytes. SHA-256 persists with metadata, and each download spools one object read and verifies the size and hash before the response can contain file content. Quarantined, unauthorized, corrupt, rejected and deleted objects are not downloadable. Private spools are bounded, closed on failure, and removed after use or age-based recovery.
+
+ClamAV must supply a clean verdict using fresh definitions. Unavailable, malformed or stale responses fail closed. A regression exposed a socket timeout gap: read timeouts do not cover a blocked upload write. A separate socket deadline now bounds both. Ingestion occurs only after clean scan; deletion retries object and AI-chunk cleanup before releasing reserved bytes.
+
+The database reserves at most 8 GB of resource bytes. It commits every S3 attempt before network I/O and disables SDK retries. Normal uploads, reads and listings share 36,000 monthly operations; 4,000 operations remain reserved for deletion. Uncertain calls never refund operation counts. These limits control this module, not other provider clients. Account backup space, inventory reconciliation after restore, and provider limits remain operator responsibilities.
+
+## Migration and release boundary
+
+V2 quarantines old metadata and preserves its storage reservation. Opt-in migration validates legacy files, persists the destination key before copying, confirms interrupted writes and scans the copy. Original files stay available to recovery operators. Legacy AI chunks must be cleared during maintenance because they predate the new scanning guarantee.
+
+Older code ignores the new states and cannot be a safe rollback target. Deployment must advance the schema epoch when adopting V2. Full recovery requires a matching metadata/file snapshot, conservative operation counters, and an inventory review before workers or uploads resume.
+
+## Evidence limits
+
+Local tests exercise the real lifecycle SQL using H2, real filesystem bytes, scanner socket protocol, concurrency and failure injection. A separate native AMD64/ARM64 workflow runs real PostgreSQL, S3Mock and ClamAV, tests EICAR rejection and restarts processes with preserved volumes. S3Mock does not establish OCI IAM behavior. Actual private-provider access, constrained full-stack workload and user-facing browser acceptance remain required before deployment completion.
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
new file mode 100644
index 00000000..85c6ec3f
--- /dev/null
+++ b/docs/operations/issue-244-change-log.md
@@ -0,0 +1,28 @@
+# Issue #244: private Course Resource storage
+
+Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: media module, isolated storage/scanner integration, migration and recovery documentation. One final issue-linked PR; no external resources provisioned. #243 owns the release package and remains a deployment dependency.
+
+## Changes
+
+- Added immutable private S3 and explicit local adapters. Production selects S3 without fallback; OCI Always Free is the planned zero-cost provider, with compatible alternate endpoints configurable.
+- Added actual byte/type/name/checksum validation, bounded spool files and verified attachment downloads under existing course authorization.
+- Added Flyway V2 lifecycle state, byte reservation, scoped upload idempotency, durable worker leases, fail-closed ClamAV scanning, delayed cleanup and orphan reconciliation.
+- Upload returns 202 with public `status` and `sha256`. Added metadata polling, instructor deletion and course usage routes. Storage keys, endpoints and scanner details stay private.
+- Added monthly attempted-operation accounting with dedicated deletion headroom. Uncertain PUT responses preserve reservations until cleanup confirms deletion.
+- AI ingestion waits for a clean scan; ingestion and deletion failures are retried by the durable worker instead of being swallowed.
+- Added native architecture CI using pinned PostgreSQL16.15, S3Mock5.2.2 and ClamAV1.5.4, actual EICAR scanning and restart durability checks.
+
+## Test evidence
+
+- Initial validation/lifecycle/scanner tests were written before their implementations and failed at compilation. The focused suite passed after implementation.
+- The blocked scanner-write regression failed after approximately 2.54 seconds against a 1-second assertion; adding the socket deadline made it pass.
+- The provider-stream closure regression failed when spool creation was unavailable; closing the provider stream around spool creation made it pass.
+- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
+- Java21 `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify` passed: 31 media tests, with four separately gated real-process integration cases skipped locally. The final legacy-index purge received an additional focused worker regression run. Exact hosted results are recorded in PR checks.
+- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
+
+## Remaining release proof
+
+The provider is unprovisioned. Native container checks must pass at the final head, followed by #243 integration, actual private bucket permissions/anonymous-denial and recovery tests, processing/failure UI browser evidence, and a measured 2 OCPU/12 GB full-stack workload. ClamAV's 4 GB container guidance cannot be added on top of the earlier 7.625 GiB base caps without reallocation. Schema V2 requires a deployment epoch boundary; old code must not be rolled back onto the new lifecycle data.
+
+See [the operator runbook](private-course-resources.md) and [the system review](../engineering/records/architecture-review-chanter-private-resources.md).
diff --git a/docs/operations/private-course-resources.md b/docs/operations/private-course-resources.md
new file mode 100644
index 00000000..32c4aeec
--- /dev/null
+++ b/docs/operations/private-course-resources.md
@@ -0,0 +1,82 @@
+# Private Course Resource operations
+
+Issue [#244](https://github.com/Vinosaamaa/chanter/issues/244) adds durable object metadata, quarantine and scanning inside `media-service`. Deployment packaging and real account/HTTPS proof depend on #243. No provider account or bucket has been provisioned. The production release must explicitly select S3 storage; local storage is for development and migration.
+
+## Provider and free limits
+
+Use one **private Standard OCI Object Storage bucket** in the same unupgraded Always Free account, with a dedicated S3-compatible customer secret key and a policy limited to that bucket. Disable public access, versioning and replication; do not add lifecycle rules that create paid storage classes. Never upgrade, subscribe to a paid fallback, enable recharge, or enter a payment flow as a workaround.
+
+[Oracle's current Always Free documentation](https://docs.oracle.com/en-us/iaas/Content/FreeTier/freetier_topic-Always_Free_Resources.htm) lists 20 GB of combined object storage and 50,000 API requests per month for free-only accounts; the trial's Standard allocation is 10 GB. This module therefore reserves at most **8,000,000,000 resource bytes** and **40,000 object-operation attempts per UTC month**, leaving separate account space and request headroom for backups and operators. Do not allocate more than 2 GB of backup data during the initial 10 GB trial allocation. Verify the actual account console before creating resources. Application limits do not control other clients or provider policy changes.
+
+The configurable adapter uses [Oracle's S3 compatibility API](https://docs.oracle.com/en-us/iaas/Content/Object/Tasks/s3compatibleapi.htm). R2-compatible endpoints remain possible, but [R2 bills usage above its included allowance](https://developers.cloudflare.com/r2/pricing/) and is not the default zero-cost path. There is no automatic provider fallback.
+
+## Runtime configuration
+
+Set these in the reviewed release's secret environment file, readable only by its operator and runtime. Do not put actual endpoints, access keys, bucket names or credentials in commits, tickets or logs.
+
+| Variable | Production value or rule |
+| --- | --- |
+| `CHANTER_MEDIA_STORAGE_BACKEND` | `s3`; startup fails if required S3 configuration is absent |
+| `CHANTER_S3_ENDPOINT` | Account-specific HTTPS S3 endpoint; no path, query or credentials in URL |
+| `CHANTER_S3_REGION` | Actual bucket region |
+| `CHANTER_S3_BUCKET` | Private resource bucket; provision separately |
+| `CHANTER_S3_ACCESS_KEY`, `CHANTER_S3_SECRET_KEY` | Dedicated least-privilege customer secret key |
+| `CHANTER_MEDIA_BYTE_LIMIT` | At most `8000000000`; lower it if account capacity is smaller |
+| `CHANTER_MEDIA_REQUEST_LIMIT` | At most `40000`; counts every attempted PUT, GET, LIST and DELETE |
+| `CHANTER_MEDIA_CLEANUP_REQUEST_RESERVE` | `4000`; only DELETE uses this protected allowance |
+| `CHANTER_MEDIA_SPOOL_DIR` | Private writable directory; bounded upload/download files, no web mount |
+| `COURSE_RESOURCE_STORAGE_DIR` | Existing local resource directory, retained during migration |
+| `CHANTER_CLAMAV_HOST`, `CHANTER_CLAMAV_PORT` | Private scanner listener, never publicly exposed; port `3310` |
+| `CHANTER_MEDIA_WORKER_ENABLED` | `true`; set `false` during recovery before reconciling backups |
+| `CHANTER_MEDIA_MIGRATE_LEGACY` | `false` normally; enable only for the reviewed legacy import |
+
+The adapter uses path-style addressing, immutable conditional PUT, content MD5 on transport, persisted SHA-256, no SDK retries, and bounded network deadlines. Each attempted call is charged before network I/O in its own committed transaction. An uncertain call never refunds the request counter. Scan GETs and reconciliation LISTs share the normal 36,000-operation allowance; only confirmed object DELETE calls can use the remaining 4,000. Once the total allowance is exhausted, cleanup waits for the next UTC month and all affected files remain inaccessible. Budget reads and reservations serialize on one database row; this is intentional for the small launch tier.
+
+Public configuration never permits HTTP object storage. The integration test enables a separate property accepting HTTP only for literal loopback hosts. S3Mock is an emulator and cannot prove provider credentials, IAM policies, anonymous-access denial, region correctness or free-account limits.
+
+## Upload and download contract
+
+All routes require the existing access-token authorization and course permissions. Browser session issuance/refresh follows #242; downloads do not accept object URLs or a refresh cookie as authorization.
+
+| Request | Result |
+| --- | --- |
+| `POST /api/v1/courses/{courseId}/course-resources` | Existing multipart fields, HTTP 202, existing resource fields plus `status` and `sha256` |
+| Optional `Idempotency-Key` | UUID scoped to the uploader; identical course/title/name/type/AI approval/bytes returns the same resource and status; a changed payload returns 409 |
+| Optional `X-Content-SHA256` | 64 hexadecimal characters; a mismatch returns 400 before storage |
+| `GET /api/v1/courses/{courseId}/course-resources` | Instructors see pending and failed entries; learners see only `AVAILABLE` |
+| `GET /api/v1/course-resources/{id}` | Poll metadata; learners cannot inspect pending or rejected entries |
+| `GET /api/v1/course-resources/{id}/content` | Ownership-checked attachment, `nosniff`, `no-store`; 409 until available, 404 after deletion |
+| `DELETE /api/v1/course-resources/{id}` | Instructor-only, idempotent HTTP 204; access stops immediately, cleanup runs durably |
+| `GET /api/v1/courses/{courseId}/course-resources/usage` | Instructor-only course `reservedBytes` and `availableBytes` |
+
+Public statuses are `PROCESSING`, `AVAILABLE`, `REJECTED` and `FAILED`. No storage key, endpoint, internal lease, scanner signature or provider error is exposed. Reusing an idempotency key never restarts a rejected or deleted upload. To intentionally submit new work after a permanent failure, use a new key. Scanner or index failures automatically retry up to five times with a 60-second delay. Failed nonlegacy objects retain their bytes for at least 24 hours before cleanup; the failure record and original key remain durable. Infected files are rejected and queued for cleanup immediately.
+
+Accepted types are UTF-8 text/Markdown, PDF, PPTX, MP3, M4A, WAV, OGG, MP4, WebM and MOV, capped at 10 MiB of actual streamed bytes. Legacy binary `.ppt` must be converted to `.pptx`. The extension, declared MIME and detected content must agree. Filenames are normalized to safe basenames; titles and filenames have length/control-character checks. PDF trailers and bounded, nonmacro PPTX archive structure are checked before storage. These checks establish supported type, not perfect document validity; ClamAV supplies the malware gate.
+
+Each download fetches once to a private file of at most 10 MiB, verifies length and SHA-256, checks that deletion has not won during the read, then streams it as an attachment. Unauthorized, corrupt, pending, rejected and deleted requests return no object bytes. An already authorized download that passed the final state check may finish while a later deletion is accepted. At most two foreground transfers run per service process; busy callers receive 429. Worker and spool cleanup do not publish files.
+
+## Scanner and capacity
+
+Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. Scan uses its real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Socket deadlines bound both reads and writes. No local or production clean-verdict bypass exists.
+
+[ClamAV's container instructions](https://docs.clamav.net/manual/Installing/Docker.html) recommend **4 GB** of RAM. The earlier #243 base caps total 7.625 GiB and exclude scanning. Adding 4 GiB would leave insufficient room for the OS on a 12 GB VM. Reallocate and measure the whole stack before deployment; passing the isolated media integration suite does not prove the full launch capacity. No extra VM or paid scan service is authorized as a fallback.
+
+## Migration and recovery
+
+1. Put resource uploads and AI resource retrieval behind maintenance before changing storage. Take an encrypted database backup and preserve the original local resource directory. Clear legacy AI resource chunks using the existing authenticated internal deletion route before allowing retrieval again; old chunks predate the quarantine guarantee.
+2. Apply media Flyway V2 through the reviewed release migration command. Existing rows become `LEGACY`, retain their byte reservation and become unavailable for downloads. Confirm their sum fits the 8 GB resource allocation. Never rewrite V1 or mark legacy rows available manually.
+3. Configure the private S3 bucket and scanner, mount original files, and set `CHANTER_MEDIA_MIGRATE_LEGACY=true`. The worker persists a new immutable object key before uploading, validates actual legacy size/type, verifies an uncertain PUT using a single GET, and then quarantines the copy for scanning. Original local files remain intact. A concurrent delete also cleans any reserved migration key.
+4. Wait until expected legacy rows become `AVAILABLE` or investigate failures. Unsupported, missing or changed files remain unavailable and reserved; restore or correct their input under maintenance, then use an operator-reviewed retry. Compare downloaded hashes and learner/instructor permissions. Turn legacy migration off. Retain old files through the backup/rollback window; remove them only through a separately reviewed exact inventory.
+5. Rollback to a release predating V2 is **not compatible**: the old server ignores quarantine and expects local storage. Treat #244 as a schema-epoch boundary in #243. Restore database and local resources together behind maintenance, or fix forward with the current lifecycle model. A database migration alone is not a safe rollback.
+
+On database restore, stop uploads and workers first. Restore the metadata and counters together, then compare a private `resources/v1/` object inventory with reserved database rows. Preserve extra objects while deciding whether they are newer accepted writes or true orphans. Rebuild accurate byte reservations and monthly attempts conservatively before reopening writes; never reset an uncertain monthly request count to zero. The account's independent backup reserve must cover the recovery plan.
+
+Ordinary reconciliation runs hourly, processes at most ten pages per run and resumes its cursor. It deletes only module-prefixed objects older than 24 hours that have no active byte reservation. Database-backed interrupted writes and rejected/deleted objects have their own retry leases. Reconciliation never scans backup prefixes. Expired private spool files are removed after one hour. Do not run reconciliation against a partially restored database.
+
+## Verification and release proof
+
+Run local module tests with `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify`. The isolated real-process suite is `.github/workflows/media-security.yml`: pinned PostgreSQL, Adobe S3Mock and ClamAV images on native AMD64 and ARM64, real EICAR rejection, immutable writes, metered attempts, and preserved-volume process restart. EICAR is the harmless standard antivirus test fixture.
+
+For a local Docker host: `docker compose -f infra/media-security/compose.yml up -d`, run `python3 scripts/media/wait-dependencies.py`, then set `MEDIA_INTEGRATION=true` and `MEDIA_RESTART_PHASE=false` for `mvn ... test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false`. Restart those three services, set `MEDIA_RESTART_PHASE=true`, and rerun the same test. The fixture uses test-only credentials, loopback ports and independent named volumes. Remove that exact test stack with its Compose `down -v` command when finished.
+
+Keep #244 open until the merged #243 package has real private-provider IAM/anonymous-denial tests, upload/download/delete/recovery proof, a measured 2 OCPU/12 GB workload including the scanner, and browser evidence for processing and failure states. No provisioned provider or production proof is claimed by this repository change.
diff --git a/infra/media-security/compose.yml b/infra/media-security/compose.yml
new file mode 100644
index 00000000..52579e2c
--- /dev/null
+++ b/infra/media-security/compose.yml
@@ -0,0 +1,45 @@
+# Ephemeral CI/development dependencies only. S3Mock does not prove provider IAM isolation.
+services:
+ postgres:
+ image: postgres:16.15-alpine3.23@sha256:421b84e07a72bb8f3715f20501a1fdbe1219aad1fa4af7786a49d9a3f2480296
+ environment:
+ POSTGRES_USER: media_test
+ POSTGRES_PASSWORD: media-test-only-password
+ POSTGRES_DB: chanter_media
+ ports: ['127.0.0.1:5544:5432']
+ volumes: ['postgres-data:/var/lib/postgresql/data']
+ mem_limit: 256m
+ healthcheck:
+ test: ['CMD-SHELL', 'pg_isready -U media_test -d chanter_media']
+ interval: 5s
+ timeout: 3s
+ retries: 30
+ s3:
+ image: adobe/s3mock:5.2.2@sha256:e7c36014dcf4c7f0f6bec9de888477d1d9b8f55eceb7d9c1186f70d7aadf81ca
+ environment:
+ COM_ADOBE_TESTING_S3MOCK_STORE_ROOT: /s3mockroot
+ COM_ADOBE_TESTING_S3MOCK_STORE_RETAIN_FILES_ON_EXIT: 'true'
+ COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: private-media-test
+ JAVA_TOOL_OPTIONS: -Xmx320m
+ ports: ['127.0.0.1:9090:9090']
+ volumes: ['s3-data:/s3mockroot']
+ mem_limit: 512m
+ clamav:
+ image: clamav/clamav:1.5.4@sha256:1fdfd24c6f0a0fb60788481487459a6d4eda8a9b448641594e04db8410d34422
+ environment:
+ TZ: UTC
+ CLAMD_STARTUP_TIMEOUT: '600'
+ FRESHCLAM_CHECKS: '12'
+ ports: ['127.0.0.1:3310:3310']
+ volumes: ['clamav-signatures:/var/lib/clamav']
+ mem_limit: 4g
+ healthcheck:
+ test: ['CMD', 'clamdcheck.sh']
+ interval: 10s
+ timeout: 5s
+ retries: 60
+ start_period: 60s
+volumes:
+ postgres-data:
+ s3-data:
+ clamav-signatures:
diff --git a/scripts/media/wait-dependencies.py b/scripts/media/wait-dependencies.py
new file mode 100644
index 00000000..8ca74b9f
--- /dev/null
+++ b/scripts/media/wait-dependencies.py
@@ -0,0 +1,28 @@
+"""Bounded readiness check for the isolated test stack, including fresh ClamAV signatures."""
+import datetime
+import socket
+import time
+import urllib.request
+
+deadline = time.monotonic() + 600
+while time.monotonic() < deadline:
+ try:
+ with urllib.request.urlopen("http://127.0.0.1:9090/private-media-test", timeout=3) as response:
+ assert response.status == 200
+ with socket.create_connection(("127.0.0.1", 3310), timeout=3) as connection:
+ connection.sendall(b"zVERSION\0")
+ data = bytearray()
+ while not data.endswith(b"\0") and len(data) < 4096:
+ chunk = connection.recv(1024)
+ if not chunk:
+ raise ValueError("scanner closed its readiness response")
+ data.extend(chunk)
+ updated = datetime.datetime.strptime(data.decode("ascii").strip("\0").split("/", 2)[2], "%a %b %d %H:%M:%S %Y").replace(tzinfo=datetime.timezone.utc)
+ age = datetime.datetime.now(datetime.timezone.utc) - updated
+ assert datetime.timedelta(minutes=-5) <= age <= datetime.timedelta(hours=72)
+ print("S3 emulator and ClamAV are ready; scanner definitions are within 72 hours")
+ break
+ except (OSError, ValueError, IndexError, AssertionError):
+ time.sleep(5)
+else:
+ raise SystemExit("Private media dependencies did not become ready with fresh scanner definitions")
From 9123a3d488d83cb250370f542a396ec91ba2fc6a Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:30:50 -0700
Subject: [PATCH 02/10] docs: record private resource storage PR318 evidence
---
docs/engineering/changes/pr-318.md | 21 +++++++++++++++++++++
1 file changed, 21 insertions(+)
create mode 100644 docs/engineering/changes/pr-318.md
diff --git a/docs/engineering/changes/pr-318.md b/docs/engineering/changes/pr-318.md
new file mode 100644
index 00000000..2b90893e
--- /dev/null
+++ b/docs/engineering/changes/pr-318.md
@@ -0,0 +1,21 @@
+---
+schemaVersion: 1
+repository: chanter
+pr: 318
+title: "Quarantine private Course Resources and verify downloads"
+classification: architecture-review
+richRecordRefs: ["architecture-review-chanter-private-resources@1"]
+reconstructed: false
+confidence: high
+unknowns: ["Actual OCI private bucket policy and recovery", "Full constrained deployment and processing-state browser acceptance"]
+headCommit: null
+mergeCommit: null
+mergedAt: null
+sources: [{"label":"Pull request #318","url":"https://github.com/Vinosaamaa/chanter/pull/318","kind":"pull-request"}]
+verification: {"state":"verified","evidenceRefs":["pull-request:318"]}
+visibility: public-safe
+publicationEligibility: eligible
+---
+# Quarantine private Course Resources and verify downloads
+
+Added durable private storage, fail-closed malware scanning, verified course-authorized downloads, idempotent recovery and conservative free-tier accounting.
From b9bb718093b694e67ffed7c42cc6ca86314407e5 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:32:32 -0700
Subject: [PATCH 03/10] fix(ci): use maintained native ARM64 ClamAV image
---
docs/operations/issue-244-change-log.md | 1 +
infra/media-security/compose.yml | 2 +-
2 files changed, 2 insertions(+), 1 deletion(-)
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index 85c6ec3f..033af3e4 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -20,6 +20,7 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
- Java21 `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify` passed: 31 media tests, with four separately gated real-process integration cases skipped locally. The final legacy-index purge received an additional focused worker regression run. Exact hosted results are recorded in PR checks.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
+- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
## Remaining release proof
diff --git a/infra/media-security/compose.yml b/infra/media-security/compose.yml
index 52579e2c..ba63f06f 100644
--- a/infra/media-security/compose.yml
+++ b/infra/media-security/compose.yml
@@ -25,7 +25,7 @@ services:
volumes: ['s3-data:/s3mockroot']
mem_limit: 512m
clamav:
- image: clamav/clamav:1.5.4@sha256:1fdfd24c6f0a0fb60788481487459a6d4eda8a9b448641594e04db8410d34422
+ image: clamav/clamav-debian:1.5.4@sha256:4c975c439fcb7ab9cbdd72162c2802f5efcb4a11dcf5b0b37da1d746474ef20e
environment:
TZ: UTC
CLAMD_STARTUP_TIMEOUT: '600'
From b3e9da872fd3781a94c385e0da7da5162f05ae71 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:34:26 -0700
Subject: [PATCH 04/10] test(media): enforce resource management permissions
over HTTP
---
.github/workflows/media-security.yml | 4 +++
.../media/api/CourseResourceSmokeTest.java | 32 +++++++++++++++++++
docs/operations/issue-244-change-log.md | 2 +-
3 files changed, 37 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/media-security.yml b/.github/workflows/media-security.yml
index f2b5d7d7..e24ad865 100644
--- a/.github/workflows/media-security.yml
+++ b/.github/workflows/media-security.yml
@@ -11,6 +11,10 @@ on:
permissions:
contents: read
+concurrency:
+ group: private-media-${{ github.ref }}
+ cancel-in-progress: true
+
jobs:
actual-storage-and-scanner:
if: github.event.repository.private == false
diff --git a/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java b/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
index 4b658e56..160a0f94 100644
--- a/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
@@ -335,4 +335,36 @@ void uploadingToUnknownCourseReturnsNotFound() throws Exception {
.param("aiApproved", "true"))
.andExpect(status().isNotFound());
}
+
+ @Test
+ void metadataDoesNotExposeStorageDetailsAndOnlyInstructorCanDeleteOrReadUsage() throws Exception {
+ UUID course = UUID.randomUUID(), teacher = UUID.randomUUID(), learner = UUID.randomUUID();
+ courseResourceAccessClient.grantInstructorUpload(course, teacher);
+ courseResourceAccessClient.grantLearnerView(course, learner);
+ var uploaded = mockMvc.perform(multipart("/api/v1/courses/{course}/course-resources", course)
+ .file(new MockMultipartFile("file", "notes.txt", "text/plain", "safe notes".getBytes(StandardCharsets.UTF_8)))
+ .header(AuthHeaders.USER_ID, teacher.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)
+ .param("aiApproved", "false")).andExpect(status().isAccepted()).andReturn().getResponse();
+ var resource = objectMapper.readValue(uploaded.getContentAsString(), CourseResourceResponse.class);
+ assertThat(uploaded.getHeader("Location")).isEqualTo("/api/v1/course-resources/" + resource.id());
+ assertThat(uploaded.getContentAsString()).doesNotContain("storageKey", "storageBackend", "lease", "endpoint");
+ mockMvc.perform(get("/api/v1/course-resources/{id}", resource.id())
+ .header(AuthHeaders.USER_ID, learner.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isNotFound());
+ mockMvc.perform(get("/api/v1/courses/{course}/course-resources/usage", course)
+ .header(AuthHeaders.USER_ID, learner.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isForbidden());
+ mockMvc.perform(org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete("/api/v1/course-resources/{id}", resource.id())
+ .header(AuthHeaders.USER_ID, learner.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isForbidden());
+ worker.runOnce();
+ mockMvc.perform(org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete("/api/v1/course-resources/{id}", resource.id())
+ .header(AuthHeaders.USER_ID, teacher.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isNoContent());
+ mockMvc.perform(get("/api/v1/course-resources/{id}/content", resource.id())
+ .header(AuthHeaders.USER_ID, learner.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isNotFound());
+ var usage = mockMvc.perform(get("/api/v1/courses/{course}/course-resources/usage", course)
+ .header(AuthHeaders.USER_ID, teacher.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isOk()).andReturn().getResponse();
+ assertThat(objectMapper.readTree(usage.getContentAsString()).get("reservedBytes").asLong()).isEqualTo(resource.byteSize());
+ worker.runOnce();
+ var deletedUsage = mockMvc.perform(get("/api/v1/courses/{course}/course-resources/usage", course)
+ .header(AuthHeaders.USER_ID, teacher.toString()).header(AuthHeaders.INTERNAL_SERVICE_TOKEN, INTERNAL_TOKEN)).andExpect(status().isOk()).andReturn().getResponse();
+ assertThat(objectMapper.readTree(deletedUsage.getContentAsString()).get("reservedBytes").asLong()).isZero();
+ }
}
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index 033af3e4..e285813d 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -18,7 +18,7 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- The blocked scanner-write regression failed after approximately 2.54 seconds against a 1-second assertion; adding the socket deadline made it pass.
- The provider-stream closure regression failed when spool creation was unavailable; closing the provider stream around spool creation made it pass.
- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
-- Java21 `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify` passed: 31 media tests, with four separately gated real-process integration cases skipped locally. The final legacy-index purge received an additional focused worker regression run. Exact hosted results are recorded in PR checks.
+- Java21 `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify` passed: 31 media tests, with four separately gated real-process integration cases skipped locally. Additional focused runs cover legacy-index purge and HTTP delete/usage authorization, bringing the media suite to32 tests. Exact hosted results are recorded in PR checks.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
From 131fa214583f0197ac92b51b50b3b6f08529fec7 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:42:01 -0700
Subject: [PATCH 05/10] fix(media): enforce scanner inspection limits and
timely signature reloads
---
.../PrivateStorageIntegrationTest.java | 11 +++-
.../application/ResourceWorkerSafetyTest.java | 2 +-
.../application/S3AdapterPolicyTest.java | 50 +++++++++++++++++++
docs/operations/issue-244-change-log.md | 3 +-
docs/operations/private-course-resources.md | 2 +-
infra/media-security/clamd.conf | 26 ++++++++++
infra/media-security/compose.yml | 4 +-
scripts/media/wait-dependencies.py | 9 +++-
8 files changed, 101 insertions(+), 6 deletions(-)
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/S3AdapterPolicyTest.java
create mode 100644 infra/media-security/clamd.conf
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java b/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
index 3a5cb9e0..856e6efe 100644
--- a/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
@@ -91,6 +91,15 @@ void realClamAvRejectsEicarInsidePresentationAndDeleteReleasesReservation() thro
assertThatThrownBy(() -> storage.open(resource.storageKey())).isInstanceOf(java.io.IOException.class);
}
@Test @Order(3) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "false")
+ void realScannerRejectsNestedContentBeyondItsInspectionLimit() throws Exception {
+ Path compressed = Path.of("target/clamav-limit-fixture.gz");
+ try (var gzip = new java.util.zip.GZIPOutputStream(Files.newOutputStream(compressed))) {
+ byte[] block = new byte[1024 * 1024];
+ for (int index = 0; index < 60; index++) gzip.write(block);
+ }
+ assertThat(scanner.scan(compressed)).isEqualTo(MalwareScanner.Verdict.INFECTED);
+ }
+ @Test @Order(4) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "false")
void conditionalWritesCannotReplaceObjectsAndAllAttemptsAreMetered() throws Exception {
String key = PrivateResourceStorage.PREFIX + UUID.randomUUID() + "/" + UUID.randomUUID() + "/" + UUID.randomUUID();
Path file = Path.of("target/s3-immutable.txt"); Files.write(file, CLEAN);
@@ -106,7 +115,7 @@ void conditionalWritesCannotReplaceObjectsAndAllAttemptsAreMetered() throws Exce
assertThat(jdbc.sql("SELECT foreground_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single() - normalBefore).isEqualTo(5);
assertThat(jdbc.sql("SELECT maintenance_requests FROM media_storage_budget WHERE id=1").query(Integer.class).single() - deletesBefore).isEqualTo(1);
}
- @Test @Order(4) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "true")
+ @Test @Order(5) @EnabledIfEnvironmentVariable(named = "MEDIA_RESTART_PHASE", matches = "true")
void databaseObjectStoreAndApplicationRestartPreserveIdentityAndContent() throws Exception {
var resource = uploadClean();
assertThat(resource.publicStatus()).isEqualTo("AVAILABLE");
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java b/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
index 228ca20c..e84b70a8 100644
--- a/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
@@ -98,7 +98,7 @@ private void notAvailable(UUID id, int status) {
var resource = upload(UUID.randomUUID()); worker.runOnce(); clearInvocations(storage);
assertThatThrownBy(() -> service.downloadCourseResource(resource.id(), UUID.randomUUID())).isInstanceOf(ResponseStatusException.class);
verify(storage, never()).open(anyString());
- doReturn(new java.io.ByteArrayInputStream("tampered bytes".getBytes())).when(storage).open(resource.storageKey());
+ doReturn(new java.io.ByteArrayInputStream("changed notes".getBytes())).when(storage).open(resource.storageKey());
notAvailable(resource.id(), 503);
try (var spools = Files.list(Path.of("target/media-spool-test"))) {
assertThat(spools.filter(p -> p.getFileName().toString().startsWith("download-")).count()).isZero();
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/S3AdapterPolicyTest.java b/backend/media-service/src/test/java/com/chanter/media/application/S3AdapterPolicyTest.java
new file mode 100644
index 00000000..b4f75057
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/S3AdapterPolicyTest.java
@@ -0,0 +1,50 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.Mockito.*;
+import com.chanter.media.infra.S3PrivateResourceStorage;
+import com.sun.net.httpserver.HttpServer;
+import java.net.InetSocketAddress;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.UUID;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+class S3AdapterPolicyTest {
+ @TempDir Path directory;
+ @Test void serverFailuresMakeExactlyOneNetworkAttemptAndConsumeTheCorrectBudget() throws Exception {
+ var counter = new AtomicInteger(); var lifecycle = mock(ResourceLifecycle.class);
+ var server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ server.createContext("/", exchange -> {
+ counter.incrementAndGet(); exchange.getRequestBody().readAllBytes();
+ byte[] error = "InternalErrorprivate-provider-detail".getBytes(java.nio.charset.StandardCharsets.UTF_8);
+ exchange.getResponseHeaders().set("Content-Type", "application/xml"); exchange.sendResponseHeaders(500, error.length);
+ try (var response = exchange.getResponseBody()) { response.write(error); }
+ }); server.start();
+ var adapter = new S3PrivateResourceStorage(lifecycle, "http://127.0.0.1:" + server.getAddress().getPort(), "us-east-1", "fixture-bucket", "fixture-key", "fixture-secret", true);
+ try {
+ String key = PrivateResourceStorage.PREFIX + UUID.randomUUID() + "/" + UUID.randomUUID() + "/" + UUID.randomUUID();
+ Path content = directory.resolve("fixture.txt"); Files.writeString(content, "fixture");
+ assertThatThrownBy(() -> adapter.put(key, content, UploadValidator.checksum(content)))
+ .isInstanceOf(java.io.IOException.class).hasMessage("Private object storage is unavailable");
+ assertThatThrownBy(() -> adapter.open(key)).isInstanceOf(java.io.IOException.class);
+ assertThatThrownBy(() -> adapter.list(null)).isInstanceOf(java.io.IOException.class);
+ assertThatThrownBy(() -> adapter.delete(key)).isInstanceOf(java.io.IOException.class);
+ assertThat(counter.get()).isEqualTo(4);
+ verify(lifecycle, times(3)).countRequest(false); verify(lifecycle).countRequest(true);
+ doThrow(new org.springframework.web.server.ResponseStatusException(org.springframework.http.HttpStatus.SERVICE_UNAVAILABLE))
+ .when(lifecycle).countRequest(false);
+ assertThatThrownBy(() -> adapter.open(key)).isInstanceOf(org.springframework.web.server.ResponseStatusException.class);
+ assertThat(counter.get()).isEqualTo(4);
+ } finally { adapter.close(); server.stop(0); }
+ }
+ @Test void insecureOrCredentialBearingProviderUrlsAreRejectedBeforeNetworkUse() {
+ var lifecycle = mock(ResourceLifecycle.class);
+ for (String endpoint : java.util.List.of("http://example.com", "https://user:secret@example.com", "https://example.com/?key=secret")) {
+ assertThatThrownBy(() -> new S3PrivateResourceStorage(lifecycle, endpoint, "region", "fixture-bucket", "key", "secret", true))
+ .isInstanceOf(IllegalArgumentException.class).hasMessage("Invalid private S3 configuration");
+ }
+ }
+}
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index e285813d..72a9ba4e 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -18,9 +18,10 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- The blocked scanner-write regression failed after approximately 2.54 seconds against a 1-second assertion; adding the socket deadline made it pass.
- The provider-stream closure regression failed when spool creation was unavailable; closing the provider stream around spool creation made it pass.
- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
-- Java21 `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify` passed: 31 media tests, with four separately gated real-process integration cases skipped locally. Additional focused runs cover legacy-index purge and HTTP delete/usage authorization, bringing the media suite to32 tests. Exact hosted results are recorded in PR checks.
+- Java21 affected-module verification passed after the secure-session rebase. Focused additions cover legacy-index purge, HTTP delete/usage authorization and a real HTTP500 server proving one attempt per S3 operation, bringing the local media suite to34 tests. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
+- Real startup logs show fresh signatures downloaded before clamd's socket was available for notification. The explicit daemon configuration rechecks definitions every60 seconds, uses bounded scan limits, rejects encrypted/over-limit content and avoids duplicate engines during reload. The native suite tests a compressed fixture beyond the scan limit as well as EICAR.
## Remaining release proof
diff --git a/docs/operations/private-course-resources.md b/docs/operations/private-course-resources.md
index 32c4aeec..1034da8d 100644
--- a/docs/operations/private-course-resources.md
+++ b/docs/operations/private-course-resources.md
@@ -57,7 +57,7 @@ Each download fetches once to a private file of at most 10 MiB, verifies length
## Scanner and capacity
-Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. Scan uses its real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Socket deadlines bound both reads and writes. No local or production clean-verdict bypass exists.
+Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. Use the reviewed `infra/media-security/clamd.conf`: one-minute signature checks recover a startup notification race; encrypted or over-limit content produces a rejection; reloads block briefly instead of holding two engines. Scan uses the real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Socket deadlines bound both reads and writes. No local or production clean-verdict bypass exists.
[ClamAV's container instructions](https://docs.clamav.net/manual/Installing/Docker.html) recommend **4 GB** of RAM. The earlier #243 base caps total 7.625 GiB and exclude scanning. Adding 4 GiB would leave insufficient room for the OS on a 12 GB VM. Reallocate and measure the whole stack before deployment; passing the isolated media integration suite does not prove the full launch capacity. No extra VM or paid scan service is authorized as a fallback.
diff --git a/infra/media-security/clamd.conf b/infra/media-security/clamd.conf
new file mode 100644
index 00000000..8fb85917
--- /dev/null
+++ b/infra/media-security/clamd.conf
@@ -0,0 +1,26 @@
+# Private INSTREAM scanner: fail closed on archives/documents the engine cannot inspect.
+User clamav
+DatabaseDirectory /var/lib/clamav
+LocalSocket /tmp/clamd.sock
+LocalSocketMode 660
+TCPSocket 3310
+TCPAddr 0.0.0.0
+LogFile /var/log/clamav/clamd.log
+LogTime yes
+SelfCheck 60
+ConcurrentDatabaseReload no
+EnableShutdownCommand no
+MaxThreads 2
+MaxQueue 8
+StreamMaxLength 10M
+MaxScanSize 40M
+MaxFileSize 10M
+MaxRecursion 16
+MaxFiles 1024
+MaxScanTime 20000
+ReadTimeout 20
+CommandReadTimeout 5
+SendBufTimeout 500
+AlertExceedsMax yes
+AlertEncryptedArchive yes
+AlertEncryptedDoc yes
diff --git a/infra/media-security/compose.yml b/infra/media-security/compose.yml
index ba63f06f..9291a9de 100644
--- a/infra/media-security/compose.yml
+++ b/infra/media-security/compose.yml
@@ -31,7 +31,9 @@ services:
CLAMD_STARTUP_TIMEOUT: '600'
FRESHCLAM_CHECKS: '12'
ports: ['127.0.0.1:3310:3310']
- volumes: ['clamav-signatures:/var/lib/clamav']
+ volumes:
+ - clamav-signatures:/var/lib/clamav
+ - ./clamd.conf:/etc/clamav/clamd.conf:ro
mem_limit: 4g
healthcheck:
test: ['CMD', 'clamdcheck.sh']
diff --git a/scripts/media/wait-dependencies.py b/scripts/media/wait-dependencies.py
index 8ca74b9f..9549f0dd 100644
--- a/scripts/media/wait-dependencies.py
+++ b/scripts/media/wait-dependencies.py
@@ -5,10 +5,13 @@
import urllib.request
deadline = time.monotonic() + 600
+last_report = None
while time.monotonic() < deadline:
+ phase = "object-store readiness"
try:
with urllib.request.urlopen("http://127.0.0.1:9090/private-media-test", timeout=3) as response:
assert response.status == 200
+ phase = "scanner connection"
with socket.create_connection(("127.0.0.1", 3310), timeout=3) as connection:
connection.sendall(b"zVERSION\0")
data = bytearray()
@@ -17,12 +20,16 @@
if not chunk:
raise ValueError("scanner closed its readiness response")
data.extend(chunk)
- updated = datetime.datetime.strptime(data.decode("ascii").strip("\0").split("/", 2)[2], "%a %b %d %H:%M:%S %Y").replace(tzinfo=datetime.timezone.utc)
+ phase = "scanner definition freshness"
+ updated = datetime.datetime.strptime(data.decode("ascii").strip("\0\r\n ").split("/", 2)[2], "%a %b %d %H:%M:%S %Y").replace(tzinfo=datetime.timezone.utc)
age = datetime.datetime.now(datetime.timezone.utc) - updated
assert datetime.timedelta(minutes=-5) <= age <= datetime.timedelta(hours=72)
print("S3 emulator and ClamAV are ready; scanner definitions are within 72 hours")
break
except (OSError, ValueError, IndexError, AssertionError):
+ if phase != last_report:
+ print("Waiting for " + phase, flush=True)
+ last_report = phase
time.sleep(5)
else:
raise SystemExit("Private media dependencies did not become ready with fresh scanner definitions")
From eb0fa246fc94aa134be7d1cba2914152ff4bce83 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:48:55 -0700
Subject: [PATCH 06/10] fix(media): preserve clean resources through AI
indexing outages
---
.../media/application/ResourceLifecycle.java | 21 ++++++++++++++++---
.../media/application/ResourceWorker.java | 15 ++++++++++++-
.../V2__private_resource_lifecycle.sql | 1 +
.../media/api/CourseResourceSmokeTest.java | 2 ++
.../application/ResourceWorkerSafetyTest.java | 19 ++++++++++++++++-
...ecture-review-chanter-private-resources.md | 2 +-
docs/operations/issue-244-change-log.md | 4 +++-
docs/operations/private-course-resources.md | 4 +++-
8 files changed, 60 insertions(+), 8 deletions(-)
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java b/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
index 02ab2241..280ba95c 100644
--- a/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
@@ -111,6 +111,7 @@ public Optional claim(boolean migrateLegacy) {
state IN ('QUARANTINED','SCANNING','DELETE_PENDING')
OR (state='SCAN_FAILED' AND byte_reservation=TRUE AND (attempts<5 OR (updated_at<:expired AND storage_backend<>'legacy')))
OR (state='REJECTED' AND byte_reservation=TRUE)
+ OR (state='AVAILABLE' AND ingestion_status IN ('PENDING','FAILED'))
OR (state='STAGING' AND created_at<:abandoned)
OR (state='LEGACY' AND :migrate=TRUE))
ORDER BY updated_at,id LIMIT 1 FOR UPDATE SKIP LOCKED
@@ -121,21 +122,22 @@ state IN ('QUARANTINED','SCANNING','DELETE_PENDING')
CourseResource r = row.get().resource();
boolean delete = List.of("DELETE_PENDING", "REJECTED", "STAGING").contains(r.state())
|| (r.state().equals("SCAN_FAILED") && row.get().attempts() >= 5);
- String state = delete ? (r.state().equals("STAGING") ? "DELETE_PENDING" : r.state()) : "SCANNING";
+ boolean index = r.state().equals("AVAILABLE");
+ String state = delete ? (r.state().equals("STAGING") ? "DELETE_PENDING" : r.state()) : index ? "AVAILABLE" : "SCANNING";
UUID lease = UUID.randomUUID();
String migrationKey = row.get().migrationKey();
if (!delete && r.storageBackend().equals("legacy") && migrationKey == null) migrationKey = PrivateResourceStorage.PREFIX + r.courseId() + "/" + r.id() + "/" + UUID.randomUUID();
jdbc.sql("UPDATE course_resources SET state=:state, lease_id=:lease, lease_until=:until, migration_key=:migration, attempts=attempts+1, updated_at=:now WHERE id=:id")
.param("state", state).param("lease", lease).param("until", time(instant.plusSeconds(180)))
.param("migration", migrationKey).param("now", time(instant)).param("id", r.id()).update();
- return Optional.of(new Job(r, lease, delete ? "DELETE" : r.storageBackend().equals("legacy") ? "MIGRATE" : "SCAN", migrationKey));
+ return Optional.of(new Job(r, lease, delete ? "DELETE" : index ? "INDEX" : r.storageBackend().equals("legacy") ? "MIGRATE" : "SCAN", migrationKey));
}
@Transactional
public void finishMigration(Job job, UploadValidator.ValidatedUpload upload, String backend) {
int changed = jdbc.sql("""
UPDATE course_resources SET storage_key=:key, storage_backend=:backend, sha256=:hash,
- file_name=:file,content_type=:type,state='QUARANTINED',lease_id=NULL,lease_until=NULL,
+ file_name=:file,content_type=:type,state='QUARANTINED',ingestion_status='NONE',lease_id=NULL,lease_until=NULL,
attempts=0,retry_at=NULL,updated_at=:now WHERE id=:id AND lease_id=:lease AND state='SCANNING'
""").param("key", job.migrationKey()).param("backend", backend).param("hash", upload.sha256())
.param("file", upload.fileName()).param("type", upload.contentType()).param("now", now())
@@ -148,6 +150,8 @@ public boolean finishScan(UUID id, UUID lease, String state) {
if (!List.of("AVAILABLE", "REJECTED", "SCAN_FAILED").contains(state)) throw new IllegalArgumentException("Invalid scan result");
int changed = jdbc.sql("""
UPDATE course_resources SET state=:state, lease_id=NULL, lease_until=NULL, updated_at=:now,
+ ingestion_status=CASE WHEN :state='AVAILABLE' AND ai_approved THEN 'PENDING' ELSE 'NONE' END,
+ attempts=CASE WHEN :state='AVAILABLE' THEN 0 ELSE attempts END,
retry_at=:retry WHERE id=:id AND lease_id=:lease AND state='SCANNING'
""").param("state", state).param("now", now())
.param("retry", state.equals("SCAN_FAILED") ? time(clock.instant().plusSeconds(60)) : null)
@@ -156,6 +160,17 @@ public boolean finishScan(UUID id, UUID lease, String state) {
return changed == 1;
}
+ @Transactional
+ public void finishIndex(UUID id, UUID lease, boolean complete) {
+ int changed = jdbc.sql("""
+ UPDATE course_resources SET ingestion_status=:status,lease_id=NULL,lease_until=NULL,
+ retry_at=:retry,updated_at=:now WHERE id=:id AND lease_id=:lease AND state='AVAILABLE'
+ """).param("status", complete ? "COMPLETE" : "FAILED")
+ .param("retry", complete ? null : time(clock.instant().plusSeconds(600)))
+ .param("now", now()).param("id", id).param("lease", lease).update();
+ if (changed == 0) releaseLease(id, lease);
+ }
+
@Transactional
public void finishDelete(UUID id, UUID lease) {
// All quota mutations take this row first, preventing inversion with upload reservations.
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java b/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java
index b9065b73..46375ecf 100644
--- a/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ResourceWorker.java
@@ -45,12 +45,14 @@ public void runOnce() {
case "DELETE" -> delete(job);
case "MIGRATE" -> migrate(job);
case "SCAN" -> scan(job);
+ case "INDEX" -> index(job);
default -> throw new IllegalStateException("Unsupported resource operation");
}
} catch (Exception failure) {
// Provider errors and scanner signatures can contain private information.
log.warn("Course Resource work deferred resourceId={} operation={}", job.resource().id(), job.operation());
if (job.operation().equals("DELETE")) lifecycle.retryJob(job.resource().id(), job.leaseId());
+ else if (job.operation().equals("INDEX")) lifecycle.finishIndex(job.resource().id(), job.leaseId(), false);
else lifecycle.finishScan(job.resource().id(), job.leaseId(), "SCAN_FAILED");
}
});
@@ -65,12 +67,23 @@ private void scan(ResourceLifecycle.Job job) throws IOException {
if (verdict == MalwareScanner.Verdict.INFECTED) {
lifecycle.finishScan(resource.id(), job.leaseId(), "REJECTED");
} else if (verdict == MalwareScanner.Verdict.CLEAN) {
- if (resource.aiApproved()) ingestion.ingestAiApprovedResource(resource.courseId(), resource.id(), resource.fileName(), Files.readAllBytes(verified));
lifecycle.finishScan(resource.id(), job.leaseId(), "AVAILABLE");
} else throw new IOException("Scanner did not return a verdict");
} finally { Files.deleteIfExists(verified); }
}
+ private void index(ResourceLifecycle.Job job) throws IOException {
+ var resource = job.resource();
+ requireBackend(resource.storageBackend());
+ Path verified = validator.verifiedDownload(storage.open(resource.storageKey()), resource.byteSize(), resource.sha256());
+ try {
+ if (lifecycle.find(resource.id()).filter(current -> current.state().equals("AVAILABLE")).isPresent()) {
+ ingestion.ingestAiApprovedResource(resource.courseId(), resource.id(), resource.fileName(), Files.readAllBytes(verified));
+ }
+ lifecycle.finishIndex(resource.id(), job.leaseId(), true);
+ } finally { Files.deleteIfExists(verified); }
+ }
+
private void delete(ResourceLifecycle.Job job) throws IOException {
var resource = job.resource();
if (resource.storageBackend().equals("legacy")) legacy.deleteLegacy(resource.id());
diff --git a/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql b/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
index 7e0d0027..0f8ed307 100644
--- a/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
+++ b/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
@@ -3,6 +3,7 @@ ALTER TABLE course_resources ADD COLUMN sha256 VARCHAR(64);
ALTER TABLE course_resources ADD COLUMN idempotency_key UUID;
ALTER TABLE course_resources ADD COLUMN storage_backend VARCHAR(16) NOT NULL DEFAULT 'legacy';
ALTER TABLE course_resources ADD COLUMN migration_key VARCHAR(512);
+ALTER TABLE course_resources ADD COLUMN ingestion_status VARCHAR(16) NOT NULL DEFAULT 'NONE';
ALTER TABLE course_resources ADD COLUMN byte_reservation BOOLEAN NOT NULL DEFAULT TRUE;
ALTER TABLE course_resources ADD COLUMN lease_id UUID;
ALTER TABLE course_resources ADD COLUMN lease_until TIMESTAMP WITH TIME ZONE;
diff --git a/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java b/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
index 160a0f94..99422a7d 100644
--- a/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/api/CourseResourceSmokeTest.java
@@ -130,6 +130,8 @@ void instructorCanUploadCourseResourceAndEnrolledLearnerCanListAndDownload() thr
assertThat(downloadResult.getResponse().getHeader("X-Content-Type-Options")).isEqualTo("nosniff");
assertThat(downloadResult.getResponse().getHeader("Cache-Control")).isEqualTo("no-store");
+ assertThat(resourceIngestionClient.ingestCalls()).isEmpty();
+ worker.runOnce();
assertThat(resourceIngestionClient.ingestCalls()).hasSize(1);
TestResourceIngestionClient.IngestCall ingestCall = resourceIngestionClient.ingestCalls().getFirst();
assertThat(ingestCall.courseId()).isEqualTo(courseId);
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java b/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
index e84b70a8..37fc7c23 100644
--- a/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/application/ResourceWorkerSafetyTest.java
@@ -32,7 +32,7 @@ class ResourceWorkerSafetyTest {
@Autowired ResourceLifecycle lifecycle;
@Autowired JdbcClient jdbc;
@Autowired TestCourseResourceAccessClient access;
- @Autowired TestResourceIngestionClient ingestion;
+ @MockitoSpyBean TestResourceIngestionClient ingestion;
@Autowired LocalCourseResourceStorage legacy;
@Autowired UploadValidator validator;
@MockitoBean MalwareScanner scanner;
@@ -151,4 +151,21 @@ INSERT INTO course_resources (id,course_id,title,file_name,content_type,byte_siz
verify(storage).delete(orphan); verify(storage, never()).delete(resource.storageKey());
assertThat(service.usage(course, teacher).reservedBytes()).isEqualTo(resource.byteSize());
}
+
+ @Test void cleanResourcesStayAvailableAndReservedAcrossIndexFailuresUntilIndexRetrySucceeds() throws Exception {
+ var resource = upload(UUID.randomUUID());
+ doThrow(new IllegalStateException("index unavailable")).when(ingestion).ingestAiApprovedResource(any(), any(), anyString(), any());
+ worker.runOnce();
+ assertThat(service.getCourseResource(resource.id(), learner).publicStatus()).isEqualTo("AVAILABLE");
+ worker.runOnce();
+ assertThat(jdbc.sql("SELECT ingestion_status FROM course_resources WHERE id=:id").param("id", resource.id()).query(String.class).single()).isEqualTo("FAILED");
+ jdbc.sql("UPDATE course_resources SET attempts=20,retry_at=NULL,updated_at=TIMESTAMP WITH TIME ZONE '2000-01-01 00:00:00Z'").update();
+ worker.runOnce();
+ assertThat(service.getCourseResource(resource.id(), learner).publicStatus()).isEqualTo("AVAILABLE");
+ assertThat(service.usage(course, teacher).reservedBytes()).isEqualTo(resource.byteSize());
+ verify(storage, never()).delete(resource.storageKey());
+ doCallRealMethod().when(ingestion).ingestAiApprovedResource(any(), any(), anyString(), any());
+ jdbc.sql("UPDATE course_resources SET retry_at=NULL").update(); worker.runOnce();
+ assertThat(jdbc.sql("SELECT ingestion_status FROM course_resources WHERE id=:id").param("id", resource.id()).query(String.class).single()).isEqualTo("COMPLETE");
+ }
}
diff --git a/docs/engineering/records/architecture-review-chanter-private-resources.md b/docs/engineering/records/architecture-review-chanter-private-resources.md
index fb98dbff..98162944 100644
--- a/docs/engineering/records/architecture-review-chanter-private-resources.md
+++ b/docs/engineering/records/architecture-review-chanter-private-resources.md
@@ -54,7 +54,7 @@ The application has four public statuses, while its internal states distinguish
Declared length, filename extension and MIME are checked against bounded actual bytes. SHA-256 persists with metadata, and each download spools one object read and verifies the size and hash before the response can contain file content. Quarantined, unauthorized, corrupt, rejected and deleted objects are not downloadable. Private spools are bounded, closed on failure, and removed after use or age-based recovery.
-ClamAV must supply a clean verdict using fresh definitions. Unavailable, malformed or stale responses fail closed. A regression exposed a socket timeout gap: read timeouts do not cover a blocked upload write. A separate socket deadline now bounds both. Ingestion occurs only after clean scan; deletion retries object and AI-chunk cleanup before releasing reserved bytes.
+ClamAV must supply a clean verdict using fresh definitions. Unavailable, malformed or stale responses fail closed. A regression exposed a socket timeout gap: read timeouts do not cover a blocked upload write. A separate socket deadline now bounds both. A clean file becomes available independently of the AI index; separate durable ingestion status retries index failures without deleting valid files. Deletion retries object and AI-chunk cleanup before releasing reserved bytes.
The database reserves at most 8 GB of resource bytes. It commits every S3 attempt before network I/O and disables SDK retries. Normal uploads, reads and listings share 36,000 monthly operations; 4,000 operations remain reserved for deletion. Uncertain calls never refund operation counts. These limits control this module, not other provider clients. Account backup space, inventory reconciliation after restore, and provider limits remain operator responsibilities.
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index 72a9ba4e..2fffb9c1 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -10,6 +10,7 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- Upload returns 202 with public `status` and `sha256`. Added metadata polling, instructor deletion and course usage routes. Storage keys, endpoints and scanner details stay private.
- Added monthly attempted-operation accounting with dedicated deletion headroom. Uncertain PUT responses preserve reservations until cleanup confirms deletion.
- AI ingestion waits for a clean scan; ingestion and deletion failures are retried by the durable worker instead of being swallowed.
+- Clean file publication is independent of indexing: durable ingestion status retries failures while preserving `AVAILABLE` content and its byte reservation. Index failure never triggers object expiry or deletion.
- Added native architecture CI using pinned PostgreSQL16.15, S3Mock5.2.2 and ClamAV1.5.4, actual EICAR scanning and restart durability checks.
## Test evidence
@@ -17,8 +18,9 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- Initial validation/lifecycle/scanner tests were written before their implementations and failed at compilation. The focused suite passed after implementation.
- The blocked scanner-write regression failed after approximately 2.54 seconds against a 1-second assertion; adding the socket deadline made it pass.
- The provider-stream closure regression failed when spool creation was unavailable; closing the provider stream around spool creation made it pass.
+- Review identified clean-file deletion risk when indexing and scan failure shared a state. The regression first failed because a learner could not read the clean resource; separate durable index work now keeps the file available across repeated index failures and retries to completion.
- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
-- Java21 affected-module verification passed after the secure-session rebase. Focused additions cover legacy-index purge, HTTP delete/usage authorization and a real HTTP500 server proving one attempt per S3 operation, bringing the local media suite to34 tests. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
+- Java21 affected-module verification passed after the secure-session rebase. The35 local media tests cover index-outage preservation, legacy-index purge, HTTP delete/usage authorization and a real HTTP500 server proving one attempt per S3 operation. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
- Real startup logs show fresh signatures downloaded before clamd's socket was available for notification. The explicit daemon configuration rechecks definitions every60 seconds, uses bounded scan limits, rejects encrypted/over-limit content and avoids duplicate engines during reload. The native suite tests a compressed fixture beyond the scan limit as well as EICAR.
diff --git a/docs/operations/private-course-resources.md b/docs/operations/private-course-resources.md
index 1034da8d..56d90221 100644
--- a/docs/operations/private-course-resources.md
+++ b/docs/operations/private-course-resources.md
@@ -49,7 +49,9 @@ All routes require the existing access-token authorization and course permission
| `DELETE /api/v1/course-resources/{id}` | Instructor-only, idempotent HTTP 204; access stops immediately, cleanup runs durably |
| `GET /api/v1/courses/{courseId}/course-resources/usage` | Instructor-only course `reservedBytes` and `availableBytes` |
-Public statuses are `PROCESSING`, `AVAILABLE`, `REJECTED` and `FAILED`. No storage key, endpoint, internal lease, scanner signature or provider error is exposed. Reusing an idempotency key never restarts a rejected or deleted upload. To intentionally submit new work after a permanent failure, use a new key. Scanner or index failures automatically retry up to five times with a 60-second delay. Failed nonlegacy objects retain their bytes for at least 24 hours before cleanup; the failure record and original key remain durable. Infected files are rejected and queued for cleanup immediately.
+Public statuses are `PROCESSING`, `AVAILABLE`, `REJECTED` and `FAILED`. No storage key, endpoint, internal lease, scanner signature or provider error is exposed. Reusing an idempotency key never restarts a rejected or deleted upload. To intentionally submit new work after a permanent failure, use a new key. Scanner failures automatically retry up to five times with a 60-second delay. Failed nonlegacy objects retain their bytes for at least 24 hours before cleanup; the failure record and original key remain durable. Infected files are rejected and queued for cleanup immediately.
+
+A clean file becomes `AVAILABLE` before AI indexing. Separate internal `ingestion_status` values (`NONE`, `PENDING`, `FAILED`, `COMPLETE`) track durable index work. Failed indexing retries every ten minutes without deleting the clean object or releasing its byte reservation. Deletion still takes priority over index completion. Richer user-facing ingestion status belongs to #246; the public upload/list shape remains unchanged here.
Accepted types are UTF-8 text/Markdown, PDF, PPTX, MP3, M4A, WAV, OGG, MP4, WebM and MOV, capped at 10 MiB of actual streamed bytes. Legacy binary `.ppt` must be converted to `.pptx`. The extension, declared MIME and detected content must agree. Filenames are normalized to safe basenames; titles and filenames have length/control-character checks. PDF trailers and bounded, nonmacro PPTX archive structure are checked before storage. These checks establish supported type, not perfect document validity; ClamAV supplies the malware gate.
From 310c79fb3d2891bd8d18b1abee668d6ca17b35cc Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 22:55:20 -0700
Subject: [PATCH 07/10] fix(media): bind durable metadata to its storage
namespace
---
.../media/application/ResourceLifecycle.java | 17 ++++++
.../infra/LocalPrivateResourceStorage.java | 11 +++-
.../media/infra/S3PrivateResourceStorage.java | 5 ++
.../V2__private_resource_lifecycle.sql | 1 +
.../application/StorageNamespaceTest.java | 53 +++++++++++++++++++
...ecture-review-chanter-private-resources.md | 2 +
docs/operations/issue-244-change-log.md | 6 ++-
docs/operations/private-course-resources.md | 4 ++
scripts/media/wait-dependencies.py | 8 ++-
9 files changed, 103 insertions(+), 4 deletions(-)
create mode 100644 backend/media-service/src/test/java/com/chanter/media/application/StorageNamespaceTest.java
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java b/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
index 280ba95c..f0798dd2 100644
--- a/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ResourceLifecycle.java
@@ -37,6 +37,23 @@ public ResourceLifecycle(JdbcClient jdbc, Clock clock,
this.jdbc = jdbc; this.clock = clock; this.byteLimit = byteLimit; this.requestLimit = requestLimit; this.cleanupReserve = cleanupReserve;
}
+ @Transactional
+ public void bindNamespace(String identity) {
+ String digest;
+ try {
+ digest = java.util.HexFormat.of().formatHex(java.security.MessageDigest.getInstance("SHA-256")
+ .digest(identity.getBytes(java.nio.charset.StandardCharsets.UTF_8)));
+ } catch (java.security.NoSuchAlgorithmException impossible) { throw new IllegalStateException(impossible); }
+ var binding = jdbc.sql("SELECT storage_namespace FROM media_storage_budget WHERE id=1 FOR UPDATE")
+ .query((rs, row) -> new Namespace(rs.getString("storage_namespace"))).single();
+ if (binding.digest() == null) {
+ jdbc.sql("UPDATE media_storage_budget SET storage_namespace=:digest WHERE id=1").param("digest", digest).update();
+ } else if (!binding.digest().equals(digest)) {
+ throw new IllegalStateException("Private storage namespace changed; reviewed data migration is required");
+ }
+ }
+ private record Namespace(String digest) {}
+
@Transactional
public CourseResource reserve(CourseResource r) {
long used = jdbc.sql("SELECT reserved_bytes FROM media_storage_budget WHERE id=1 FOR UPDATE").query(Long.class).single();
diff --git a/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java b/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java
index db61989d..feaf4cc2 100644
--- a/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java
+++ b/backend/media-service/src/main/java/com/chanter/media/infra/LocalPrivateResourceStorage.java
@@ -1,20 +1,29 @@
package com.chanter.media.infra;
import com.chanter.media.application.PrivateResourceStorage;
+import com.chanter.media.application.ResourceLifecycle;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
import org.springframework.beans.factory.annotation.Value;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.sql.init.dependency.DependsOnDatabaseInitialization;
import org.springframework.stereotype.Component;
@Component
+@DependsOnDatabaseInitialization
@ConditionalOnProperty(name = "chanter.media.storage-backend", havingValue = "local")
public class LocalPrivateResourceStorage implements PrivateResourceStorage {
private final Path root;
- public LocalPrivateResourceStorage(@Value("${chanter.media.storage-dir}") String root) throws IOException {
+ @Autowired
+ public LocalPrivateResourceStorage(@Value("${chanter.media.storage-dir}") String root, ResourceLifecycle lifecycle) throws IOException {
+ this(root);
+ lifecycle.bindNamespace("local\n" + this.root.toRealPath());
+ }
+ public LocalPrivateResourceStorage(String root) throws IOException {
this.root = Path.of(root).toAbsolutePath().normalize();
Files.createDirectories(this.root);
if (Files.getFileStore(this.root).supportsFileAttributeView("posix")) Files.setPosixFilePermissions(this.root, java.nio.file.attribute.PosixFilePermissions.fromString("rwx------"));
diff --git a/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java b/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java
index 27555c25..4a32b201 100644
--- a/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java
+++ b/backend/media-service/src/main/java/com/chanter/media/infra/S3PrivateResourceStorage.java
@@ -14,6 +14,7 @@
import java.util.Map;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.boot.sql.init.dependency.DependsOnDatabaseInitialization;
import org.springframework.stereotype.Component;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
@@ -28,6 +29,7 @@
import software.amazon.awssdk.services.s3.model.*;
@Component
+@DependsOnDatabaseInitialization
@ConditionalOnProperty(name = "chanter.media.storage-backend", havingValue = "s3")
public class S3PrivateResourceStorage implements PrivateResourceStorage {
private final S3Client client;
@@ -51,6 +53,9 @@ public S3PrivateResourceStorage(ResourceLifecycle lifecycle,
throw new IllegalArgumentException("Invalid private S3 configuration");
}
this.lifecycle = lifecycle; this.bucket = bucket;
+ int port = uri.getPort() == -1 ? ("https".equals(uri.getScheme()) ? 443 : 80) : uri.getPort();
+ lifecycle.bindNamespace("s3\n" + uri.getScheme().toLowerCase(java.util.Locale.ROOT) + "://"
+ + uri.getHost().toLowerCase(java.util.Locale.ROOT) + ":" + port + "\n" + bucket);
this.client = S3Client.builder().endpointOverride(uri).region(Region.of(region))
.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)))
.httpClientBuilder(UrlConnectionHttpClient.builder().connectionTimeout(Duration.ofSeconds(3)).socketTimeout(Duration.ofSeconds(15)))
diff --git a/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql b/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
index 0f8ed307..061d0986 100644
--- a/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
+++ b/backend/media-service/src/main/resources/db/migration/V2__private_resource_lifecycle.sql
@@ -16,6 +16,7 @@ CREATE INDEX idx_resource_processing ON course_resources(state, retry_at, lease_
CREATE TABLE media_storage_budget (
id INTEGER PRIMARY KEY CHECK (id = 1),
+ storage_namespace VARCHAR(64),
reserved_bytes BIGINT NOT NULL CHECK (reserved_bytes >= 0),
request_month VARCHAR(7) NOT NULL,
foreground_requests INTEGER NOT NULL,
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/StorageNamespaceTest.java b/backend/media-service/src/test/java/com/chanter/media/application/StorageNamespaceTest.java
new file mode 100644
index 00000000..f2462a91
--- /dev/null
+++ b/backend/media-service/src/test/java/com/chanter/media/application/StorageNamespaceTest.java
@@ -0,0 +1,53 @@
+package com.chanter.media.application;
+
+import static org.assertj.core.api.Assertions.*;
+import com.chanter.media.infra.S3PrivateResourceStorage;
+import com.sun.net.httpserver.HttpServer;
+import java.net.InetSocketAddress;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.DynamicPropertyRegistry;
+import org.springframework.test.context.DynamicPropertySource;
+
+@SpringBootTest
+@ActiveProfiles("test")
+class StorageNamespaceTest {
+ private static final AtomicInteger REQUESTS = new AtomicInteger();
+ private static final HttpServer SERVER = server();
+ private static final String ENDPOINT = "http://127.0.0.1:" + SERVER.getAddress().getPort();
+ @Autowired ResourceLifecycle lifecycle;
+ @Autowired S3PrivateResourceStorage configured;
+ private static HttpServer server() {
+ try {
+ var server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0);
+ server.createContext("/", exchange -> { REQUESTS.incrementAndGet(); exchange.sendResponseHeaders(500, -1); exchange.close(); });
+ server.start(); return server;
+ } catch (java.io.IOException failure) { throw new IllegalStateException(failure); }
+ }
+ @DynamicPropertySource static void properties(DynamicPropertyRegistry registry) {
+ registry.add("spring.datasource.url", () -> "jdbc:h2:mem:storage-namespace;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE;DB_CLOSE_DELAY=-1");
+ registry.add("chanter.media.storage-backend", () -> "s3");
+ registry.add("chanter.media.s3.endpoint", () -> ENDPOINT);
+ registry.add("chanter.media.s3.region", () -> "us-east-1");
+ registry.add("chanter.media.s3.bucket", () -> "original-private-bucket");
+ registry.add("chanter.media.s3.access-key", () -> "fixture-access");
+ registry.add("chanter.media.s3.secret-key", () -> "fixture-secret");
+ registry.add("chanter.media.s3.allow-local-http", () -> true);
+ }
+ @AfterAll static void stop() { SERVER.stop(0); }
+ @Test void changedBucketOrEndpointCannotStartOrMakeAnyObjectRequest() {
+ for (String[] changed : new String[][] {{ENDPOINT, "different-private-bucket"}, {ENDPOINT.replace("127.0.0.1", "localhost"), "original-private-bucket"}}) {
+ assertThatThrownBy(() -> new S3PrivateResourceStorage(lifecycle, changed[0], "us-east-1", changed[1], "fixture-access", "fixture-secret", true))
+ .isInstanceOf(IllegalStateException.class).hasMessageContaining("namespace");
+ }
+ assertThat(REQUESTS.get()).isZero();
+ }
+ @Test void credentialRotationAndNormalizedEndpointKeepTheSameBinding() {
+ var rotated = new S3PrivateResourceStorage(lifecycle, ENDPOINT + "/", "us-east-1", "original-private-bucket", "rotated-access", "rotated-secret", true);
+ rotated.close(); assertThat(REQUESTS.get()).isZero();
+ }
+}
diff --git a/docs/engineering/records/architecture-review-chanter-private-resources.md b/docs/engineering/records/architecture-review-chanter-private-resources.md
index 98162944..5472e838 100644
--- a/docs/engineering/records/architecture-review-chanter-private-resources.md
+++ b/docs/engineering/records/architecture-review-chanter-private-resources.md
@@ -58,6 +58,8 @@ ClamAV must supply a clean verdict using fresh definitions. Unavailable, malform
The database reserves at most 8 GB of resource bytes. It commits every S3 attempt before network I/O and disables SDK retries. Normal uploads, reads and listings share 36,000 monthly operations; 4,000 operations remain reserved for deletion. Uncertain calls never refund operation counts. These limits control this module, not other provider clients. Account backup space, inventory reconciliation after restore, and provider limits remain operator responsibilities.
+A persisted namespace fingerprint binds the database to the normalized endpoint and bucket, or the local root, before the adapter can issue requests. Credentials do not affect the identity. This prevents a configuration change from treating missing objects in a different bucket as successful deletion and releasing real reservations. Changing the namespace requires a reviewed copy, checksum inventory and explicit binding update under maintenance; startup never rebinds automatically.
+
## Migration and release boundary
V2 quarantines old metadata and preserves its storage reservation. Opt-in migration validates legacy files, persists the destination key before copying, confirms interrupted writes and scans the copy. Original files stay available to recovery operators. Legacy AI chunks must be cleared during maintenance because they predate the new scanning guarantee.
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index 2fffb9c1..a9cc223f 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -5,6 +5,7 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
## Changes
- Added immutable private S3 and explicit local adapters. Production selects S3 without fallback; OCI Always Free is the planned zero-cost provider, with compatible alternate endpoints configurable.
+- Bound storage metadata to its endpoint/bucket or local-root identity before object access. Credential rotation preserves the binding; namespace changes require an explicit reviewed migration.
- Added actual byte/type/name/checksum validation, bounded spool files and verified attachment downloads under existing course authorization.
- Added Flyway V2 lifecycle state, byte reservation, scoped upload idempotency, durable worker leases, fail-closed ClamAV scanning, delayed cleanup and orphan reconciliation.
- Upload returns 202 with public `status` and `sha256`. Added metadata polling, instructor deletion and course usage routes. Storage keys, endpoints and scanner details stay private.
@@ -20,10 +21,11 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- The provider-stream closure regression failed when spool creation was unavailable; closing the provider stream around spool creation made it pass.
- Review identified clean-file deletion risk when indexing and scan failure shared a state. The regression first failed because a learner could not read the clean resource; separate durable index work now keeps the file available across repeated index failures and retries to completion.
- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
-- Java21 affected-module verification passed after the secure-session rebase. The35 local media tests cover index-outage preservation, legacy-index purge, HTTP delete/usage authorization and a real HTTP500 server proving one attempt per S3 operation. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
+- Java 21 affected-module verification passed after the secure-session rebase. The 37 local media tests cover index-outage preservation, legacy-index purge, HTTP delete/usage authorization and a real HTTP 500 server proving one attempt per S3 operation. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
+- The namespace-change regression first failed because a changed bucket could construct a client. Startup binding now rejects bucket or endpoint changes before any network request, while normalized endpoints and rotated credentials pass.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
-- Real startup logs show fresh signatures downloaded before clamd's socket was available for notification. The explicit daemon configuration rechecks definitions every60 seconds, uses bounded scan limits, rejects encrypted/over-limit content and avoids duplicate engines during reload. The native suite tests a compressed fixture beyond the scan limit as well as EICAR.
+- Real startup logs show signatures downloaded before clamd's socket was available for notification. The explicit daemon configuration rechecks definitions every 60 seconds, uses bounded scan limits, rejects encrypted/over-limit content and avoids duplicate engines during reload. Readiness still failed the freshness check; a bounded version-response diagnostic records the actual scanner metadata without weakening the gate. The native suite tests a compressed fixture beyond the scan limit as well as EICAR.
## Remaining release proof
diff --git a/docs/operations/private-course-resources.md b/docs/operations/private-course-resources.md
index 56d90221..7bd4f988 100644
--- a/docs/operations/private-course-resources.md
+++ b/docs/operations/private-course-resources.md
@@ -34,6 +34,8 @@ The adapter uses path-style addressing, immutable conditional PUT, content MD5 o
Public configuration never permits HTTP object storage. The integration test enables a separate property accepting HTTP only for literal loopback hosts. S3Mock is an emulator and cannot prove provider credentials, IAM policies, anonymous-access denial, region correctness or free-account limits.
+Startup binds the database to a SHA-256 fingerprint of the backend and normalized endpoint plus bucket, or the canonical local root. Endpoint trailing slashes and default ports do not change the fingerprint. Credentials are excluded so keys can rotate. A changed namespace stops startup before any object request, including cleanup. The binding persists even when the bucket is empty. Never clear it to work around a startup failure.
+
## Upload and download contract
All routes require the existing access-token authorization and course permissions. Browser session issuance/refresh follows #242; downloads do not accept object URLs or a refresh cookie as authorization.
@@ -73,6 +75,8 @@ Run maintained ClamAV with a persistent signature directory, UTC timezone and Fr
On database restore, stop uploads and workers first. Restore the metadata and counters together, then compare a private `resources/v1/` object inventory with reserved database rows. Preserve extra objects while deciding whether they are newer accepted writes or true orphans. Rebuild accurate byte reservations and monthly attempts conservatively before reopening writes; never reset an uncertain monthly request count to zero. The account's independent backup reserve must cover the recovery plan.
+To migrate a bound namespace, stop every media instance and keep resource and AI access behind maintenance. Copy the exact reserved object and migration-key inventory without removing originals; validate every destination object's length and SHA-256 against the database. Reconcile failed and interrupted writes and account for copy attempts against both provider budgets. Have the operator review the inventory and destination before updating the singleton `media_storage_budget.storage_namespace` fingerprint and matching release configuration together. Preserve the original binding and object inventory for recovery. Start with workers disabled, verify authorized reads and anonymous denial, then resume workers. A backend change also requires a reviewed metadata migration; changing only the binding is insufficient. There is no automatic rebinding, copy or deletion path.
+
Ordinary reconciliation runs hourly, processes at most ten pages per run and resumes its cursor. It deletes only module-prefixed objects older than 24 hours that have no active byte reservation. Database-backed interrupted writes and rejected/deleted objects have their own retry leases. Reconciliation never scans backup prefixes. Expired private spool files are removed after one hour. Do not run reconciliation against a partially restored database.
## Verification and release proof
diff --git a/scripts/media/wait-dependencies.py b/scripts/media/wait-dependencies.py
index 9549f0dd..abc24fde 100644
--- a/scripts/media/wait-dependencies.py
+++ b/scripts/media/wait-dependencies.py
@@ -6,6 +6,7 @@
deadline = time.monotonic() + 600
last_report = None
+last_version = None
while time.monotonic() < deadline:
phase = "object-store readiness"
try:
@@ -21,7 +22,12 @@
raise ValueError("scanner closed its readiness response")
data.extend(chunk)
phase = "scanner definition freshness"
- updated = datetime.datetime.strptime(data.decode("ascii").strip("\0\r\n ").split("/", 2)[2], "%a %b %d %H:%M:%S %Y").replace(tzinfo=datetime.timezone.utc)
+ version = data.decode("ascii").strip("\0\r\n ")
+ if version != last_version:
+ # Isolated public CI fixture: bounded, escaped scanner metadata contains no application data.
+ print("Scanner VERSION: " + repr(version[:256]), flush=True)
+ last_version = version
+ updated = datetime.datetime.strptime(version.split("/", 2)[2], "%a %b %d %H:%M:%S %Y").replace(tzinfo=datetime.timezone.utc)
age = datetime.datetime.now(datetime.timezone.utc) - updated
assert datetime.timedelta(minutes=-5) <= age <= datetime.timedelta(hours=72)
print("S3 emulator and ClamAV are ready; scanner definitions are within 72 hours")
From 736301f6ee51b66d7671e10be33c04a5b8803ff6 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 23:07:59 -0700
Subject: [PATCH 08/10] fix(media): refresh signatures before scanner startup
and gate product seeding
---
docs/operations/issue-244-change-log.md | 5 +--
docs/operations/private-course-resources.md | 6 ++--
infra/docker-compose.yml | 7 ++++
infra/media-security/compose.yml | 3 ++
scripts/media/wait-dependencies.py | 23 ++++++++----
scripts/product/down.sh | 4 +--
scripts/product/health.sh | 2 ++
scripts/product/lib.sh | 5 +--
scripts/seed-workable-product-demo.sh | 40 +++++++++++++++++----
9 files changed, 74 insertions(+), 21 deletions(-)
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index a9cc223f..cf634e83 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -13,6 +13,7 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- AI ingestion waits for a clean scan; ingestion and deletion failures are retried by the durable worker instead of being swallowed.
- Clean file publication is independent of indexing: durable ingestion status retries failures while preserving `AVAILABLE` content and its byte reservation. Index failure never triggers object expiry or deletion.
- Added native architecture CI using pinned PostgreSQL16.15, S3Mock5.2.2 and ClamAV1.5.4, actual EICAR scanning and restart durability checks.
+- The product stack inherits that scanner configuration and checks definition freshness. Demo seeding supplies an explicit safe filename and waits for clean availability and usable index chunks before installing grants.
## Test evidence
@@ -25,10 +26,10 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- The namespace-change regression first failed because a changed bucket could construct a client. Startup binding now rejects bucket or endpoint changes before any network request, while normalized endpoints and rotated credentials pass.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
-- Real startup logs show signatures downloaded before clamd's socket was available for notification. The explicit daemon configuration rechecks definitions every 60 seconds, uses bounded scan limits, rejects encrypted/over-limit content and avoids duplicate engines during reload. Readiness still failed the freshness check; a bounded version-response diagnostic records the actual scanner metadata without weakening the gate. The native suite tests a compressed fixture beyond the scan limit as well as EICAR.
+- Native AMD64 and ARM64 diagnostics proved clamd retained September 7 signatures after FreshClam downloaded September 10 definitions during engine startup. One-minute self-checks reported no change and did not repair the missed notification. Startup now completes a foreground update before handing control back to the upstream daemon entrypoint. The 72-hour gate stays unchanged. Explicit scan limits reject encrypted/over-limit content and avoid duplicate engines during reload; the native suite tests a compressed fixture beyond the scan limit as well as EICAR.
## Remaining release proof
-The provider is unprovisioned. Native container checks must pass at the final head, followed by #243 integration, actual private bucket permissions/anonymous-denial and recovery tests, processing/failure UI browser evidence, and a measured 2 OCPU/12 GB full-stack workload. ClamAV's 4 GB container guidance cannot be added on top of the earlier 7.625 GiB base caps without reallocation. Schema V2 requires a deployment epoch boundary; old code must not be rolled back onto the new lifecycle data.
+The provider is unprovisioned. Native container checks must pass at the final head, followed by #243 integration, actual private bucket permissions/anonymous-denial and recovery tests, processing/failure UI browser evidence, and a measured 2 OCPU/12 GB full-stack workload. ClamAV's 4 GB container guidance cannot be added on top of the earlier 7.625 GiB base caps without reallocation. Schema V2 requires deployment epoch 4, following #319's authentication epoch 3; old code must not be rolled back onto the new lifecycle data.
See [the operator runbook](private-course-resources.md) and [the system review](../engineering/records/architecture-review-chanter-private-resources.md).
diff --git a/docs/operations/private-course-resources.md b/docs/operations/private-course-resources.md
index 7bd4f988..05186fa0 100644
--- a/docs/operations/private-course-resources.md
+++ b/docs/operations/private-course-resources.md
@@ -61,7 +61,7 @@ Each download fetches once to a private file of at most 10 MiB, verifies length
## Scanner and capacity
-Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. Use the reviewed `infra/media-security/clamd.conf`: one-minute signature checks recover a startup notification race; encrypted or over-limit content produces a rejection; reloads block briefly instead of holding two engines. Scan uses the real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Socket deadlines bound both reads and writes. No local or production clean-verdict bypass exists.
+Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. The reviewed Compose service completes an update before starting clamd, preventing an update during engine loading from missing its notification. Startup fails if that update fails. Use `infra/media-security/clamd.conf`: encrypted or over-limit content produces a rejection; reloads block briefly instead of holding two engines. Scan uses the real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Socket deadlines bound both reads and writes. No local or production clean-verdict bypass exists.
[ClamAV's container instructions](https://docs.clamav.net/manual/Installing/Docker.html) recommend **4 GB** of RAM. The earlier #243 base caps total 7.625 GiB and exclude scanning. Adding 4 GiB would leave insufficient room for the OS on a 12 GB VM. Reallocate and measure the whole stack before deployment; passing the isolated media integration suite does not prove the full launch capacity. No extra VM or paid scan service is authorized as a fallback.
@@ -71,7 +71,7 @@ Run maintained ClamAV with a persistent signature directory, UTC timezone and Fr
2. Apply media Flyway V2 through the reviewed release migration command. Existing rows become `LEGACY`, retain their byte reservation and become unavailable for downloads. Confirm their sum fits the 8 GB resource allocation. Never rewrite V1 or mark legacy rows available manually.
3. Configure the private S3 bucket and scanner, mount original files, and set `CHANTER_MEDIA_MIGRATE_LEGACY=true`. The worker persists a new immutable object key before uploading, validates actual legacy size/type, verifies an uncertain PUT using a single GET, and then quarantines the copy for scanning. Original local files remain intact. A concurrent delete also cleans any reserved migration key.
4. Wait until expected legacy rows become `AVAILABLE` or investigate failures. Unsupported, missing or changed files remain unavailable and reserved; restore or correct their input under maintenance, then use an operator-reviewed retry. Compare downloaded hashes and learner/instructor permissions. Turn legacy migration off. Retain old files through the backup/rollback window; remove them only through a separately reviewed exact inventory.
-5. Rollback to a release predating V2 is **not compatible**: the old server ignores quarantine and expects local storage. Treat #244 as a schema-epoch boundary in #243. Restore database and local resources together behind maintenance, or fix forward with the current lifecycle model. A database migration alone is not a safe rollback.
+5. Rollback to a release predating V2 is **not compatible**: the old server ignores quarantine and expects local storage. Advance the release schema epoch from 3 to 4 when #243 adopts #244. Epoch 3 already includes #319's password encoding transition. Restore database and local resources together behind maintenance, or fix forward with the current lifecycle model. A database migration alone is not a safe rollback.
On database restore, stop uploads and workers first. Restore the metadata and counters together, then compare a private `resources/v1/` object inventory with reserved database rows. Preserve extra objects while deciding whether they are newer accepted writes or true orphans. Rebuild accurate byte reservations and monthly attempts conservatively before reopening writes; never reset an uncertain monthly request count to zero. The account's independent backup reserve must cover the recovery plan.
@@ -85,4 +85,6 @@ Run local module tests with `mvn -s backend/.mvn/settings.xml -f backend/pom.xml
For a local Docker host: `docker compose -f infra/media-security/compose.yml up -d`, run `python3 scripts/media/wait-dependencies.py`, then set `MEDIA_INTEGRATION=true` and `MEDIA_RESTART_PHASE=false` for `mvn ... test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false`. Restart those three services, set `MEDIA_RESTART_PHASE=true`, and rerun the same test. The fixture uses test-only credentials, loopback ports and independent named volumes. Remove that exact test stack with its Compose `down -v` command when finished.
+The local product stack inherits the same scanner service and daemon configuration. `make product-up` waits for fresh definitions; `make product-health` checks them again. `make product-demo-seed` uses a safe explicit multipart filename and waits for `AVAILABLE` metadata plus actual index chunks before installing Study Assistant grants. The 14 existing product browser journeys remain enabled. This development stack does not establish production capacity.
+
Keep #244 open until the merged #243 package has real private-provider IAM/anonymous-denial tests, upload/download/delete/recovery proof, a measured 2 OCPU/12 GB workload including the scanner, and browser evidence for processing and failure states. No provisioned provider or production proof is claimed by this repository change.
diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml
index 0548d236..dc2a055e 100644
--- a/infra/docker-compose.yml
+++ b/infra/docker-compose.yml
@@ -1,4 +1,10 @@
services:
+ clamav:
+ extends:
+ file: media-security/compose.yml
+ service: clamav
+ profiles: [product]
+
mailpit:
image: axllent/mailpit:v1.31.1
profiles: [product]
@@ -141,5 +147,6 @@ services:
- "127.0.0.1:7882:7882/udp"
volumes:
+ clamav-signatures:
postgres-data:
minio-data:
diff --git a/infra/media-security/compose.yml b/infra/media-security/compose.yml
index 9291a9de..682b84ea 100644
--- a/infra/media-security/compose.yml
+++ b/infra/media-security/compose.yml
@@ -26,6 +26,9 @@ services:
mem_limit: 512m
clamav:
image: clamav/clamav-debian:1.5.4@sha256:4c975c439fcb7ab9cbdd72162c2802f5efcb4a11dcf5b0b37da1d746474ef20e
+ # The upstream entrypoint prepares volume ownership before executing this command.
+ # Refresh before starting clamd: an update during engine loading can miss its reload notification.
+ command: ['sh', '-ec', 'unset CLAMAV_NO_CLAMD; freshclam --foreground --stdout; exec /init']
environment:
TZ: UTC
CLAMD_STARTUP_TIMEOUT: '600'
diff --git a/scripts/media/wait-dependencies.py b/scripts/media/wait-dependencies.py
index abc24fde..bce42df3 100644
--- a/scripts/media/wait-dependencies.py
+++ b/scripts/media/wait-dependencies.py
@@ -1,19 +1,28 @@
-"""Bounded readiness check for the isolated test stack, including fresh ClamAV signatures."""
+"""Bounded readiness for fresh ClamAV signatures and the optional isolated S3 test store."""
+import argparse
import datetime
+import os
import socket
import time
import urllib.request
-deadline = time.monotonic() + 600
+parser = argparse.ArgumentParser()
+parser.add_argument("--scanner-only", action="store_true")
+parser.add_argument("--timeout", type=int, default=600)
+arguments = parser.parse_args()
+if not 1 <= arguments.timeout <= 600:
+ parser.error("timeout must be between 1 and 600 seconds")
+deadline = time.monotonic() + arguments.timeout
last_report = None
last_version = None
while time.monotonic() < deadline:
phase = "object-store readiness"
try:
- with urllib.request.urlopen("http://127.0.0.1:9090/private-media-test", timeout=3) as response:
- assert response.status == 200
+ if not arguments.scanner_only:
+ with urllib.request.urlopen("http://127.0.0.1:9090/private-media-test", timeout=3) as response:
+ assert response.status == 200
phase = "scanner connection"
- with socket.create_connection(("127.0.0.1", 3310), timeout=3) as connection:
+ with socket.create_connection((os.getenv("CHANTER_CLAMAV_HOST", "127.0.0.1"), int(os.getenv("CHANTER_CLAMAV_PORT", "3310"))), timeout=3) as connection:
connection.sendall(b"zVERSION\0")
data = bytearray()
while not data.endswith(b"\0") and len(data) < 4096:
@@ -24,13 +33,13 @@
phase = "scanner definition freshness"
version = data.decode("ascii").strip("\0\r\n ")
if version != last_version:
- # Isolated public CI fixture: bounded, escaped scanner metadata contains no application data.
+ # Bounded, escaped version metadata contains no application or scanned-file data.
print("Scanner VERSION: " + repr(version[:256]), flush=True)
last_version = version
updated = datetime.datetime.strptime(version.split("/", 2)[2], "%a %b %d %H:%M:%S %Y").replace(tzinfo=datetime.timezone.utc)
age = datetime.datetime.now(datetime.timezone.utc) - updated
assert datetime.timedelta(minutes=-5) <= age <= datetime.timedelta(hours=72)
- print("S3 emulator and ClamAV are ready; scanner definitions are within 72 hours")
+ print("ClamAV is ready; scanner definitions are within 72 hours")
break
except (OSError, ValueError, IndexError, AssertionError):
if phase != last_report:
diff --git a/scripts/product/down.sh b/scripts/product/down.sh
index 137f03b0..607d9c62 100755
--- a/scripts/product/down.sh
+++ b/scripts/product/down.sh
@@ -18,9 +18,9 @@ done < <(product_java_modules)
product_stop_module frontend
-echo "Stopping product Docker services (realtime, LiveKit, local email)..."
+echo "Stopping product Docker services (realtime, LiveKit, scanner, local email)..."
docker compose -f "$ROOT/infra/docker-compose.yml" --env-file "$ROOT/.env" --profile product \
- stop realtime-service livekit mailpit >/dev/null 2>&1 || true
+ stop realtime-service livekit clamav mailpit >/dev/null 2>&1 || true
echo "Product app processes stopped. Core infra (Postgres, Redis, broker, MinIO) is still running."
echo "Run 'make infra-down' to stop infrastructure."
diff --git a/scripts/product/health.sh b/scripts/product/health.sh
index 09bb3500..d137b5b6 100755
--- a/scripts/product/health.sh
+++ b/scripts/product/health.sh
@@ -37,4 +37,6 @@ if [ "$failures" -gt 0 ]; then
exit 1
fi
+python3 "$(product_repo_root)/scripts/media/wait-dependencies.py" --scanner-only --timeout 5
+
echo "Product stack health checks passed."
diff --git a/scripts/product/lib.sh b/scripts/product/lib.sh
index 37dc4bc1..14347b19 100755
--- a/scripts/product/lib.sh
+++ b/scripts/product/lib.sh
@@ -427,8 +427,9 @@ product_prepare_infrastructure() {
mail_services+=(mailpit)
fi
docker compose -f "$compose_file" --env-file "$root/.env" --profile product stop realtime-service >/dev/null 2>&1 || true
- docker compose -f "$compose_file" --env-file "$root/.env" --profile product up -d --wait --wait-timeout 180 \
- postgres redis redpanda livekit "${mail_services[@]}"
+ docker compose -f "$compose_file" --env-file "$root/.env" --profile product up -d --wait --wait-timeout 600 \
+ postgres redis redpanda livekit clamav "${mail_services[@]}"
+ python3 "$root/scripts/media/wait-dependencies.py" --scanner-only
product_ensure_databases
echo "Infrastructure is healthy."
}
diff --git a/scripts/seed-workable-product-demo.sh b/scripts/seed-workable-product-demo.sh
index 429e54fd..960745bf 100755
--- a/scripts/seed-workable-product-demo.sh
+++ b/scripts/seed-workable-product-demo.sh
@@ -162,13 +162,13 @@ echo "==> Upload AI-approved course resource for Study Assistant grounding"
RESOURCE_TITLE="Homework Help Guide"
EXISTING_RESOURCES=$(curl -sf "$GATEWAY/api/v1/courses/$COURSE_ID/course-resources" \
-H "Authorization: Bearer $OWNER_TOKEN")
-HAS_RESOURCE=$(echo "$EXISTING_RESOURCES" | python3 -c "
+RESOURCE_ID=$(echo "$EXISTING_RESOURCES" | python3 -c "
import sys, json
data = json.load(sys.stdin)
resources = data.get('courseResources', data.get('resources', []))
-print('yes' if any(r.get('title') == '$RESOURCE_TITLE' for r in resources) else 'no')
+print(next((r['id'] for r in resources if r.get('title') == '$RESOURCE_TITLE'), ''))
")
-if [[ "$HAS_RESOURCE" == "yes" ]]; then
+if [[ -n "$RESOURCE_ID" ]]; then
echo " reusing existing $RESOURCE_TITLE"
else
RESOURCE_FILE="$ROOT/scripts/.workable-demo-ai-resource.txt"
@@ -179,14 +179,42 @@ Submit homework assignments through the course portal before Friday at 11:59 PM.
Late submissions receive a ten percent penalty per day unless you request an extension
from your instructor in the questions channel.
EOF
- curl -sf -X POST "$GATEWAY/api/v1/courses/$COURSE_ID/course-resources" \
+ RESOURCE_JSON=$(curl -sf -X POST "$GATEWAY/api/v1/courses/$COURSE_ID/course-resources" \
-H "Authorization: Bearer $OWNER_TOKEN" \
-F "title=$RESOURCE_TITLE" \
-F "aiApproved=true" \
- -F "file=@$RESOURCE_FILE;type=text/plain" >/dev/null
- echo " uploaded $RESOURCE_TITLE (aiApproved=true)"
+ -F "file=@$RESOURCE_FILE;type=text/plain;filename=homework-help-guide.txt")
+ RESOURCE_ID=$(echo "$RESOURCE_JSON" | json_field "['id']")
+ echo " queued $RESOURCE_TITLE for validation and scanning"
fi
+echo "==> Wait for a clean resource and usable Study Assistant index"
+RESOURCE_READY=false
+RESOURCE_DEADLINE=$((SECONDS + 180))
+while ((SECONDS < RESOURCE_DEADLINE)); do
+ RESOURCE_STATE=$(curl -sf --max-time 10 "$GATEWAY/api/v1/course-resources/$RESOURCE_ID" \
+ -H "Authorization: Bearer $OWNER_TOKEN" | json_field "['status']")
+ if [[ "$RESOURCE_STATE" == "REJECTED" || "$RESOURCE_STATE" == "FAILED" ]]; then
+ echo "error: demo resource did not pass scanning ($RESOURCE_STATE)" >&2
+ exit 1
+ fi
+ if [[ "$RESOURCE_STATE" == "AVAILABLE" ]]; then
+ if CHUNK_COUNT=$(curl -sf --max-time 10 \
+ "${AGENT_SERVICE_URL:-http://localhost:${AGENT_PORT:-8085}}/api/v1/internal/resource-chunks/$RESOURCE_ID" \
+ -H "X-Chanter-Internal-Service-Token: $CHANTER_INTERNAL_SERVICE_TOKEN" \
+ | python3 -c 'import sys,json; print(len(json.load(sys.stdin)["chunks"]))' 2>/dev/null) && [[ "$CHUNK_COUNT" -gt 0 ]]; then
+ RESOURCE_READY=true
+ break
+ fi
+ fi
+ sleep 3
+done
+if [[ "$RESOURCE_READY" != "true" ]]; then
+ echo "error: demo resource scan/index did not become ready within the bounded wait" >&2
+ exit 1
+fi
+echo " resource is available and indexed"
+
echo "==> Install AI Study Assistant (idempotent)"
ASSISTANT_INSTALLED=$(curl -sf "$GATEWAY/api/v1/study-servers/$SERVER_ID/study-assistant" \
-H "Authorization: Bearer $OWNER_TOKEN" \
From 2042bd4cd69f6b751046c923ad640dad7b94aaa2 Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 23:14:31 -0700
Subject: [PATCH 09/10] test(media): isolate scanner readiness in product
script fixtures
---
docs/operations/issue-244-change-log.md | 2 ++
scripts/product/lib.test.sh | 9 ++++++---
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index cf634e83..6948df40 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -27,6 +27,8 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
- Native AMD64 and ARM64 diagnostics proved clamd retained September 7 signatures after FreshClam downloaded September 10 definitions during engine startup. One-minute self-checks reported no change and did not repair the missed notification. Startup now completes a foreground update before handing control back to the upstream daemon entrypoint. The 72-hour gate stays unchanged. Explicit scan limits reject encrypted/over-limit content and avoid duplicate engines during reload; the native suite tests a compressed fixture beyond the scan limit as well as EICAR.
+- Native run [34677339518](https://github.com/Vinosaamaa/chanter/actions/runs/34677339518) passed at `736301f` on AMD64 and ARM64: five actual cases across initial and preserved-volume restart phases. The [product job](https://github.com/Vinosaamaa/chanter/actions/runs/34677339521/job/103509437120) also passed all 14 browser journeys and observed the seed resource becoming available and indexed. These receipts do not establish actual OCI policy or production capacity.
+- The infrastructure unit fixture originally stubbed Docker but contacted the new real scanner readiness process. It now captures that invocation within the fixture and asserts scanner service selection. Its old service-list assertion was reproduced failing before the expected list was updated. Real scanner and browser proof remain separate hosted jobs.
## Remaining release proof
diff --git a/scripts/product/lib.test.sh b/scripts/product/lib.test.sh
index 8b6e516d..6f3a363f 100755
--- a/scripts/product/lib.test.sh
+++ b/scripts/product/lib.test.sh
@@ -140,20 +140,23 @@ assert_contains "gateway health" "http://localhost:8080/actuator/health" "$healt
assert_contains "auth health via gateway" "http://localhost:8080/api/v1/auth/health" "$health_checks"
assert_contains "realtime health" "http://localhost:8087/actuator/health" "$health_checks"
-# Exercise the selected service list without requiring Docker. Mailpit is local-only;
+# Exercise service selection and readiness invocation without requiring Docker. Mailpit is local-only;
# the unused object-store image must not block account recovery or product startup.
prepare_services() (
export CHANTER_EMAIL_LOCAL_SINK="$1"
docker() { printf '%s\n' "$*"; }
+ python3() { printf 'readiness: %s\n' "$*"; }
product_ensure_databases() { :; }
product_prepare_infrastructure
)
local_start="$(prepare_services true)"
production_start="$(prepare_services false)"
assert_contains "local SMTP inbox starts with product services" \
- "postgres redis redpanda livekit mailpit" "$local_start"
+ "postgres redis redpanda livekit clamav mailpit" "$local_start"
assert_contains "external SMTP starts only runtime dependencies" \
- "postgres redis redpanda livekit" "$production_start"
+ "postgres redis redpanda livekit clamav" "$production_start"
+assert_contains "scanner definitions gate infrastructure readiness" \
+ "scripts/media/wait-dependencies.py --scanner-only" "$local_start"
assert_eq "external SMTP does not start a test inbox" "false" \
"$(if [[ "$production_start" == *mailpit* ]]; then echo true; else echo false; fi)"
assert_eq "unused object store does not block startup" "false" \
From 01c5b4426c04d8f116b5ef9089bc938593ae60fe Mon Sep 17 00:00:00 2001
From: Vinosaamaa <12794431+Vinosaamaa@users.noreply.github.com>
Date: Fri, 11 Sep 2026 23:30:33 -0700
Subject: [PATCH 10/10] fix(media): isolate scanner traffic on a private Unix
socket
---
.github/workflows/media-security.yml | 8 +++
.../media/application/ClamAvScanner.java | 61 +++++++++++++------
.../src/main/resources/application.yml | 4 +-
.../media/application/ClamAvScannerTest.java | 26 ++++++--
.../PrivateStorageIntegrationTest.java | 2 +-
.../src/test/resources/application-test.yml | 2 +
...ecture-review-chanter-private-resources.md | 2 +-
docs/operations/issue-244-change-log.md | 4 +-
docs/operations/private-course-resources.md | 10 ++-
infra/docker-compose.yml | 2 +
infra/media-security/clamd.conf | 6 +-
infra/media-security/compose.yml | 10 +--
infra/media-security/start-scanner.sh | 18 ++++++
scripts/media/wait-dependencies.py | 4 +-
scripts/product/lib.sh | 3 +
15 files changed, 123 insertions(+), 39 deletions(-)
create mode 100644 infra/media-security/start-scanner.sh
diff --git a/.github/workflows/media-security.yml b/.github/workflows/media-security.yml
index e24ad865..9f614884 100644
--- a/.github/workflows/media-security.yml
+++ b/.github/workflows/media-security.yml
@@ -38,8 +38,16 @@ jobs:
java-version: '21.0.12+8.0.LTS'
- name: Start real PostgreSQL, S3 emulator and malware scanner
run: |
+ export CHANTER_SCANNER_CLIENT_GID="$(id -g)"
+ export CHANTER_CLAMAV_SOCKET_PATH="$GITHUB_WORKSPACE/.cache/media-socket/clamd.sock"
+ echo "CHANTER_SCANNER_CLIENT_GID=$CHANTER_SCANNER_CLIENT_GID" >> "$GITHUB_ENV"
+ echo "CHANTER_CLAMAV_SOCKET_PATH=$CHANTER_CLAMAV_SOCKET_PATH" >> "$GITHUB_ENV"
docker compose -f infra/media-security/compose.yml up -d
python3 scripts/media/wait-dependencies.py
+ test "$(stat -c '%a' .cache/media-socket/clamd.sock)" = 660
+ test "$(stat -c '%g' .cache/media-socket/clamd.sock)" = "$(id -g)"
+ test "$(stat -c '%u' .cache/media-socket/clamd.sock)" = 1000
+ test "$(stat -c '%a' .cache/media-socket)" = 2770
- name: Verify private upload, real malware rejection and metered object operations
run: scripts/java21.sh mvn -B -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false
- name: Restart database and storage processes while preserving their volumes
diff --git a/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java b/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java
index f7db10fc..e5071660 100644
--- a/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java
+++ b/backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java
@@ -3,7 +3,10 @@
import java.io.DataOutputStream;
import java.io.IOException;
import java.net.InetSocketAddress;
-import java.net.Socket;
+import java.net.StandardProtocolFamily;
+import java.net.UnixDomainSocketAddress;
+import java.nio.channels.Channels;
+import java.nio.channels.SocketChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -21,21 +24,31 @@ public class ClamAvScanner implements MalwareScanner {
private static final java.util.concurrent.ScheduledExecutorService DEADLINES = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(
runnable -> { var thread = new Thread(runnable, "clamav-socket-deadlines"); thread.setDaemon(true); return thread; });
private final String host;
+ private final Path socketPath;
+ private final boolean tcpDevelopment;
private final int port;
private final Duration timeout;
private final Duration maximumAge;
private final Clock clock;
- public ClamAvScanner(@Value("${chanter.media.scanner.host:localhost}") String host,
+ public ClamAvScanner(@Value("${chanter.media.scanner.socket-path:/run/clamav/clamd.sock}") String socketPath,
+ @Value("${chanter.media.scanner.tcp-development:false}") boolean tcpDevelopment,
+ @Value("${chanter.media.scanner.host:127.0.0.1}") String host,
@Value("${chanter.media.scanner.port:3310}") int port,
@Value("${chanter.media.scanner.timeout:20s}") Duration timeout,
@Value("${chanter.media.scanner.maximum-definition-age:72h}") Duration maximumAge, Clock clock) {
if (port < 1 || port > 65535 || timeout.toMillis() < 1 || timeout.compareTo(Duration.ofSeconds(30)) > 0
|| maximumAge.isNegative() || maximumAge.isZero() || maximumAge.compareTo(Duration.ofDays(7)) > 0) throw new IllegalArgumentException("Invalid scanner policy");
+ if (tcpDevelopment && !java.util.Set.of("127.0.0.1", "::1", "localhost").contains(host)) {
+ throw new IllegalArgumentException("Development scanner TCP must use a literal loopback host");
+ }
+ this.socketPath = tcpDevelopment ? null : Path.of(socketPath);
+ if (!tcpDevelopment && !this.socketPath.isAbsolute()) throw new IllegalArgumentException("Scanner socket path must be absolute");
+ this.tcpDevelopment = tcpDevelopment;
this.host = host; this.port = port; this.timeout = timeout; this.maximumAge = maximumAge; this.clock = clock;
}
@Override public Verdict scan(Path file) throws IOException {
try (var socket = connect()) {
- socket.getOutputStream().write("zVERSION\0".getBytes(StandardCharsets.US_ASCII));
+ socket.output().write("zVERSION\0".getBytes(StandardCharsets.US_ASCII));
String[] version = response(socket).split("/", 3);
if (version.length != 3) throw new IOException("Scanner definitions are unavailable");
try {
@@ -46,8 +59,8 @@ public ClamAvScanner(@Value("${chanter.media.scanner.host:localhost}") String ho
} catch (java.time.format.DateTimeParseException exception) { throw new IOException("Scanner definitions are unavailable"); }
}
try (var socket = connect(); var source = Files.newInputStream(file)) {
- socket.getOutputStream().write("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII));
- var output = new DataOutputStream(socket.getOutputStream());
+ socket.output().write("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII));
+ var output = new DataOutputStream(socket.output());
byte[] buffer = new byte[8192]; int count; long size = 0;
while ((count = source.read(buffer)) != -1) {
if ((size += count) > 10L * 1024 * 1024) throw new IOException("Scanner stream limit exceeded");
@@ -60,23 +73,33 @@ public ClamAvScanner(@Value("${chanter.media.scanner.host:localhost}") String ho
throw new IOException("Scanner did not verify the resource");
}
}
- private Socket connect() throws IOException {
- // SO_TIMEOUT covers reads only. Closing the socket also bounds a blocked INSTREAM write.
- Socket socket = new Socket() {
- private final java.util.concurrent.ScheduledFuture deadline = DEADLINES.schedule(() -> {
- try { close(); } catch (IOException ignored) { }
- }, timeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);
- @Override public void close() throws IOException { if (deadline != null) deadline.cancel(false); super.close(); }
- };
+ private Connection connect() throws IOException {
+ SocketChannel channel;
+ try { channel = SocketChannel.open(tcpDevelopment
+ ? (host.equals("::1") ? StandardProtocolFamily.INET6 : StandardProtocolFamily.INET) : StandardProtocolFamily.UNIX); }
+ catch (UnsupportedOperationException unavailable) { throw new IOException("Local scanner transport is unavailable"); }
+ // Closing the channel bounds connection, blocked writes and reads on both supported transports.
+ var deadline = DEADLINES.schedule(() -> {
+ try { channel.close(); } catch (IOException ignored) { }
+ }, timeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);
+ var connection = new Connection(channel, deadline);
try {
- socket.connect(new InetSocketAddress(host, port), Math.min(3000, Math.toIntExact(timeout.toMillis())));
- socket.setSoTimeout(Math.toIntExact(timeout.toMillis()));
- return socket;
- } catch (IOException exception) { socket.close(); throw new IOException("Scanner is unavailable"); }
+ if (tcpDevelopment) {
+ String loopback = host.equals("localhost") ? "127.0.0.1" : host;
+ channel.connect(new InetSocketAddress(loopback, port));
+ } else channel.connect(UnixDomainSocketAddress.of(socketPath));
+ return connection;
+ } catch (IOException | RuntimeException exception) { connection.close(); throw new IOException("Scanner is unavailable"); }
+ }
+ private record Connection(SocketChannel channel, java.util.concurrent.ScheduledFuture deadline) implements AutoCloseable {
+ java.io.InputStream input() { return Channels.newInputStream(channel); }
+ java.io.OutputStream output() { return Channels.newOutputStream(channel); }
+ @Override public void close() throws IOException { deadline.cancel(false); channel.close(); }
}
- private static String response(Socket socket) throws IOException {
+ private static String response(Connection socket) throws IOException {
var bytes = new java.io.ByteArrayOutputStream(); int value;
- while ((value = socket.getInputStream().read()) != 0) {
+ var input = socket.input();
+ while ((value = input.read()) != 0) {
if (value == -1 || bytes.size() >= 4096) throw new IOException("Invalid scanner response");
bytes.write(value);
}
diff --git a/backend/media-service/src/main/resources/application.yml b/backend/media-service/src/main/resources/application.yml
index 2b67efb7..93a016c1 100644
--- a/backend/media-service/src/main/resources/application.yml
+++ b/backend/media-service/src/main/resources/application.yml
@@ -44,7 +44,9 @@ chanter:
access-key: ${CHANTER_S3_ACCESS_KEY:}
secret-key: ${CHANTER_S3_SECRET_KEY:}
scanner:
- host: ${CHANTER_CLAMAV_HOST:localhost}
+ socket-path: ${CHANTER_CLAMAV_SOCKET_PATH:/run/clamav/clamd.sock}
+ tcp-development: ${CHANTER_CLAMAV_TCP_DEVELOPMENT:false}
+ host: ${CHANTER_CLAMAV_HOST:127.0.0.1}
port: ${CHANTER_CLAMAV_PORT:3310}
timeout: 20s
maximum-definition-age: 72h
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java b/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java
index fa641c68..8220eb3a 100644
--- a/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/application/ClamAvScannerTest.java
@@ -17,6 +17,24 @@
import org.junit.jupiter.api.Test;
class ClamAvScannerTest {
+ @Test
+ void remotePlaintextScannerIsRejectedEvenWhenDevelopmentTcpIsEnabled() {
+ assertThatThrownBy(() -> new ClamAvScanner("", true, "scanner.example.test", 3310,
+ Duration.ofSeconds(1), Duration.ofHours(72), Clock.systemUTC()))
+ .isInstanceOf(IllegalArgumentException.class).hasMessageContaining("loopback");
+ }
+
+ @Test
+ void missingUnixSocketFailsWithoutFallingBackToTcp() throws Exception {
+ try (var server = new ServerSocket(0)) {
+ server.setSoTimeout(300);
+ var scanner = new ClamAvScanner(Path.of("target/missing-scanner.sock").toAbsolutePath().toString(), false,
+ "127.0.0.1", server.getLocalPort(), Duration.ofMillis(100), Duration.ofHours(72), Clock.systemUTC());
+ assertThatThrownBy(() -> scanner.scan(Path.of("target/missing.txt"))).isInstanceOf(java.io.IOException.class);
+ assertThatThrownBy(server::accept).isInstanceOf(java.net.SocketTimeoutException.class);
+ }
+ }
+
@org.junit.jupiter.params.ParameterizedTest
@org.junit.jupiter.params.provider.ValueSource(booleans = {true, false})
void streamsBytesAndDistinguishesMalwareFromClean(boolean infected) throws Exception {
@@ -38,7 +56,7 @@ void streamsBytesAndDistinguishesMalwareFromClean(boolean infected) throws Excep
} catch (Exception exception) { throw new RuntimeException(exception); }
});
Path file = Path.of("target/clam-stream-fixture.txt"); Files.createDirectories(file.getParent()); Files.writeString(file, "scan me");
- var scanner = new ClamAvScanner("127.0.0.1", server.getLocalPort(), Duration.ofSeconds(2), Duration.ofHours(72), Clock.systemUTC());
+ var scanner = new ClamAvScanner("", true, "127.0.0.1", server.getLocalPort(), Duration.ofSeconds(2), Duration.ofHours(72), Clock.systemUTC());
assertThat(scanner.scan(file)).isEqualTo(infected ? MalwareScanner.Verdict.INFECTED : MalwareScanner.Verdict.CLEAN);
serving.get(5, java.util.concurrent.TimeUnit.SECONDS);
}
@@ -47,7 +65,7 @@ void streamsBytesAndDistinguishesMalwareFromClean(boolean infected) throws Excep
@Test
void refusesUnavailableScannerInsteadOfDeclaringTheFileClean() throws Exception {
int port; try (var server = new ServerSocket(0)) { port = server.getLocalPort(); }
- var scanner = new ClamAvScanner("127.0.0.1", port, Duration.ofMillis(100), Duration.ofHours(72), Clock.systemUTC());
+ var scanner = new ClamAvScanner("", true, "127.0.0.1", port, Duration.ofMillis(100), Duration.ofHours(72), Clock.systemUTC());
assertThatThrownBy(() -> scanner.scan(Path.of("target/missing.txt"))).isInstanceOf(java.io.IOException.class);
}
@@ -60,7 +78,7 @@ void staleDefinitionsFailClosedBeforeSendingResourceBytes() throws Exception {
socket.getOutputStream().write("ClamAV 1.5.2/28000/Mon Jan 1 00:00:00 2024\0".getBytes(StandardCharsets.US_ASCII));
} catch (Exception exception) { throw new RuntimeException(exception); }
});
- var scanner = new ClamAvScanner("127.0.0.1", server.getLocalPort(), Duration.ofSeconds(1), Duration.ofHours(72), Clock.systemUTC());
+ var scanner = new ClamAvScanner("", true, "127.0.0.1", server.getLocalPort(), Duration.ofSeconds(1), Duration.ofHours(72), Clock.systemUTC());
assertThatThrownBy(() -> scanner.scan(Path.of("target/missing.txt"))).isInstanceOf(java.io.IOException.class).hasMessageContaining("stale");
serving.get(2, java.util.concurrent.TimeUnit.SECONDS);
}
@@ -83,7 +101,7 @@ void blockedScannerWritesHaveABoundedDeadline() throws Exception {
} catch (Exception exception) { throw new RuntimeException(exception); }
});
Path file = Path.of("target/clam-blocked-fixture.bin"); Files.createDirectories(file.getParent()); Files.write(file, new byte[10 * 1024 * 1024]);
- var scanner = new ClamAvScanner("127.0.0.1", server.getLocalPort(), Duration.ofMillis(150), Duration.ofHours(72), Clock.systemUTC());
+ var scanner = new ClamAvScanner("", true, "127.0.0.1", server.getLocalPort(), Duration.ofMillis(150), Duration.ofHours(72), Clock.systemUTC());
long started = System.nanoTime();
assertThatThrownBy(() -> scanner.scan(file)).isInstanceOf(java.io.IOException.class);
assertThat(Duration.ofNanos(System.nanoTime() - started)).isLessThan(Duration.ofSeconds(1));
diff --git a/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java b/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
index 856e6efe..302c7b81 100644
--- a/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
+++ b/backend/media-service/src/test/java/com/chanter/media/application/PrivateStorageIntegrationTest.java
@@ -50,7 +50,7 @@ class PrivateStorageIntegrationTest {
registry.add("chanter.media.s3.access-key", () -> "emulator-only");
registry.add("chanter.media.s3.secret-key", () -> "emulator-only");
registry.add("chanter.media.s3.allow-local-http", () -> true);
- registry.add("chanter.media.scanner.host", () -> "127.0.0.1");
+ registry.add("chanter.media.scanner.socket-path", () -> System.getenv("CHANTER_CLAMAV_SOCKET_PATH"));
}
@BeforeAll void setup() {
access.grantInstructorUpload(COURSE, TEACHER); access.grantLearnerView(COURSE, LEARNER);
diff --git a/backend/media-service/src/test/resources/application-test.yml b/backend/media-service/src/test/resources/application-test.yml
index 93412c8f..e51f8230 100644
--- a/backend/media-service/src/test/resources/application-test.yml
+++ b/backend/media-service/src/test/resources/application-test.yml
@@ -17,3 +17,5 @@ chanter:
spool-dir: target/media-spool-test
storage-backend: local
worker-enabled: false
+ scanner:
+ socket-path: ${user.dir}/target/unused-scanner.sock
diff --git a/docs/engineering/records/architecture-review-chanter-private-resources.md b/docs/engineering/records/architecture-review-chanter-private-resources.md
index 5472e838..200952e5 100644
--- a/docs/engineering/records/architecture-review-chanter-private-resources.md
+++ b/docs/engineering/records/architecture-review-chanter-private-resources.md
@@ -54,7 +54,7 @@ The application has four public statuses, while its internal states distinguish
Declared length, filename extension and MIME are checked against bounded actual bytes. SHA-256 persists with metadata, and each download spools one object read and verifies the size and hash before the response can contain file content. Quarantined, unauthorized, corrupt, rejected and deleted objects are not downloadable. Private spools are bounded, closed on failure, and removed after use or age-based recovery.
-ClamAV must supply a clean verdict using fresh definitions. Unavailable, malformed or stale responses fail closed. A regression exposed a socket timeout gap: read timeouts do not cover a blocked upload write. A separate socket deadline now bounds both. A clean file becomes available independently of the AI index; separate durable ingestion status retries index failures without deleting valid files. Deletion retries object and AI-chunk cleanup before releasing reserved bytes.
+ClamAV must supply a clean verdict using fresh definitions. Unavailable, malformed or stale responses fail closed. A regression exposed a socket timeout gap: read timeouts do not cover a blocked upload write. Closing the channel on a total deadline now bounds connection, reads and writes. The scanner defaults to Unix IPC because its TCP protocol lacks encryption and authentication. Only the media and scanner processes share the socket directory, with mode 2770 and a group-restricted 0660 socket. Optional development TCP is limited to loopback and disabled by default; Unix errors never trigger a network fallback. A clean file becomes available independently of the AI index; separate durable ingestion status retries index failures without deleting valid files. Deletion retries object and AI-chunk cleanup before releasing reserved bytes.
The database reserves at most 8 GB of resource bytes. It commits every S3 attempt before network I/O and disables SDK retries. Normal uploads, reads and listings share 36,000 monthly operations; 4,000 operations remain reserved for deletion. Uncertain calls never refund operation counts. These limits control this module, not other provider clients. Account backup space, inventory reconciliation after restore, and provider limits remain operator responsibilities.
diff --git a/docs/operations/issue-244-change-log.md b/docs/operations/issue-244-change-log.md
index 6948df40..6c4a70a0 100644
--- a/docs/operations/issue-244-change-log.md
+++ b/docs/operations/issue-244-change-log.md
@@ -14,6 +14,7 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- Clean file publication is independent of indexing: durable ingestion status retries failures while preserving `AVAILABLE` content and its byte reservation. Index failure never triggers object expiry or deletion.
- Added native architecture CI using pinned PostgreSQL16.15, S3Mock5.2.2 and ClamAV1.5.4, actual EICAR scanning and restart durability checks.
- The product stack inherits that scanner configuration and checks definition freshness. Demo seeding supplies an explicit safe filename and waits for clean availability and usable index chunks before installing grants.
+- CodeAnt identified plaintext scanner transport. The default now uses a Unix socket shared only by scanner and media processes, with explicit directory/socket ownership and permissions. The optional TCP client is development-only, defaults off and rejects remote hosts; there is no fallback from Unix failure.
## Test evidence
@@ -22,13 +23,14 @@ Owning issue: [#244](https://github.com/Vinosaamaa/chanter/issues/244). Lane: me
- The provider-stream closure regression failed when spool creation was unavailable; closing the provider stream around spool creation made it pass.
- Review identified clean-file deletion risk when indexing and scan failure shared a state. The regression first failed because a learner could not read the clean resource; separate durable index work now keeps the file available across repeated index failures and retries to completion.
- Local module verification covers concurrent idempotency and request caps, stale leases, deletion versus scanning/reading, real local immutable writes, corrupt content, unavailable/infected scan outcomes, uncertain PUT cleanup, preserved legacy bytes and reconciliation.
-- Java 21 affected-module verification passed after the secure-session rebase. The 37 local media tests cover index-outage preservation, legacy-index purge, HTTP delete/usage authorization and a real HTTP 500 server proving one attempt per S3 operation. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
+- Java 21 affected-module verification passed after the secure-session rebase. The 39 local media tests cover index-outage preservation, legacy-index purge, HTTP delete/usage authorization, remote scanner TCP rejection, no Unix-to-TCP fallback and a real HTTP 500 server proving one attempt per S3 operation. Five separately gated real-process cases require hosted containers. Exact hosted results are recorded in PR checks.
- The namespace-change regression first failed because a changed bucket could construct a client. Startup binding now rejects bucket or endpoint changes before any network request, while normalized endpoints and rotated credentials pass.
- The new Compose file validates with Docker Compose; the new workflow passes actionlint. Container execution is delegated to hosted CI because the implementation host has no Docker daemon. No mock result is described as real scanner/provider evidence.
- The first native ARM64 job exposed that the ClamAV Alpine image has no ARM64 manifest. The suite now pins ClamAV's official Debian multi-architecture image, with AMD64 and ARM64 digests verified from the registry. The native gate remains enabled.
- Native AMD64 and ARM64 diagnostics proved clamd retained September 7 signatures after FreshClam downloaded September 10 definitions during engine startup. One-minute self-checks reported no change and did not repair the missed notification. Startup now completes a foreground update before handing control back to the upstream daemon entrypoint. The 72-hour gate stays unchanged. Explicit scan limits reject encrypted/over-limit content and avoid duplicate engines during reload; the native suite tests a compressed fixture beyond the scan limit as well as EICAR.
- Native run [34677339518](https://github.com/Vinosaamaa/chanter/actions/runs/34677339518) passed at `736301f` on AMD64 and ARM64: five actual cases across initial and preserved-volume restart phases. The [product job](https://github.com/Vinosaamaa/chanter/actions/runs/34677339521/job/103509437120) also passed all 14 browser journeys and observed the seed resource becoming available and indexed. These receipts do not establish actual OCI policy or production capacity.
- The infrastructure unit fixture originally stubbed Docker but contacted the new real scanner readiness process. It now captures that invocation within the fixture and asserts scanner service selection. Its old service-list assertion was reproduced failing before the expected list was updated. Real scanner and browser proof remain separate hosted jobs.
+- The Unix transport follow-up keeps the existing deadline and protocol regressions, adds remote-TCP and fallback rejection, and requires native AMD64/ARM64 tests to use the actual Unix socket. The workflow checks its UID, group and permission bits before scanning. Earlier TCP receipts above remain historical and do not substitute for the new transport's final-head gates.
## Remaining release proof
diff --git a/docs/operations/private-course-resources.md b/docs/operations/private-course-resources.md
index 05186fa0..c01806f1 100644
--- a/docs/operations/private-course-resources.md
+++ b/docs/operations/private-course-resources.md
@@ -26,7 +26,9 @@ Set these in the reviewed release's secret environment file, readable only by it
| `CHANTER_MEDIA_CLEANUP_REQUEST_RESERVE` | `4000`; only DELETE uses this protected allowance |
| `CHANTER_MEDIA_SPOOL_DIR` | Private writable directory; bounded upload/download files, no web mount |
| `COURSE_RESOURCE_STORAGE_DIR` | Existing local resource directory, retained during migration |
-| `CHANTER_CLAMAV_HOST`, `CHANTER_CLAMAV_PORT` | Private scanner listener, never publicly exposed; port `3310` |
+| `CHANTER_CLAMAV_SOCKET_PATH` | Absolute shared Unix socket path, `/run/clamav/clamd.sock` inside the media container |
+| `CHANTER_SCANNER_CLIENT_GID` | `10001` for the production media process; the native test/product stack uses its caller's group |
+| `CHANTER_CLAMAV_TCP_DEVELOPMENT` | `false`; production uses Unix IPC and never falls back to TCP |
| `CHANTER_MEDIA_WORKER_ENABLED` | `true`; set `false` during recovery before reconciling backups |
| `CHANTER_MEDIA_MIGRATE_LEGACY` | `false` normally; enable only for the reviewed legacy import |
@@ -61,7 +63,9 @@ Each download fetches once to a private file of at most 10 MiB, verifies length
## Scanner and capacity
-Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. The reviewed Compose service completes an update before starting clamd, preventing an update during engine loading from missing its notification. Startup fails if that update fails. Use `infra/media-security/clamd.conf`: encrypted or over-limit content produces a rejection; reloads block briefly instead of holding two engines. Scan uses the real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Socket deadlines bound both reads and writes. No local or production clean-verdict bypass exists.
+Run maintained ClamAV with a persistent signature directory, UTC timezone and FreshClam updates. The reviewed Compose service completes an update before starting clamd, preventing an update during engine loading from missing its notification. Startup fails if that update fails. Use `infra/media-security/clamd.conf`: encrypted or over-limit content produces a rejection; reloads block briefly instead of holding two engines. Scan uses the real INSTREAM protocol; missing, malformed, stale (older than 72 hours), or unavailable definitions fail closed. Channel-close deadlines bound connection, reads and writes. No local or production clean-verdict bypass exists.
+
+[ClamAV does not encrypt or authenticate TCP traffic](https://docs.clamav.net/manual/Usage/ClamdProtocol.html). The deployed scanner therefore listens only on a Unix socket. Mount its socket directory only into the scanner and media containers, with directory mode `2770`, socket mode `0660`, owner UID `1000`, and the media process's group. The startup script prepares that group and directory before the daemon drops privileges. A missing or denied socket fails closed. The optional development TCP client requires `CHANTER_CLAMAV_TCP_DEVELOPMENT=true` and accepts only explicit loopback hosts via `CHANTER_CLAMAV_HOST`/`CHANTER_CLAMAV_PORT`; it is never an automatic fallback. Native tests and the product stack use Unix IPC, not this development client.
[ClamAV's container instructions](https://docs.clamav.net/manual/Installing/Docker.html) recommend **4 GB** of RAM. The earlier #243 base caps total 7.625 GiB and exclude scanning. Adding 4 GiB would leave insufficient room for the OS on a 12 GB VM. Reallocate and measure the whole stack before deployment; passing the isolated media integration suite does not prove the full launch capacity. No extra VM or paid scan service is authorized as a fallback.
@@ -83,7 +87,7 @@ Ordinary reconciliation runs hourly, processes at most ten pages per run and res
Run local module tests with `mvn -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am verify`. The isolated real-process suite is `.github/workflows/media-security.yml`: pinned PostgreSQL, Adobe S3Mock and ClamAV images on native AMD64 and ARM64, real EICAR rejection, immutable writes, metered attempts, and preserved-volume process restart. EICAR is the harmless standard antivirus test fixture.
-For a local Docker host: `docker compose -f infra/media-security/compose.yml up -d`, run `python3 scripts/media/wait-dependencies.py`, then set `MEDIA_INTEGRATION=true` and `MEDIA_RESTART_PHASE=false` for `mvn ... test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false`. Restart those three services, set `MEDIA_RESTART_PHASE=true`, and rerun the same test. The fixture uses test-only credentials, loopback ports and independent named volumes. Remove that exact test stack with its Compose `down -v` command when finished.
+On a Linux Docker host, set `CHANTER_SCANNER_CLIENT_GID` to `id -g` and `CHANTER_CLAMAV_SOCKET_PATH` to the repository's absolute `.cache/media-socket/clamd.sock` path. Run `docker compose -f infra/media-security/compose.yml up -d`, then `python3 scripts/media/wait-dependencies.py`. Set `MEDIA_INTEGRATION=true` and `MEDIA_RESTART_PHASE=false` for `mvn ... test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false`. Restart those three services, set `MEDIA_RESTART_PHASE=true`, and rerun the same test. The fixture uses test-only credentials, loopback database/object-store ports, a private Unix socket and independent named volumes. Remove that exact test stack with its Compose `down -v` command when finished.
The local product stack inherits the same scanner service and daemon configuration. `make product-up` waits for fresh definitions; `make product-health` checks them again. `make product-demo-seed` uses a safe explicit multipart filename and waits for `AVAILABLE` metadata plus actual index chunks before installing Study Assistant grants. The 14 existing product browser journeys remain enabled. This development stack does not establish production capacity.
diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml
index dc2a055e..f6342728 100644
--- a/infra/docker-compose.yml
+++ b/infra/docker-compose.yml
@@ -4,6 +4,8 @@ services:
file: media-security/compose.yml
service: clamav
profiles: [product]
+ volumes:
+ - ${CHANTER_CLAMAV_SOCKET_DIR:-../.product/clamav}:/run/clamav
mailpit:
image: axllent/mailpit:v1.31.1
diff --git a/infra/media-security/clamd.conf b/infra/media-security/clamd.conf
index 8fb85917..6a42794d 100644
--- a/infra/media-security/clamd.conf
+++ b/infra/media-security/clamd.conf
@@ -1,10 +1,10 @@
# Private INSTREAM scanner: fail closed on archives/documents the engine cannot inspect.
User clamav
DatabaseDirectory /var/lib/clamav
-LocalSocket /tmp/clamd.sock
+LocalSocket /run/clamav/clamd.sock
+LocalSocketGroup chanter-media
LocalSocketMode 660
-TCPSocket 3310
-TCPAddr 0.0.0.0
+FixStaleSocket yes
LogFile /var/log/clamav/clamd.log
LogTime yes
SelfCheck 60
diff --git a/infra/media-security/compose.yml b/infra/media-security/compose.yml
index 682b84ea..232cd0d5 100644
--- a/infra/media-security/compose.yml
+++ b/infra/media-security/compose.yml
@@ -26,20 +26,20 @@ services:
mem_limit: 512m
clamav:
image: clamav/clamav-debian:1.5.4@sha256:4c975c439fcb7ab9cbdd72162c2802f5efcb4a11dcf5b0b37da1d746474ef20e
- # The upstream entrypoint prepares volume ownership before executing this command.
- # Refresh before starting clamd: an update during engine loading can miss its reload notification.
- command: ['sh', '-ec', 'unset CLAMAV_NO_CLAMD; freshclam --foreground --stdout; exec /init']
+ command: ['sh', '/etc/clamav/start-scanner.sh']
environment:
TZ: UTC
CLAMD_STARTUP_TIMEOUT: '600'
FRESHCLAM_CHECKS: '12'
- ports: ['127.0.0.1:3310:3310']
+ CHANTER_SCANNER_CLIENT_GID: ${CHANTER_SCANNER_CLIENT_GID:-10001}
volumes:
- clamav-signatures:/var/lib/clamav
- ./clamd.conf:/etc/clamav/clamd.conf:ro
+ - ./start-scanner.sh:/etc/clamav/start-scanner.sh:ro
+ - ../../.cache/media-socket:/run/clamav
mem_limit: 4g
healthcheck:
- test: ['CMD', 'clamdcheck.sh']
+ test: ['CMD-SHELL', "printf 'zPING\\000' | nc -U /run/clamav/clamd.sock | tr -d '\\000' | grep -qx PONG"]
interval: 10s
timeout: 5s
retries: 60
diff --git a/infra/media-security/start-scanner.sh b/infra/media-security/start-scanner.sh
new file mode 100644
index 00000000..a3437fdc
--- /dev/null
+++ b/infra/media-security/start-scanner.sh
@@ -0,0 +1,18 @@
+#!/bin/sh
+set -eu
+
+# /init has already prepared signature volume ownership before invoking this command.
+unset CLAMAV_NO_CLAMD
+client_gid="${CHANTER_SCANNER_CLIENT_GID:-10001}"
+case "$client_gid" in ''|*[!0-9]*) echo "Invalid scanner client group" >&2; exit 1 ;; esac
+[ "$client_gid" -ge 1 ] && [ "$client_gid" -le 65535 ]
+if ! getent group chanter-media >/dev/null; then
+ groupadd --non-unique --gid "$client_gid" chanter-media
+fi
+[ "$(getent group chanter-media | cut -d: -f3)" = "$client_gid" ]
+usermod --append --groups chanter-media clamav
+install -d -m 2770 -o clamav -g chanter-media /run/clamav
+
+# Updating during engine startup can miss clamd's notification. Finish first.
+freshclam --foreground --stdout
+exec /init
diff --git a/scripts/media/wait-dependencies.py b/scripts/media/wait-dependencies.py
index bce42df3..9158b185 100644
--- a/scripts/media/wait-dependencies.py
+++ b/scripts/media/wait-dependencies.py
@@ -22,7 +22,9 @@
with urllib.request.urlopen("http://127.0.0.1:9090/private-media-test", timeout=3) as response:
assert response.status == 200
phase = "scanner connection"
- with socket.create_connection((os.getenv("CHANTER_CLAMAV_HOST", "127.0.0.1"), int(os.getenv("CHANTER_CLAMAV_PORT", "3310"))), timeout=3) as connection:
+ with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection:
+ connection.settimeout(3)
+ connection.connect(os.getenv("CHANTER_CLAMAV_SOCKET_PATH", "/run/clamav/clamd.sock"))
connection.sendall(b"zVERSION\0")
data = bytearray()
while not data.endswith(b"\0") and len(data) < 4096:
diff --git a/scripts/product/lib.sh b/scripts/product/lib.sh
index 14347b19..21712d55 100755
--- a/scripts/product/lib.sh
+++ b/scripts/product/lib.sh
@@ -151,6 +151,9 @@ product_load_env() {
set +a
export LIVEKIT_URL="${LIVEKIT_URL:-ws://localhost:7880}"
export LIVEKIT_HTTP_URL="${LIVEKIT_HTTP_URL:-http://localhost:7880}"
+ export CHANTER_SCANNER_CLIENT_GID="${CHANTER_SCANNER_CLIENT_GID:-$(id -g)}"
+ export CHANTER_CLAMAV_SOCKET_DIR="${CHANTER_CLAMAV_SOCKET_DIR:-$root/.product/clamav}"
+ export CHANTER_CLAMAV_SOCKET_PATH="${CHANTER_CLAMAV_SOCKET_PATH:-$CHANTER_CLAMAV_SOCKET_DIR/clamd.sock}"
product_validate_runtime_secrets "$env_file"
}