Quarantine private Course Resources and verify downloads

.github/workflows/media-security.yml

@@ -0,0 +1,69 @@
+name: Private media storage
+
+on:
+  pull_request:
+    paths: ['backend/media-service/**', 'infra/media-security/**', 'scripts/media/**', '.github/workflows/media-security.yml']
+  push:
+    branches: [main]
+    paths: ['backend/media-service/**', 'infra/media-security/**', 'scripts/media/**', '.github/workflows/media-security.yml']
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: private-media-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  actual-storage-and-scanner:
+    if: github.event.repository.private == false
+    strategy:
+      fail-fast: false
+      matrix:
+        runner: [ubuntu-24.04, ubuntu-24.04-arm]
+    runs-on: ${{ matrix.runner }}
+    timeout-minutes: 35
+    env:
+      MEDIA_INTEGRATION: 'true'
+      MEDIA_RESTART_PHASE: 'false'
+      COMPOSE_PROJECT_NAME: media-security-test
+    steps:
+      - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803
+        with:
+          persist-credentials: false
+      - uses: actions/setup-java@b6effb05e454b25005698d916606bdc6ffcbf961
+        with:
+          distribution: temurin
+          java-version: '21.0.12+8.0.LTS'
+      - name: Start real PostgreSQL, S3 emulator and malware scanner
+        run: |
+          export CHANTER_SCANNER_CLIENT_GID="$(id -g)"
+          export CHANTER_CLAMAV_SOCKET_PATH="$GITHUB_WORKSPACE/.cache/media-socket/clamd.sock"
+          echo "CHANTER_SCANNER_CLIENT_GID=$CHANTER_SCANNER_CLIENT_GID" >> "$GITHUB_ENV"
+          echo "CHANTER_CLAMAV_SOCKET_PATH=$CHANTER_CLAMAV_SOCKET_PATH" >> "$GITHUB_ENV"
+          docker compose -f infra/media-security/compose.yml up -d
+          python3 scripts/media/wait-dependencies.py
+          test "$(stat -c '%a' .cache/media-socket/clamd.sock)" = 660
+          test "$(stat -c '%g' .cache/media-socket/clamd.sock)" = "$(id -g)"
+          test "$(stat -c '%u' .cache/media-socket/clamd.sock)" = 1000
+          test "$(stat -c '%a' .cache/media-socket)" = 2770
+      - name: Verify private upload, real malware rejection and metered object operations
+        run: scripts/java21.sh mvn -B -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false
+      - name: Restart database and storage processes while preserving their volumes
+        run: |
+          docker compose -f infra/media-security/compose.yml restart postgres s3 clamav
+          python3 scripts/media/wait-dependencies.py
+      - name: Verify durable metadata, object bytes, idempotency and deletion after restart
+        env:
+          MEDIA_RESTART_PHASE: 'true'
+        run: scripts/java21.sh mvn -B -s backend/.mvn/settings.xml -f backend/pom.xml -pl media-service -am test -Dtest=PrivateStorageIntegrationTest -Dsurefire.failIfNoSpecifiedTests=false
+      - name: Capture resource limits and actual container use
+        if: always()
+        run: |
+          docker stats --no-stream
+          docker compose -f infra/media-security/compose.yml ps
+          docker compose -f infra/media-security/compose.yml logs --tail 60
+      - name: Remove only the ephemeral test stack
+        if: always()
+        run: docker compose -f infra/media-security/compose.yml down -v

backend/media-service/pom.xml

@@ -13,7 +13,36 @@
     <artifactId>media-service</artifactId>
     <name>Chanter Media Service</name>
 
+    <dependencyManagement>
+        <dependencies>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>bom</artifactId>
+                <version>2.54.17</version>
+                <type>pom</type>
+                <scope>import</scope>
+            </dependency>
+        </dependencies>
+    </dependencyManagement>
+
     <dependencies>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+            <exclusions>
+                <exclusion><groupId>software.amazon.awssdk</groupId><artifactId>apache-client</artifactId></exclusion>
+                <exclusion><groupId>software.amazon.awssdk</groupId><artifactId>netty-nio-client</artifactId></exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>url-connection-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-core</artifactId>
+            <version>4.0.0</version>
+        </dependency>
         <dependency>
             <groupId>com.chanter</groupId>
             <artifactId>common</artifactId>

backend/media-service/src/main/java/com/chanter/media/MediaServiceApplication.java

@@ -2,8 +2,10 @@
 
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.scheduling.annotation.EnableScheduling;
 
 @SpringBootApplication
+@EnableScheduling
 public class MediaServiceApplication {
 
     public static void main(String[] args) {

backend/media-service/src/main/java/com/chanter/media/api/CourseResourceController.java

@@ -12,6 +12,10 @@
 import org.springframework.http.HttpHeaders;
 import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
+import org.springframework.core.io.InputStreamResource;
+import org.springframework.core.io.Resource;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.RequestHeader;
 import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
@@ -21,7 +25,6 @@
 import org.springframework.web.bind.annotation.RequestPart;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
-import org.springframework.web.servlet.support.ServletUriComponentsBuilder;
 
 @RestController
 @RequestMapping(ServiceInfo.API_V1_PREFIX)
@@ -39,21 +42,20 @@ public ResponseEntity<CourseResourceResponse> uploadCourseResource(
             @RequestAttribute(AuthRequestAttributes.USER_ID) UUID uploaderUserId,
             @RequestParam(required = false) String title,
             @RequestParam boolean aiApproved,
-            @RequestPart("file") MultipartFile file
+            @RequestPart("file") MultipartFile file,
+            @RequestHeader(value = "Idempotency-Key", required = false) UUID idempotencyKey,
+            @RequestHeader(value = "X-Content-SHA256", required = false) String checksum
     ) {
         CourseResource courseResource = courseResourceService.uploadCourseResource(
                 courseId,
                 uploaderUserId,
                 title,
                 aiApproved,
-                file
+                file, idempotencyKey, checksum
         );
-        URI location = ServletUriComponentsBuilder.fromCurrentRequest()
-                .path("/{resourceId}")
-                .buildAndExpand(courseResource.id())
-                .toUri();
+        URI location = URI.create(ServiceInfo.API_V1_PREFIX + "/course-resources/" + courseResource.id());
 
-        return ResponseEntity.created(location).body(CourseResourceResponse.from(courseResource));
+        return ResponseEntity.accepted().location(location).body(CourseResourceResponse.from(courseResource));
     }
 
     @GetMapping("/courses/{courseId}/course-resources")
@@ -71,7 +73,7 @@ public CourseResourceListResponse listCourseResources(
     }
 
     @GetMapping("/course-resources/{resourceId}/content")
-    public ResponseEntity<byte[]> downloadCourseResource(
+    public ResponseEntity<Resource> downloadCourseResource(
             @PathVariable UUID resourceId,
             @RequestAttribute(AuthRequestAttributes.USER_ID) UUID viewerUserId
     ) {
@@ -86,8 +88,27 @@ public ResponseEntity<byte[]> downloadCourseResource(
 
         return ResponseEntity.ok()
                 .header(HttpHeaders.CONTENT_DISPOSITION, disposition.toString())
+                .header("X-Content-Type-Options", "nosniff")
+                .header(HttpHeaders.CACHE_CONTROL, "no-store")
                 .contentType(safeContentType(stored.courseResource().contentType()))
-                .body(stored.content());
+                .contentLength(stored.courseResource().byteSize())
+                .body(new InputStreamResource(stored.content()));
+    }
+
+    @GetMapping("/course-resources/{resourceId}")
+    public CourseResourceResponse resource(@PathVariable UUID resourceId, @RequestAttribute(AuthRequestAttributes.USER_ID) UUID user) {
+        return CourseResourceResponse.from(courseResourceService.getCourseResource(resourceId, user));
+    }
+
+    @DeleteMapping("/course-resources/{resourceId}")
+    public ResponseEntity<Void> delete(@PathVariable UUID resourceId, @RequestAttribute(AuthRequestAttributes.USER_ID) UUID user) {
+        courseResourceService.deleteCourseResource(resourceId, user);
+        return ResponseEntity.noContent().build();
+    }
+
+    @GetMapping("/courses/{courseId}/course-resources/usage")
+    public com.chanter.media.application.ResourceLifecycle.Usage usage(@PathVariable UUID courseId, @RequestAttribute(AuthRequestAttributes.USER_ID) UUID user) {
+        return courseResourceService.usage(courseId, user);
     }
 
     /** Parse stored content type for download; fall back if missing/invalid (SEC-17). */

backend/media-service/src/main/java/com/chanter/media/api/CourseResourceResponse.java

@@ -13,7 +13,9 @@ public record CourseResourceResponse(
         long byteSize,
         boolean aiApproved,
         UUID uploadedByUserId,
-        Instant createdAt
+        Instant createdAt,
+        String status,
+        String sha256
 ) {
 
     public static CourseResourceResponse from(CourseResource courseResource) {
@@ -26,7 +28,9 @@ public static CourseResourceResponse from(CourseResource courseResource) {
                 courseResource.byteSize(),
                 courseResource.aiApproved(),
                 courseResource.uploadedByUserId(),
-                courseResource.createdAt()
+                courseResource.createdAt(),
+                courseResource.publicStatus(),
+                courseResource.sha256()
         );
     }
 }

backend/media-service/src/main/java/com/chanter/media/application/ClamAvScanner.java

@@ -0,0 +1,108 @@
+package com.chanter.media.application;
+
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.StandardProtocolFamily;
+import java.net.UnixDomainSocketAddress;
+import java.nio.channels.Channels;
+import java.nio.channels.SocketChannel;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Duration;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.util.Locale;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+@Component
+public class ClamAvScanner implements MalwareScanner {
+    private static final java.util.concurrent.ScheduledExecutorService DEADLINES = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(
+            runnable -> { var thread = new Thread(runnable, "clamav-socket-deadlines"); thread.setDaemon(true); return thread; });
+    private final String host;
+    private final Path socketPath;
+    private final boolean tcpDevelopment;
+    private final int port;
+    private final Duration timeout;
+    private final Duration maximumAge;
+    private final Clock clock;
+    public ClamAvScanner(@Value("${chanter.media.scanner.socket-path:/run/clamav/clamd.sock}") String socketPath,
+            @Value("${chanter.media.scanner.tcp-development:false}") boolean tcpDevelopment,
+            @Value("${chanter.media.scanner.host:127.0.0.1}") String host,
+            @Value("${chanter.media.scanner.port:3310}") int port,
+            @Value("${chanter.media.scanner.timeout:20s}") Duration timeout,
+            @Value("${chanter.media.scanner.maximum-definition-age:72h}") Duration maximumAge, Clock clock) {
+        if (port < 1 || port > 65535 || timeout.toMillis() < 1 || timeout.compareTo(Duration.ofSeconds(30)) > 0
+                || maximumAge.isNegative() || maximumAge.isZero() || maximumAge.compareTo(Duration.ofDays(7)) > 0) throw new IllegalArgumentException("Invalid scanner policy");
+        if (tcpDevelopment && !java.util.Set.of("127.0.0.1", "::1", "localhost").contains(host)) {
+            throw new IllegalArgumentException("Development scanner TCP must use a literal loopback host");
+        }
+        this.socketPath = tcpDevelopment ? null : Path.of(socketPath);
+        if (!tcpDevelopment && !this.socketPath.isAbsolute()) throw new IllegalArgumentException("Scanner socket path must be absolute");
+        this.tcpDevelopment = tcpDevelopment;
+        this.host = host; this.port = port; this.timeout = timeout; this.maximumAge = maximumAge; this.clock = clock;
+    }
+    @Override public Verdict scan(Path file) throws IOException {
+        try (var socket = connect()) {
+            socket.output().write("zVERSION\0".getBytes(StandardCharsets.US_ASCII));
+            String[] version = response(socket).split("/", 3);
+            if (version.length != 3) throw new IOException("Scanner definitions are unavailable");
+            try {
+                var updated = LocalDateTime.parse(version[2].strip().replaceAll("\\s+", " "), DateTimeFormatter.ofPattern("EEE MMM d HH:mm:ss yyyy", Locale.US)).toInstant(ZoneOffset.UTC);
+                if (updated.isBefore(clock.instant().minus(maximumAge)) || updated.isAfter(clock.instant().plusSeconds(300))) {
+                    throw new IOException("Scanner definitions are stale");
+                }
+            } catch (java.time.format.DateTimeParseException exception) { throw new IOException("Scanner definitions are unavailable"); }
+        }
+        try (var socket = connect(); var source = Files.newInputStream(file)) {
+            socket.output().write("zINSTREAM\0".getBytes(StandardCharsets.US_ASCII));
+            var output = new DataOutputStream(socket.output());
+            byte[] buffer = new byte[8192]; int count; long size = 0;
+            while ((count = source.read(buffer)) != -1) {
+                if ((size += count) > 10L * 1024 * 1024) throw new IOException("Scanner stream limit exceeded");
+                output.writeInt(count); output.write(buffer, 0, count);
+            }
+            output.writeInt(0); output.flush();
+            String result = response(socket);
+            if (result.equals("stream: OK")) return Verdict.CLEAN;
+            if (result.startsWith("stream: ") && result.endsWith(" FOUND")) return Verdict.INFECTED;
+            throw new IOException("Scanner did not verify the resource");
+        }
+    }
+    private Connection connect() throws IOException {
+        SocketChannel channel;
+        try { channel = SocketChannel.open(tcpDevelopment
+                ? (host.equals("::1") ? StandardProtocolFamily.INET6 : StandardProtocolFamily.INET) : StandardProtocolFamily.UNIX); }
+        catch (UnsupportedOperationException unavailable) { throw new IOException("Local scanner transport is unavailable"); }
+        // Closing the channel bounds connection, blocked writes and reads on both supported transports.
+        var deadline = DEADLINES.schedule(() -> {
+            try { channel.close(); } catch (IOException ignored) { }
+        }, timeout.toMillis(), java.util.concurrent.TimeUnit.MILLISECONDS);
+        var connection = new Connection(channel, deadline);
+        try {
+            if (tcpDevelopment) {
+                String loopback = host.equals("localhost") ? "127.0.0.1" : host;
+                channel.connect(new InetSocketAddress(loopback, port));
+            } else channel.connect(UnixDomainSocketAddress.of(socketPath));
+            return connection;
+        } catch (IOException | RuntimeException exception) { connection.close(); throw new IOException("Scanner is unavailable"); }
+    }
+    private record Connection(SocketChannel channel, java.util.concurrent.ScheduledFuture<?> deadline) implements AutoCloseable {
+        java.io.InputStream input() { return Channels.newInputStream(channel); }
+        java.io.OutputStream output() { return Channels.newOutputStream(channel); }
+        @Override public void close() throws IOException { deadline.cancel(false); channel.close(); }
+    }
+    private static String response(Connection socket) throws IOException {
+        var bytes = new java.io.ByteArrayOutputStream(); int value;
+        var input = socket.input();
+        while ((value = input.read()) != 0) {
+            if (value == -1 || bytes.size() >= 4096) throw new IOException("Invalid scanner response");
+            bytes.write(value);
+        }
+        return bytes.toString(StandardCharsets.US_ASCII).strip();
+    }
+}