Encoder testing

.github/workflows/build-fork.yaml

@@ -0,0 +1,41 @@
+name: Build and push BOOM image
+
+on:
+  push:
+    branches: [feat/public-filter-endpoints, feat/rtf-mtan-encoders]
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: keshavmajithia/boom
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest
+          platforms: linux/amd64
+          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache
+          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-cache,mode=max

Dockerfile

@@ -11,7 +11,7 @@ RUN apt-get update && \
     ca-certificates curl bash tar xz-utils gcc g++ python3 python3-venv libhdf5-dev \
     perl make libsasl2-dev libsasl2-2 default-jre-headless pkg-config clang libclang-dev && \
     apt-get clean && rm -rf /var/lib/apt/lists/* && \
-    curl -fsSL https://dlcdn.apache.org/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -o /tmp/kafka.tgz && \
+    curl -fsSL https://archive.apache.org/dist/kafka/${KAFKA_VERSION}/kafka_${SCALA_VERSION}-${KAFKA_VERSION}.tgz -o /tmp/kafka.tgz && \
     tar -xzf /tmp/kafka.tgz -C /opt && \
     ln -s /opt/kafka_${SCALA_VERSION}-${KAFKA_VERSION} /opt/kafka && \
     rm -f /tmp/kafka.tgz

k8s/08-boom-scheduler-ztf.yaml

@@ -0,0 +1,105 @@
+##############################################
+# BOOM Scheduler ZTF — Deployment
+##############################################
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: boom-scheduler-ztf
+  namespace: umn-babamul
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: boom-scheduler-ztf
+  template:
+    metadata:
+      labels:
+        app: boom-scheduler-ztf
+    spec:
+      initContainers:
+        # Downloads all ONNX model files from HuggingFace into a shared
+        # emptyDir volume before the main scheduler container starts.
+        # Models are loaded from /app/data/models/ at runtime.
+        - name: download-models
+          image: curlimages/curl:8.7.1
+          command:
+            - sh
+            - -c
+            - |
+              set -e
+              HF_BASE="https://huggingface.co/boom-astro/boomEncoders/resolve/main"
+
+              # ACAI classifiers (~35 KB each)
+              for variant in acai_h acai_n acai_v acai_o acai_b; do
+                echo "Downloading ${variant}.d1_dnn_20201130.onnx ..."
+                curl -fsSL "${HF_BASE}/${variant}.d1_dnn_20201130.onnx" -o "/models/${variant}.d1_dnn_20201130.onnx"
+              done
+
+              # BTSBot classifier (~900 KB)
+              echo "Downloading btsbot-v1.0.1.onnx ..."
+              curl -fsSL "${HF_BASE}/btsbot-v1.0.1.onnx" -o /models/btsbot-v1.0.1.onnx
+
+              # RTF encoder (8.7 MB)
+              echo "Downloading rtf_embed.onnx ..."
+              curl -fsSL "${HF_BASE}/rtf_embed.onnx" -o /models/rtf_embed.onnx
+
+              # mTAN encoder (383 KB)
+              echo "Downloading mtan_embed.onnx ..."
+              curl -fsSL "${HF_BASE}/mtan_embed.onnx" -o /models/mtan_embed.onnx
+
+              echo "All models downloaded."
+              ls -lh /models/
+          volumeMounts:
+            - name: models
+              mountPath: /models
+      containers:
+        - name: scheduler-ztf
+          image: ghcr.io/keshavmajithia/boom:latest
+          command: ["/app/scheduler", "ztf"]
+          env:
+            - name: BOOM_DATABASE__PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: boom-secrets
+                  key: BOOM_DATABASE__PASSWORD
+            - name: BOOM_API__AUTH__SECRET_KEY
+              valueFrom:
+                secretKeyRef:
+                  name: boom-secrets
+                  key: BOOM_API__AUTH__SECRET_KEY
+            - name: BOOM_API__AUTH__ADMIN_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: boom-secrets
+                  key: BOOM_API__AUTH__ADMIN_PASSWORD
+            - name: BOOM_DATABASE__HOST
+              value: "mongo"
+            - name: BOOM_DATABASE__USERNAME
+              value: "mongoadmin"
+            - name: BOOM_REDIS__HOST
+              value: "valkey"
+            - name: BOOM_BABAMUL__ENABLED
+              value: "false"
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: "http://otel-collector:4317"
+          volumeMounts:
+            - name: boom-config
+              mountPath: /app/config.yaml
+              subPath: config.yaml
+              readOnly: true
+            - name: models
+              mountPath: /app/data/models
+              readOnly: true
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "1Gi"
+            limits:
+              cpu: "2"
+              memory: "4Gi"
+      volumes:
+        - name: boom-config
+          configMap:
+            name: boom-config
+        - name: models
+          emptyDir: {}

src/api/auth.rs

@@ -183,7 +183,7 @@ pub async fn get_test_auth(db: &Database) -> Result<AuthProvider, std::io::Error
     AuthProvider::new(&app_config, &db).await
 }
 
-pub const PUBLIC_ROUTES: &[&str] = &["/docs", "/auth", "/"];
+pub const PUBLIC_ROUTES: &[&str] = &["/docs", "/auth", "/", "/filters/test", "/filters/test/count", "/surveys/ZTF/schemas", "/filters/schemas/ZTF", "/filters/schemas/LSST"];
 
 pub async fn auth_middleware(
     req: ServiceRequest,

src/api/routes/filters.rs

@@ -853,14 +853,7 @@ impl FilterTestResponse {
 pub async fn post_filter_test(
     db: web::Data<Database>,
     body: web::Json<FilterTestRequest>,
-    current_user: Option<web::ReqData<User>>,
 ) -> HttpResponse {
-    let _current_user = match current_user {
-        Some(user) => user,
-        None => {
-            return HttpResponse::Unauthorized().body("Unauthorized");
-        }
-    };
     let body = body.clone();
     let survey = body.survey;
     let permissions = body.permissions;
@@ -987,14 +980,7 @@ impl FilterTestCountResponse {
 pub async fn post_filter_test_count(
     db: web::Data<Database>,
     body: web::Json<FilterTestCountRequest>,
-    current_user: Option<web::ReqData<User>>,
 ) -> HttpResponse {
-    let _current_user = match current_user {
-        Some(user) => user,
-        None => {
-            return HttpResponse::Unauthorized().body("Unauthorized");
-        }
-    };
     let body = body.clone();
     let survey = body.survey;
     let permissions = body.permissions;

src/enrichment/models/mod.rs

@@ -1,10 +1,14 @@
 mod acai;
 mod base;
 mod btsbot;
+mod mtan;
+mod rtf;
 
 pub use acai::AcaiModel;
 pub use base::{load_model, load_model_on_device, Model, ModelError};
 pub use btsbot::BtsBotModel;
+pub use mtan::MtanModel;
+pub use rtf::RtfModel;
 
 use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
@@ -24,6 +28,8 @@ pub struct SharedModels {
     pub acai_o: Mutex<AcaiModel>,
     pub acai_b: Mutex<AcaiModel>,
     pub btsbot: Mutex<BtsBotModel>,
+    pub rtf_embed: Mutex<RtfModel>,
+    pub mtan_embed: Mutex<MtanModel>,
 }
 
 impl std::fmt::Debug for SharedModels {
@@ -63,6 +69,14 @@ impl SharedModels {
                     "data/models/btsbot-v1.0.1.onnx",
                     id,
                 )?),
+                rtf_embed: Mutex::new(RtfModel::new_on_device(
+                    "data/models/rtf_embed.onnx",
+                    id,
+                )?),
+                mtan_embed: Mutex::new(MtanModel::new_on_device(
+                    "data/models/mtan_embed.onnx",
+                    id,
+                )?),
             },
             None => Self {
                 acai_h: Mutex::new(AcaiModel::new("data/models/acai_h.d1_dnn_20201130.onnx")?),
@@ -71,6 +85,8 @@ impl SharedModels {
                 acai_o: Mutex::new(AcaiModel::new("data/models/acai_o.d1_dnn_20201130.onnx")?),
                 acai_b: Mutex::new(AcaiModel::new("data/models/acai_b.d1_dnn_20201130.onnx")?),
                 btsbot: Mutex::new(BtsBotModel::new("data/models/btsbot-v1.0.1.onnx")?),
+                rtf_embed: Mutex::new(RtfModel::new("data/models/rtf_embed.onnx")?),
+                mtan_embed: Mutex::new(MtanModel::new("data/models/mtan_embed.onnx")?),
             },
         };
         info!("all ONNX models loaded successfully");