Merge branch 'dev' of https://github.com/MaibornWolff/SecObserve into stackable

Author: dervoeti

.github/workflows/build_push_dev.yml

@@ -37,7 +37,7 @@ jobs:
       -
         name: Build and push backend
         id: build-and-push-backend
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
         with:
           context: .
           file: ./docker/backend/prod/django/Dockerfile
@@ -78,7 +78,7 @@ jobs:
       -
         name: Build and push frontend
         id: build-and-push-frontend
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
         with:
           context: .
           file: ./docker/frontend/prod/Dockerfile

.github/workflows/build_push_release.yml

@@ -36,7 +36,7 @@ jobs:
         run: echo "CREATED=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
       -
         name: Build and push backend
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
         with:
           context: .
           file: ./docker/backend/prod/django/Dockerfile
@@ -74,7 +74,7 @@ jobs:
         run: echo "CREATED=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
       -
         name: Build and push frontend
-        uses: docker/build-push-action@14487ce63c7a62a4a324b0bfb37086795e31c6c1 # v6.16.0
+        uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0
         with:
           context: .
           file: ./docker/frontend/prod/Dockerfile
@@ -98,13 +98,13 @@ jobs:
           ref: 'v${{ github.event.inputs.release }}'
       - 
         name: Run vulnerability scanners for images
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_configuration: 'so_configuration_sca_current.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}
       - 
         name: Run vulnerability scanners for endpoints
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_configuration: 'so_configuration_endpoints.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/check_backend.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - name: Set up Python 3.12
+      - name: Set up Python
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
           python-version: 3.12

.github/workflows/check_licenses_dev.yml

@@ -37,7 +37,7 @@ jobs:
           cdxgen ./frontend --type npm --no-babel --required-only --profile license-compliance --no-auto-compositions --project-name secobserve --output sbom_frontend_application.json
       -
         name: Import backend SBOM
-        uses: MaibornWolff/secobserve_actions_templates/actions/upload_sbom@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/upload_sbom@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_product_name: 'SecObserve'
           so_file_name: 'sbom_backend_application.json'
@@ -46,7 +46,7 @@ jobs:
           so_api_token: ${{ secrets.SO_API_TOKEN }}
       -
         name: Import frontend SBOM
-        uses: MaibornWolff/secobserve_actions_templates/actions/upload_sbom@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/upload_sbom@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_product_name: 'SecObserve'
           so_file_name: 'sbom_frontend_application.json'

.github/workflows/check_vulnerabilities.yml

@@ -14,7 +14,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       -
         name: Run vulnerability scanners for code
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_configuration: 'so_configuration_code.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/scan_sca_current.yml

@@ -16,16 +16,16 @@ jobs:
         name: Checkout
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
-          ref: 'v1.31.0'
+          ref: 'v1.32.1'
       -
         name: Run SCA vulnerability scanners
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_configuration: 'so_configuration_sca_current.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}
       - 
         name: Run endpoint vulnerability scanners
-        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@32595f58f393dbc99507c7ec574e47c3443808f1 # main
+        uses: MaibornWolff/secobserve_actions_templates/actions/vulnerability_scanner@03881bede1d05a40887bf26d8dfd7a1a37be892d # main
         with:
           so_configuration: 'so_configuration_endpoints.yml'
           SO_API_TOKEN: ${{ secrets.SO_API_TOKEN }}

.github/workflows/scorecard.yml

@@ -67,6 +67,6 @@ jobs:
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@60168efe1c415ce0f5521ea06d5c2062adbeed1b # v3.28.17
+        uses: github/codeql-action/upload-sarif@ff0a06e83cb2de871e5a09832bc6a81e7276941f # v3.28.18
         with:
           sarif_file: results.sarif

backend/application/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "1.31.0"
+__version__ = "1.32.1"
 
 import pymysql
 

backend/application/commons/api/views.py

@@ -1,5 +1,6 @@
 from typing import Union
 
+import environ
 from rest_framework.decorators import action
 from rest_framework.exceptions import ValidationError
 from rest_framework.permissions import IsAuthenticated
@@ -66,6 +67,10 @@ def get(self, request: Request) -> Response:
             if settings.feature_exploit_information:
                 features.append("feature_exploit_information")
 
+            env = environ.Env()
+            if env("EMAIL_HOST", default="") or env("EMAIL_PORT", default=""):
+                features.append("feature_email")
+
         content: dict[str, Union[int, list[str]]] = {
             "features": features,
         }

backend/application/core/api/serializers_product.py

@@ -606,7 +606,7 @@ class Meta:
         model = Branch
         fields = ["id", "name", "name_with_product"]
 
-    def get_name_with_product(self, obj: Service) -> str:
+    def get_name_with_product(self, obj: Branch) -> str:
         return f"{obj.name} ({obj.product.name})"
 
 
@@ -618,6 +618,11 @@ class ServiceSerializer(ModelSerializer):
     open_low_observation_count = SerializerMethodField()
     open_none_observation_count = SerializerMethodField()
     open_unknown_observation_count = SerializerMethodField()
+    forbidden_licenses_count = SerializerMethodField()
+    review_required_licenses_count = SerializerMethodField()
+    unknown_licenses_count = SerializerMethodField()
+    allowed_licenses_count = SerializerMethodField()
+    ignored_licenses_count = SerializerMethodField()
 
     class Meta:
         model = Service
@@ -644,6 +649,32 @@ def get_open_none_observation_count(self, obj: Service) -> int:
     def get_open_unknown_observation_count(self, obj: Service) -> int:
         return obj.open_unknown_observation_count
 
+    def get_forbidden_licenses_count(self, obj: Branch) -> int:
+        return obj.forbidden_licenses_count
+
+    def get_review_required_licenses_count(self, obj: Branch) -> int:
+        return obj.review_required_licenses_count
+
+    def get_unknown_licenses_count(self, obj: Branch) -> int:
+        return obj.unknown_licenses_count
+
+    def get_allowed_licenses_count(self, obj: Branch) -> int:
+        return obj.allowed_licenses_count
+
+    def get_ignored_licenses_count(self, obj: Branch) -> int:
+        return obj.ignored_licenses_count
+
+
+class ServiceNameSerializer(ModelSerializer):
+    name_with_product = SerializerMethodField()
+
+    class Meta:
+        model = Branch
+        fields = ["id", "name", "name_with_product"]
+
+    def get_name_with_product(self, obj: Service) -> str:
+        return f"{obj.name} ({obj.product.name})"
+
 
 class PURLTypeElementSerializer(Serializer):
     id = CharField()

backend/application/core/api/views.py

@@ -75,6 +75,7 @@
     ProductSerializer,
     PURLTypeElementSerializer,
     PURLTypeSerializer,
+    ServiceNameSerializer,
     ServiceSerializer,
 )
 from application.core.models import (
@@ -471,6 +472,18 @@ def get_queryset(self) -> QuerySet[Service]:
         return get_services().select_related("product")
 
 
+class ServiceNameViewSet(GenericViewSet, ListModelMixin, RetrieveModelMixin):
+    serializer_class = ServiceNameSerializer
+    filterset_class = ServiceFilter
+    permission_classes = (IsAuthenticated, UserHasServicePermission)
+    queryset = Service.objects.none()
+    filter_backends = [SearchFilter, DjangoFilterBackend]
+    search_fields = ["name"]
+
+    def get_queryset(self) -> QuerySet[Service]:
+        return get_services().select_related("product")
+
+
 class ObservationViewSet(ModelViewSet):
     serializer_class = ObservationSerializer
     filterset_class = ObservationFilter

backend/application/core/models.py

@@ -330,6 +330,51 @@ def open_unknown_observation_count(self) -> int:
             current_status=Status.STATUS_OPEN,
         ).count()
 
+    @property
+    def forbidden_licenses_count(self) -> int:
+        License_Component = apps.get_model("licenses", "License_Component")
+        return License_Component.objects.filter(
+            origin_service=self,
+            branch=self.product.repository_default_branch,
+            evaluation_result=License_Policy_Evaluation_Result.RESULT_FORBIDDEN,
+        ).count()
+
+    @property
+    def review_required_licenses_count(self) -> int:
+        License_Component = apps.get_model("licenses", "License_Component")
+        return License_Component.objects.filter(
+            origin_service=self,
+            branch=self.product.repository_default_branch,
+            evaluation_result=License_Policy_Evaluation_Result.RESULT_REVIEW_REQUIRED,
+        ).count()
+
+    @property
+    def unknown_licenses_count(self) -> int:
+        License_Component = apps.get_model("licenses", "License_Component")
+        return License_Component.objects.filter(
+            origin_service=self,
+            branch=self.product.repository_default_branch,
+            evaluation_result=License_Policy_Evaluation_Result.RESULT_UNKNOWN,
+        ).count()
+
+    @property
+    def allowed_licenses_count(self) -> int:
+        License_Component = apps.get_model("licenses", "License_Component")
+        return License_Component.objects.filter(
+            origin_service=self,
+            branch=self.product.repository_default_branch,
+            evaluation_result=License_Policy_Evaluation_Result.RESULT_ALLOWED,
+        ).count()
+
+    @property
+    def ignored_licenses_count(self) -> int:
+        License_Component = apps.get_model("licenses", "License_Component")
+        return License_Component.objects.filter(
+            origin_service=self,
+            branch=self.product.repository_default_branch,
+            evaluation_result=License_Policy_Evaluation_Result.RESULT_IGNORED,
+        ).count()
+
 
 class Product_Member(Model):
     product = ForeignKey(Product, on_delete=CASCADE)

backend/application/core/queries/observation.py

@@ -67,12 +67,31 @@ def get_observations_for_vulnerability_check(
     branch: Optional[Branch],
     filename: str,
     api_configuration_name: str,
+    service: Optional[str],
 ) -> QuerySet[Observation]:
+    if filename or api_configuration_name:
+        return Observation.objects.filter(
+            product=product,
+            branch=branch,
+            upload_filename=filename,
+            api_configuration_name=api_configuration_name,
+        )
+
+    if service:
+        return Observation.objects.filter(
+            product=product,
+            branch=branch,
+            upload_filename="",
+            api_configuration_name="",
+            origin_service__name=service,
+        )
+
     return Observation.objects.filter(
         product=product,
         branch=branch,
-        upload_filename=filename,
-        api_configuration_name=api_configuration_name,
+        upload_filename="",
+        api_configuration_name="",
+        origin_service__isnull=True,
     )
 
 

backend/application/import_observations/api/serializers.py

@@ -49,12 +49,14 @@ class FileUploadSBOMByIdRequestSerializer(Serializer):
     file = FileField(max_length=255)
     product = IntegerField(validators=[MinValueValidator(0)])
     branch = IntegerField(validators=[MinValueValidator(0)], required=False)
+    service = CharField(max_length=255, required=False)
 
 
 class FileUploadSBOMByNameRequestSerializer(Serializer):
     file = FileField(max_length=255)
     product_name = CharField(max_length=255)
     branch_name = CharField(max_length=255, required=False)
+    service = CharField(max_length=255, required=False)
 
 
 class ApiImportObservationsByIdRequestSerializer(Serializer):

backend/application/import_observations/api/views.py

@@ -331,12 +331,13 @@ def _file_upload_observations(
 
 def _file_upload_sbom(request_serializer: Serializer, product: Product, branch: Optional[Branch]) -> dict[str, int]:
     file = request_serializer.validated_data.get("file")
+    service = request_serializer.validated_data.get("service")
 
     file_upload_parameters = FileUploadParameters(
         product=product,
         branch=branch,
         file=file,
-        service="",
+        service=service,
         docker_image_name_tag="",
         endpoint_url="",
         kubernetes_cluster="",

backend/application/import_observations/parsers/azure_defender/parser.py

@@ -28,7 +28,9 @@ def check_format(self, data: list[dict]) -> bool:
             return True
         return False
 
-    def get_observations(self, data: list[dict], product: Product, branch: Optional[Branch]) -> list[Observation]:
+    def get_observations(
+        self, data: list[dict], product: Product, branch: Optional[Branch]
+    ) -> tuple[list[Observation], str]:
         observations = []
 
         for row in data:
@@ -64,7 +66,7 @@ def get_observations(self, data: list[dict], product: Product, branch: Optional[
 
                 observations.append(observation)
 
-        return observations
+        return observations, self.get_name()
 
     def format_markdown(self, string: str) -> str:
         string = self.replace_string_with_newlines(string, r"\.[A-Z]")

backend/application/import_observations/parsers/base_parser.py

@@ -14,12 +14,14 @@ def get_name(cls) -> str:
     def get_type(cls) -> str:
         raise NotImplementedError("get_type() must be overridden")
 
-    def get_observations(self, data: Any, product: Product, branch: Optional[Branch]) -> list[Observation]:
+    def get_observations(self, data: Any, product: Product, branch: Optional[Branch]) -> tuple[list[Observation], str]:
         raise NotImplementedError("get_observations() must be overridden")
 
-    def get_license_components(self, data: Any) -> list[License_Component]:  # pylint: disable=unused-argument
+    def get_license_components(
+        self, data: Any  # pylint: disable=unused-argument
+    ) -> tuple[list[License_Component], str]:
         # data is used in the child classes
-        return []
+        return [], ""
 
     def get_int_or_none(self, value: Optional[str]) -> int | None:
         if value:

backend/application/import_observations/parsers/cryptolyzer/parser.py

@@ -137,7 +137,7 @@ def check_format(self, data: Any) -> bool:
             return True
         return False
 
-    def get_observations(self, data: dict, product: Product, branch: Optional[Branch]) -> list[Observation]:
+    def get_observations(self, data: dict, product: Product, branch: Optional[Branch]) -> tuple[list[Observation], str]:
         observations = []
 
         observation = self.check_weak_protocols(data)
@@ -159,7 +159,7 @@ def get_observations(self, data: dict, product: Product, branch: Optional[Branch
         if observation:
             observations.append(observation)
 
-        return observations
+        return observations, self.get_name()
 
     def check_weak_protocols(self, data: dict) -> Optional[Observation]:
         endpoint_url = self.get_endpoint_url(data.get("versions", {}).get("target", {}))