Merge pull request #213 from nexB/update-search

Remove search_vector field and index and create SearchVector per request

Author: JonoYang

Diff for: packagedb/api.py

@@ -8,49 +8,37 @@
 #
 
 import logging
+
+import django_filters
 from django.core.exceptions import ValidationError
-from django.db.models import OuterRef
-from django.db.models import Q
-from django.db.models import Subquery
+from django.db.models import OuterRef, Q, Subquery
+from django_filters.filters import Filter, OrderingFilter
 from django_filters.rest_framework import FilterSet
-from django_filters.filters import Filter
-from django_filters.filters import OrderingFilter
-import django_filters
-
 from packageurl import PackageURL
 from packageurl.contrib.django.utils import purl_to_lookups
-from rest_framework import status
-from rest_framework import viewsets
+from rest_framework import status, viewsets
 from rest_framework.decorators import action
 from rest_framework.response import Response
+from univers.version_constraint import InvalidConstraintsError
+from univers.version_range import RANGE_CLASS_BY_SCHEMES, VersionRange
+from univers.versions import InvalidVersion
 
-from matchcode.api import MultipleCharFilter
-from matchcode.api import MultipleCharInFilter
+from matchcode.api import MultipleCharFilter, MultipleCharInFilter
 # UnusedImport here!
 # But importing the mappers and visitors module triggers routes registration
 from minecode import visitors  # NOQA
 from minecode import priority_router
-from minecode.models import PriorityResourceURI
-from minecode.models import ScannableURI
+from minecode.models import PriorityResourceURI, ScannableURI
 from minecode.route import NoRouteAvailable
-from packagedb.models import Package
-from packagedb.models import PackageContentType
-from packagedb.models import PackageSet
-from packagedb.models import Resource
-from packagedb.serializers import DependentPackageSerializer
-from packagedb.serializers import ResourceAPISerializer
-from packagedb.serializers import PackageAPISerializer
-from packagedb.serializers import PackageSetAPISerializer
-from packagedb.serializers import PartySerializer
-from packagedb.package_managers import get_api_package_name
-from packagedb.package_managers import get_version_fetcher
-from packagedb.package_managers import VERSION_API_CLASSES_BY_PACKAGE_TYPE
-
-from univers import versions
-from univers.version_range import RANGE_CLASS_BY_SCHEMES
-from univers.versions import InvalidVersion
-from univers.version_range import VersionRange
-from univers.version_constraint import InvalidConstraintsError
+from packagedb.filters import PackageSearchFilter
+from packagedb.models import Package, PackageContentType, PackageSet, Resource
+from packagedb.package_managers import (VERSION_API_CLASSES_BY_PACKAGE_TYPE,
+                                        get_api_package_name,
+                                        get_version_fetcher)
+from packagedb.serializers import (DependentPackageSerializer,
+                                   PackageAPISerializer,
+                                   PackageSetAPISerializer, PartySerializer,
+                                   ResourceAPISerializer)
 
 logger = logging.getLogger(__name__)
 
@@ -84,21 +72,21 @@ def filter(self, qs, value):
         return qs.filter(package=package)
 
 
-class ResourceFilter(FilterSet):
+class ResourceFilterSet(FilterSet):
     package = PackageResourceUUIDFilter(label='Package UUID')
     purl = PackageResourcePurlFilter(label='Package pURL')
     md5 = MultipleCharInFilter(
-        help_text="Exact MD5. Multi-value supported.",
+        help_text='Exact MD5. Multi-value supported.',
     )
     sha1 = MultipleCharInFilter(
-        help_text="Exact SHA1. Multi-value supported.",
+        help_text='Exact SHA1. Multi-value supported.',
     )
 
 
 class ResourceViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Resource.objects.select_related('package')
     serializer_class = ResourceAPISerializer
-    filterset_class = ResourceFilter
+    filterset_class = ResourceFilterSet
     lookup_field = 'sha1'
 
     @action(detail=False, methods=['post'])
@@ -169,70 +157,63 @@ def filter_by_checksums(self, request, *args, **kwargs):
         return self.get_paginated_response(serializer.data)
 
 
-class MultiplePackageURLFilter(Filter):
+class MultiplePackageURLFilter(MultipleCharFilter):
     def filter(self, qs, value):
-        try:
-            request = self.parent.request
-        except AttributeError:
-            return None
+        if not value:
+            # Even though not a noop, no point filtering if empty.
+            return qs
 
-        values = request.GET.getlist(self.field_name)
-        if all(v == '' for v in values):
+        if self.is_noop(qs, value):
             return qs
 
-        values = {item for item in values}
+        if all(v == '' for v in value):
+            return qs
 
         q = Q()
-        for val in values:
+        for val in value:
             lookups = purl_to_lookups(val)
             if not lookups:
                 continue
-
             q.add(Q(**lookups), Q.OR)
 
-        if not q:
-            return qs.none()
-
-        return qs.filter(q)
-
-
-class PackageSearchFilter(Filter):
-    def filter(self, qs, value):
-        try:
-            request = self.parent.request
-        except AttributeError:
-            return None
-
-        if not value:
-            return qs
+        if q:
+            qs = self.get_method(qs)(q)
+        else:
+            qs = qs.none()
 
-        return Package.objects.filter(search_vector=value)
+        return qs.distinct() if self.distinct else qs
 
 
-class PackageFilter(FilterSet):
+class PackageFilterSet(FilterSet):
     type = django_filters.CharFilter(
-        lookup_expr="iexact",
-        help_text="Exact type. (case-insensitive)",
+        lookup_expr='iexact',
+        help_text='Exact type. (case-insensitive)',
     )
     namespace = django_filters.CharFilter(
-        lookup_expr="iexact",
-        help_text="Exact namespace. (case-insensitive)",
+        lookup_expr='iexact',
+        help_text='Exact namespace. (case-insensitive)',
     )
     name = MultipleCharFilter(
-        lookup_expr="iexact",
-        help_text="Exact name. Multi-value supported. (case-insensitive)",
+        lookup_expr='iexact',
+        help_text='Exact name. Multi-value supported. (case-insensitive)',
     )
     version = MultipleCharFilter(
-        help_text="Exact version. Multi-value supported.",
+        help_text='Exact version. Multi-value supported.',
     )
     md5 = MultipleCharInFilter(
-        help_text="Exact MD5. Multi-value supported.",
+        help_text='Exact MD5. Multi-value supported.',
     )
     sha1 = MultipleCharInFilter(
-        help_text="Exact SHA1. Multi-value supported.",
+        help_text='Exact SHA1. Multi-value supported.',
+    )
+    purl = MultiplePackageURLFilter(
+        label='Package URL',
+    )
+    search = PackageSearchFilter(
+        label='Search',
+        field_name='name',
+        lookup_expr='icontains',
     )
-    purl = MultiplePackageURLFilter(label='Package URL')
-    search = PackageSearchFilter(label='Search')
 
     sort = OrderingFilter(fields=[
             'type',
@@ -250,6 +231,7 @@ class PackageFilter(FilterSet):
     class Meta:
         model = Package
         fields = (
+            'search',
             'type',
             'namespace',
             'name',
@@ -270,7 +252,7 @@ class PackageViewSet(viewsets.ReadOnlyModelViewSet):
     queryset = Package.objects.prefetch_related('dependencies', 'parties')
     serializer_class = PackageAPISerializer
     lookup_field = 'uuid'
-    filterset_class = PackageFilter
+    filterset_class = PackageFilterSet
 
     @action(detail=True, methods=['get'])
     def latest_version(self, request, *args, **kwargs):
@@ -429,7 +411,7 @@ def index_packages(self, request, *args, **kwargs):
         packages = request.data.get('packages') or []
         queued_packages = []
         unqueued_packages = []
-        supported_ecosystems = ["maven", "npm"]
+        supported_ecosystems = ['maven', 'npm']
 
         unique_purls, unsupported_packages, unsupported_vers = get_resolved_purls(packages, supported_ecosystems)
 

Diff for: packagedb/filters.py

@@ -0,0 +1,92 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# purldb is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/purldb for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import shlex
+
+import django_filters
+from django.core.exceptions import FieldError
+from django.db.models import Q
+
+# The function and Classes in this file are from https://github.com/nexB/scancode.io/blob/main/scanpipe/filters.py
+
+
+def parse_query_string_to_lookups(query_string, default_lookup_expr, default_field):
+    """Parse a query string and convert it into queryset lookups using Q objects."""
+    lookups = Q()
+    terms = shlex.split(query_string)
+
+    lookup_types = {
+        "=": "iexact",
+        "^": "istartswith",
+        "$": "iendswith",
+        "~": "icontains",
+        ">": "gt",
+        "<": "lt",
+    }
+
+    for term in terms:
+        lookup_expr = default_lookup_expr
+        negated = False
+
+        if ":" in term:
+            field_name, search_value = term.split(":", maxsplit=1)
+            if field_name.endswith(tuple(lookup_types.keys())):
+                lookup_symbol = field_name[-1]
+                lookup_expr = lookup_types.get(lookup_symbol)
+                field_name = field_name[:-1]
+
+            if field_name.startswith("-"):
+                field_name = field_name[1:]
+                negated = True
+
+        else:
+            search_value = term
+            field_name = default_field
+
+        lookups &= Q(**{f"{field_name}__{lookup_expr}": search_value}, _negated=negated)
+
+    return lookups
+
+
+class QuerySearchFilter(django_filters.CharFilter):
+    """Add support for complex query syntax in search filter."""
+
+    def filter(self, qs, value):
+        if not value:
+            return qs
+
+        lookups = parse_query_string_to_lookups(
+            query_string=value,
+            default_lookup_expr=self.lookup_expr,
+            default_field=self.field_name,
+        )
+
+        try:
+            return qs.filter(lookups)
+        except FieldError:
+            return qs.none()
+
+
+class PackageSearchFilter(QuerySearchFilter):
+    def filter(self, qs, value):
+        if not value:
+            return qs
+
+        if value.startswith("pkg:"):
+            return qs.for_package_url(value)
+
+        if "://" not in value and ":" in value:
+            return super().filter(qs, value)
+
+        search_fields = ["type", "namespace", "name", "version", "download_url"]
+        lookups = Q()
+        for field_names in search_fields:
+            lookups |= Q(**{f"{field_names}__{self.lookup_expr}": value})
+
+        return qs.filter(lookups)

Diff for: packagedb/migrations/0080_remove_package_packagedb_p_search__8d33bb_gin_and_more.py

@@ -0,0 +1,20 @@
+# Generated by Django 4.1.2 on 2023-11-07 00:32
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("packagedb", "0079_alter_package_name_alter_package_namespace_and_more"),
+    ]
+
+    operations = [
+        migrations.RemoveIndex(
+            model_name="package",
+            name="packagedb_p_search__8d33bb_gin",
+        ),
+        migrations.RemoveField(
+            model_name="package",
+            name="search_vector",
+        ),
+    ]

Diff for: packagedb/models.py

@@ -531,8 +531,6 @@ class Package(
         ),
     )
 
-    search_vector = SearchVectorField(null=True)
-
     objects = PackageQuerySet.as_manager()
 
     # TODO: Think about ordering, unique together, indexes, etc.
@@ -550,8 +548,6 @@ class Meta:
             )
         ]
         indexes = [
-            # GIN index for search performance increase
-            GinIndex(fields=['search_vector']),
             # multicolumn index for search on a whole `purl`
             models.Index(fields=[
                 'type', 'namespace', 'name', 'version', 'qualifiers', 'subpath'