Merge remote-tracking branch 'origin/master' into andrewmcknight/reveng-221-admin-show-of-invoices-that-are-1-difference

# Conflicts:
#	static/gsAdmin/views/invoiceComparison.tsx

Author: Andrew McKnight

migrations_lockfile.txt

@@ -7,7 +7,7 @@ will then be regenerated, and you should be able to merge without conflicts.
 
 discover: 0003_discover_json_field
 
-explore: 0008_add_trace_item_attribute_context
+explore: 0009_add_trace_item_attribute_value_context
 
 feedback: 0007_cleanup_failed_safe_deletes
 

src/sentry/api/endpoints/organization_events_validate.py

@@ -35,21 +35,30 @@ class Validation:
 
 
 @dataclass(kw_only=True)
-class AttributeValidation(Validation):
+class NamedValidation(Validation):
     name: str
+
+
+@dataclass(kw_only=True)
+class AttributeValidation(NamedValidation):
     # None when its an error
     attrType: str | None
 
 
+@dataclass(kw_only=True)
+class QueryValidation(Validation):
+    fields: list[AttributeValidation] = dataclass_field(default_factory=list)
+
+
 @dataclass(kw_only=True)
 class ValidationResponse:
     valid: bool
-    dataset: list[Validation] = dataclass_field(default_factory=list)
+    dataset: list[NamedValidation] = dataclass_field(default_factory=list)
     environment: list[Validation] = dataclass_field(default_factory=list)
     field: list[AttributeValidation] = dataclass_field(default_factory=list)
     orderby: list[AttributeValidation] = dataclass_field(default_factory=list)
     projects: list[Validation] = dataclass_field(default_factory=list)
-    query: list[Validation | AttributeValidation] = dataclass_field(default_factory=list)
+    query: QueryValidation
 
 
 def serialize_type(search_type: constants.SearchType) -> str:
@@ -183,7 +192,7 @@ def get(self, request: Request, organization: Organization) -> Response:
         if not self.has_feature(organization, request):
             return Response(status=400)
 
-        response = ValidationResponse(valid=True)
+        response = ValidationResponse(valid=True, query=QueryValidation(valid=True, error=None))
 
         try:
             snuba_params = self.get_snuba_params(
@@ -205,12 +214,17 @@ def get(self, request: Request, organization: Organization) -> Response:
             dataset = self.get_dataset(request, organization)
         except ParseError as error:
             response.valid = False
-            response.dataset.append(Validation(valid=False, error=str(error)))
+            response.dataset.append(
+                NamedValidation(
+                    name=request.GET.get("dataset", "discover"), valid=False, error=str(error)
+                )
+            )
             return self.serialize_response(response)
 
         if dataset not in RPC_DATASETS:
             response.dataset.append(
-                Validation(
+                NamedValidation(
+                    name=request.GET.get("dataset", "discover"),
                     valid=True,
                     error="This dataset is not compatible with the validate endpoint, your request may still be valid",
                 )
@@ -230,23 +244,36 @@ def get(self, request: Request, organization: Organization) -> Response:
 
         # Validate query
         query_string = request.GET.get("query", "")
-        query_validity: list[AttributeValidation] = []
         query_attributes_to_lookup: dict[AttributeKey.Type.ValueType, list[ResolvedAttribute]] = {}
         query_columns = []
         try:
-            # While resolve_query also runs parse_search_query, we don't need the resolved_query just want to dry-run it
-            # to get any errors
-            resolver.resolve_query(query_string)
-            parsed_terms = resolver.parse_search_query(query_string)
+            try:
+                parsed_terms = resolver.parse_search_query(query_string)
+            except InvalidSearchQuery as err:
+                # If we fail to parse, try again but truncate the query to hopefully get some terms
+                if err.extra is not None:
+                    try:
+                        parsed_terms = resolver.parse_search_query(
+                            query_string[: err.extra.get("idx", 0) - 1]
+                        )
+                    except InvalidSearchQuery:
+                        # If we fail again don't bubble the error up
+                        parsed_terms = []
+                else:
+                    parsed_terms = []
             query_columns = resolver.collect_terms(parsed_terms)
-            query_validity, query_attributes_to_lookup, valid = self.validate_columns(
+            response.query.fields, query_attributes_to_lookup, valid = self.validate_columns(
                 query_columns, resolver
             )
             if not valid:
                 response.valid = valid
+            # While resolve_query also runs parse_search_query, we don't need the resolved_query just want to dry-run it
+            # to get any errors
+            resolver.resolve_query(query_string)
         except InvalidSearchQuery as error:
             response.valid = False
-            response.query.append(Validation(error=str(error), valid=False))
+            response.query.error = str(error)
+            response.query.valid = False
 
         # Lookup unknown fields and add to validities
         # Combine the lookup dictionaries
@@ -288,12 +315,18 @@ def get(self, request: Request, organization: Organization) -> Response:
                             column_validity.append(validity)
                         if (
                             resolved.public_alias in query_columns
-                            and validity not in query_validity
+                            and validity not in response.query.fields
                         ):
-                            query_validity.append(validity)
+                            response.query.fields.append(validity)
 
         response.field.extend(column_validity)
-        response.query.extend(query_validity)
+        # If the response is still valid check if there's a field validity we wanna use
+        if response.query.valid:
+            for field in response.query.fields:
+                if not field.valid:
+                    response.query.valid = False
+                    response.query.error = field.error
+                    break
 
         # Validate orderby
         orderby_validity = []

src/sentry/api/endpoints/organization_trace_item_attributes.py

@@ -9,6 +9,7 @@
 from rest_framework import serializers
 from rest_framework.request import Request
 from rest_framework.response import Response
+from sentry_conventions.attributes import ATTRIBUTE_METADATA
 from sentry_protos.snuba.v1.endpoint_trace_item_attributes_pb2 import (
     TraceItemAttributeNamesRequest,
     TraceItemAttributeNamesResponse,
@@ -34,6 +35,7 @@
 from sentry.api.base import cell_silo_endpoint
 from sentry.api.bases import NoProjects, OrganizationEventsEndpointBase
 from sentry.api.endpoints.organization_trace_item_attributes_types import (
+    TraceItemAttributeContext,
     TraceItemAttributeKey,
     TraceItemAttributeSource,
 )
@@ -185,6 +187,23 @@ def get_result(self, limit, cursor=None):
     description="Sentry [search syntax](https://docs.sentry.io/concepts/search/) to filter trace items before computing attributes.",
 )
 
+EXPAND_QUERY_PARAM = OpenApiParameter(
+    name="expand",
+    location="query",
+    required=False,
+    many=True,
+    type=str,
+    enum=["context"],
+    # Internal-only for now: gated behind the data-browsing-attribute-context
+    # feature, so exclude it from the public OpenAPI spec.
+    exclude=True,
+    description=(
+        "Optional fields to expand. Pass `context` to include the sentry "
+        "conventions metadata (brief, examples, deprecation, etc.) for "
+        "attributes that map to a known convention."
+    ),
+)
+
 
 class OrganizationTraceItemAttributesEndpointBase(OrganizationEventsEndpointBase):
     publish_status = {
@@ -221,6 +240,7 @@ class OrganizationTraceItemAttributesEndpointSerializer(serializers.Serializer):
     )
     substringMatch = serializers.CharField(required=False, source="substring_match")
     query = serializers.CharField(required=False)
+    expand = serializers.MultipleChoiceField(choices=["context"], required=False)
 
     def validate(self, attrs: Any) -> Any:
         if attrs.get("item_type") is None and attrs.get("dataset") is None:
@@ -277,11 +297,60 @@ def resolve_attribute_values_referrer(item_type: str) -> Referrer:
         raise ValueError(f"Invalid item type: {item_type}")
 
 
+def build_sentry_convention_context(
+    public_name: str, internal_name: str
+) -> TraceItemAttributeContext | None:
+    """
+    Build the sentry conventions context for an attribute, if it maps to a known
+    convention. Only fields actually present in the conventions metadata are
+    included.
+
+    A convention may be keyed in ``ATTRIBUTE_METADATA`` by either the public
+    alias or the internal name (see
+    ``_update_attribute_definitions_with_deprecations`` in
+    ``search/eap/spans/attributes.py``), so we try the public name first and
+    fall back to the internal name.
+    """
+    metadata = ATTRIBUTE_METADATA.get(public_name) or ATTRIBUTE_METADATA.get(internal_name)
+    if metadata is None:
+        return None
+
+    deprecation = metadata.deprecation
+
+    # brief and isDeprecated are always present for a known convention.
+    context: TraceItemAttributeContext = {
+        "brief": metadata.brief,
+        "isDeprecated": bool(
+            deprecation is not None and (deprecation.status is not None or deprecation.replacement)
+        ),
+    }
+
+    if metadata.additional_context:
+        context["details"] = (
+            list(metadata.additional_context)
+            if isinstance(metadata.additional_context, (list, tuple))
+            else [metadata.additional_context]
+        )
+
+    if metadata.example is not None:
+        context["examples"] = (
+            list(metadata.example)
+            if isinstance(metadata.example, (list, tuple))
+            else [metadata.example]
+        )
+
+    if deprecation is not None and deprecation.replacement:
+        context["replacementAttribute"] = deprecation.replacement
+
+    return context
+
+
 def as_attribute_key(
     name: str,
     attr_type: Literal["string", "number", "boolean"],
     item_type: SupportedTraceItemType,
     is_proxy: bool = False,
+    include_context: bool = False,
 ) -> TraceItemAttributeKey:
     public_key, public_name, attribute_source = translate_internal_to_public_alias(
         name, attr_type, item_type
@@ -326,6 +395,11 @@ def as_attribute_key(
     if secondary_aliases:
         attribute_key["secondaryAliases"] = sorted(secondary_aliases)
 
+    if include_context and serialized_source["source_type"] == AttributeSourceType.SENTRY.value:
+        context = build_sentry_convention_context(public_name, name)
+        if context is not None:
+            attribute_key["context"] = context
+
     return attribute_key
 
 
@@ -365,6 +439,7 @@ class OrganizationTraceItemAttributesEndpoint(OrganizationTraceItemAttributesEnd
             ATTRIBUTE_TYPE_QUERY_PARAM,
             SUBSTRING_MATCH_QUERY_PARAM,
             SEARCH_QUERY_PARAM,
+            EXPAND_QUERY_PARAM,
             CursorQueryParam,
         ],
         responses={
@@ -438,6 +513,15 @@ def get(
         debug = request.user.is_superuser and request.GET.get("debug", False)
         debug_infos: list[dict] = []
 
+        # Only expand the sentry conventions context when explicitly requested
+        # via `expand=context` and the feature is enabled for the org. When the
+        # feature is disabled this is a no-op even if `expand=context` is passed.
+        include_context = "context" in serialized.get("expand", set()) and features.has(
+            "organizations:data-browsing-attribute-context",
+            organization,
+            actor=request.user,
+        )
+
         def data_fn(offset: int, limit: int) -> list[TraceItemAttributeKey]:
             futures = []
             with ContextPropagatingThreadPoolExecutor(
@@ -457,6 +541,7 @@ def data_fn(offset: int, limit: int) -> list[TraceItemAttributeKey]:
                             column_definitions,
                             trace_item_type,
                             include_internal,
+                            include_context,
                             debug=debug,
                         )
                     )
@@ -490,6 +575,7 @@ def query_trace_attributes(
         column_definitions: ColumnDefinitions,
         trace_item_type: SupportedTraceItemType,
         include_internal: bool,
+        include_context: bool = False,
         debug: str | bool = False,
     ) -> tuple[list[TraceItemAttributeKey], dict | None]:
         debug_info: dict | None = None
@@ -580,6 +666,7 @@ def query_trace_attributes(
                 substring_match,
                 aliased_attributes,
                 all_aliased_attributes,
+                include_context,
             )
 
             sentry_sdk.set_context("api_response", {"attributes": attributes})
@@ -597,6 +684,7 @@ def serialize_trace_attributes(
         substring_match: str,
         aliased_attributes: list[ResolvedAttribute | ProxyResolvedAttribute],
         exclude_attributes: list[ResolvedAttribute | ProxyResolvedAttribute],
+        include_context: bool = False,
     ) -> list[TraceItemAttributeKey]:
         attribute_keys = {}
         for attribute in rpc_response.attributes:
@@ -609,6 +697,7 @@ def serialize_trace_attributes(
                     attribute.name,
                     attribute_type,
                     trace_item_type,
+                    include_context=include_context,
                 )
                 if (
                     not is_sentry_convention_replacement_attribute(
@@ -645,6 +734,7 @@ def serialize_trace_attributes(
                     attribute_type,
                     trace_item_type,
                     is_proxy=isinstance(aliased_attr, ProxyResolvedAttribute),
+                    include_context=include_context,
                 )
                 if can_expose_attribute_to_api(
                     aliased_attr.internal_name,

src/sentry/api/endpoints/organization_trace_item_attributes_types.py

@@ -1,14 +1,45 @@
-from typing import Literal, NotRequired, TypedDict
+from typing import Any, Literal, NotRequired, TypedDict
 
 
 class TraceItemAttributeSource(TypedDict):
     source_type: Literal["sentry", "user"]
     is_transformed_alias: NotRequired[bool]
 
 
+class TraceItemAttributeContext(TypedDict):
+    """
+    Additional, mostly-static metadata about an attribute sourced from the
+    sentry conventions (``sentry_conventions.attributes.ATTRIBUTE_METADATA``).
+
+    Only attributes that map to a known sentry convention have context, and
+    within the context only the fields actually present in the conventions
+    metadata are included. This is only attached when the caller passes
+    ``expand=context`` and the ``data-browsing-attribute-context`` feature is
+    enabled.
+    """
+
+    # A short, human-readable description of the attribute. Always present for a
+    # known convention.
+    brief: str
+    # Whether the convention has been deprecated. Always present for a known
+    # convention.
+    isDeprecated: bool
+    # Longer-form notes that add nuance beyond the brief (e.g. caveats,
+    # double-counting warnings). Sourced from the convention's
+    # ``additional_context``.
+    details: NotRequired[list[str]]
+    # Example value(s) for the attribute, normalized to a list.
+    examples: NotRequired[list[Any]]
+    # The attribute that replaces this one, when deprecated.
+    replacementAttribute: NotRequired[str]
+
+
 class TraceItemAttributeKey(TypedDict):
     key: str
     name: str
     secondaryAliases: NotRequired[list[str]]
     attributeSource: TraceItemAttributeSource
     attributeType: Literal["string", "number", "boolean"]
+    # Sentry conventions metadata, only present when requested via
+    # ``expand=context`` (and gated behind the feature flag).
+    context: NotRequired[TraceItemAttributeContext]

src/sentry/api/event_search.py

@@ -2018,12 +2018,14 @@ def parse_search_query(
         idx = e.column()
         prefix = query[max(0, idx - 5) : idx]
         suffix = query[idx : (idx + 5)]
-        raise InvalidSearchQuery(
+        err = InvalidSearchQuery(
             "{} {}".format(
                 f"Parse error at '{prefix}{suffix}' (column {e.column():d}).",
                 "This is commonly caused by unmatched parentheses. Enclose any text in double quotes.",
-            )
+            ),
         )
+        err.extra = {"idx": idx}
+        raise err
 
     return SearchVisitor(
         config,