Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 89 additions & 39 deletions products/error_tracking/backend/facade/query_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,27 +42,57 @@ def normalize_volume_resolution(volume_resolution: int) -> int:


CONTEXT_EVENT_SELECTS = ["properties.$exception_list", "properties.$exception_releases"]
EVENT_PROPERTY_SELECTS = [
"properties.$exception_types",
"properties.$exception_values",
"properties.$exception_list",
"properties.$exception_fingerprint",
"properties.$exception_issue_id",
"properties.$session_id",
"properties.$lib",
"properties.$browser",
"properties.$browser_version",
"properties.$os",
"properties.$os_version",
"properties.$current_url",
]
DEFAULT_EVENT_CONTEXT_INCLUDES = ["exception", "environment", "navigation", "correlation"]
EVENT_CONTEXT_PROPERTY_SELECTS = {
"exception": [
"properties.$exception_types",
"properties.$exception_values",
"properties.$exception_list",
"properties.$exception_fingerprint",
"properties.$exception_level",
"properties.$exception_handled",
],
"stacktrace": ["properties.$exception_list"],
"code_variables": ["properties.$exception_list"],
"environment": [
"properties.$lib",
"properties.$lib_version",
"properties.$browser",
"properties.$browser_version",
"properties.$os",
"properties.$os_version",
"properties.$app_namespace",
"properties.$app_version",
"properties.$device_type",
],
"release": ["properties.$exception_releases"],
"navigation": ["properties.$current_url", "properties.$screen_name", "properties.$referrer"],
"correlation": [

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

adding few ids to link with other resources

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

💙

"properties.$session_id",
"properties.$trace_id",
"properties.$span_id",
"properties.$ai_trace_id",
"properties.$ai_span_id",
],
"diagnostics": ["properties.$cymbal_errors"],
}
EVENT_PROPERTY_SELECTS = list(
dict.fromkeys(select for context_group in EVENT_CONTEXT_PROPERTY_SELECTS.values() for select in context_group)
)
EVENT_SELECTS = ["uuid", "timestamp", "distinct_id", *EVENT_PROPERTY_SELECTS]
EVENT_SEARCH_PROPERTIES = ["properties.$exception_types", "properties.$exception_values", "properties.$current_url"]
PROPERTY_COLUMN_NAMES = {
select.removeprefix("properties.") for select in [*CONTEXT_EVENT_SELECTS, *EVENT_PROPERTY_SELECTS]
}


def build_event_selects(includes: list[str]) -> list[str]:
property_selects = list(
dict.fromkeys(select for include in includes for select in EVENT_CONTEXT_PROPERTY_SELECTS.get(include, []))
)
return ["uuid", "timestamp", "distinct_id", *property_selects]


def compact_dict(record: dict[str, object]) -> dict[str, object]:
"""Remove empty response fields while intentionally preserving 0 and false values."""
return {
Expand Down Expand Up @@ -178,8 +208,8 @@ def parse_jsonish(value: object) -> object:
return parsed


def truncate_text(value: object, verbosity: str) -> object:
if verbosity == "raw" or not isinstance(value, str) or len(value) <= MAX_NORMALIZED_TEXT_CHARS:
def truncate_text(value: object) -> object:
if not isinstance(value, str) or len(value) <= MAX_NORMALIZED_TEXT_CHARS:
return value
suffix = f"… [truncated from {len(value)} chars]"
return f"{value[: MAX_NORMALIZED_TEXT_CHARS - len(suffix)]}{suffix}"
Expand All @@ -193,13 +223,17 @@ def strip_non_raw_fields(record: dict[str, object]) -> dict[str, object]:
return {key: value for key, value in record.items() if key not in {"junk_drawer", "raw_id"}}


def normalize_frame(frame: object, verbosity: str, only_app_frames: bool) -> dict[str, object] | None:
def normalize_frame(
frame: object, only_app_frames: bool, include_code_variables: bool = True
) -> dict[str, object] | None:
frame_record = as_record(parse_jsonish(frame))
if frame_record is None:
return None
if only_app_frames and frame_record.get("in_app") is not True:
return None
base_frame = frame_record if verbosity == "raw" else strip_non_raw_fields(frame_record)
base_frame = strip_non_raw_fields(frame_record)
if not include_code_variables:
base_frame = {key: value for key, value in base_frame.items() if key != "code_variables"}
normalized = {
**base_frame,
"mangled_name": frame_record.get("mangled_name") or frame_record.get("function") or frame_record.get("name"),
Expand All @@ -210,7 +244,9 @@ def normalize_frame(frame: object, verbosity: str, only_app_frames: bool) -> dic
return compact_dict(normalized)


def normalize_stacktrace(stacktrace: object, verbosity: str, only_app_frames: bool) -> dict[str, object] | None:
def normalize_stacktrace(
stacktrace: object, only_app_frames: bool, include_code_variables: bool = True
) -> dict[str, object] | None:
stacktrace_record = as_record(parse_jsonish(stacktrace))
if stacktrace_record is None:
return None
Expand All @@ -219,16 +255,18 @@ def normalize_stacktrace(stacktrace: object, verbosity: str, only_app_frames: bo
[
normalized
for frame in cast(list[object], raw_frames)
if (normalized := normalize_frame(frame, verbosity, only_app_frames)) is not None
if (normalized := normalize_frame(frame, only_app_frames, include_code_variables)) is not None
]
if isinstance(raw_frames, list)
else None
)
base_stacktrace = stacktrace_record if verbosity == "raw" else strip_non_raw_fields(stacktrace_record)
base_stacktrace = strip_non_raw_fields(stacktrace_record)
return compact_dict({**base_stacktrace, "frames": frames})


def normalize_exception(exception: object, verbosity: str, only_app_frames: bool) -> dict[str, object] | None:
def normalize_exception(
exception: object, include_stacktrace: bool, only_app_frames: bool, include_code_variables: bool = True
) -> dict[str, object] | None:
exception_record = as_record(parse_jsonish(exception))
if exception_record is None:
return None
Expand All @@ -238,22 +276,21 @@ def normalize_exception(exception: object, verbosity: str, only_app_frames: bool
"value": truncate_text(
exception_record.get("value")
or exception_record.get("message")
or exception_record.get("exception_message"),
verbosity,
or exception_record.get("exception_message")
),
"module": exception_record.get("module"),
"mechanism": exception_record.get("mechanism"),
}
)
if verbosity == "summary":
if not include_stacktrace:
return summary
stacktrace = normalize_stacktrace(exception_record.get("stacktrace"), verbosity, only_app_frames)
if verbosity == "raw":
return compact_dict({**exception_record, "stacktrace": stacktrace})
stacktrace = normalize_stacktrace(exception_record.get("stacktrace"), only_app_frames, include_code_variables)
return compact_dict({**summary, "stacktrace": stacktrace})


def normalize_exception_list(value: object, verbosity: str, only_app_frames: bool) -> object:
def normalize_exception_list(
value: object, include_stacktrace: bool, only_app_frames: bool, include_code_variables: bool = True
) -> object:
parsed = parse_jsonish(value)
record = as_record(parsed)
exceptions = parsed if isinstance(parsed, list) else record.get("values") if record else None
Expand All @@ -262,26 +299,31 @@ def normalize_exception_list(value: object, verbosity: str, only_app_frames: boo
return [
normalized
for exception in exceptions
if (normalized := normalize_exception(exception, verbosity, only_app_frames)) is not None
if (normalized := normalize_exception(exception, include_stacktrace, only_app_frames, include_code_variables))
is not None
]


def normalize_string_array(value: object, verbosity: str = "summary", truncate_items: bool = False) -> object:
def normalize_string_array(value: object, truncate_items: bool = False) -> object:
parsed = parse_jsonish(value)
if not isinstance(parsed, list):
return parsed
return [truncate_text(item, verbosity) for item in parsed] if truncate_items else parsed
return [truncate_text(item) for item in parsed] if truncate_items else parsed


def normalize_error_property(name: str, value: object, verbosity: str, only_app_frames: bool) -> object:
def normalize_error_property(
name: str, value: object, include_stacktrace: bool, only_app_frames: bool, include_code_variables: bool = True
) -> object:
if name == "$exception_list":
return normalize_exception_list(value, verbosity, only_app_frames)
if name == "$exception_releases":
return normalize_exception_list(value, include_stacktrace, only_app_frames, include_code_variables)
if name in {"$exception_releases", "$cymbal_errors"}:
return parse_jsonish(value)
if name == "$exception_handled" and isinstance(value, str) and value.lower() in {"true", "false"}:
return value.lower() == "true"
if name == "$exception_types":
return normalize_string_array(value)
Comment thread
hpouillot marked this conversation as resolved.
if name == "$exception_values":
return normalize_string_array(value, verbosity, True)
return normalize_string_array(value, True)
return value


Expand All @@ -291,7 +333,13 @@ def property_name(select: str) -> str | None:
return select if select in PROPERTY_COLUMN_NAMES else None


def map_event_row(row: object, columns: list[str], verbosity: str, only_app_frames: bool) -> dict[str, object]:
def map_event_row(
row: object,
columns: list[str],
include_stacktrace: bool,
only_app_frames: bool,
include_code_variables: bool = True,
) -> dict[str, object]:
if isinstance(row, list):
values = row
else:
Expand All @@ -305,7 +353,9 @@ def map_event_row(row: object, columns: list[str], verbosity: str, only_app_fram
continue
prop = property_name(column)
if prop:
properties[prop] = normalize_error_property(prop, value, verbosity, only_app_frames)
properties[prop] = normalize_error_property(
prop, value, include_stacktrace, only_app_frames, include_code_variables
)
else:
event[column] = value
return event
Expand All @@ -318,7 +368,7 @@ def map_context_event_properties(data: dict[str, object]) -> dict[str, object]:
return {}
raw_columns = data.get("columns")
columns = [str(column) for column in raw_columns] if isinstance(raw_columns, list) else CONTEXT_EVENT_SELECTS
return cast(dict[str, object], map_event_row(row, columns, "stack", True)["properties"])
return cast(dict[str, object], map_event_row(row, columns, True, True)["properties"])


def get_frames(exception_list: object) -> list[dict[str, object]]:
Expand Down
45 changes: 36 additions & 9 deletions products/error_tracking/backend/presentation/views/query.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,13 @@
import structlog
from drf_spectacular.utils import OpenApiResponse
from rest_framework import status, viewsets
from rest_framework.exceptions import ValidationError
from rest_framework.response import Response

from posthog.schema import DateRange, ErrorTrackingIssueAssignee, ErrorTrackingQuery, EventsQuery

from posthog.hogql.errors import ResolutionError

from posthog.api.mixins import ValidatedRequest, validated_request
from posthog.api.routing import TeamAndOrgViewSetMixin
from posthog.api.utils import action
Expand All @@ -21,10 +24,11 @@
)
from products.error_tracking.backend.facade.query_utils import (
CONTEXT_EVENT_SELECTS,
EVENT_SELECTS,
DEFAULT_EVENT_CONTEXT_INCLUDES,
ISSUE_FIELDS,
LIST_ISSUE_FIELDS,
build_date_range,
build_event_selects,
build_fingerprint_event_where,
build_fingerprint_where,
build_impact,
Expand Down Expand Up @@ -181,9 +185,14 @@ def issue(self, request: ValidatedRequest, **kwargs: object) -> Response:
tags={"productKey": "error_tracking"},
)
with tags_context(product=Product.ERROR_TRACKING, feature=Feature.QUERY):
event_data = (
EventsQueryRunner(team=self.team, query=context_event_query).calculate().model_dump(mode="json")
)
try:
event_data = (
EventsQueryRunner(team=self.team, query=context_event_query, user=request.user)
.calculate()
.model_dump(mode="json")
)
except ResolutionError as error:
raise ValidationError(str(error)) from error
if event_data.get("error"):
logger.warning(
"error_tracking_issue_context_query_failed",
Expand Down Expand Up @@ -230,13 +239,20 @@ def issue_events(self, request: ValidatedRequest, **kwargs: object) -> Response:
if not facade_api.issue_exists_by_id(self.team.id, issue_id):
return Response(status=status.HTTP_404_NOT_FOUND)
date_range = build_date_range(params.get("dateRange"))
requested_includes = params.get("include")
includes = (
cast(list[str], requested_includes)
if isinstance(requested_includes, list)
else DEFAULT_EVENT_CONTEXT_INCLUDES
)
event_selects = build_event_selects(includes)
Comment thread
hpouillot marked this conversation as resolved.
Comment thread
hpouillot marked this conversation as resolved.
fingerprints = facade_api.resolve_fingerprints(self.team.pk, [issue_id])
if not fingerprints:
return Response({"results": [], "hasMore": False, "limit": limit, "offset": offset})
query = EventsQuery(
kind="EventsQuery",
event="$exception",
select=EVENT_SELECTS,
select=event_selects,
where=build_fingerprint_event_where(fingerprints, cast(str | None, params.get("searchQuery"))),
properties=cast(list[dict[str, object]], params.get("filterGroup", [])),
filterTestAccounts=cast(bool, params.get("filterTestAccounts", True)),
Expand All @@ -248,14 +264,25 @@ def issue_events(self, request: ValidatedRequest, **kwargs: object) -> Response:
tags={"productKey": "error_tracking"},
)
with tags_context(product=Product.ERROR_TRACKING, feature=Feature.QUERY):
data = EventsQueryRunner(team=self.team, query=query).calculate().model_dump(mode="json")
try:
data = (
EventsQueryRunner(team=self.team, query=query, user=request.user)
.calculate()
.model_dump(mode="json")
)
except ResolutionError as error:
raise ValidationError(str(error)) from error
raw_columns = data.get("columns")
columns = [str(column) for column in raw_columns] if isinstance(raw_columns, list) else EVENT_SELECTS
columns = [str(column) for column in raw_columns] if isinstance(raw_columns, list) else event_selects
raw_results_value = data.get("results")
raw_results: list[object] = raw_results_value if isinstance(raw_results_value, list) else []
verbosity = cast(str, params.get("verbosity", "summary"))

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Dropping the verbosity param in favor of include. It won't be backward compatible but this endpoint is only used by agents from what I saw

include_stacktrace = "stacktrace" in includes or "code_variables" in includes
only_app_frames = cast(bool, params.get("onlyAppFrames", True))
results = [map_event_row(row, columns, verbosity, only_app_frames) for row in raw_results[:limit]]
include_code_variables = "code_variables" in includes
results = [
map_event_row(row, columns, include_stacktrace, only_app_frames, include_code_variables)
for row in raw_results[:limit]
]
has_more, next_offset = get_page_info(data, limit, offset)
payload: dict[str, object] = {"results": results, "hasMore": has_more, "limit": limit, "offset": offset}
if next_offset is not None:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -199,11 +199,25 @@ class ErrorTrackingIssueEventsQueryRequestSerializer(serializers.Serializer):
)
limit = serializers.IntegerField(required=False, min_value=1, max_value=20, default=1, help_text="Page size.")
offset = serializers.IntegerField(required=False, min_value=0, default=0, help_text="Pagination offset.")
verbosity = serializers.ChoiceField(
choices=["summary", "stack", "raw"],
include = serializers.ListField(
child=serializers.ChoiceField(
choices=[
"exception",
"stacktrace",
"code_variables",
"environment",
"release",
"navigation",
"correlation",
"diagnostics",
]
),
required=False,
default="summary",
help_text="Controls exception detail size: summary, stack, or raw. Defaults to summary.",
help_text=(
"Context groups to return. Defaults to exception, environment, navigation, and correlation. "
"Request stacktrace for frames, code_variables for captured and SDK-masked frame variables, release for "
"release metadata, or diagnostics for ingestion errors. code_variables implies stacktrace."
),
)
onlyAppFrames = serializers.BooleanField(
required=False,
Expand Down
Loading
Loading