From 81df085aeb6030ce0553371d6213a1670c82a14a Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Tue, 23 Jun 2026 17:58:44 +0200 Subject: [PATCH 01/17] chore(deps): upgrade posthog-python --- common/ingestion/acceptance_tests/requirements.txt | 2 +- pyproject.toml | 2 +- uv.lock | 9 ++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/common/ingestion/acceptance_tests/requirements.txt b/common/ingestion/acceptance_tests/requirements.txt index a91f25a9528a..a0954f4b0c1f 100644 --- a/common/ingestion/acceptance_tests/requirements.txt +++ b/common/ingestion/acceptance_tests/requirements.txt @@ -5,4 +5,4 @@ requests-toolbelt==1.0.0 boto3==1.29.7 docker==6.1.3 python-multipart==0.0.6 -posthog==6.7.6 +posthog==7.20.2 diff --git a/pyproject.toml b/pyproject.toml index 492b53ddc6fc..5eba4bc54fd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,7 +100,7 @@ dependencies = [ "paramiko~=3.5.0", "pillow==12.2.0", "protobuf~=5.29.6", - "posthoganalytics==7.13.0", + "posthoganalytics==7.20.2", "polars==1.37.1", "psycopg2-binary==2.9.10", "psycopg[binary]==3.2.4", diff --git a/uv.lock b/uv.lock index 857dd7adf780..f683b1a1c091 100644 --- a/uv.lock +++ b/uv.lock @@ -5913,7 +5913,7 @@ requires-dist = [ { name = "pixelhog", specifier = "~=1.2.0" }, { name = "playwright", specifier = "~=1.60.0" }, { name = "polars", specifier = "==1.37.1" }, - { name = "posthoganalytics", specifier = "==7.13.0" }, + { name = "posthoganalytics", specifier = "==7.20.2" }, { name = "protobuf", specifier = "~=5.29.6" }, { name = "psycopg", extras = ["binary"], specifier = "==3.2.4" }, { name = "psycopg2-binary", specifier = "==2.9.10" }, @@ -6052,18 +6052,17 @@ sentiment = [ [[package]] name = "posthoganalytics" -version = "7.13.0" +version = "7.20.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, { name = "distro" }, - { name = "python-dateutil" }, { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/05/27/34aa6241cdc0145d6a7ceedfd590425d9f35ca17983f6f812f83ffce5e08/posthoganalytics-7.13.0.tar.gz", hash = "sha256:a1dc24da926f016df3c50cd6846eb899326bf83eedaa2486d1c6956a41250f16", size = 193851, upload-time = "2026-04-21T09:12:15.658Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a9/f1/7b3eede03afb2079ebcc5e229643b32cd1bc4b00c231efcd7e2c79407bcd/posthoganalytics-7.20.2.tar.gz", hash = "sha256:d5d387816a36f5458e6fcc583621e3e6d742683fff361db909ff13f3111bbcff", size = 256052, upload-time = "2026-06-22T15:36:56.593Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/28/703fc06f56231f81c522cb895ebdcd9d14bfbe38f1b8d4f07a358b6696ab/posthoganalytics-7.13.0-py3-none-any.whl", hash = "sha256:87d8f04f9255645efd058caf027b7f6a68642186d91de5f174f369c8e1819f1d", size = 229183, upload-time = "2026-04-21T09:12:13.981Z" }, + { url = "https://files.pythonhosted.org/packages/58/4c/868e2f7e033eaadad5c2be9df76dc14bca5fa66d620c02e54c08f8f4b44b/posthoganalytics-7.20.2-py3-none-any.whl", hash = "sha256:d82c9152dcf44ae08ce303d975d8a6d5848d8eeb73aabadf02acc9aeea70277e", size = 298975, upload-time = "2026-06-22T15:36:54.971Z" }, ] [[package]] From 9bcbe5a939c83562226a195e4b9bf675ccf215a4 Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Tue, 23 Jun 2026 18:12:52 +0200 Subject: [PATCH 02/17] fix(deps): handle nullable feature flag results --- ee/clickhouse/views/groups.py | 4 +-- ee/hogai/utils/feature_flags.py | 25 ++++++++------- posthog/api/email_verification.py | 4 +-- posthog/hogql_queries/ai/ai_table_resolver.py | 16 ++++++++++ posthog/hogql_queries/hogql_cohort_query.py | 6 ++-- .../insights/funnels/funnels_query_runner.py | 4 +-- .../insights/trends/trends_query_builder.py | 5 ++- .../insights/trends/trends_query_runner.py | 4 +-- .../legacy_compatibility/feature_flag.py | 4 +-- posthog/models/team/team.py | 6 ++-- posthog/ph_client.py | 31 +++++++++++++++++++ .../queries/sub_queries/events_subquery.py | 5 +-- .../activities/materialize_view_duckgres.py | 4 +-- .../ducklake_copy_data_imports_workflow.py | 4 +-- .../ducklake_copy_data_modeling_workflow.py | 4 +-- .../backend/api/score_definitions.py | 28 +++++++++++++++-- .../backend/api/trace_reviews.py | 25 ++++++++++++++- .../ai_observability/backend/feature_flags.py | 5 ++- products/business_knowledge/backend/logic.py | 4 +-- products/data_modeling/backend/api/node.py | 4 +-- .../backend/api/data_modeling_job.py | 4 +-- .../backend/data_load/service.py | 9 ++---- .../experiments/backend/presentation/views.py | 4 +-- .../feature_flags/backend/api/feature_flag.py | 10 +++--- .../marketing_analytics_config.py | 7 +++-- .../product_analytics/backend/api/insight.py | 6 ++-- .../src/llm_gateway/callbacks/__init__.py | 2 +- 27 files changed, 163 insertions(+), 71 deletions(-) diff --git a/ee/clickhouse/views/groups.py b/ee/clickhouse/views/groups.py index 7ef4ee19578e..de1016256813 100644 --- a/ee/clickhouse/views/groups.py +++ b/ee/clickhouse/views/groups.py @@ -8,7 +8,6 @@ from django.utils import timezone import structlog -import posthoganalytics from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter from loginas.utils import is_impersonated_session @@ -44,6 +43,7 @@ ) from posthog.models.user import User from posthog.personhog_client.converters import GroupTypeMappingResult +from posthog.ph_client import feature_enabled_or_false from posthog.rbac.user_access_control import UserAccessControlSerializerMixin from posthog.utils import str_to_bool @@ -871,7 +871,7 @@ def property_values(self, request: request.Request, **kw): ) def _is_crm_enabled(self, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "crm-iteration-one", str(user.distinct_id), groups={"organization": str(self.team.organization.id)}, diff --git a/ee/hogai/utils/feature_flags.py b/ee/hogai/utils/feature_flags.py index 3382bfdd032f..3087069460f3 100644 --- a/ee/hogai/utils/feature_flags.py +++ b/ee/hogai/utils/feature_flags.py @@ -3,6 +3,7 @@ import posthoganalytics from posthog.models import Team, User +from posthog.ph_client import feature_enabled_or_false from products.business_knowledge.backend.logic import has_feature_flag as bk_has_feature_flag @@ -14,7 +15,7 @@ def is_privacy_mode_enabled(team: Team) -> bool: """ Check if privacy mode is enabled for a team's organization. """ - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-privacy-mode", str(team.organization_id), groups={"organization": str(team.organization_id)}, @@ -24,7 +25,7 @@ def is_privacy_mode_enabled(team: Team) -> bool: def has_phai_tasks_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-tasks", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -34,7 +35,7 @@ def has_phai_tasks_feature_flag(team: Team, user: User) -> bool: def has_task_tool_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-task-tool", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -44,7 +45,7 @@ def has_task_tool_feature_flag(team: Team, user: User) -> bool: def has_conversation_topic_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "posthog-ai-web-analytics-nudge", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -54,7 +55,7 @@ def has_conversation_topic_feature_flag(team: Team, user: User) -> bool: def has_memory_tool_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-memory-tool", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -64,7 +65,7 @@ def has_memory_tool_feature_flag(team: Team, user: User) -> bool: def has_plan_mode_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-plan-mode", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -74,7 +75,7 @@ def has_plan_mode_feature_flag(team: Team, user: User) -> bool: def has_experiment_summary_tool_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "experiment-ai-summary", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -84,7 +85,7 @@ def has_experiment_summary_tool_feature_flag(team: Team, user: User) -> bool: def is_core_memory_disabled(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-core-mem-disabled", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -94,7 +95,7 @@ def is_core_memory_disabled(team: Team, user: User) -> bool: def has_mcp_servers_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "mcp-servers", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -104,7 +105,7 @@ def has_mcp_servers_feature_flag(team: Team, user: User) -> bool: def has_sandbox_mode_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "phai-sandbox-mode", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -114,7 +115,7 @@ def has_sandbox_mode_feature_flag(team: Team, user: User) -> bool: def has_user_interview_mode_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "user-interviews", str(user.distinct_id), groups={"organization": str(team.organization_id)}, @@ -124,7 +125,7 @@ def has_user_interview_mode_feature_flag(team: Team, user: User) -> bool: def has_customer_analytics_mode_feature_flag(team: Team, user: User) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "customer-analytics-csp", str(user.distinct_id), groups={"organization": str(team.organization_id)}, diff --git a/posthog/api/email_verification.py b/posthog/api/email_verification.py index 5057c4c288c2..9d4c7ffcf02e 100644 --- a/posthog/api/email_verification.py +++ b/posthog/api/email_verification.py @@ -1,11 +1,11 @@ from django.contrib.auth.models import AbstractBaseUser from django.contrib.auth.tokens import PasswordResetTokenGenerator -import posthoganalytics from rest_framework import exceptions from posthog.exceptions_capture import capture_exception from posthog.models.user import User +from posthog.ph_client import feature_enabled_or_false from posthog.tasks.email import send_email_verification VERIFICATION_DISABLED_FLAG = "email-verification-disabled" @@ -13,7 +13,7 @@ def is_email_verification_disabled(user: User) -> bool: # using disabled here so that the default state (if no flag exists) is that verification defaults to ON. - return user.organization is not None and posthoganalytics.feature_enabled( + return user.organization is not None and feature_enabled_or_false( VERIFICATION_DISABLED_FLAG, str(user.organization.id), groups={"organization": str(user.organization.id)}, diff --git a/posthog/hogql_queries/ai/ai_table_resolver.py b/posthog/hogql_queries/ai/ai_table_resolver.py index d98971398632..61b9a3b64427 100644 --- a/posthog/hogql_queries/ai/ai_table_resolver.py +++ b/posthog/hogql_queries/ai/ai_table_resolver.py @@ -10,6 +10,7 @@ from posthog.clickhouse.query_tagging import Product, tag_queries, tags_context from posthog.hogql_queries.ai.ai_column_rewriter import rewrite_expr_for_events_table, rewrite_query_for_events_table from posthog.hogql_queries.ai.ai_property_rewriter import rewrite_expr_for_ai_events_table +from posthog.ph_client import feature_enabled_or_false AI_EVENTS_QUERY_TOTAL = Counter( "posthog_ai_events_query_total", @@ -38,6 +39,21 @@ class AIEventsUnavailableError(Exception): and the caller opted out of the events fallback (``fall_back_to_events=False``).""" +def is_ai_events_enabled(team: Team) -> bool: + """Kill switch for ai_events table reads. + + When disabled, all single-trace runners skip the ai_events attempt + and query the events table directly. + """ + return feature_enabled_or_false( + "ai-events-table-rollout", + str(team.id), + groups={"organization": str(team.organization_id)}, + group_properties={"organization": {"id": str(team.organization_id)}}, + send_feature_flag_events=False, + ) + + class AIEventsExpiredError(AIEventsUnavailableError): """The requested AI events exist in the shared events table but have aged out of ai_events (past its retention TTL).""" diff --git a/posthog/hogql_queries/hogql_cohort_query.py b/posthog/hogql_queries/hogql_cohort_query.py index 986e1b471b95..90904d3334c6 100644 --- a/posthog/hogql_queries/hogql_cohort_query.py +++ b/posthog/hogql_queries/hogql_cohort_query.py @@ -4,7 +4,6 @@ from numbers import Number from typing import Any, Literal, Optional, Union, cast -import posthoganalytics from rest_framework.exceptions import ValidationError from posthog.schema import ( @@ -49,6 +48,7 @@ from posthog.hogql_queries.utils.query_date_range import QueryDateRange from posthog.models import Filter, Property, Team from posthog.models.property import OperatorInterval, PropertyGroup +from posthog.ph_client import feature_enabled_or_false from posthog.types import AnyPropertyFilter from products.cohorts.backend.models.cohort import Cohort @@ -589,7 +589,7 @@ def _get_condition_for_property(self, prop: Property) -> ast.SelectQuery | ast.S raise ValueError(f"Invalid property type for Cohort queries: {prop.type}") def _should_combine_person_properties_and(self) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "hogql-cohort-combine-person-properties", str(self.team.uuid), groups={ @@ -609,7 +609,7 @@ def _should_combine_person_properties_and(self) -> bool: ) def _should_combine_person_properties_or(self) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "hogql-cohort-combine-person-properties-or", str(self.team.uuid), groups={ diff --git a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py index 1c3af9300779..ea736ba5b510 100644 --- a/posthog/hogql_queries/insights/funnels/funnels_query_runner.py +++ b/posthog/hogql_queries/insights/funnels/funnels_query_runner.py @@ -7,7 +7,6 @@ from django.conf import settings import structlog -import posthoganalytics from posthog.schema import ( CachedFunnelsQueryResponse, @@ -52,6 +51,7 @@ from posthog.models import Team from posthog.models.filters.mixins.utils import cached_property from posthog.models.user import User +from posthog.ph_client import feature_enabled_or_false logger = structlog.get_logger(__name__) @@ -494,7 +494,7 @@ def _is_compare_active(self) -> bool: return self._team_flag_funnels_compare() def _team_flag_funnels_compare(self) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "product-analytics-funnels-compare", str(self.team.uuid), groups={ diff --git a/posthog/hogql_queries/insights/trends/trends_query_builder.py b/posthog/hogql_queries/insights/trends/trends_query_builder.py index 6f12e9abb312..207a52aba116 100644 --- a/posthog/hogql_queries/insights/trends/trends_query_builder.py +++ b/posthog/hogql_queries/insights/trends/trends_query_builder.py @@ -1,7 +1,5 @@ from typing import cast -import posthoganalytics - from posthog.schema import ( ActionsNode, Breakdown as BreakdownSchema, @@ -28,6 +26,7 @@ from posthog.hogql_queries.utils.query_date_range import QueryDateRange from posthog.models.filters.mixins.utils import cached_property from posthog.models.team.team import Team +from posthog.ph_client import feature_enabled_or_false from products.actions.backend.models.action import Action @@ -1012,7 +1011,7 @@ def _breakdown_outer_query_filter(self, breakdown: Breakdown): ) def _team_flag_fewer_array_ops(self) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "trends-breakdown-fewer-array-ops", str(self.team.uuid), groups={ diff --git a/posthog/hogql_queries/insights/trends/trends_query_runner.py b/posthog/hogql_queries/insights/trends/trends_query_runner.py index f17291e027ab..249a93c0df99 100644 --- a/posthog/hogql_queries/insights/trends/trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/trends_query_runner.py @@ -10,7 +10,6 @@ from django.db import models from django.db.models.functions import Coalesce -import posthoganalytics from natsort import natsorted, ns from posthog.schema import ( @@ -81,6 +80,7 @@ from posthog.models import Team from posthog.models.filters.mixins.utils import cached_property from posthog.models.user import User +from posthog.ph_client import feature_enabled_or_false from posthog.queries.util import correct_result_for_sampling from posthog.utils import multisort @@ -884,7 +884,7 @@ def _has_session_breakdown(self) -> bool: return any(breakdown.type == "session" for breakdown in (filter.breakdowns or [])) def _team_flag_session_property_pre_aggregation(self) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "trends-session-property-pre-aggregation", str(self.team.uuid), groups={ diff --git a/posthog/hogql_queries/legacy_compatibility/feature_flag.py b/posthog/hogql_queries/legacy_compatibility/feature_flag.py index 4a069b6336ee..65c18f588246 100644 --- a/posthog/hogql_queries/legacy_compatibility/feature_flag.py +++ b/posthog/hogql_queries/legacy_compatibility/feature_flag.py @@ -1,16 +1,16 @@ from typing import Literal -import posthoganalytics from rest_framework.request import Request from posthog.models import Team +from posthog.ph_client import feature_enabled_or_false def insight_api_use_legacy_queries(team: Team) -> bool: """ Use the legacy implementation of insight api calculation endpoints. """ - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "insight-api-use-legacy-queries", str(team.uuid), groups={ diff --git a/posthog/models/team/team.py b/posthog/models/team/team.py index 220434180c0b..22b740e19098 100644 --- a/posthog/models/team/team.py +++ b/posthog/models/team/team.py @@ -14,7 +14,6 @@ import pytz import pydantic -import posthoganalytics from posthog.clickhouse.query_tagging import Feature, Product, tag_queries, tags_context from posthog.cloud_utils import is_cloud @@ -33,6 +32,7 @@ sane_repr, validate_rate_limit, ) +from posthog.ph_client import feature_enabled_or_false from posthog.rbac.decorators import field_access_control from posthog.session_recordings.models.session_recording_playlist import SessionRecordingPlaylist from posthog.settings.utils import get_list @@ -796,7 +796,7 @@ def _person_on_events_person_id_no_override_properties_on_events(self) -> bool: # on PostHog Cloud, use the feature flag if is_cloud(): - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "persons-on-events-person-id-no-override-properties-on-events", str(self.uuid), groups={"project": str(self.id)}, @@ -822,7 +822,7 @@ def _person_on_events_person_id_override_properties_on_events(self) -> bool: # on PostHog Cloud, use the feature flag if is_cloud(): - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "persons-on-events-v2-reads-enabled", str(self.uuid), groups={"organization": str(self.organization_id)}, diff --git a/posthog/ph_client.py b/posthog/ph_client.py index ca0e768cdced..814d41173ccb 100644 --- a/posthog/ph_client.py +++ b/posthog/ph_client.py @@ -1,7 +1,11 @@ +from collections.abc import Mapping from contextlib import contextmanager +from numbers import Number from typing import Any +from uuid import UUID import structlog +import posthoganalytics from posthog.cloud_utils import is_cloud from posthog.utils import get_instance_region @@ -15,6 +19,33 @@ logger = structlog.get_logger(__name__) +def feature_enabled_or_false( + key: str, + distinct_id: Number | str | UUID | int, + groups: Mapping[str, str | int] | None = None, + person_properties: dict[str, Any] | None = None, + group_properties: dict[str, dict[str, Any]] | None = None, + only_evaluate_locally: bool = False, + send_feature_flag_events: bool = True, + disable_geoip: bool | None = None, + device_id: str | None = None, +) -> bool: + return ( + posthoganalytics.feature_enabled( + key, + distinct_id, + groups=groups, + person_properties=person_properties, + group_properties=group_properties, + only_evaluate_locally=only_evaluate_locally, + send_feature_flag_events=send_feature_flag_events, + disable_geoip=disable_geoip, + device_id=device_id, + ) + is True + ) + + def get_regional_ph_client(**kwargs: Any): if not is_cloud(): return diff --git a/posthog/session_recordings/queries/sub_queries/events_subquery.py b/posthog/session_recordings/queries/sub_queries/events_subquery.py index 72e9ae6b68e6..1227a987d8b9 100644 --- a/posthog/session_recordings/queries/sub_queries/events_subquery.py +++ b/posthog/session_recordings/queries/sub_queries/events_subquery.py @@ -21,6 +21,7 @@ from posthog.clickhouse.query_tagging import Feature, Product, tag_queries from posthog.hogql_queries.legacy_compatibility.filter_to_query import MathAvailability, legacy_entity_to_node from posthog.models import Entity, EventProperty, Team +from posthog.ph_client import feature_enabled_or_false from posthog.session_recordings.queries.sub_queries.base_query import SessionRecordingsListingBaseQuery from posthog.session_recordings.queries.utils import ( INVERSE_OPERATOR_FOR, @@ -120,7 +121,7 @@ def _select_from_events( # with a bad experience filtering out 60% of results incorrectly # We use a feature flag to control this behavior for safe rollout. - remove_order_by = posthoganalytics.feature_enabled( + remove_order_by = feature_enabled_or_false( "remove-order-by-for-event-subquery", str(self._team.organization.id), send_feature_flag_events=False, @@ -151,7 +152,7 @@ def _is_hybrid_query_mode_enabled(self) -> bool: This solves the "late identification problem" where filtering by person properties in standard PoE mode only finds sessions where those properties existed at event time. """ - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "enable-hybrid-poe-replay-filtering", str(self._team.id), send_feature_flag_events=False, diff --git a/posthog/temporal/data_modeling/activities/materialize_view_duckgres.py b/posthog/temporal/data_modeling/activities/materialize_view_duckgres.py index 9dd8d32d3931..802ff59f771f 100644 --- a/posthog/temporal/data_modeling/activities/materialize_view_duckgres.py +++ b/posthog/temporal/data_modeling/activities/materialize_view_duckgres.py @@ -3,13 +3,13 @@ import datetime as dt import dataclasses -import posthoganalytics from structlog.contextvars import bind_contextvars from temporalio import activity from posthog.ducklake.common import get_duckgres_server_for_organization, is_dev_mode from posthog.exceptions_capture import capture_exception from posthog.models import Team +from posthog.ph_client import feature_enabled_or_false from posthog.sync import database_sync_to_async_pool from posthog.temporal.common.logger import get_logger @@ -63,7 +63,7 @@ def _is_duckgres_shadow_enabled(team: Team) -> bool: return False try: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( FEATURE_FLAG, str(team.pk), groups={ diff --git a/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py b/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py index 591324536887..20f5ec5d56f8 100644 --- a/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py +++ b/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py @@ -8,7 +8,6 @@ import duckdb import deltalake -import posthoganalytics from structlog.contextvars import bind_contextvars from temporalio import activity, workflow from temporalio.common import RetryPolicy @@ -42,6 +41,7 @@ ) from posthog.exceptions_capture import capture_exception from posthog.models import Team +from posthog.ph_client import feature_enabled_or_false from posthog.sync import database_sync_to_async from posthog.temporal.common.base import PostHogWorkflow from posthog.temporal.common.heartbeat_sync import HeartbeaterSync @@ -180,7 +180,7 @@ async def ducklake_copy_data_imports_gate_activity(inputs: DuckLakeCopyWorkflowG capture_exception(error) try: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "ducklake-data-imports-copy-workflow", str(team.uuid), groups={ diff --git a/posthog/temporal/ducklake/ducklake_copy_data_modeling_workflow.py b/posthog/temporal/ducklake/ducklake_copy_data_modeling_workflow.py index c9c9a6b97c58..e8d091bbb022 100644 --- a/posthog/temporal/ducklake/ducklake_copy_data_modeling_workflow.py +++ b/posthog/temporal/ducklake/ducklake_copy_data_modeling_workflow.py @@ -4,7 +4,6 @@ import duckdb import deltalake -import posthoganalytics from structlog.contextvars import bind_contextvars from temporalio import activity, workflow from temporalio.common import RetryPolicy @@ -37,6 +36,7 @@ ) from posthog.exceptions_capture import capture_exception from posthog.models import Team +from posthog.ph_client import feature_enabled_or_false from posthog.sync import database_sync_to_async from posthog.temporal.common.base import PostHogWorkflow from posthog.temporal.common.heartbeat_sync import HeartbeaterSync @@ -104,7 +104,7 @@ async def ducklake_copy_workflow_gate_activity(inputs: DuckLakeCopyWorkflowGateI return False try: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "ducklake-data-modeling-copy-workflow", str(team.uuid), groups={ diff --git a/products/ai_observability/backend/api/score_definitions.py b/products/ai_observability/backend/api/score_definitions.py index a470df58ed6c..7078d8d6a0fe 100644 --- a/products/ai_observability/backend/api/score_definitions.py +++ b/products/ai_observability/backend/api/score_definitions.py @@ -11,6 +11,7 @@ from drf_spectacular.utils import OpenApiExample, OpenApiParameter, OpenApiResponse from rest_framework import mixins, serializers, status, viewsets from rest_framework.decorators import action +from rest_framework.permissions import BasePermission from rest_framework.request import Request from rest_framework.response import Response @@ -19,13 +20,36 @@ from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.shared import UserBasicSerializer from posthog.event_usage import report_user_action -from posthog.models import User +from posthog.models import Team, User from posthog.permissions import AccessControlPermission +from posthog.ph_client import feature_enabled_or_false from posthog.rbac.access_control_api_mixin import AccessControlViewSetMixin from products.ai_observability.backend.models.score_definitions import ScoreDefinition, StaleScoreDefinitionVersion from products.ai_observability.backend.score_definition_configs import ScoreDefinitionConfigField +HUMAN_REVIEWS_FEATURE_FLAG = "llma-trace-review" + + +def is_human_reviews_feature_enabled(user: User, team: Team) -> bool: + distinct_id = user.distinct_id or str(user.uuid) + organization_id = str(team.organization_id) + project_id = str(team.id) + + return feature_enabled_or_false( + HUMAN_REVIEWS_FEATURE_FLAG, + distinct_id, + groups={"organization": organization_id, "project": project_id}, + group_properties={"organization": {"id": organization_id}, "project": {"id": project_id}}, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) + + +class HumanReviewsFeatureFlagPermission(BasePermission): + def has_permission(self, request, view) -> bool: + return is_human_reviews_feature_enabled(cast(User, request.user), view.team) + class ScoreDefinitionSerializer(serializers.ModelSerializer): created_by = UserBasicSerializer( @@ -177,7 +201,7 @@ class ScoreDefinitionViewSet( viewsets.GenericViewSet, ): scope_object = "llm_analytics" - permission_classes = [AccessControlPermission] + permission_classes = [HumanReviewsFeatureFlagPermission, AccessControlPermission] serializer_class = ScoreDefinitionSerializer queryset = ScoreDefinition.objects.all() filter_backends = [DjangoFilterBackend] diff --git a/products/ai_observability/backend/api/trace_reviews.py b/products/ai_observability/backend/api/trace_reviews.py index e13c0dbe6b37..57498e34983c 100644 --- a/products/ai_observability/backend/api/trace_reviews.py +++ b/products/ai_observability/backend/api/trace_reviews.py @@ -13,6 +13,7 @@ from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiExample, OpenApiParameter, OpenApiResponse, extend_schema_field from rest_framework import serializers, status +from rest_framework.permissions import BasePermission from rest_framework.request import Request from rest_framework.response import Response from rest_framework.viewsets import ModelViewSet @@ -24,6 +25,7 @@ from posthog.event_usage import report_user_action from posthog.models import Team, User from posthog.permissions import AccessControlPermission +from posthog.ph_client import feature_enabled_or_false from posthog.rbac.access_control_api_mixin import AccessControlViewSetMixin from products.ai_observability.backend.api.metrics import llma_track_latency @@ -35,9 +37,30 @@ normalize_score_definition_key, ) +TRACE_REVIEW_FEATURE_FLAG = "llma-trace-review" TRACE_REVIEW_SCORE_VALUE_FIELDS = ("categorical_values", "numeric_value", "boolean_value") +def is_trace_review_feature_enabled(user: User, team: Team) -> bool: + distinct_id = user.distinct_id or str(user.uuid) + organization_id = str(team.organization_id) + project_id = str(team.id) + + return feature_enabled_or_false( + TRACE_REVIEW_FEATURE_FLAG, + distinct_id, + groups={"organization": organization_id, "project": project_id}, + group_properties={"organization": {"id": organization_id}, "project": {"id": project_id}}, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) + + +class TraceReviewFeatureFlagPermission(BasePermission): + def has_permission(self, request, view) -> bool: + return is_trace_review_feature_enabled(cast(User, request.user), view.team) + + class TraceReviewScoreSerializer(serializers.ModelSerializer): categorical_values = serializers.ListField( read_only=True, @@ -566,7 +589,7 @@ def filter_search(self, queryset: QuerySet, _name: str, value: str) -> QuerySet: class TraceReviewViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, ModelViewSet): scope_object = "llm_analytics" - permission_classes = [AccessControlPermission] + permission_classes = [TraceReviewFeatureFlagPermission, AccessControlPermission] serializer_class = TraceReviewSerializer queryset = TraceReview.objects.all() filter_backends = [DjangoFilterBackend] diff --git a/products/ai_observability/backend/feature_flags.py b/products/ai_observability/backend/feature_flags.py index 396c93555881..591cacd5b4f9 100644 --- a/products/ai_observability/backend/feature_flags.py +++ b/products/ai_observability/backend/feature_flags.py @@ -1,8 +1,7 @@ from __future__ import annotations -import posthoganalytics - from posthog.models import Team, User +from posthog.ph_client import feature_enabled_or_false SENTIMENT_EVALUATIONS_FEATURE_FLAG = "llm-analytics-sentiment-evaluations" @@ -12,7 +11,7 @@ def is_sentiment_evaluations_enabled(user: User, team: Team) -> bool: organization_id = str(team.organization_id) project_id = str(team.id) - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( SENTIMENT_EVALUATIONS_FEATURE_FLAG, distinct_id, groups={"organization": organization_id, "project": project_id}, diff --git a/products/business_knowledge/backend/logic.py b/products/business_knowledge/backend/logic.py index 1cbdae1cee20..aba8e6e40dca 100644 --- a/products/business_knowledge/backend/logic.py +++ b/products/business_knowledge/backend/logic.py @@ -25,7 +25,6 @@ from django.utils import timezone import structlog -import posthoganalytics from langchain_core.messages import HumanMessage, SystemMessage from posthog.api.embedding_worker import generate_embedding @@ -34,6 +33,7 @@ from posthog.models.scoping import with_team_scope from posthog.models.team.team import Team from posthog.models.user import User +from posthog.ph_client import feature_enabled_or_false from posthog.security.url_validation import is_url_allowed from ee.hogai.llm import MaxChatAnthropic @@ -1625,7 +1625,7 @@ def has_feature_flag(team: Team) -> bool: check — `ee/hogai/utils/feature_flags.py` delegates here.""" if settings.DEBUG: return True - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "product-business-knowledge", str(team.organization_id), groups={"organization": str(team.organization_id)}, diff --git a/products/data_modeling/backend/api/node.py b/products/data_modeling/backend/api/node.py index 3ab2eb16b32f..a4a85d1da826 100644 --- a/products/data_modeling/backend/api/node.py +++ b/products/data_modeling/backend/api/node.py @@ -8,7 +8,6 @@ from django.db import models from django.db.models import OuterRef, Subquery -import posthoganalytics from rest_framework import filters, request, response, serializers, status, viewsets from rest_framework.decorators import action from rest_framework.pagination import PageNumberPagination @@ -17,6 +16,7 @@ from posthog.api.routing import TeamAndOrgViewSetMixin from posthog.api.scoped_related_fields import TeamScopedPrimaryKeyRelatedField from posthog.models import Team, User +from posthog.ph_client import feature_enabled_or_false from posthog.temporal.common.client import sync_connect from posthog.temporal.data_modeling.run_workflow import RunWorkflowInputs, Selector from posthog.temporal.data_modeling.workflows.execute_dag import ExecuteDAGInputs @@ -108,7 +108,7 @@ class NodePagination(PageNumberPagination): def _is_v2_backend_enabled(user: User, team: Team) -> bool: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "data-modeling-backend-v2", str(user.distinct_id), groups={ diff --git a/products/data_warehouse/backend/api/data_modeling_job.py b/products/data_warehouse/backend/api/data_modeling_job.py index dda59dd73c53..086b1012d855 100644 --- a/products/data_warehouse/backend/api/data_modeling_job.py +++ b/products/data_warehouse/backend/api/data_modeling_job.py @@ -1,10 +1,10 @@ -import posthoganalytics from django_filters.rest_framework import DjangoFilterBackend from rest_framework import pagination, serializers, viewsets from rest_framework.decorators import action from rest_framework.response import Response from posthog.api.routing import TeamAndOrgViewSetMixin +from posthog.ph_client import feature_enabled_or_false from products.data_modeling.backend.models.data_modeling_job import DataModelingJob, DataModelingJobEngine @@ -51,7 +51,7 @@ class DataModelingJobViewSet(TeamAndOrgViewSetMixin, viewsets.ReadOnlyModelViewS def _is_duckgres_shadow_enabled(self) -> bool: try: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( DUCKGRES_SHADOW_FLAG, str(self.team.pk), groups={ diff --git a/products/data_warehouse/backend/data_load/service.py b/products/data_warehouse/backend/data_load/service.py index e6efa5a08453..af7672a3f882 100644 --- a/products/data_warehouse/backend/data_load/service.py +++ b/products/data_warehouse/backend/data_load/service.py @@ -25,6 +25,7 @@ ) from temporalio.common import RetryPolicy +from posthog.ph_client import feature_enabled_or_false from posthog.temporal.common.client import async_connect, sync_connect from posthog.temporal.common.schedule import ( a_create_schedule, @@ -390,9 +391,7 @@ def is_any_external_data_schema_paused(team_id: int) -> bool: def is_cdc_enabled_for_team(team: Team) -> bool: - import posthoganalytics - - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "dwh-postgres-cdc", str(team.organization_id), groups={"organization": str(team.organization_id)}, @@ -401,9 +400,7 @@ def is_cdc_enabled_for_team(team: Team) -> bool: def is_xmin_enabled_for_team(team: Team) -> bool: - import posthoganalytics - - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "dwh-postgres-xmin", str(team.organization_id), groups={"organization": str(team.organization_id)}, diff --git a/products/experiments/backend/presentation/views.py b/products/experiments/backend/presentation/views.py index 254c4bdb1884..4f946e96a4ae 100644 --- a/products/experiments/backend/presentation/views.py +++ b/products/experiments/backend/presentation/views.py @@ -16,7 +16,6 @@ from django.db.models import BooleanField, Case, Exists, OuterRef, Prefetch, Q, QuerySet, Value, When from django.utils.text import slugify -import posthoganalytics from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema, extend_schema_view from opentelemetry import trace from rest_framework import serializers, viewsets @@ -35,6 +34,7 @@ from posthog.models.organization import OrganizationMembership from posthog.models.team.team import Team from posthog.models.user import User +from posthog.ph_client import feature_enabled_or_false from posthog.rbac.access_control_api_mixin import AccessControlViewSetMixin from posthog.temporal.common.client import sync_connect from posthog.temporal.experiments.models import ExperimentTimeseriesRecalculationWorkflowInputs @@ -154,7 +154,7 @@ def _is_prompt_experiments_feature_enabled(user: User, team: Team) -> bool: distinct_id = user.distinct_id or str(user.uuid) organization_id = str(team.organization_id) project_id = str(team.id) - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( PROMPT_EXPERIMENTS_FEATURE_FLAG, distinct_id, groups={"organization": organization_id, "project": project_id}, diff --git a/products/feature_flags/backend/api/feature_flag.py b/products/feature_flags/backend/api/feature_flag.py index a4bcd985351c..75ad6f19c118 100644 --- a/products/feature_flags/backend/api/feature_flag.py +++ b/products/feature_flags/backend/api/feature_flag.py @@ -16,7 +16,6 @@ from django.db.models import Count, Prefetch, Q, QuerySet, deletion import structlog -import posthoganalytics from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiExample, OpenApiParameter, OpenApiResponse, extend_schema_field from rest_framework import exceptions, request, serializers, status, viewsets @@ -61,6 +60,7 @@ ) from posthog.models.property import Property from posthog.permissions import TeamSecretTokenPermission, get_authenticator_scopes +from posthog.ph_client import feature_enabled_or_false from posthog.queries.base import determine_parsed_date_for_property_matching from posthog.rate_limit import BurstRateThrottle, ClickHouseBurstRateThrottle, ClickHouseSustainedRateThrottle from posthog.rbac.access_control_api_mixin import AccessControlViewSetMixin @@ -187,7 +187,7 @@ def _is_enforce_feature_flag_write_scope_enabled(request, *, team_id: int | None return False try: organization_id = str(Team.objects.values_list("organization_id", flat=True).get(pk=team_id)) - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( ENFORCE_FEATURE_FLAG_WRITE_SCOPE_FLAG, user.distinct_id, groups={"organization": organization_id}, @@ -265,7 +265,7 @@ def _is_realtime_cohort_flag_targeting_enabled(request) -> bool: user = getattr(request, "user", None) if user is None or user.is_anonymous: return False - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( REALTIME_COHORT_FLAG_TARGETING_FLAG, user.distinct_id, groups={"organization": str(user.organization.id)}, @@ -585,7 +585,7 @@ def is_enabled(request) -> bool: # Check FLAG_EVALUATION_TAGS feature flag try: - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( "flag-evaluation-tags", request.user.distinct_id, groups={"organization": str(request.user.organization.id)}, @@ -1563,7 +1563,7 @@ def _is_early_exit_enabled(self) -> bool: user = getattr(request, "user", None) if user is None or user.is_anonymous: return False - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( EARLY_EXIT_FLAG, user.distinct_id, groups={"organization": str(user.organization.id)}, diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py index 2774894fba4e..df7d60ad5f38 100644 --- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py +++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py @@ -3,7 +3,6 @@ from typing import TYPE_CHECKING import structlog -import posthoganalytics from posthog.schema import ( AttributionMode, @@ -12,6 +11,8 @@ MarketingAnalyticsDrillDownLevel, ) +from posthog.ph_client import feature_enabled_or_false + if TYPE_CHECKING: from posthog.models.team import Team @@ -116,7 +117,7 @@ def from_team(cls, team: "Team") -> "MarketingAnalyticsConfig": config.attribution_mode = AttributionMode(ma_config.attribution_mode) # Gate precomputation behind feature flag - config.conversion_goal_precomputation_enabled = posthoganalytics.feature_enabled( + config.conversion_goal_precomputation_enabled = feature_enabled_or_false( "marketing-analytics-precomputation", str(team.uuid), groups={"organization": str(team.organization.id)}, @@ -134,7 +135,7 @@ def from_team(cls, team: "Team") -> "MarketingAnalyticsConfig": # Gate multi-touch attribution behind feature flag if config.attribution_mode in MULTI_TOUCH_MODES: - has_multi_touch = posthoganalytics.feature_enabled( + has_multi_touch = feature_enabled_or_false( "marketing-analytics-multi-touch-attribution", str(team.uuid), groups={"organization": str(team.organization.id)}, diff --git a/products/product_analytics/backend/api/insight.py b/products/product_analytics/backend/api/insight.py index d4839b5fa7af..a3c24751b418 100644 --- a/products/product_analytics/backend/api/insight.py +++ b/products/product_analytics/backend/api/insight.py @@ -13,7 +13,6 @@ from django.utils.timezone import now import structlog -import posthoganalytics from django_filters.rest_framework import DjangoFilterBackend from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema_view @@ -99,6 +98,7 @@ from posthog.models.organization import Organization from posthog.models.team.team import Team from posthog.models.utils import UUIDT +from posthog.ph_client import feature_enabled_or_false from posthog.rate_limit import ( AIObservabilitySummarizationBurstThrottle, AIObservabilitySummarizationDailyThrottle, @@ -209,7 +209,7 @@ def is_legacy_insight_endpoint_blocked(user: Any, team: Team) -> bool: if not distinct_id: return False - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( LEGACY_INSIGHT_ENDPOINTS_BLOCKED_FLAG, str(distinct_id), groups={ @@ -229,7 +229,7 @@ def is_legacy_insight_filters_blocked(user: Any, team: Team) -> bool: if not distinct_id: return False - return posthoganalytics.feature_enabled( + return feature_enabled_or_false( LEGACY_INSIGHT_FILTERS_BLOCKED_FLAG, str(distinct_id), groups={ diff --git a/services/llm-gateway/src/llm_gateway/callbacks/__init__.py b/services/llm-gateway/src/llm_gateway/callbacks/__init__.py index 4a23b2f06f81..679415e2db7f 100644 --- a/services/llm-gateway/src/llm_gateway/callbacks/__init__.py +++ b/services/llm-gateway/src/llm_gateway/callbacks/__init__.py @@ -26,4 +26,4 @@ def init_callbacks() -> None: callbacks.append(RateLimitCallback()) callbacks.append(PrometheusCallback()) - litellm.callbacks = callbacks # ty: ignore[invalid-assignment] + litellm.callbacks = callbacks From 11eb8b3965d243efd4dce7682d69f84a85e3103a Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Tue, 23 Jun 2026 18:26:05 +0200 Subject: [PATCH 03/17] fix(deps): normalize ducklake feature flag check --- .../temporal/ducklake/ducklake_copy_data_imports_workflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py b/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py index 20f5ec5d56f8..e363f864efd2 100644 --- a/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py +++ b/posthog/temporal/ducklake/ducklake_copy_data_imports_workflow.py @@ -161,7 +161,7 @@ async def ducklake_copy_data_imports_gate_activity(inputs: DuckLakeCopyWorkflowG # same posthog_data_imports_team_{id} tables with zero coordination, so a # team must never have both enabled. The sink wins. try: - if posthoganalytics.feature_enabled( + if feature_enabled_or_false( DUCKGRES_BATCH_SINK_FLAG, str(team.uuid), groups={ From 60ec7896001d5c97e8aaa7317ff80bd81ed81a3e Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Tue, 23 Jun 2026 18:56:12 +0200 Subject: [PATCH 04/17] fix(deps): normalize remaining feature flag checks --- posthog/hogql/database/database.py | 6 +++--- posthog/tasks/email.py | 4 ++-- .../test_ducklake_copy_data_imports_workflow.py | 16 ++++++++-------- .../test_ducklake_copy_data_modeling_workflow.py | 12 ++++++------ 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/posthog/hogql/database/database.py b/posthog/hogql/database/database.py index 416b9977b543..f7cad6bcfb92 100644 --- a/posthog/hogql/database/database.py +++ b/posthog/hogql/database/database.py @@ -14,7 +14,6 @@ from django.db.models import Prefetch, Q import structlog -import posthoganalytics from opentelemetry import trace from pydantic import BaseModel, ConfigDict @@ -143,6 +142,7 @@ from posthog.models.group_type_mapping import get_group_types_for_project from posthog.models.organization import OrganizationMembership from posthog.models.team.team import Team, WeekStartDay +from posthog.ph_client import feature_enabled_or_false from posthog.rbac.user_access_control import NO_ACCESS_LEVEL, UserAccessControl from posthog.schema_enums import DatabaseSerializedFieldType, PersonsOnEventsMode, SessionTableVersion from posthog.synthetic_user import SyntheticUser @@ -1089,7 +1089,7 @@ def _fetch_sources( is_direct_query = connection_id is not None with timings.measure("feature_flags", emit_span=True): - is_managed_viewset_enabled = posthoganalytics.feature_enabled( + is_managed_viewset_enabled = feature_enabled_or_false( "managed-viewsets", str(team.uuid), groups={ @@ -1131,7 +1131,7 @@ def _fetch_sources( team, user, user_access_control ) - is_hogql_warehouse_access_control_enabled = posthoganalytics.feature_enabled( + is_hogql_warehouse_access_control_enabled = feature_enabled_or_false( "hogql-warehouse-access-control", str(team.uuid), groups={"organization": str(team.organization_id), "project": str(team.id)}, diff --git a/posthog/tasks/email.py b/posthog/tasks/email.py index e71129f131fe..19a421150a2c 100644 --- a/posthog/tasks/email.py +++ b/posthog/tasks/email.py @@ -27,7 +27,7 @@ from posthog.models.comment.utils import build_comment_item_url from posthog.models.messaging import MessagingRecord, get_email_hashes from posthog.models.utils import UUIDT -from posthog.ph_client import get_client +from posthog.ph_client import feature_enabled_or_false, get_client from posthog.scoping_audit import skip_team_scope_audit from posthog.user_permissions import UserPermissions @@ -1107,7 +1107,7 @@ def login_from_new_device_notification( elif user.current_organization is None: enabled = False else: - enabled = posthoganalytics.feature_enabled( + enabled = feature_enabled_or_false( key="login-from-new-device-notification", distinct_id=str(user.distinct_id), groups={"organization": str(user.current_organization.id)}, diff --git a/posthog/temporal/tests/ducklake/test_ducklake_copy_data_imports_workflow.py b/posthog/temporal/tests/ducklake/test_ducklake_copy_data_imports_workflow.py index 7018567d1a26..ee576ac8e461 100644 --- a/posthog/temporal/tests/ducklake/test_ducklake_copy_data_imports_workflow.py +++ b/posthog/temporal/tests/ducklake/test_ducklake_copy_data_imports_workflow.py @@ -101,7 +101,7 @@ def fake_feature_enabled( return flag_enabled monkeypatch.setattr( - "posthog.temporal.ducklake.ducklake_copy_data_imports_workflow.posthoganalytics.feature_enabled", + "posthog.temporal.ducklake.ducklake_copy_data_imports_workflow.feature_enabled_or_false", fake_feature_enabled, ) @@ -125,7 +125,7 @@ def fake_feature_enabled(key, distinct_id, **kwargs): return True monkeypatch.setattr( - "posthog.temporal.ducklake.ducklake_copy_data_imports_workflow.posthoganalytics.feature_enabled", + "posthog.temporal.ducklake.ducklake_copy_data_imports_workflow.feature_enabled_or_false", fake_feature_enabled, ) @@ -1039,8 +1039,8 @@ async def copy_stub(inputs: DuckLakeCopyDataImportsActivityInputs): call_counts["copy"] += 1 monkeypatch.setattr( - ducklake_module.posthoganalytics, - "feature_enabled", + ducklake_module, + "feature_enabled_or_false", lambda *args, **kwargs: False, ) monkeypatch.setattr(ducklake_module, "prepare_data_imports_ducklake_metadata_activity", metadata_stub) @@ -1106,8 +1106,8 @@ async def verify_stub(inputs: DuckLakeCopyDataImportsActivityInputs): return [] monkeypatch.setattr( - ducklake_module.posthoganalytics, - "feature_enabled", + ducklake_module, + "feature_enabled_or_false", # Key-aware: the gate checks the duckgres-batch-sink exclusion first, # and a catch-all True would wrongly trip it. lambda key, *args, **kwargs: key == "ducklake-data-imports-copy-workflow", @@ -1198,8 +1198,8 @@ async def cleanup_stub(inputs: DuckLakeDataImportsStagingCleanupInputs): call_counts["cleanup"] += 1 monkeypatch.setattr( - ducklake_module.posthoganalytics, - "feature_enabled", + ducklake_module, + "feature_enabled_or_false", # Key-aware: the gate checks the duckgres-batch-sink exclusion first, # and a catch-all True would wrongly trip it. lambda key, *args, **kwargs: key == "ducklake-data-imports-copy-workflow", diff --git a/posthog/temporal/tests/ducklake/test_ducklake_copy_data_modeling_workflow.py b/posthog/temporal/tests/ducklake/test_ducklake_copy_data_modeling_workflow.py index 5535c246ec49..2d3d167a40d0 100644 --- a/posthog/temporal/tests/ducklake/test_ducklake_copy_data_modeling_workflow.py +++ b/posthog/temporal/tests/ducklake/test_ducklake_copy_data_modeling_workflow.py @@ -361,8 +361,8 @@ async def copy_stub(inputs: DuckLakeCopyActivityInputs): call_counts["copy"] += 1 monkeypatch.setattr( - ducklake_module.posthoganalytics, - "feature_enabled", + ducklake_module, + "feature_enabled_or_false", lambda *args, **kwargs: False, ) monkeypatch.setattr(ducklake_module, "prepare_data_modeling_ducklake_metadata_activity", metadata_stub) @@ -847,8 +847,8 @@ async def verify_stub(inputs: DuckLakeCopyActivityInputs): return [] monkeypatch.setattr( - ducklake_module.posthoganalytics, - "feature_enabled", + ducklake_module, + "feature_enabled_or_false", lambda *args, **kwargs: True, ) @@ -930,8 +930,8 @@ async def cleanup_stub(inputs: DuckLakeDataModelingStagingCleanupInputs): call_counts["cleanup"] += 1 monkeypatch.setattr( - ducklake_module.posthoganalytics, - "feature_enabled", + ducklake_module, + "feature_enabled_or_false", lambda *args, **kwargs: True, ) monkeypatch.setattr(ducklake_module, "prepare_data_modeling_ducklake_metadata_activity", metadata_stub) From cb4bf84452dff051649a844e978cc45aec22b618 Mon Sep 17 00:00:00 2001 From: "tests-posthog[bot]" <250237707+tests-posthog[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 14:22:57 +0000 Subject: [PATCH 05/17] test(backend): update query snapshots --- .../test_paths_query_runner_ee.ambr | 3028 +---------------- 1 file changed, 1 insertion(+), 3027 deletions(-) diff --git a/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr b/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr index 0078fbc0b7eb..793352a38c98 100644 --- a/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr +++ b/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr @@ -1,1062 +1,4 @@ # serializer version: 1 -# name: TestClickhousePaths.test_end - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/about') AS target_index, - if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, -5) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, -5) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE ifNull(greater(target_index, 0), 0))) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_end.1 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/about') AS target_index, - if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, -5) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, -5) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE ifNull(greater(target_index, 0), 0))) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - ['/bar/*/foo'] AS groupings, - multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 1))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups.1 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - ['/xxx/invalid/*'] AS groupings, - multiMatchAnyIndex(path_item_ungrouped, ['/xxx/invalid/.*']) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 1))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.1 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), events.event), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$screen')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.2 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(events.event, '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), not(startsWith(events.event, '$'))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.3 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), and(or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$'))), ifNull(notIn(path_item, ['/custom1', '/1', '/2', '/3']), 1))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_event_ordering - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 5) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 5) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(events.event, '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-03 23:59:59', 'UTC')))), not(startsWith(events.event, '$'))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_groups_filtering_person_on_events - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) - WHERE and(equals(events.team_id, 99999), ifNull(equals(events__group_0.properties___industry, 'finance'), 0), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_groups_filtering_person_on_events.1 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.person_id AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 0)) - GROUP BY groups.group_type_index, - groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) - WHERE and(equals(events.team_id, 99999), ifNull(equals(events__group_0.properties___industry, 'technology'), 0), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) - ORDER BY events.person_id ASC, - toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_groups_filtering_person_on_events.2 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - events.person_id AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT JOIN - (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, - groups.group_type_index AS index, - groups.group_key AS key - FROM groups - WHERE and(equals(groups.team_id, 99999), equals(index, 1)) - GROUP BY groups.group_type_index, - groups.group_key) AS events__group_1 ON equals(events.`$group_1`, events__group_1.key) - WHERE and(equals(events.team_id, 99999), ifNull(equals(events__group_1.properties___industry, 'technology'), 0), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) - ORDER BY events.person_id ASC, - toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- # name: TestClickhousePaths.test_person_dropoffs ''' SELECT source.id AS id @@ -2146,103 +1088,7 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhousePaths.test_person_on_events_v2 - ''' - - SELECT DISTINCT person_id - FROM events - WHERE team_id = 99999 - AND distinct_id = 'poev2_p2' - ''' -# --- -# name: TestClickhousePaths.test_person_on_events_v2.1 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_recording +# name: TestClickhousePaths.test_recording ''' SELECT persons.id AS id, persons.created_at AS created_at, @@ -3515,1875 +2361,3 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhousePaths.test_respect_session_limits - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 5) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 5) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_start_and_end - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/about') AS target_index, - indexOf(compact_path, '/5') AS start_target_index, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, - indexOf(start_filtered_path, '/about') AS end_target_index, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, - if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, - if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_start_and_end.1 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/about') AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - indexOf(compact_path, '/5') AS start_target_index, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, - indexOf(start_filtered_path, '/about') AS end_target_index, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, - if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, - if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, - if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, - if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, - if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, - if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) - WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), equals(path_key, '2_/about')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_start_and_end.2 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/about') AS target_index, - indexOf(compact_path, '/2') AS start_target_index, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, - indexOf(start_filtered_path, '/about') AS end_target_index, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, - if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, - if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_start_and_end.3 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/about') AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - indexOf(compact_path, '/2') AS start_target_index, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, - indexOf(start_filtered_path, '/about') AS end_target_index, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, - if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, - if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, - if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, - if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, - if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, - if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, - if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, - if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) - WHERE and(ifNull(equals(last_path_key, '3_...'), 0), equals(path_key, '4_/5')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_start_dropping_orphaned_edges - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, '/2') AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 5) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 5) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index - WHERE ifNull(greater(target_index, 0), 0))) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 6 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_conversion_times - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 2) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 2) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.1 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 2) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 2) AS limited_timings, - if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, - arraySlice(filtered_uuid, 1, 2) AS limited_uuid, - if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, - arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, - if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, - arraySlice(filtered_session_id, 1, 2) AS limited_session_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, - arraySlice(filtered_window_id, 1, 2) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), equals(path_key, '2_/2')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.2 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 2) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 2) AS limited_timings, - if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, - arraySlice(filtered_uuid, 1, 2) AS limited_uuid, - if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, - arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, - if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, - arraySlice(filtered_session_id, 1, 2) AS limited_session_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, - arraySlice(filtered_window_id, 1, 2) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), equals(path_key, '3_/3')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.3 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 3) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 3) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.4 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 3) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 3) AS limited_timings, - if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, - arraySlice(filtered_uuid, 1, 3) AS limited_uuid, - if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, - arraySlice(filtered_timestamp, 1, 3) AS limited_timestamp, - if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, - arraySlice(filtered_session_id, 1, 3) AS limited_session_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, - arraySlice(filtered_window_id, 1, 3) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), equals(path_key, '3_/3')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.5 - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.6 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, - arraySlice(filtered_uuid, 1, 4) AS limited_uuid, - if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, - arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, - if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, - arraySlice(filtered_session_id, 1, 4) AS limited_session_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, - arraySlice(filtered_window_id, 1, 4) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), equals(path_key, '2_/2')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.7 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, - arraySlice(filtered_uuid, 1, 4) AS limited_uuid, - if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, - arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, - if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, - arraySlice(filtered_session_id, 1, 4) AS limited_session_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, - arraySlice(filtered_window_id, 1, 4) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), equals(path_key, '3_/3')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_step_limit.8 - ''' - SELECT source.id AS id - FROM - (SELECT person_id AS actor_id, - groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, - count(*) AS event_count, - actor_id AS id - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key, - final_uuid AS uuid, - final_timestamp AS timestamp, - final_session_id AS session_id, - final_window_id AS window_id - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - joined_path_tuple.4 AS final_uuid, - joined_path_tuple.5 AS final_timestamp, - joined_path_tuple.6 AS final_session_id, - joined_path_tuple.7 AS final_window_id, - arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, - arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, - arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, - arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, - arraySlice(filtered_uuid, 1, 4) AS limited_uuid, - if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, - arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, - if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, - arraySlice(filtered_session_id, 1, 4) AS limited_session_id, - if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, - arraySlice(filtered_window_id, 1, 4) AS limited_window_id, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, - path_time_tuple.4 AS uuid_items, - path_time_tuple.5 AS timestamp_items, - path_time_tuple.6 AS session_id_items, - path_time_tuple.7 AS window_id_items - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list, - groupArray(uuid) AS uuid_list, - groupArray(timestamp) AS timestamp_list, - groupArray(session_id) AS session_id_list, - groupArray(window_id) AS window_id_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, - events.uuid AS uuid, - toTimeZone(events.timestamp, 'UTC') AS timestamp, - ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, - ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, - NULL AS groupings, - multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE and(ifNull(equals(last_path_key, '3_/3'), 0), equals(path_key, '4_/4')) - GROUP BY person_id) AS source - ORDER BY source.id ASC - LIMIT 101 - OFFSET 0 SETTINGS optimize_aggregation_in_order=1, - join_algorithm='auto', - readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=0, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_wildcard_groups_across_people - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 4) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 4) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - ['/bar/*/foo'] AS groupings, - multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- -# name: TestClickhousePaths.test_wildcard_groups_evil_input - ''' - SELECT last_path_key AS source_event, - path_key AS target_event, - count(*) AS event_count, - avg(conversion_time) AS average_conversion_time - FROM - (SELECT person_id AS person_id, - path AS path, - conversion_time AS conversion_time, - event_in_session_index AS event_in_session_index, - concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, - if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, - path_dropoff_key AS path_dropoff_key - FROM - (SELECT person_id AS person_id, - joined_path_tuple.1 AS path, - joined_path_tuple.2 AS conversion_time, - joined_path_tuple.3 AS prev_path, - event_in_session_index, - session_index AS session_index, - arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, - arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) - and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, - arrayFilter((x, y) -> y, time, mapping) AS timings, - arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, - indexOf(compact_path, NULL) AS target_index, - if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, - arraySlice(filtered_path, 1, 5) AS limited_path, - if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, - arraySlice(filtered_timings, 1, 5) AS limited_timings, - arrayDifference(limited_timings) AS timings_diff, - concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, - arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings - FROM - (SELECT person_id AS person_id, - path_time_tuple.1 AS path_basic, - path_time_tuple.2 AS time, - session_index, - arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, - arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths - FROM - (SELECT person_id AS person_id, - groupArray(timestamp) AS timing_list, - groupArray(path_item) AS path_list - FROM - (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, - if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, - ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, - ['(a+)+', - '[aaa|aaaa]+', - '1.*', - '.*', - '/3?q=1', - '/3*'] AS groupings, - multiMatchAnyIndex(path_item_ungrouped, ['\\(a\\+\\)\\+', '\\[aaa\\|aaaa\\]\\+', '1\\..*', '\\..*', '/3\\?q=1', '/3.*']) AS group_index, - (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item - FROM events - LEFT OUTER JOIN - (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, - person_distinct_id_overrides.distinct_id AS distinct_id - FROM person_distinct_id_overrides - WHERE equals(person_distinct_id_overrides.team_id, 99999) - GROUP BY person_distinct_id_overrides.distinct_id - HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) - WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) - ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) - GROUP BY person_id) ARRAY - JOIN session_paths AS path_time_tuple, - arrayEnumerate(session_paths) AS session_index) ARRAY - JOIN limited_path_timings AS joined_path_tuple, - arrayEnumerate(limited_path_timings) AS event_in_session_index)) - WHERE isNotNull(source_event) - GROUP BY source_event, - target_event - ORDER BY event_count DESC, - source_event ASC, - target_event ASC - LIMIT 50 SETTINGS readonly=2, - max_execution_time=60, - allow_experimental_object_type=1, - max_ast_elements=4000000, - max_expanded_ast_elements=4000000, - max_bytes_before_external_group_by=23622320128, - transform_null_in=1, - optimize_min_equality_disjunction_chain_length=4294967295, - optimize_rewrite_aggregate_function_with_if=0, - optimize_min_inequality_conjunction_chain_length=4294967295, - allow_experimental_join_condition=1, - use_hive_partitioning=0 - ''' -# --- From 60dc1cb114a445980c7847f7945062f01349142c Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Wed, 24 Jun 2026 18:28:04 +0200 Subject: [PATCH 06/17] chore: bump posthog python sdk --- pyproject.toml | 2 +- uv.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 5eba4bc54fd8..993c42ad9628 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -100,7 +100,7 @@ dependencies = [ "paramiko~=3.5.0", "pillow==12.2.0", "protobuf~=5.29.6", - "posthoganalytics==7.20.2", + "posthoganalytics==7.20.4", "polars==1.37.1", "psycopg2-binary==2.9.10", "psycopg[binary]==3.2.4", diff --git a/uv.lock b/uv.lock index f683b1a1c091..47d0f7aaae09 100644 --- a/uv.lock +++ b/uv.lock @@ -5913,7 +5913,7 @@ requires-dist = [ { name = "pixelhog", specifier = "~=1.2.0" }, { name = "playwright", specifier = "~=1.60.0" }, { name = "polars", specifier = "==1.37.1" }, - { name = "posthoganalytics", specifier = "==7.20.2" }, + { name = "posthoganalytics", specifier = "==7.20.4" }, { name = "protobuf", specifier = "~=5.29.6" }, { name = "psycopg", extras = ["binary"], specifier = "==3.2.4" }, { name = "psycopg2-binary", specifier = "==2.9.10" }, @@ -6052,7 +6052,7 @@ sentiment = [ [[package]] name = "posthoganalytics" -version = "7.20.2" +version = "7.20.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff" }, @@ -6060,9 +6060,9 @@ dependencies = [ { name = "requests" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/f1/7b3eede03afb2079ebcc5e229643b32cd1bc4b00c231efcd7e2c79407bcd/posthoganalytics-7.20.2.tar.gz", hash = "sha256:d5d387816a36f5458e6fcc583621e3e6d742683fff361db909ff13f3111bbcff", size = 256052, upload-time = "2026-06-22T15:36:56.593Z" } +sdist = { url = "https://files.pythonhosted.org/packages/21/ee/a2e18df3178f3cd05512bb65b87b3037b145c2889477820b1a5b243b1bce/posthoganalytics-7.20.4.tar.gz", hash = "sha256:89f364d1796ca4e414977e67f6dae94136fb41385f68f37d4c3c85bd4278fb74", size = 262180, upload-time = "2026-06-24T15:48:16.262Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/4c/868e2f7e033eaadad5c2be9df76dc14bca5fa66d620c02e54c08f8f4b44b/posthoganalytics-7.20.2-py3-none-any.whl", hash = "sha256:d82c9152dcf44ae08ce303d975d8a6d5848d8eeb73aabadf02acc9aeea70277e", size = 298975, upload-time = "2026-06-22T15:36:54.971Z" }, + { url = "https://files.pythonhosted.org/packages/d1/de/81f6ffaae7ab5e95121b960bb98a2958b5ab7c36c10e1933da49d791b239/posthoganalytics-7.20.4-py3-none-any.whl", hash = "sha256:fcbfc2f9db527abeea8ba06caede720ee5fa5115f89b48827527314418c6a33c", size = 305163, upload-time = "2026-06-24T15:48:14.397Z" }, ] [[package]] From 575cd63915624b01e6f4c911117739109feb7107 Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Wed, 24 Jun 2026 18:31:46 +0200 Subject: [PATCH 07/17] fix(tracing): fix invalid noqa comments --- products/tracing/backend/logic.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/products/tracing/backend/logic.py b/products/tracing/backend/logic.py index 2cd80a517f83..676c9d9e5841 100644 --- a/products/tracing/backend/logic.py +++ b/products/tracing/backend/logic.py @@ -1071,8 +1071,7 @@ def run_aggregation_query( service_names: list[str] | None = None, ) -> TraceSpansAggregationQueryResponse | CachedTraceSpansAggregationQueryResponse: """Facade-friendly entry point for running a flat span aggregation query.""" - # noqa-justified: the runners import `translate_span_filter` from this module, so a - # module-level import here is circular and only resolves when this module loads first. + # The runners import `translate_span_filter` from this module, so a module-level import here is circular. from .aggregation_query_runner import TraceSpansAggregationQueryRunner # noqa: PLC0415 query = TraceSpansAggregationQuery( @@ -1098,7 +1097,7 @@ def run_tree_query( service_names: list[str] | None = None, ) -> TraceSpansTreeQueryResponse | CachedTraceSpansTreeQueryResponse: """Facade-friendly entry point for running a span call-tree aggregation query.""" - # noqa-justified: same circular import as run_aggregation_query above. + # Same circular import as run_aggregation_query above. from .aggregation_query_runner import TraceSpansTreeQueryRunner # noqa: PLC0415 query = TraceSpansTreeQuery( From a9c40cbcb1cde6cc12d2957d2183e5e964f9f0f8 Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Wed, 24 Jun 2026 18:38:59 +0200 Subject: [PATCH 08/17] fix(marketing): normalize cost flag check --- .../backend/hogql_queries/marketing_analytics_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py index df7d60ad5f38..fe4553e147fa 100644 --- a/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py +++ b/products/marketing_analytics/backend/hogql_queries/marketing_analytics_config.py @@ -126,7 +126,7 @@ def from_team(cls, team: "Team") -> "MarketingAnalyticsConfig": # Gate cost precomputation (reads native CH instead of the S3 cost tables) behind its own flag, # so it rolls out independently of the conversion/touchpoint precompute. - config.costs_precomputation_enabled = posthoganalytics.feature_enabled( + config.costs_precomputation_enabled = feature_enabled_or_false( "marketing-analytics-costs-precomputation", str(team.uuid), groups={"organization": str(team.organization.id)}, From 1e5f9e9f7c53f12000014568459d777b9e19241a Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Wed, 24 Jun 2026 18:48:23 +0200 Subject: [PATCH 09/17] chore(deps): update ingestion sdk pin --- common/ingestion/acceptance_tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/ingestion/acceptance_tests/requirements.txt b/common/ingestion/acceptance_tests/requirements.txt index a0954f4b0c1f..1e0145defc9d 100644 --- a/common/ingestion/acceptance_tests/requirements.txt +++ b/common/ingestion/acceptance_tests/requirements.txt @@ -5,4 +5,4 @@ requests-toolbelt==1.0.0 boto3==1.29.7 docker==6.1.3 python-multipart==0.0.6 -posthog==7.20.2 +posthog==7.20.4 From e4645925d4e5c3d47f97c42b6d7b7168a0bf03e8 Mon Sep 17 00:00:00 2001 From: "tests-posthog[bot]" <250237707+tests-posthog[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 17:09:44 +0000 Subject: [PATCH 10/17] test(backend): update query snapshots --- .../test_paths_query_runner_ee.ambr | 3034 ++++++++++++++++- 1 file changed, 3030 insertions(+), 4 deletions(-) diff --git a/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr b/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr index 793352a38c98..0078fbc0b7eb 100644 --- a/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr +++ b/products/product_analytics/backend/hogql_queries/paths/test/__snapshots__/test_paths_query_runner_ee.ambr @@ -1,4 +1,1062 @@ # serializer version: 1 +# name: TestClickhousePaths.test_end + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_end.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + if(ifNull(greater(target_index, 0), 0), arrayResize(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, -5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arrayResize(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, -5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/bar/*/foo'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 1))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_exclusion_filters_with_wildcard_groups.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/xxx/invalid/*'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/xxx/invalid/.*']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), and(equals(events.event, '$pageview'), ifNull(notIn(path_item, ['/bar/*/foo']), 1))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$screen')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_inclusion_exclusion_filters.3 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), and(or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$'))), ifNull(notIn(path_item, ['/custom1', '/1', '/2', '/3']), 1))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_event_ordering + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(events.event, '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-03 23:59:59', 'UTC')))), not(startsWith(events.event, '$'))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_groups_filtering_person_on_events + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + LEFT JOIN + (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 99999), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) + WHERE and(equals(events.team_id, 99999), ifNull(equals(events__group_0.properties___industry, 'finance'), 0), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_groups_filtering_person_on_events.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + events.person_id AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT JOIN + (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 99999), equals(index, 0)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_0 ON equals(events.`$group_0`, events__group_0.key) + WHERE and(equals(events.team_id, 99999), ifNull(equals(events__group_0.properties___industry, 'technology'), 0), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY events.person_id ASC, + toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_groups_filtering_person_on_events.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + events.person_id AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT JOIN + (SELECT tupleElement(argMax(tuple(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(groups.group_properties, 'industry'), ''), 'null'), '^"|"$', '')), toTimeZone(groups._timestamp, 'UTC')), 1) AS properties___industry, + groups.group_type_index AS index, + groups.group_key AS key + FROM groups + WHERE and(equals(groups.team_id, 99999), equals(index, 1)) + GROUP BY groups.group_type_index, + groups.group_key) AS events__group_1 ON equals(events.`$group_1`, events__group_1.key) + WHERE and(equals(events.team_id, 99999), ifNull(equals(events__group_1.properties___industry, 'technology'), 0), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY events.person_id ASC, + toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- # name: TestClickhousePaths.test_person_dropoffs ''' SELECT source.id AS id @@ -1088,11 +2146,107 @@ use_hive_partitioning=0 ''' # --- -# name: TestClickhousePaths.test_recording +# name: TestClickhousePaths.test_person_on_events_v2 ''' - SELECT persons.id AS id, - persons.created_at AS created_at, - source.event_count AS event_count, + + SELECT DISTINCT person_id + FROM events + WHERE team_id = 99999 + AND distinct_id = 'poev2_p2' + ''' +# --- +# name: TestClickhousePaths.test_person_on_events_v2.1 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-02-01 23:59:59', 'UTC')))), or(equals(events.event, '$pageview'), equals(events.event, '$screen'), not(startsWith(events.event, '$')))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_recording + ''' + SELECT persons.id AS id, + persons.created_at AS created_at, + source.event_count AS event_count, source.matching_events AS matching_events FROM (SELECT person_id AS actor_id, @@ -2361,3 +3515,1875 @@ use_hive_partitioning=0 ''' # --- +# name: TestClickhousePaths.test_respect_session_limits + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end.1 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/5') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 5), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(5, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 5), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(5, 2)), [filtered_timings[plus(1, intDiv(5, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 5), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(5, 2)), [filtered_uuid[plus(1, intDiv(5, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 5), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(5, 2)), [filtered_timestamp[plus(1, intDiv(5, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 5), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(5, 2)), [filtered_session_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 5), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(5, 2)), [filtered_window_id[plus(1, intDiv(5, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(5, 2)), intDiv(5, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '1_/5'), 0), equals(path_key, '2_/about')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end.2 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_start_and_end.3 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/about') AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + indexOf(compact_path, '/2') AS start_target_index, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(compact_path, start_target_index), compact_path) AS start_filtered_path, + indexOf(start_filtered_path, '/about') AS end_target_index, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_path, end_target_index), start_filtered_path) AS filtered_path, + if(ifNull(greater(length(filtered_path), 4), 0), arrayConcat(arraySlice(filtered_path, 1, intDiv(4, 2)), ['...'], arraySlice(filtered_path, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_path) AS limited_path, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timings, start_target_index), timings) AS start_filtered_timings, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timings, end_target_index), start_filtered_timings) AS filtered_timings, + if(ifNull(greater(length(filtered_timings), 4), 0), arrayConcat(arraySlice(filtered_timings, 1, intDiv(4, 2)), [filtered_timings[plus(1, intDiv(4, 2))]], arraySlice(filtered_timings, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timings) AS limited_timings, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(uuid, start_target_index), uuid) AS start_filtered_uuid, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_uuid, end_target_index), start_filtered_uuid) AS filtered_uuid, + if(ifNull(greater(length(filtered_uuid), 4), 0), arrayConcat(arraySlice(filtered_uuid, 1, intDiv(4, 2)), [filtered_uuid[plus(1, intDiv(4, 2))]], arraySlice(filtered_uuid, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_uuid) AS limited_uuid, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(timestamp, start_target_index), timestamp) AS start_filtered_timestamp, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_timestamp, end_target_index), start_filtered_timestamp) AS filtered_timestamp, + if(ifNull(greater(length(filtered_timestamp), 4), 0), arrayConcat(arraySlice(filtered_timestamp, 1, intDiv(4, 2)), [filtered_timestamp[plus(1, intDiv(4, 2))]], arraySlice(filtered_timestamp, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_timestamp) AS limited_timestamp, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(session_id, start_target_index), session_id) AS start_filtered_session_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_session_id, end_target_index), start_filtered_session_id) AS filtered_session_id, + if(ifNull(greater(length(filtered_session_id), 4), 0), arrayConcat(arraySlice(filtered_session_id, 1, intDiv(4, 2)), [filtered_session_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_session_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_session_id) AS limited_session_id, + if(ifNull(greater(start_target_index, 0), 0), arraySlice(window_id, start_target_index), window_id) AS start_filtered_window_id, + if(ifNull(greater(end_target_index, 0), 0), arrayResize(start_filtered_window_id, end_target_index), start_filtered_window_id) AS filtered_window_id, + if(ifNull(greater(length(filtered_window_id), 4), 0), arrayConcat(arraySlice(filtered_window_id, 1, intDiv(4, 2)), [filtered_window_id[plus(1, intDiv(4, 2))]], arraySlice(filtered_window_id, multiply(-1, intDiv(4, 2)), intDiv(4, 2))), filtered_window_id) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE and(ifNull(greater(start_target_index, 0), 0), ifNull(greater(end_target_index, 0), 0)))) + WHERE and(ifNull(equals(last_path_key, '3_...'), 0), equals(path_key, '4_/5')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_start_dropping_orphaned_edges + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, '/2') AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2021-05-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2021-05-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index + WHERE ifNull(greater(target_index, 0), 0))) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 6 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_conversion_times + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.1 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 2) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 2) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 2) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), equals(path_key, '2_/2')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.2 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 2) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 2) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 2) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 2) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 2) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 2) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), equals(path_key, '3_/3')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.3 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 3) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 3) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.4 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 3) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 3) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 3) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 3) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 3) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 3) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), equals(path_key, '3_/3')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.5 + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.6 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '1_/1'), 0), equals(path_key, '2_/2')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.7 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '2_/2'), 0), equals(path_key, '3_/3')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_step_limit.8 + ''' + SELECT source.id AS id + FROM + (SELECT person_id AS actor_id, + groupUniqArray(100)(tuple(timestamp, uuid, session_id, window_id)) AS matching_events, + count(*) AS event_count, + actor_id AS id + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key, + final_uuid AS uuid, + final_timestamp AS timestamp, + final_session_id AS session_id, + final_window_id AS window_id + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + joined_path_tuple.4 AS final_uuid, + joined_path_tuple.5 AS final_timestamp, + joined_path_tuple.6 AS final_session_id, + joined_path_tuple.7 AS final_window_id, + arrayFilter((x, y) -> y, uuid_items, mapping) AS uuid, + arrayFilter((x, y) -> y, timestamp_items, mapping) AS timestamp, + arrayFilter((x, y) -> y, session_id_items, mapping) AS session_id, + arrayFilter((x, y) -> y, window_id_items, mapping) AS window_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + if(ifNull(greater(target_index, 0), 0), arraySlice(uuid, target_index), uuid) AS filtered_uuid, + arraySlice(filtered_uuid, 1, 4) AS limited_uuid, + if(ifNull(greater(target_index, 0), 0), arraySlice(timestamp, target_index), timestamp) AS filtered_timestamp, + arraySlice(filtered_timestamp, 1, 4) AS limited_timestamp, + if(ifNull(greater(target_index, 0), 0), arraySlice(session_id, target_index), session_id) AS filtered_session_id, + arraySlice(filtered_session_id, 1, 4) AS limited_session_id, + if(ifNull(greater(target_index, 0), 0), arraySlice(window_id, target_index), window_id) AS filtered_window_id, + arraySlice(filtered_window_id, 1, 4) AS limited_window_id, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, '')), limited_uuid, limited_timestamp, limited_session_id, limited_window_id) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list), uuid_list, timestamp_list, session_id_list, window_id_list) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths, + path_time_tuple.4 AS uuid_items, + path_time_tuple.5 AS timestamp_items, + path_time_tuple.6 AS session_id_items, + path_time_tuple.7 AS window_id_items + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list, + groupArray(uuid) AS uuid_list, + groupArray(timestamp) AS timestamp_list, + groupArray(session_id) AS session_id_list, + groupArray(window_id) AS window_id_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$screen'), replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$screen_name'), ''), 'null'), '^"|"$', ''), if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event)), '') AS path_item_ungrouped, + events.uuid AS uuid, + toTimeZone(events.timestamp, 'UTC') AS timestamp, + ifNull(nullIf(nullIf(events.`$session_id`, ''), 'null'), '') AS session_id, + ifNull(nullIf(nullIf(events.`$window_id`, ''), 'null'), '') AS window_id, + NULL AS groupings, + multiMatchAnyIndex(path_item_ungrouped, NULL) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2011-12-31 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2012-01-07 23:59:59', 'UTC'))))) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE and(ifNull(equals(last_path_key, '3_/3'), 0), equals(path_key, '4_/4')) + GROUP BY person_id) AS source + ORDER BY source.id ASC + LIMIT 101 + OFFSET 0 SETTINGS optimize_aggregation_in_order=1, + join_algorithm='auto', + readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=0, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_wildcard_groups_across_people + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 4) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 4) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['/bar/*/foo'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['/bar/.*/foo']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- +# name: TestClickhousePaths.test_wildcard_groups_evil_input + ''' + SELECT last_path_key AS source_event, + path_key AS target_event, + count(*) AS event_count, + avg(conversion_time) AS average_conversion_time + FROM + (SELECT person_id AS person_id, + path AS path, + conversion_time AS conversion_time, + event_in_session_index AS event_in_session_index, + concat(ifNull(toString(event_in_session_index), ''), '_', ifNull(toString(path), '')) AS path_key, + if(ifNull(greater(event_in_session_index, 1), 0), concat(ifNull(toString(minus(event_in_session_index, 1)), ''), '_', ifNull(toString(prev_path), '')), NULL) AS last_path_key, + path_dropoff_key AS path_dropoff_key + FROM + (SELECT person_id AS person_id, + joined_path_tuple.1 AS path, + joined_path_tuple.2 AS conversion_time, + joined_path_tuple.3 AS prev_path, + event_in_session_index, + session_index AS session_index, + arrayPopFront(arrayPushBack(path_basic, '')) AS path_basic_0, + arrayMap((x, y) -> if(ifNull(equals(x, y), isNull(x) + and isNull(y)), 0, 1), path_basic, path_basic_0) AS mapping, + arrayFilter((x, y) -> y, time, mapping) AS timings, + arrayFilter((x, y) -> y, path_basic, mapping) AS compact_path, + indexOf(compact_path, NULL) AS target_index, + if(ifNull(greater(target_index, 0), 0), arraySlice(compact_path, target_index), compact_path) AS filtered_path, + arraySlice(filtered_path, 1, 5) AS limited_path, + if(ifNull(greater(target_index, 0), 0), arraySlice(timings, target_index), timings) AS filtered_timings, + arraySlice(filtered_timings, 1, 5) AS limited_timings, + arrayDifference(limited_timings) AS timings_diff, + concat(ifNull(toString(length(limited_path)), ''), '_', ifNull(toString(limited_path[-1]), '')) AS path_dropoff_key, + arrayZip(limited_path, timings_diff, arrayPopBack(arrayPushFront(limited_path, ''))) AS limited_path_timings + FROM + (SELECT person_id AS person_id, + path_time_tuple.1 AS path_basic, + path_time_tuple.2 AS time, + session_index, + arrayZip(path_list, timing_list, arrayDifference(timing_list)) AS paths_tuple, + arraySplit(x -> if(ifNull(less(x.3, 1800), 0), 0, 1), paths_tuple) AS session_paths + FROM + (SELECT person_id AS person_id, + groupArray(timestamp) AS timing_list, + groupArray(path_item) AS path_list + FROM + (SELECT toTimeZone(events.timestamp, 'UTC') AS timestamp, + if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) AS person_id, + ifNull(if(equals(events.event, '$pageview'), replaceRegexpAll(ifNull(replaceRegexpAll(nullIf(nullIf(JSONExtractRaw(events.properties, '$current_url'), ''), 'null'), '^"|"$', ''), ''), '(.)/$', '\\1'), events.event), '') AS path_item_ungrouped, + ['(a+)+', + '[aaa|aaaa]+', + '1.*', + '.*', + '/3?q=1', + '/3*'] AS groupings, + multiMatchAnyIndex(path_item_ungrouped, ['\\(a\\+\\)\\+', '\\[aaa\\|aaaa\\]\\+', '1\\..*', '\\..*', '/3\\?q=1', '/3.*']) AS group_index, + (if(ifNull(greater(group_index, 0), 0), groupings[group_index], path_item_ungrouped) AS path_item) AS path_item + FROM events + LEFT OUTER JOIN + (SELECT argMax(person_distinct_id_overrides.person_id, person_distinct_id_overrides.version) AS person_id, + person_distinct_id_overrides.distinct_id AS distinct_id + FROM person_distinct_id_overrides + WHERE equals(person_distinct_id_overrides.team_id, 99999) + GROUP BY person_distinct_id_overrides.distinct_id + HAVING equals(argMax(person_distinct_id_overrides.is_deleted, person_distinct_id_overrides.version), 0) SETTINGS optimize_aggregation_in_order=1) AS events__override ON equals(events.distinct_id, events__override.distinct_id) + WHERE and(equals(events.team_id, 99999), and(greaterOrEquals(events.timestamp, toStartOfInterval(assumeNotNull(toDateTime('2012-01-01 00:00:00', 'UTC')), toIntervalDay(1))), lessOrEquals(events.timestamp, assumeNotNull(toDateTime('2023-05-23 23:59:59', 'UTC')))), equals(events.event, '$pageview')) + ORDER BY if(not(empty(events__override.distinct_id)), events__override.person_id, events.person_id) ASC, toTimeZone(events.timestamp, 'UTC') ASC) + GROUP BY person_id) ARRAY + JOIN session_paths AS path_time_tuple, + arrayEnumerate(session_paths) AS session_index) ARRAY + JOIN limited_path_timings AS joined_path_tuple, + arrayEnumerate(limited_path_timings) AS event_in_session_index)) + WHERE isNotNull(source_event) + GROUP BY source_event, + target_event + ORDER BY event_count DESC, + source_event ASC, + target_event ASC + LIMIT 50 SETTINGS readonly=2, + max_execution_time=60, + allow_experimental_object_type=1, + max_ast_elements=4000000, + max_expanded_ast_elements=4000000, + max_bytes_before_external_group_by=23622320128, + transform_null_in=1, + optimize_min_equality_disjunction_chain_length=4294967295, + optimize_rewrite_aggregate_function_with_if=0, + optimize_min_inequality_conjunction_chain_length=4294967295, + allow_experimental_join_condition=1, + use_hive_partitioning=0 + ''' +# --- From a5d08ae9405b0dc29b1198e632c066fc287f9a8e Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Wed, 24 Jun 2026 22:21:04 +0200 Subject: [PATCH 11/17] test: patch feature flag helper --- .../views/test/test_clickhouse_groups.py | 8 ++--- .../ai/test/test_ai_table_resolver.py | 33 ++++++++++++++++++- .../trends/test/test_trends_query_runner.py | 4 +-- .../test/test_session_recording_playlist.py | 2 +- .../test/test_session_recordings.py | 8 ++--- .../test_ai_observability_access_control.py | 11 +++++++ .../backend/api/test/test_evaluations.py | 24 ++++---------- .../api/test/test_score_definitions.py | 8 +++++ .../backend/api/test/test_trace_reviews.py | 9 +++++ .../backend/api/test/test_node_api.py | 8 ++--- .../backend/test/test_create_from_prompt.py | 2 +- .../backend/api/test/test_feature_flag.py | 10 +++--- .../api/test/test_scope_enforcement.py | 4 +-- .../test_marketing_analytics_config.py | 8 ++--- .../backend/api/test/test_insight.py | 6 ++-- 15 files changed, 96 insertions(+), 49 deletions(-) diff --git a/ee/clickhouse/views/test/test_clickhouse_groups.py b/ee/clickhouse/views/test/test_clickhouse_groups.py index c5d9021acd73..fa51cd7175ac 100644 --- a/ee/clickhouse/views/test/test_clickhouse_groups.py +++ b/ee/clickhouse/views/test/test_clickhouse_groups.py @@ -179,7 +179,7 @@ def test_find_missing_group_key(self): self.assertEqual(response.status_code, 400) @freeze_time("2021-05-02") - @patch(f"{PATH}.posthoganalytics.feature_enabled", return_value=False) + @patch(f"{PATH}.feature_enabled_or_false", return_value=False) def test_retrieve_group_crm_disabled(self, _): index: GroupTypeIndex = 0 key = "key" @@ -207,7 +207,7 @@ def test_retrieve_group_crm_disabled(self, _): self.assertEqual(0, Notebook.objects.filter(team=self.team).count()) @freeze_time("2021-05-02") - @patch(f"{PATH}.posthoganalytics.feature_enabled", return_value=True) + @patch(f"{PATH}.feature_enabled_or_false", return_value=True) def test_retrieve_group_crm_enabled(self, _): index: GroupTypeIndex = 0 key = "key" @@ -246,7 +246,7 @@ def test_retrieve_group_crm_enabled(self, _): self.assertEqual(notebook.content[1]["type"], "text") @freeze_time("2021-05-02") - @patch(f"{PATH}.posthoganalytics.feature_enabled", return_value=True) + @patch(f"{PATH}.feature_enabled_or_false", return_value=True) def test_find_with_skip_create_notebook_does_not_create_notebook(self, _): index: GroupTypeIndex = 0 key = "key" @@ -295,7 +295,7 @@ def test_retrieve_group_with_notebook(self): @freeze_time("2021-05-02") @patch("products.notebooks.backend.logic.ResourceNotebook.objects.create", side_effect=IntegrityError) - @patch(f"{PATH}.posthoganalytics.feature_enabled", return_value=True) + @patch(f"{PATH}.feature_enabled_or_false", return_value=True) def test_retrieve_group_notebook_transaction_rollback(self, _, mock_relationship_create): index: GroupTypeIndex = 0 key = "key" diff --git a/posthog/hogql_queries/ai/test/test_ai_table_resolver.py b/posthog/hogql_queries/ai/test/test_ai_table_resolver.py index 75ff9b44f452..8d783e2c92ad 100644 --- a/posthog/hogql_queries/ai/test/test_ai_table_resolver.py +++ b/posthog/hogql_queries/ai/test/test_ai_table_resolver.py @@ -3,7 +3,38 @@ from posthog.hogql import ast -from posthog.hogql_queries.ai.ai_table_resolver import AIEventsExpiredError, AIEventsNotFoundError, query_ai_events +from posthog.hogql_queries.ai.ai_table_resolver import ( + AIEventsExpiredError, + AIEventsNotFoundError, + is_ai_events_enabled, + query_ai_events, +) + + +class TestIsAiEventsEnabled: + @patch("posthog.hogql_queries.ai.ai_table_resolver.feature_enabled_or_false", return_value=True) + def test_returns_true_when_flag_enabled(self, mock_flag): + team = Mock(id=123, organization_id="org_abc") + assert is_ai_events_enabled(team) is True + mock_flag.assert_called_once_with( + "ai-events-table-rollout", + "123", + groups={"organization": "org_abc"}, + group_properties={"organization": {"id": "org_abc"}}, + send_feature_flag_events=False, + ) + + @patch("posthog.hogql_queries.ai.ai_table_resolver.feature_enabled_or_false", return_value=False) + def test_returns_false_when_flag_disabled(self, mock_flag): + team = Mock(id=456, organization_id="org_xyz") + assert is_ai_events_enabled(team) is False + mock_flag.assert_called_once_with( + "ai-events-table-rollout", + "456", + groups={"organization": "org_xyz"}, + group_properties={"organization": {"id": "org_xyz"}}, + send_feature_flag_events=False, + ) class TestQueryAiEvents: diff --git a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py index 549cf37a56ba..4906dca96991 100644 --- a/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py +++ b/posthog/hogql_queries/insights/trends/test/test_trends_query_runner.py @@ -3091,7 +3091,7 @@ def test_cohort_modifier_with_too_few_cohorts(self, patch_create_default_modifie ), ] ) - @patch("posthog.hogql_queries.insights.trends.trends_query_runner.posthoganalytics.feature_enabled") + @patch("posthog.hogql_queries.insights.trends.trends_query_runner.feature_enabled_or_false") def test_session_property_pre_aggregation_modifier_gate( self, _name: str, @@ -3107,7 +3107,7 @@ def test_session_property_pre_aggregation_modifier_gate( ) assert runner.modifiers.sessionPropertyPreAggregation is expected - @patch("posthog.hogql_queries.insights.trends.trends_query_runner.posthoganalytics.feature_enabled") + @patch("posthog.hogql_queries.insights.trends.trends_query_runner.feature_enabled_or_false") def test_session_property_pre_aggregation_modifier_clears_on_dashboard_reapply(self, patch_feature_enabled): # apply_dashboard_filters re-runs __post_init__. The modifier must reflect the *current* # query state, not the initial one — so a session-breakdown query that gets overridden diff --git a/posthog/session_recordings/test/test_session_recording_playlist.py b/posthog/session_recordings/test/test_session_recording_playlist.py index 093698870d5d..aa75e9745d3c 100644 --- a/posthog/session_recordings/test/test_session_recording_playlist.py +++ b/posthog/session_recordings/test/test_session_recording_playlist.py @@ -903,7 +903,7 @@ def test_add_remove_static_playlist_items(self): ) @patch( - "posthog.hogql.database.database.posthoganalytics.feature_enabled", + "posthog.hogql.database.database.feature_enabled_or_false", new=MagicMock(return_value=False), ) @snapshot_postgres_queries diff --git a/posthog/session_recordings/test/test_session_recordings.py b/posthog/session_recordings/test/test_session_recordings.py index a11be9e71153..823b708d69a0 100644 --- a/posthog/session_recordings/test/test_session_recordings.py +++ b/posthog/session_recordings/test/test_session_recordings.py @@ -105,7 +105,7 @@ def produce_replay_summary( ] ) @patch( - "posthog.hogql.database.database.posthoganalytics.feature_enabled", + "posthog.hogql.database.database.feature_enabled_or_false", new=MagicMock(return_value=False), ) @snapshot_postgres_queries @@ -155,7 +155,7 @@ def test_get_session_recordings_scenarios( assert len(results[0]["person"]["distinct_ids"]) == expected_distinct_ids_count @patch( - "posthog.hogql.database.database.posthoganalytics.feature_enabled", + "posthog.hogql.database.database.feature_enabled_or_false", new=MagicMock(return_value=False), ) @snapshot_postgres_queries @@ -222,7 +222,7 @@ def test_get_session_recordings_includes_person_data(self) -> None: ] ) @patch( - "posthog.hogql.database.database.posthoganalytics.feature_enabled", + "posthog.hogql.database.database.feature_enabled_or_false", new=MagicMock(return_value=False), ) @snapshot_postgres_queries @@ -352,7 +352,7 @@ def test_listing_recordings_is_not_nplus1_for_persons(self): with ( freeze_time("2022-06-03T12:00:00.000Z"), patch( - "posthog.hogql.database.database.posthoganalytics.feature_enabled", + "posthog.hogql.database.database.feature_enabled_or_false", return_value=False, ), snapshot_postgres_queries_context(self), diff --git a/products/ai_observability/backend/api/test/test_ai_observability_access_control.py b/products/ai_observability/backend/api/test/test_ai_observability_access_control.py index 2f9186dc2720..71ede47b46ac 100644 --- a/products/ai_observability/backend/api/test/test_ai_observability_access_control.py +++ b/products/ai_observability/backend/api/test/test_ai_observability_access_control.py @@ -37,6 +37,17 @@ def setUp(self): ] self.organization.save() + self.score_definitions_flag_patcher = patch( + "products.ai_observability.backend.api.score_definitions.feature_enabled_or_false", return_value=True + ) + self.trace_reviews_flag_patcher = patch( + "products.ai_observability.backend.api.trace_reviews.feature_enabled_or_false", return_value=True + ) + self.score_definitions_flag_patcher.start() + self.trace_reviews_flag_patcher.start() + self.addCleanup(self.score_definitions_flag_patcher.stop) + self.addCleanup(self.trace_reviews_flag_patcher.stop) + AccessControl.objects.create( team=self.team, resource="project", diff --git a/products/ai_observability/backend/api/test/test_evaluations.py b/products/ai_observability/backend/api/test/test_evaluations.py index 48b7abd9d275..701d7f072193 100644 --- a/products/ai_observability/backend/api/test/test_evaluations.py +++ b/products/ai_observability/backend/api/test/test_evaluations.py @@ -114,9 +114,7 @@ def test_evaluation_rollback_when_auto_report_fails(self): self.assertEqual(EvaluationReport.objects.count(), 0) def test_can_create_sentiment_evaluation_without_default_report(self): - with patch( - "products.ai_observability.backend.feature_flags.posthoganalytics.feature_enabled", return_value=True - ): + with patch("products.ai_observability.backend.feature_flags.feature_enabled_or_false", return_value=True): response = self.client.post( f"/api/environments/{self.team.id}/evaluations/", { @@ -139,9 +137,7 @@ def test_can_create_sentiment_evaluation_without_default_report(self): self.assertEqual(EvaluationReport.objects.filter(evaluation=evaluation).count(), 0) def test_create_sentiment_evaluation_requires_feature_flag(self): - with patch( - "products.ai_observability.backend.feature_flags.posthoganalytics.feature_enabled", return_value=False - ): + with patch("products.ai_observability.backend.feature_flags.feature_enabled_or_false", return_value=False): response = self.client.post( f"/api/environments/{self.team.id}/evaluations/", { @@ -172,9 +168,7 @@ def test_re_enable_sentiment_evaluation_requires_feature_flag(self): created_by=self.user, ) - with patch( - "products.ai_observability.backend.feature_flags.posthoganalytics.feature_enabled", return_value=False - ): + with patch("products.ai_observability.backend.feature_flags.feature_enabled_or_false", return_value=False): response = self.client.patch( f"/api/environments/{self.team.id}/evaluations/{evaluation.id}/", {"enabled": True}, @@ -198,9 +192,7 @@ def test_update_existing_sentiment_evaluation_allows_unchanged_type_when_feature created_by=self.user, ) - with patch( - "products.ai_observability.backend.feature_flags.posthoganalytics.feature_enabled", return_value=False - ): + with patch("products.ai_observability.backend.feature_flags.feature_enabled_or_false", return_value=False): response = self.client.patch( f"/api/environments/{self.team.id}/evaluations/{evaluation.id}/", { @@ -218,9 +210,7 @@ def test_update_existing_sentiment_evaluation_allows_unchanged_type_when_feature self.assertEqual(evaluation.name, "Updated Sentiment Evaluation") def test_sentiment_evaluation_rejects_model_configuration(self): - with patch( - "products.ai_observability.backend.feature_flags.posthoganalytics.feature_enabled", return_value=True - ): + with patch("products.ai_observability.backend.feature_flags.feature_enabled_or_false", return_value=True): response = self.client.post( f"/api/environments/{self.team.id}/evaluations/", { @@ -252,9 +242,7 @@ def test_sentiment_evaluation_rejects_model_configuration(self): def test_rejects_unsupported_evaluation_output_type_combinations( self, _name, evaluation_type, output_type, evaluation_config, output_config ): - with patch( - "products.ai_observability.backend.feature_flags.posthoganalytics.feature_enabled", return_value=True - ): + with patch("products.ai_observability.backend.feature_flags.feature_enabled_or_false", return_value=True): response = self.client.post( f"/api/environments/{self.team.id}/evaluations/", { diff --git a/products/ai_observability/backend/api/test/test_score_definitions.py b/products/ai_observability/backend/api/test/test_score_definitions.py index 6a680cf4e06b..beca27d01814 100644 --- a/products/ai_observability/backend/api/test/test_score_definitions.py +++ b/products/ai_observability/backend/api/test/test_score_definitions.py @@ -12,6 +12,14 @@ class TestScoreDefinitionsApi(APIBaseTest): + def setUp(self) -> None: + super().setUp() + self.feature_flag_patcher = patch( + "products.ai_observability.backend.api.score_definitions.feature_enabled_or_false", return_value=True + ) + self.feature_flag_patcher.start() + self.addCleanup(self.feature_flag_patcher.stop) + def _endpoint(self) -> str: return f"/api/environments/{self.team.id}/llm_analytics/score_definitions/" diff --git a/products/ai_observability/backend/api/test/test_trace_reviews.py b/products/ai_observability/backend/api/test/test_trace_reviews.py index 4f4aa2463baa..be79375820d6 100644 --- a/products/ai_observability/backend/api/test/test_trace_reviews.py +++ b/products/ai_observability/backend/api/test/test_trace_reviews.py @@ -1,4 +1,5 @@ from posthog.test.base import APIBaseTest +from unittest.mock import patch from django.test import override_settings @@ -11,6 +12,14 @@ class TestTraceReviewsApi(APIBaseTest): + def setUp(self) -> None: + super().setUp() + self.feature_flag_patcher = patch( + "products.ai_observability.backend.api.trace_reviews.feature_enabled_or_false", return_value=True + ) + self.feature_flag_patcher.start() + self.addCleanup(self.feature_flag_patcher.stop) + def _endpoint(self) -> str: return f"/api/environments/{self.team.id}/llm_analytics/trace_reviews/" diff --git a/products/data_modeling/backend/api/test/test_node_api.py b/products/data_modeling/backend/api/test/test_node_api.py index 7a59efab9148..40bb3119f0f4 100644 --- a/products/data_modeling/backend/api/test/test_node_api.py +++ b/products/data_modeling/backend/api/test/test_node_api.py @@ -205,7 +205,7 @@ def test_materialize_starts_workflow(self, mock_sync_connect): self.assertEqual(response.status_code, status.HTTP_200_OK) mock_client.start_workflow.assert_called_once() - @patch("products.data_modeling.backend.api.node.posthoganalytics.feature_enabled", return_value=True) + @patch("products.data_modeling.backend.api.node.feature_enabled_or_false", return_value=True) @patch("products.data_modeling.backend.api.node.sync_connect") def test_run_uses_execute_dag_when_v2_enabled(self, mock_sync_connect, mock_feature_flag): mock_client = AsyncMock() @@ -220,7 +220,7 @@ def test_run_uses_execute_dag_when_v2_enabled(self, mock_sync_connect, mock_feat call_args = mock_client.start_workflow.call_args self.assertEqual(call_args[0][0], "data-modeling-execute-dag") - @patch("products.data_modeling.backend.api.node.posthoganalytics.feature_enabled", return_value=False) + @patch("products.data_modeling.backend.api.node.feature_enabled_or_false", return_value=False) @patch("products.data_modeling.backend.api.node.sync_connect") def test_run_uses_run_workflow_when_v2_disabled(self, mock_sync_connect, mock_feature_flag): mock_client = AsyncMock() @@ -235,7 +235,7 @@ def test_run_uses_run_workflow_when_v2_disabled(self, mock_sync_connect, mock_fe call_args = mock_client.start_workflow.call_args self.assertEqual(call_args[0][0], "data-modeling-run") - @patch("products.data_modeling.backend.api.node.posthoganalytics.feature_enabled", return_value=True) + @patch("products.data_modeling.backend.api.node.feature_enabled_or_false", return_value=True) @patch("products.data_modeling.backend.api.node.sync_connect") def test_materialize_uses_materialize_view_when_v2_enabled(self, mock_sync_connect, mock_feature_flag): mock_client = AsyncMock() @@ -249,7 +249,7 @@ def test_materialize_uses_materialize_view_when_v2_enabled(self, mock_sync_conne call_args = mock_client.start_workflow.call_args self.assertEqual(call_args[0][0], "data-modeling-materialize-view") - @patch("products.data_modeling.backend.api.node.posthoganalytics.feature_enabled", return_value=False) + @patch("products.data_modeling.backend.api.node.feature_enabled_or_false", return_value=False) @patch("products.data_modeling.backend.api.node.sync_connect") def test_materialize_uses_run_workflow_when_v2_disabled(self, mock_sync_connect, mock_feature_flag): mock_client = AsyncMock() diff --git a/products/experiments/backend/test/test_create_from_prompt.py b/products/experiments/backend/test/test_create_from_prompt.py index 3866b9432756..b44e097d2d62 100644 --- a/products/experiments/backend/test/test_create_from_prompt.py +++ b/products/experiments/backend/test/test_create_from_prompt.py @@ -29,7 +29,7 @@ class TestExperimentsCreateFromPrompt(APILicensedTest): def setUp(self) -> None: super().setUp() self.feature_flag_patcher = patch( - "products.experiments.backend.presentation.views.posthoganalytics.feature_enabled", + "products.experiments.backend.presentation.views.feature_enabled_or_false", return_value=True, ) self.mock_feature_enabled = self.feature_flag_patcher.start() diff --git a/products/feature_flags/backend/api/test/test_feature_flag.py b/products/feature_flags/backend/api/test/test_feature_flag.py index cbf4d1c367e4..eaecb859119e 100644 --- a/products/feature_flags/backend/api/test/test_feature_flag.py +++ b/products/feature_flags/backend/api/test/test_feature_flag.py @@ -632,7 +632,7 @@ def test_string_group_variant_preserved(self): ("false", False), ] ) - @patch("products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled") + @patch("products.feature_flags.backend.api.feature_flag.feature_enabled_or_false") def test_boolean_early_exit_accepted(self, _name, value, mock_feature_enabled): mock_feature_enabled.return_value = True response = self.client.post( @@ -651,7 +651,7 @@ def test_boolean_early_exit_accepted(self, _name, value, mock_feature_enabled): flag = FeatureFlag.objects.get(key=f"early-exit-{_name}", team=self.team) self.assertEqual(flag.filters["early_exit"], value) - @patch("products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled") + @patch("products.feature_flags.backend.api.feature_flag.feature_enabled_or_false") def test_early_exit_rejected_without_feature_flag(self, mock_feature_enabled): mock_feature_enabled.return_value = False response = self.client.post( @@ -669,7 +669,7 @@ def test_early_exit_rejected_without_feature_flag(self, mock_feature_enabled): self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) self.assertIn("early_exit is not available", response.json()["detail"]) - @patch("products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled") + @patch("products.feature_flags.backend.api.feature_flag.feature_enabled_or_false") def test_early_exit_false_accepted_without_feature_flag(self, mock_feature_enabled): mock_feature_enabled.return_value = False response = self.client.post( @@ -686,7 +686,7 @@ def test_early_exit_false_accepted_without_feature_flag(self, mock_feature_enabl ) self.assertEqual(response.status_code, status.HTTP_201_CREATED) - @patch("products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled") + @patch("products.feature_flags.backend.api.feature_flag.feature_enabled_or_false") def test_early_exit_unchanged_truthy_allowed_when_flag_disabled(self, mock_feature_enabled): # A flag created while the feature was enabled keeps working if access is later revoked, # as long as the PATCH doesn't newly turn early_exit on. @@ -5154,7 +5154,7 @@ def test_creating_feature_flag_with_nested_behavioral_cohort(self): ), ] ) - @patch("products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled") + @patch("products.feature_flags.backend.api.feature_flag.feature_enabled_or_false") def test_behavioral_cohort_flag_validation( self, _name, diff --git a/products/feature_flags/backend/api/test/test_scope_enforcement.py b/products/feature_flags/backend/api/test/test_scope_enforcement.py index c1719267827c..a7128719e474 100644 --- a/products/feature_flags/backend/api/test/test_scope_enforcement.py +++ b/products/feature_flags/backend/api/test/test_scope_enforcement.py @@ -83,7 +83,7 @@ def test_gate_evaluates_against_target_team_org_not_actor_org(self): request = SimpleNamespace(user=self.user) with patch( - "products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled", + "products.feature_flags.backend.api.feature_flag.feature_enabled_or_false", return_value=True, ) as mock_feature_enabled: result = _is_enforce_feature_flag_write_scope_enabled(request, team_id=other_team.id) @@ -112,7 +112,7 @@ def test_gate_fails_closed_and_logs_on_error(self): def test_gate_returns_feature_enabled_result(self): request = SimpleNamespace(user=self.user) with patch( - "products.feature_flags.backend.api.feature_flag.posthoganalytics.feature_enabled", + "products.feature_flags.backend.api.feature_flag.feature_enabled_or_false", return_value=False, ): assert _is_enforce_feature_flag_write_scope_enabled(request, team_id=self.team.id) is False diff --git a/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_config.py b/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_config.py index 450b63fec22a..67b8dc0e2c1b 100644 --- a/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_config.py +++ b/products/marketing_analytics/backend/hogql_queries/test_marketing_analytics_config.py @@ -38,7 +38,7 @@ def test_from_team_defaults_to_last_touch(self): assert not config.is_multi_touch @patch( - "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.posthoganalytics.feature_enabled", + "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false", return_value=True, ) def test_from_team_multi_touch_flag_enabled_keeps_mode(self, mock_ff): @@ -50,7 +50,7 @@ def test_from_team_multi_touch_flag_enabled_keeps_mode(self, mock_ff): assert len(self._multi_touch_flag_calls(mock_ff)) == 1 @patch( - "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.posthoganalytics.feature_enabled", + "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false", return_value=False, ) def test_from_team_multi_touch_flag_disabled_falls_back_to_last_touch(self, mock_ff): @@ -61,7 +61,7 @@ def test_from_team_multi_touch_flag_disabled_falls_back_to_last_touch(self, mock assert not config.is_multi_touch @patch( - "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.posthoganalytics.feature_enabled", + "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false", return_value=None, ) def test_from_team_multi_touch_flag_returns_none_falls_back(self, mock_ff): @@ -77,7 +77,7 @@ def test_from_team_single_touch_modes_skip_flag_check(self): self._set_attribution_mode(AttributionMode.FIRST_TOUCH) with patch( - "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.posthoganalytics.feature_enabled", + "products.marketing_analytics.backend.hogql_queries.marketing_analytics_config.feature_enabled_or_false", ) as mock_ff: config = MarketingAnalyticsConfig.from_team(self.team) diff --git a/products/product_analytics/backend/api/test/test_insight.py b/products/product_analytics/backend/api/test/test_insight.py index aef86fcf52bd..a3cd74208f07 100644 --- a/products/product_analytics/backend/api/test/test_insight.py +++ b/products/product_analytics/backend/api/test/test_insight.py @@ -81,7 +81,7 @@ def setUp(self) -> None: ) def test_legacy_insight_endpoints_blocked_with_feature_flag(self, _name: str, path: str) -> None: with patch( - "products.product_analytics.backend.api.insight.posthoganalytics.feature_enabled", return_value=True + "products.product_analytics.backend.api.insight.feature_enabled_or_false", return_value=True ) as mock_feature_enabled: response = self.client.get(path.format(team_id=self.team.id)) @@ -94,7 +94,7 @@ def test_legacy_insight_endpoints_blocked_with_feature_flag(self, _name: str, pa def test_creating_legacy_filter_insight_blocked_with_feature_flag(self) -> None: with patch( - "products.product_analytics.backend.api.insight.posthoganalytics.feature_enabled", return_value=True + "products.product_analytics.backend.api.insight.feature_enabled_or_false", return_value=True ) as mock_feature_enabled: response = self.client.post( f"/api/projects/{self.team.id}/insights/", @@ -113,7 +113,7 @@ def test_creating_legacy_filter_insight_blocked_with_feature_flag(self) -> None: def test_creating_query_insight_not_blocked_by_legacy_filter_flag(self) -> None: with patch( - "products.product_analytics.backend.api.insight.posthoganalytics.feature_enabled", return_value=True + "products.product_analytics.backend.api.insight.feature_enabled_or_false", return_value=True ) as mock_feature_enabled: response = self.client.post( f"/api/projects/{self.team.id}/insights/", From c1e08e42322cec134789065d0102da7a792ae524 Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Thu, 25 Jun 2026 10:45:29 +0200 Subject: [PATCH 12/17] test: keep warehouse temporal path --- products/warehouse_sources/backend/temporal/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 products/warehouse_sources/backend/temporal/.gitkeep diff --git a/products/warehouse_sources/backend/temporal/.gitkeep b/products/warehouse_sources/backend/temporal/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 From 8b2929bf5ac594b94fd87d9d803d40cd62d415ce Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Thu, 25 Jun 2026 11:03:49 +0200 Subject: [PATCH 13/17] test: keep signals emission path --- products/signals/backend/emission/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 products/signals/backend/emission/.gitkeep diff --git a/products/signals/backend/emission/.gitkeep b/products/signals/backend/emission/.gitkeep new file mode 100644 index 000000000000..e69de29bb2d1 From 2ded947a5abdbbe0c45c4563f03693e3e1fd6730 Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Thu, 25 Jun 2026 11:38:21 +0200 Subject: [PATCH 14/17] test: update sdk compatibility assertions --- posthog/temporal/common/test_scoped.py | 21 ++++++++++++++------- posthog/test/test_team.py | 3 +++ 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/posthog/temporal/common/test_scoped.py b/posthog/temporal/common/test_scoped.py index 00a78a8bb871..cbe4c6e7450a 100644 --- a/posthog/temporal/common/test_scoped.py +++ b/posthog/temporal/common/test_scoped.py @@ -76,15 +76,16 @@ async def my_activity(x: int) -> int: def test_upstream_scoped_breaks_iscoroutinefunction_on_async_fn() -> None: - """Documents the upstream bug: posthoganalytics.scoped() wraps an async function - in a synchronous wrapper, defeating the iscoroutinefunction check Temporal uses - to dispatch activities.""" + """Documents the upstream bug in older SDKs while tolerating fixed SDKs.""" @posthoganalytics.scoped() async def my_activity(x: int) -> int: return x + 1 - assert not inspect.iscoroutinefunction(my_activity) + if inspect.iscoroutinefunction(my_activity): + assert asyncio.run(my_activity(1)) == 2 + return + coro = my_activity(1) try: assert inspect.iscoroutine(coro) @@ -93,15 +94,21 @@ async def my_activity(x: int) -> int: def test_upstream_scoped_result_fails_json_encoding() -> None: - """Reproduce the exact production failure: an upstream-scoped async activity - returns an unawaited coroutine, and json.dumps raises the same TypeError that - appears in Temporal's payload-encoder traceback.""" + """Reproduce the old SDK failure while tolerating async-aware scoped().""" @posthoganalytics.scoped() async def my_activity(x: int) -> int: return x + 1 result = my_activity(1) + if inspect.iscoroutinefunction(my_activity): + try: + assert asyncio.run(result) == 2 + finally: + if inspect.iscoroutine(result): + result.close() + return + try: with pytest.raises(TypeError, match="coroutine"): json.dumps(result) diff --git a/posthog/test/test_team.py b/posthog/test/test_team.py index 1b2ad7fad648..18dcccf937dd 100644 --- a/posthog/test/test_team.py +++ b/posthog/test/test_team.py @@ -211,6 +211,7 @@ def test_team_on_cloud_uses_feature_flag_to_determine_person_on_events(self, moc "persons-on-events-v2-reads-enabled", str(team.uuid), groups={"organization": str(self.organization.id)}, + person_properties=None, group_properties={ "organization": { "id": str(self.organization.id), @@ -219,6 +220,8 @@ def test_team_on_cloud_uses_feature_flag_to_determine_person_on_events(self, moc }, only_evaluate_locally=True, send_feature_flag_events=False, + disable_geoip=None, + device_id=None, ) @mock.patch("posthoganalytics.feature_enabled", return_value=False) From e8696cc1a00cd9f88fc7f8c2fbd41863b9f4799a Mon Sep 17 00:00:00 2001 From: "tests-posthog[bot]" <250237707+tests-posthog[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:05:47 +0000 Subject: [PATCH 15/17] test(backend): update query snapshots --- .../__snapshots__/test_upgrade_queries_workflow.ambr | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr b/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr index 0839c952c78f..a1c0325ab25a 100644 --- a/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr +++ b/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr @@ -11,6 +11,14 @@ OR query @? '$.** ? ( @.kind == "InsightVizNode" && (!exists(@.version) || @.version == null || @.version < 4) + )' + OR query @? '$.** ? ( + @.kind == "RetentionQuery" && + (!exists(@.version) || @.version == null || @.version < 2) + )' + OR query @? '$.** ? ( + @.kind == "StickinessQuery" && + (!exists(@.version) || @.version == null || @.version < 2) )' OR query @? '$.** ? ( @.kind == "TrendsQuery" && From a1d94d7f339a19ebffff740486079fe291469186 Mon Sep 17 00:00:00 2001 From: Hugues Pouillot Date: Thu, 25 Jun 2026 12:09:12 +0200 Subject: [PATCH 16/17] test: remove empty path placeholders --- products/signals/backend/emission/.gitkeep | 0 products/warehouse_sources/backend/temporal/.gitkeep | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 products/signals/backend/emission/.gitkeep delete mode 100644 products/warehouse_sources/backend/temporal/.gitkeep diff --git a/products/signals/backend/emission/.gitkeep b/products/signals/backend/emission/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/products/warehouse_sources/backend/temporal/.gitkeep b/products/warehouse_sources/backend/temporal/.gitkeep deleted file mode 100644 index e69de29bb2d1..000000000000 From 58a493f48f9b86a375a9c182dbd10308dec56e46 Mon Sep 17 00:00:00 2001 From: "tests-posthog[bot]" <250237707+tests-posthog[bot]@users.noreply.github.com> Date: Thu, 25 Jun 2026 10:41:04 +0000 Subject: [PATCH 17/17] test(backend): update query snapshots --- .../__snapshots__/test_upgrade_queries_workflow.ambr | 8 -------- 1 file changed, 8 deletions(-) diff --git a/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr b/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr index a1c0325ab25a..0839c952c78f 100644 --- a/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr +++ b/posthog/temporal/tests/product_analytics/__snapshots__/test_upgrade_queries_workflow.ambr @@ -11,14 +11,6 @@ OR query @? '$.** ? ( @.kind == "InsightVizNode" && (!exists(@.version) || @.version == null || @.version < 4) - )' - OR query @? '$.** ? ( - @.kind == "RetentionQuery" && - (!exists(@.version) || @.version == null || @.version < 2) - )' - OR query @? '$.** ? ( - @.kind == "StickinessQuery" && - (!exists(@.version) || @.version == null || @.version < 2) )' OR query @? '$.** ? ( @.kind == "TrendsQuery" &&