diff --git a/System-Tables-Demo/Lakeflow/Lakeflow Pipeline Events System Tables Dashboard.lvdash.json b/System-Tables-Demo/Lakeflow/Lakeflow Pipeline Events System Tables Dashboard.lvdash.json
new file mode 100644
index 0000000..8aadbd5
--- /dev/null
+++ b/System-Tables-Demo/Lakeflow/Lakeflow Pipeline Events System Tables Dashboard.lvdash.json
@@ -0,0 +1,10034 @@
+{
+ "datasets": [
+ {
+ "name": "5e3eb18f",
+ "displayName": "Event Times",
+ "queryLines": [
+ "SELECT event_time AS event_timestamp\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ "WHERE (:pipeline_id = 'All' OR pipeline_id = :pipeline_id)\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "82c4e3b5",
+ "displayName": "Pipelines Runs Errors",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
+ "ev AS (SELECT *, coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state) AS ec FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR')\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " e.origin.dataset_name AS `Target Table`,\n",
+ " r.create_time AS `Pipeline Run Start Time`,\n",
+ " e.event_time AS `Error Time`,\n",
+ " e.message AS `Error Message in Log`,\n",
+ " nullif(concat_ws('; ', transform(coalesce(e.error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), '') AS `Error Message`,\n",
+ " (CASE WHEN e.ec IS NULL THEN '' WHEN e.ec = '' THEN '' ELSE e.ec END) AS `Error Code`,\n",
+ " to_json(e.error) AS `Error Details`\n",
+ "FROM ev e\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ " LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
+ "WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY e.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "c91619f8",
+ "displayName": "Pipelines Metrics Freshness",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT pipeline_link AS `Pipeline`,\n",
+ " latest_event_timestamp AS `Observability data freshness`,\n",
+ " current_timestamp() AS `Check time`,\n",
+ " timestampdiff(MINUTE, latest_event_timestamp, current_timestamp()) AS `Latency at check time (minutes)`\n",
+ "FROM (\n",
+ " SELECT e.pipeline_id, p.pipeline_link, p.tags_array, max(e.event_time) AS latest_event_timestamp\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p USING (pipeline_id)\n",
+ " WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id) AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ " GROUP BY 1, 2, 3)\n",
+ "ORDER BY latest_event_timestamp\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "a189e440",
+ "displayName": "Pipelines Metrics Freshness Summary",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT current_timestamp() AS `Observability data check time`,\n",
+ " min_latest AS `Oldest observability data freshness`,\n",
+ " max_latest AS `Latest observability data freshness`,\n",
+ " timestampdiff(MINUTE, min_latest, current_timestamp()) AS `Largest latency (minutes)`,\n",
+ " timestampdiff(MINUTE, max_latest, current_timestamp()) AS `Smallest latency (minutes)`\n",
+ "FROM (\n",
+ " SELECT min(latest_event_timestamp) AS min_latest, max(latest_event_timestamp) AS max_latest\n",
+ " FROM (\n",
+ " SELECT e.pipeline_id, max(e.event_time) AS latest_event_timestamp\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p USING (pipeline_id)\n",
+ " WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id) AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ " GROUP BY 1))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "6740610b",
+ "displayName": "Pipeline Ids",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT pipeline_id\n",
+ "FROM pipes\n",
+ "WHERE (:tag_value = '')\n",
+ " OR (:tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0))\n",
+ " OR array_contains(tags_array, :tag_value)\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "tag_value",
+ "keyword": "tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "19d796af",
+ "displayName": "Pipelines Runs Status",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (\n",
+ " SELECT pipeline_id, update_id AS pipeline_run_id,\n",
+ " MIN(period_start_time) AS create_time,\n",
+ " MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time,\n",
+ " MAX(result_state) AS latest_state\n",
+ " FROM system.lakeflow.pipeline_update_timeline\n",
+ " GROUP BY pipeline_id, update_id),\n",
+ "phases AS (\n",
+ " SELECT pipeline_id, update_id,\n",
+ " min(CASE WHEN st = 'INITIALIZING' THEN event_time END) AS init_start,\n",
+ " min(CASE WHEN st = 'RUNNING' THEN event_time END) AS run_start\n",
+ " FROM (SELECT pipeline_id, update_id, event_time, upper(details:update_progress.state::string) AS st\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE event_type = 'update_progress')\n",
+ " GROUP BY 1, 2),\n",
+ "errs AS (\n",
+ " SELECT pipeline_id, update_id,\n",
+ " max_by(message, event_time) AS err_log,\n",
+ " max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
+ " max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
+ " max_by(to_json(error), event_time) AS err_full,\n",
+ " max(event_time) AS err_time\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ " WHERE level = 'ERROR'\n",
+ " GROUP BY 1, 2\n",
+ ")\n",
+ "SELECT p.pipeline_link AS `Pipeline`,\n",
+ " p.pipeline_name AS `Pipeline Name`,\n",
+ " concat('', r.pipeline_run_id, '') AS `Pipeline run`,\n",
+ " r.create_time AS `Create time`,\n",
+ " r.end_time AS `End time`,\n",
+ " r.latest_state AS `Status String`,\n",
+ " concat('', coalesce(r.latest_state, '-'), '') AS `Status`,\n",
+ " ifnull(er.err_log, '') AS `Latest error log message`,\n",
+ " ifnull(er.err_msg, '') AS `Latest error message`,\n",
+ " ifnull(er.err_code, '') AS `Latest error code`,\n",
+ " er.err_full AS `Latest full error`,\n",
+ " timestampdiff(second, r.create_time, ifnull(r.end_time, current_timestamp())) AS `Total seconds`,\n",
+ " timestampdiff(second, r.create_time, ph.init_start) AS `Starting seconds`,\n",
+ " timestampdiff(second, ph.init_start, ph.run_start) AS `Initialization seconds`,\n",
+ " timestampdiff(second, ph.run_start, ifnull(r.end_time, current_timestamp())) AS `Running seconds`\n",
+ "FROM runs r\n",
+ " LEFT JOIN pipes p USING (pipeline_id)\n",
+ " LEFT JOIN phases ph ON ph.pipeline_id = r.pipeline_id AND ph.update_id = r.pipeline_run_id\n",
+ " LEFT JOIN errs er ON er.pipeline_id = r.pipeline_id AND er.update_id = r.pipeline_run_id\n",
+ "WHERE (:pipeline_id = 'All' OR r.pipeline_id = :pipeline_id)\n",
+ " AND ((r.create_time >= :event_range.min AND r.create_time <= :event_range.max)\n",
+ " OR (r.end_time >= :event_range.min AND r.end_time <= :event_range.max))\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "5ef35e45",
+ "displayName": "Pipeline Tags",
+ "queryLines": [
+ "SELECT '' AS tag_value\n",
+ "UNION ALL\n",
+ "SELECT DISTINCT tv AS tag_value\n",
+ "FROM (\n",
+ " SELECT explode(transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END))) AS tv\n",
+ " FROM (SELECT tags FROM system.lakeflow.pipelines WHERE tags IS NOT NULL AND cardinality(tags) > 0\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1))\n"
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "8897fc81",
+ "displayName": "Table Warnings",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id)\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " e.origin.dataset_name AS `Target Table`,\n",
+ " r.create_time AS `Pipeline Start Time`,\n",
+ " e.event_time AS `Warning Time`,\n",
+ " e.message AS `Warning Log Message`\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ " LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
+ "WHERE e.level IN ('WARN', 'WARNING')\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
+ " AND e.origin.dataset_name IS NOT NULL\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY e.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "2faba057",
+ "displayName": "Pipelines Hourly Error Rate",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT e.pipeline_id AS `Pipeline Id`,\n",
+ " p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " date_trunc('HOUR', e.event_time) AS `Hour`,\n",
+ " count(*) AS `Number of Errors`\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.level = 'ERROR'\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (date_trunc('HOUR', e.event_time) >= :event_range.min AND date_trunc('HOUR', e.event_time) <= :event_range.max)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "GROUP BY 1, 2, 3, 4\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "2660d018",
+ "displayName": "Table Changes",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id)\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " e.origin.flow_name AS `Flow`,\n",
+ " e.event_time AS `Event Time`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.num_output_rows AS BIGINT), 0) AS `Appended Rows`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.num_upserted_rows AS BIGINT), 0) AS `Upserted Rows`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.num_deleted_rows AS BIGINT), 0) AS `Deleted Rows`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.num_output_bytes AS BIGINT), 0) AS `Appended Bytes`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.num_output_rows AS BIGINT), 0)\n",
+ " + ifnull(TRY_CAST(e.details:flow_progress.metrics.num_upserted_rows AS BIGINT), 0)\n",
+ " + ifnull(TRY_CAST(e.details:flow_progress.metrics.num_deleted_rows AS BIGINT), 0) AS `Total Changes`\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.event_type = 'flow_progress'\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND (e.details:flow_progress.metrics.num_output_rows IS NOT NULL\n",
+ " OR e.details:flow_progress.metrics.num_upserted_rows IS NOT NULL\n",
+ " OR e.details:flow_progress.metrics.num_deleted_rows IS NOT NULL)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "248b74c6",
+ "displayName": "Table Status",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time, MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time, MAX(result_state) AS result_state FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
+ "fp AS (\n",
+ " SELECT pipeline_id, update_id, origin.dataset_name AS dataset_name, event_time,\n",
+ " details:flow_progress.status::string AS status\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ " WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL),\n",
+ "latest_tbl AS (\n",
+ " SELECT pipeline_id, dataset_name,\n",
+ " max_by(update_id, event_time) AS update_id,\n",
+ " max_by(status, event_time) AS status,\n",
+ " max(event_time) AS updated_at\n",
+ " FROM fp GROUP BY 1, 2),\n",
+ "latest_change AS (\n",
+ " SELECT pipeline_id, origin.dataset_name AS dataset_name, max(event_time) AS changes_time\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ " WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL\n",
+ " AND TRY_CAST(details:flow_progress.metrics.num_output_rows AS BIGINT) > 0\n",
+ " GROUP BY 1, 2),\n",
+ "tbl_err AS (\n",
+ " SELECT pipeline_id, origin.dataset_name AS dataset_name,\n",
+ " max_by(update_id, event_time) AS update_id,\n",
+ " max_by(message, event_time) AS err_log,\n",
+ " max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
+ " max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
+ " max(event_time) AS err_time\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR' AND origin.dataset_name IS NOT NULL\n",
+ " GROUP BY 1, 2)\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " lt.dataset_name AS `Target Table`,\n",
+ " concat('', lt.update_id, '') AS `Latest Run`,\n",
+ " concat('', coalesce(lt.status, '-'), '') AS `Latest State`,\n",
+ " lt.status AS `Latest State String`,\n",
+ " CAST(NULL AS STRING) AS `Latest Schema`,\n",
+ " current_timestamp() AS `Last Data Refresh Time`,\n",
+ " lc.changes_time AS `Latest Changes Time`,\n",
+ " timestampdiff(SECOND, lc.changes_time, current_timestamp()) AS `Seconds Since Latest Change`,\n",
+ " te.err_time AS `Time of Latest Error`,\n",
+ " te.err_code AS `Code of Latest Error`,\n",
+ " te.err_log AS `Log Message of Latest Error`,\n",
+ " te.err_msg AS `Message of Latest Error`,\n",
+ " concat('', te.update_id, '') AS `Latest Pipeline Run with Error`,\n",
+ " lt.updated_at AS `Last Updated`,\n",
+ " timestampdiff(SECOND, lt.updated_at, current_timestamp()) AS `Seconds Since Latest Status Update`\n",
+ "FROM latest_tbl lt\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = lt.pipeline_id\n",
+ " LEFT JOIN latest_change lc ON lc.pipeline_id = lt.pipeline_id AND lc.dataset_name = lt.dataset_name\n",
+ " LEFT JOIN tbl_err te ON te.pipeline_id = lt.pipeline_id AND te.dataset_name = lt.dataset_name\n",
+ "WHERE (:pipeline_id = 'All' OR lt.pipeline_id = :pipeline_id)\n",
+ " AND (lt.updated_at >= :event_range.min AND lt.updated_at <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR lt.dataset_name = :table_name)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "0ac0bb93",
+ "displayName": "Selected Pipelines",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT pipeline_id AS `Pipeline Id`,\n",
+ " pipeline_link AS `Pipeline Name`,\n",
+ " pipeline_type AS `Pipeline Type`,\n",
+ " tags_map AS `Pipeline Tags`\n",
+ "FROM pipes\n",
+ "WHERE (:pipeline_id = 'All' OR pipeline_id = :pipeline_id)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "7103e200",
+ "displayName": "Table Status Per Pipeline Run",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time, MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time, MAX(result_state) AS result_state FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
+ "fp AS (\n",
+ " SELECT pipeline_id, update_id, origin.dataset_name AS dataset_name, event_time,\n",
+ " details:flow_progress.status::string AS status\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ " WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL),\n",
+ "latest_tbl AS (\n",
+ " SELECT pipeline_id, dataset_name,\n",
+ " max_by(update_id, event_time) AS update_id,\n",
+ " max_by(status, event_time) AS status,\n",
+ " max(event_time) AS updated_at\n",
+ " FROM fp GROUP BY 1, 2),\n",
+ "latest_change AS (\n",
+ " SELECT pipeline_id, origin.dataset_name AS dataset_name, max(event_time) AS changes_time\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ " WHERE event_type = 'flow_progress' AND origin.dataset_name IS NOT NULL\n",
+ " AND TRY_CAST(details:flow_progress.metrics.num_output_rows AS BIGINT) > 0\n",
+ " GROUP BY 1, 2),\n",
+ "tbl_err AS (\n",
+ " SELECT pipeline_id, origin.dataset_name AS dataset_name,\n",
+ " max_by(update_id, event_time) AS update_id,\n",
+ " max_by(message, event_time) AS err_log,\n",
+ " max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
+ " max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
+ " max(event_time) AS err_time\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR' AND origin.dataset_name IS NOT NULL\n",
+ " GROUP BY 1, 2),\n",
+ "fp_run AS (\n",
+ " SELECT pipeline_id, update_id, dataset_name,\n",
+ " max_by(status, event_time) AS status, max(event_time) AS updated_at\n",
+ " FROM fp GROUP BY 1, 2, 3)\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " fr.dataset_name AS `Target Table`,\n",
+ " concat('', fr.update_id, '') AS `Pipeline Run`,\n",
+ " concat('', coalesce(fr.status, '-'), '') AS `Table State`,\n",
+ " ifnull(te.err_code, '') AS `Error Code`,\n",
+ " te.err_time AS `Error Time`,\n",
+ " ifnull(te.err_msg, '') AS `Error Message`,\n",
+ " ifnull(te.err_log, '') AS `Error Message in Log`,\n",
+ " CAST(NULL AS STRING) AS `Latest Schema`,\n",
+ " rn.create_time AS `Pipeline Run Create Time`,\n",
+ " rn.end_time AS `Pipeline Run End Time`,\n",
+ " concat('', coalesce(rn.result_state, '-'), '') AS `Pipeline Run State`,\n",
+ " fr.updated_at AS `Last Updated`\n",
+ "FROM fp_run fr\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = fr.pipeline_id\n",
+ " LEFT JOIN runs rn ON rn.pipeline_id = fr.pipeline_id AND rn.update_id = fr.update_id\n",
+ " LEFT JOIN tbl_err te ON te.pipeline_id = fr.pipeline_id AND te.dataset_name = fr.dataset_name AND te.update_id = fr.update_id\n",
+ "WHERE (:pipeline_id = 'All' OR fr.pipeline_id = :pipeline_id)\n",
+ " AND (fr.updated_at >= :event_range.min AND fr.updated_at <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR fr.dataset_name = :table_name)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "c0688ffc",
+ "displayName": "Pipelines Runs Warnings",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id)\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " r.create_time AS `Pipeline Run Start Time`,\n",
+ " e.event_time AS `Warning Time`,\n",
+ " e.message AS `Warning Message`\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ " LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
+ "WHERE e.level IN ('WARN', 'WARNING')\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY e.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "8db826d7",
+ "displayName": "Table Names",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT DISTINCT e.origin.dataset_name AS table_name\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.origin.dataset_name IS NOT NULL\n",
+ " AND e.origin.dataset_name NOT LIKE '%.cdc_staging_table'\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "b18ee129",
+ "displayName": "Selected Tables",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "tabs AS (SELECT DISTINCT pipeline_id, origin.dataset_name AS table_name FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE origin.dataset_name IS NOT NULL)\n",
+ "SELECT p.pipeline_link AS `Pipeline`,\n",
+ " t.table_name AS `Target Table`\n",
+ "FROM tabs t LEFT JOIN pipes p ON p.pipeline_id = t.pipeline_id\n",
+ "WHERE (:pipeline_id = 'All' OR t.pipeline_id = :pipeline_id)\n",
+ " AND (:table_name = 'All' OR t.table_name = :table_name)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "3b8b7d86",
+ "displayName": "Table Errors",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
+ "ev AS (SELECT *, coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state) AS ec FROM system.lakeflow_pipeline_events_preview.pipeline_events WHERE level = 'ERROR')\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " e.origin.dataset_name AS `Target Table`,\n",
+ " r.create_time AS `Pipeline Start Time`,\n",
+ " e.event_time AS `Error Time`,\n",
+ " e.message AS `Error Log Message`,\n",
+ " nullif(concat_ws('; ', transform(coalesce(e.error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), '') AS `Error Message`,\n",
+ " (CASE WHEN e.ec IS NULL THEN '' WHEN e.ec = '' THEN '' ELSE e.ec END) AS `Error Code`,\n",
+ " to_json(e.error) AS `Error Details`\n",
+ "FROM ev e\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ " LEFT JOIN runs r ON r.pipeline_id = e.pipeline_id AND r.update_id = e.update_id\n",
+ "WHERE (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
+ " AND e.origin.dataset_name IS NOT NULL\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY e.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "62d03d54",
+ "displayName": "Pipelines Status",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (\n",
+ " SELECT pipeline_id, update_id, create_time, end_time, result_state,\n",
+ " row_number() OVER (PARTITION BY pipeline_id ORDER BY create_time DESC) AS rn\n",
+ " FROM (\n",
+ " SELECT pipeline_id, update_id,\n",
+ " MIN(period_start_time) AS create_time,\n",
+ " MAX(CASE WHEN result_state IS NOT NULL THEN period_end_time END) AS end_time,\n",
+ " MAX(result_state) AS result_state\n",
+ " FROM system.lakeflow.pipeline_update_timeline\n",
+ " GROUP BY pipeline_id, update_id)),\n",
+ "cnts AS (\n",
+ " SELECT pipeline_id, update_id,\n",
+ " sum(CASE WHEN level = 'ERROR' THEN 1 ELSE 0 END) AS n_err,\n",
+ " sum(CASE WHEN level IN ('WARN', 'WARNING') THEN 1 ELSE 0 END) AS n_warn\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events GROUP BY 1, 2),\n",
+ "errs AS (\n",
+ " SELECT pipeline_id, update_id,\n",
+ " max_by(message, event_time) AS err_log,\n",
+ " max_by(nullif(concat_ws('; ', transform(coalesce(error.exceptions, array()), x -> coalesce(x.error_class, x.sql_state))), ''), event_time) AS err_msg,\n",
+ " max_by(coalesce(try_element_at(error.exceptions, 1).error_class, try_element_at(error.exceptions, 1).sql_state), event_time) AS err_code,\n",
+ " max_by(to_json(error), event_time) AS err_full,\n",
+ " max(event_time) AS err_time\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events\n",
+ " WHERE level = 'ERROR'\n",
+ " GROUP BY 1, 2\n",
+ ")\n",
+ "SELECT p.pipeline_link AS `Pipeline`,\n",
+ " p.pipeline_name AS `Pipeline Name`,\n",
+ " concat('', r.update_id, '') AS `Latest Pipeline Run`,\n",
+ " r.create_time AS `Latest Create time`,\n",
+ " r.end_time AS `Latest End time`,\n",
+ " r.result_state AS `Latest Status String`,\n",
+ " concat('', coalesce(r.result_state, '-'), '') AS `Latest Status`,\n",
+ " ifnull(er.err_log, '') AS `Latest error log message`,\n",
+ " ifnull(er.err_msg, '') AS `Latest error message`,\n",
+ " ifnull(er.err_code, '') AS `Latest error code`,\n",
+ " ifnull(c.n_err, 0) AS `Latest run errors`,\n",
+ " ifnull(c.n_warn, 0) AS `Latest run warnings`,\n",
+ " timestampdiff(second, r.create_time, ifnull(r.end_time, current_timestamp())) AS `Latest Total Seconds`,\n",
+ " r.end_time AS `Last Updated`\n",
+ "FROM runs r\n",
+ " LEFT JOIN pipes p USING (pipeline_id)\n",
+ " LEFT JOIN cnts c ON c.pipeline_id = r.pipeline_id AND c.update_id = r.update_id\n",
+ " LEFT JOIN errs er ON er.pipeline_id = r.pipeline_id AND er.update_id = r.update_id\n",
+ "WHERE r.rn = 1\n",
+ " AND (:pipeline_id = 'All' OR r.pipeline_id = :pipeline_id)\n",
+ " AND ((r.create_time >= :event_range.min AND r.create_time <= :event_range.max)\n",
+ " OR (r.end_time >= :event_range.min AND r.end_time <= :event_range.max))\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "b72dbc41",
+ "displayName": "Table Backlog",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " e.origin.dataset_name AS `Target Table`,\n",
+ " e.event_time AS `Event Time`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_bytes AS BIGINT), 0) AS `Backlog Bytes`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_records AS BIGINT), 0) AS `Backlog Rows`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_files AS BIGINT), 0) AS `Backlog Files`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.metrics.backlog_seconds AS BIGINT), 0) AS `Backlog Seconds`\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.event_type = 'flow_progress'\n",
+ " AND e.origin.dataset_name IS NOT NULL\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "e9270b22",
+ "displayName": "Table Expectation Checks",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
+ "exploded AS (\n",
+ " SELECT e.pipeline_id, e.update_id, e.origin.dataset_name AS dataset_name, e.event_time,\n",
+ " ex.name AS expectation_name, ex.passed_records, ex.failed_records\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
+ " LATERAL VIEW explode(CAST(e.details:flow_progress.data_quality.expectations\n",
+ " AS ARRAY>)) AS ex\n",
+ " WHERE e.event_type = 'flow_progress')\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " ex.update_id AS `Pipeline Run Id`,\n",
+ " concat('', ex.update_id, '') AS `Pipeline Run`,\n",
+ " ex.dataset_name AS `Target Table`,\n",
+ " r.create_time AS `Pipeline Start Time`,\n",
+ " ex.expectation_name AS `Expectation Name`,\n",
+ " ex.passed_records AS `Rows Passed`,\n",
+ " ex.failed_records AS `Rows Failed`,\n",
+ " round(100.0 * ex.failed_records / NULLIF(ex.passed_records + ex.failed_records, 0), 2) AS `Failure Percentage`,\n",
+ " ex.event_time AS `Check Time`\n",
+ "FROM exploded ex\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = ex.pipeline_id\n",
+ " LEFT JOIN runs r ON r.pipeline_id = ex.pipeline_id AND r.update_id = ex.update_id\n",
+ "WHERE ex.dataset_name IS NOT NULL\n",
+ " AND (:pipeline_id = 'All' OR ex.pipeline_id = :pipeline_id)\n",
+ " AND (ex.event_time >= :event_range.min AND ex.event_time <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR ex.dataset_name = :table_name)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY ex.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "f786fd7e",
+ "displayName": "Table Expectation Failures",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ "),\n",
+ "runs AS (SELECT pipeline_id, update_id, MIN(period_start_time) AS create_time FROM system.lakeflow.pipeline_update_timeline GROUP BY pipeline_id, update_id),\n",
+ "exploded AS (\n",
+ " SELECT e.pipeline_id, e.update_id, e.origin.dataset_name AS dataset_name, e.event_time,\n",
+ " ex.name AS expectation_name, ex.passed_records, ex.failed_records\n",
+ " FROM system.lakeflow_pipeline_events_preview.pipeline_events e\n",
+ " LATERAL VIEW explode(CAST(e.details:flow_progress.data_quality.expectations\n",
+ " AS ARRAY>)) AS ex\n",
+ " WHERE e.event_type = 'flow_progress')\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " ex.update_id AS `Pipeline Run Id`,\n",
+ " concat('', ex.update_id, '') AS `Pipeline Run`,\n",
+ " ex.dataset_name AS `Target Table`,\n",
+ " r.create_time AS `Pipeline Start Time`,\n",
+ " ex.expectation_name AS `Expectation Name`,\n",
+ " ex.passed_records AS `Rows Passed`,\n",
+ " ex.failed_records AS `Rows Failed`,\n",
+ " round(100.0 * ex.failed_records / NULLIF(ex.passed_records + ex.failed_records, 0), 2) AS `Failure Percentage`,\n",
+ " ex.event_time AS `Check Time`\n",
+ "FROM exploded ex\n",
+ " LEFT JOIN pipes p ON p.pipeline_id = ex.pipeline_id\n",
+ " LEFT JOIN runs r ON r.pipeline_id = ex.pipeline_id AND r.update_id = ex.update_id\n",
+ "WHERE ex.dataset_name IS NOT NULL\n",
+ " AND (:pipeline_id = 'All' OR ex.pipeline_id = :pipeline_id)\n",
+ " AND (ex.event_time >= :event_range.min AND ex.event_time <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR ex.dataset_name = :table_name)\n",
+ " AND ex.failed_records > 0\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY ex.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "bc1ad993",
+ "displayName": "Table Expectations Dropped Rows",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT p.pipeline_name AS `Pipeline Name`,\n",
+ " p.pipeline_link AS `Pipeline Link`,\n",
+ " e.update_id AS `Pipeline Run Id`,\n",
+ " concat('', e.update_id, '') AS `Pipeline Run`,\n",
+ " e.origin.dataset_name AS `Target Table`,\n",
+ " e.event_time AS `Event Time`,\n",
+ " ifnull(TRY_CAST(e.details:flow_progress.data_quality.dropped_records AS BIGINT), 0) AS `Expectations Dropped Rows`\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.event_type = 'flow_progress'\n",
+ " AND e.origin.dataset_name IS NOT NULL\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND (:table_name = 'All' OR e.origin.dataset_name = :table_name)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "ORDER BY e.event_time DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "table_name",
+ "keyword": "table_name",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "catalog": "main",
+ "schema": "databricks_ingestion_monitoring"
+ },
+ {
+ "name": "flow_outcomes",
+ "displayName": "Flow outcomes",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT e.details:flow_progress.status::string AS status,\n",
+ " approx_count_distinct(coalesce(e.update_id, '') || '|' || coalesce(e.origin.flow_id, e.origin.flow_name, '')) AS flow_runs\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.event_type = 'flow_progress'\n",
+ " AND e.details:flow_progress.status::string IN ('COMPLETED', 'FAILED', 'SKIPPED', 'STOPPED', 'EXCLUDED')\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "GROUP BY 1 ORDER BY flow_runs DESC\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "hygiene",
+ "displayName": "Config hygiene",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT e.event_type || ': ' || COALESCE(e.details:deprecation.feature::string, e.details:unsupported_operation.operation::string, e.details:advisory.subject::string, '-') AS item,\n",
+ " count(*) AS events\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.event_type IN ('deprecation', 'unsupported_operation', 'advisory')\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "GROUP BY 1 ORDER BY events DESC LIMIT 12\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "dd_util",
+ "displayName": "Cluster utilization",
+ "queryLines": [
+ "WITH pipes AS (\n",
+ " SELECT pipeline_id,\n",
+ " name AS pipeline_name,\n",
+ " concat('', coalesce(name, pipeline_id), '') AS pipeline_link,\n",
+ " pipeline_type,\n",
+ " CASE WHEN tags IS NULL OR cardinality(tags) = 0 THEN NULL\n",
+ " ELSE transform(map_entries(tags), e -> concat(e.key, CASE WHEN e.value IS NULL OR e.value = '' THEN '' ELSE concat(':', e.value) END)) END AS tags_array,\n",
+ " tags AS tags_map\n",
+ " FROM system.lakeflow.pipelines\n",
+ " QUALIFY row_number() OVER (PARTITION BY pipeline_id ORDER BY change_time DESC) = 1\n",
+ ")\n",
+ "SELECT CASE WHEN :param_util_grain = 'hour' THEN date_trunc('HOUR', e.event_time) ELSE date_trunc('MINUTE', e.event_time) END AS ts,\n",
+ " CASE WHEN :param_util_groupby = 'per cluster (pod)' THEN COALESCE(e.origin.cluster_id, '(unknown)') ELSE 'all clusters' END AS cluster,\n",
+ " avg(TRY_CAST(e.details:cluster_resources.task_slot_metrics.avg_task_slot_utilization AS DOUBLE)) * 100 AS slot_utilization_pct\n",
+ "FROM system.lakeflow_pipeline_events_preview.pipeline_events e LEFT JOIN pipes p ON p.pipeline_id = e.pipeline_id\n",
+ "WHERE e.event_type = 'cluster_resources'\n",
+ " AND (:pipeline_id = 'All' OR e.pipeline_id = :pipeline_id)\n",
+ " AND (e.event_time >= :event_range.min AND e.event_time <= :event_range.max)\n",
+ " AND ((:pipeline_tag_value = '') OR (:pipeline_tag_value = '' AND (tags_array IS NULL OR array_size(tags_array) = 0)) OR array_contains(tags_array, :pipeline_tag_value))\n",
+ "GROUP BY 1, 2 ORDER BY ts\n"
+ ],
+ "parameters": [
+ {
+ "displayName": "pipeline_id",
+ "keyword": "pipeline_id",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "All"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "event_range",
+ "keyword": "event_range",
+ "dataType": "DATETIME",
+ "complexType": "RANGE",
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-7d/d"
+ },
+ "max": {
+ "value": "now/d"
+ }
+ }
+ }
+ },
+ {
+ "displayName": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": ""
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "Utilization grain",
+ "keyword": "param_util_grain",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "hour"
+ }
+ ]
+ }
+ }
+ },
+ {
+ "displayName": "Cluster grouping",
+ "keyword": "param_util_groupby",
+ "dataType": "STRING",
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "all clusters"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ {
+ "name": "util_grain",
+ "displayName": "Utilization grain options",
+ "queryLines": [
+ "SELECT grain FROM VALUES ('minute', 0), ('hour', 1) AS t(grain, ord) ORDER BY ord\n"
+ ]
+ },
+ {
+ "name": "util_groupby",
+ "displayName": "Cluster grouping options",
+ "queryLines": [
+ "SELECT opt FROM VALUES ('all clusters', 0), ('per cluster (pod)', 1) AS t(opt, ord) ORDER BY ord\n"
+ ]
+ }
+ ],
+ "pages": [
+ {
+ "name": "cd63ec52",
+ "displayName": "Filtered",
+ "layout": [
+ {
+ "widget": {
+ "name": "df5934b3",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "0ac0bb93",
+ "fields": [
+ {
+ "name": "Pipeline Id",
+ "expression": "`Pipeline Id`"
+ },
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "Pipeline Type",
+ "expression": "`Pipeline Type`"
+ },
+ {
+ "name": "Pipeline Tags",
+ "expression": "`Pipeline Tags`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Id",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100000,
+ "title": "Pipeline Id",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Pipeline Name",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100001,
+ "title": "Pipeline Name",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 400
+ },
+ {
+ "fieldName": "Pipeline Type",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100002,
+ "title": "Pipeline Type",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Pipeline Tags",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "complex",
+ "displayAs": "json",
+ "visible": true,
+ "order": 100003,
+ "title": "Pipeline Tags",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Selected Pipelines",
+ "description": "Pipelines selected using the global filters"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 6,
+ "width": 6,
+ "height": 5
+ }
+ },
+ {
+ "widget": {
+ "name": "665329d1",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "b18ee129",
+ "fields": [
+ {
+ "name": "Pipeline",
+ "expression": "`Pipeline`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100000,
+ "title": "Pipeline",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 400
+ },
+ {
+ "fieldName": "Target Table",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100001,
+ "title": "Target Table",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Selected Tables",
+ "description": "Tables selected using the global filters"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 11,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "1fcb6a9c",
+ "multilineTextboxSpec": {
+ "lines": [
+ "***Pipelines, pipeline runs and tables selected using the Global filters***"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 4,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "5693a89a",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "19d796af",
+ "fields": [
+ {
+ "name": "Pipeline",
+ "expression": "`Pipeline`"
+ },
+ {
+ "name": "Pipeline run",
+ "expression": "`Pipeline run`"
+ },
+ {
+ "name": "Status",
+ "expression": "`Status`"
+ },
+ {
+ "name": "Create time",
+ "expression": "`Create time`"
+ },
+ {
+ "name": "End time",
+ "expression": "`End time`"
+ },
+ {
+ "name": "Latest error log message",
+ "expression": "`Latest error log message`"
+ },
+ {
+ "name": "Latest error message",
+ "expression": "`Latest error message`"
+ },
+ {
+ "name": "Total seconds",
+ "expression": "`Total seconds`"
+ },
+ {
+ "name": "Starting seconds",
+ "expression": "`Starting seconds`"
+ },
+ {
+ "name": "Initialization seconds",
+ "expression": "`Initialization seconds`"
+ },
+ {
+ "name": "Running seconds",
+ "expression": "`Running seconds`"
+ },
+ {
+ "name": "Latest full error",
+ "expression": "`Latest full error`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 0,
+ "title": "Pipeline",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Pipeline run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 1,
+ "title": "Pipeline run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Status",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 2,
+ "title": "Status",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 150
+ },
+ {
+ "fieldName": "Create time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 3,
+ "title": "Create time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "End time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 4,
+ "title": "End time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest error log message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 5,
+ "title": "Latest error log message",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest error message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 6,
+ "title": "Latest error message",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Total seconds",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 7,
+ "title": "Total seconds",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Starting seconds",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 8,
+ "title": "Starting seconds",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Initialization seconds",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 10,
+ "title": "Initialization seconds",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Running seconds",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 11,
+ "title": "Running seconds",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest full error",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "complex",
+ "displayAs": "json",
+ "visible": true,
+ "order": 12,
+ "title": "Latest full error",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 100001,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Status String",
+ "type": "string",
+ "displayAs": "string",
+ "order": 100005,
+ "title": "Status String",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Latest error code",
+ "type": "string",
+ "displayAs": "string",
+ "order": 100009,
+ "title": "Latest error code",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Selected pipeline runs",
+ "description": "Pipeline runs selected using the global filters"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 17,
+ "width": 6,
+ "height": 9
+ }
+ },
+ {
+ "widget": {
+ "name": "observability-data-updated",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "a189e440",
+ "fields": [
+ {
+ "name": "minutely(Observability data check time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Observability data check time`)"
+ }
+ ],
+ "disaggregated": true,
+ "orders": [
+ {
+ "direction": "DESC",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Observability data check time`)"
+ }
+ ]
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "period": {
+ "fieldName": "minutely(Observability data check time)"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Observability Data Last Updated At"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "oldest-observability-data-latency-minutes",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "a189e440",
+ "fields": [
+ {
+ "name": "Largest latency (minutes)",
+ "expression": "`Largest latency (minutes)`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "Largest latency (minutes)"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Oldest Observability Data Latency (in minutes)",
+ "showDescription": true,
+ "description": "See \"Observability Data Freshness\" in \"Pipelines Health\" tab for more info"
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "newest-observability-data-latency-minutes",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "a189e440",
+ "fields": [
+ {
+ "name": "Smallest latency (minutes)",
+ "expression": "`Smallest latency (minutes)`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "Smallest latency (minutes)"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Newest Observability Data Latency (in minutes)",
+ "showDescription": true,
+ "description": "See \"Observability Data Freshness\" in \"Pipelines Health\" tab for more info"
+ }
+ }
+ },
+ "position": {
+ "x": 4,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "selected-pipelines",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "0ac0bb93",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Number of Selected Pipelines",
+ "showDescription": true,
+ "description": "See \"Selected Pipelines\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 2,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "number-selected-tables",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "b18ee129",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Number of Selected Tables",
+ "description": "See \"Selected Tables\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 2,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "number-selected-pipeline-runs",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "19d796af",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Number of Selected Pipeline Runs",
+ "description": "See \"Selected Pipeline Runs\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 4,
+ "y": 2,
+ "width": 2,
+ "height": 2
+ }
+ }
+ ],
+ "pageType": "PAGE_TYPE_CANVAS"
+ },
+ {
+ "name": "a44ce042",
+ "displayName": "Pipelines Health",
+ "layout": [
+ {
+ "widget": {
+ "name": "dd476651",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "c91619f8",
+ "fields": [
+ {
+ "name": "Pipeline",
+ "expression": "`Pipeline`"
+ },
+ {
+ "name": "Observability data freshness",
+ "expression": "`Observability data freshness`"
+ },
+ {
+ "name": "Check time",
+ "expression": "`Check time`"
+ },
+ {
+ "name": "Latency at check time (minutes)",
+ "expression": "`Latency at check time (minutes)`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100000,
+ "title": "Pipeline",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Observability data freshness",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 100001,
+ "title": "Observability data freshness",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Check time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 100002,
+ "title": "Check time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latency at check time (minutes)",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100003,
+ "title": "Latency at check time (minutes)",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Observability Data Freshness",
+ "description": "Time and latency of the latest processed event by pipeline (oldest to newest)"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 24,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "6820f9d5",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "82c4e3b5",
+ "fields": [
+ {
+ "name": "Pipeline Link",
+ "expression": "`Pipeline Link`"
+ },
+ {
+ "name": "Pipeline Run",
+ "expression": "`Pipeline Run`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "Error Time",
+ "expression": "`Error Time`"
+ },
+ {
+ "name": "Error Code",
+ "expression": "`Error Code`"
+ },
+ {
+ "name": "Error Message in Log",
+ "expression": "`Error Message in Log`"
+ },
+ {
+ "name": "Error Message",
+ "expression": "`Error Message`"
+ },
+ {
+ "name": "Error Details",
+ "expression": "`Error Details`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Link",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 1,
+ "title": "Pipeline Link",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Pipeline Run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 3,
+ "title": "Pipeline Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 300
+ },
+ {
+ "fieldName": "Target Table",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 4,
+ "title": "Target Table",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 6,
+ "title": "Error Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Code",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 7,
+ "title": "Error Code",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Message in Log",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 8,
+ "title": "Error Message in Log",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 9,
+ "title": "Error Message",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Details",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "complex",
+ "displayAs": "json",
+ "visible": true,
+ "order": 10,
+ "title": "Error Details",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 0,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Run Id",
+ "type": "string",
+ "displayAs": "string",
+ "order": 2,
+ "title": "Pipeline Run Id",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Run Start Time",
+ "type": "datetime",
+ "displayAs": "datetime",
+ "order": 5,
+ "title": "Pipeline Run Start Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Errors in Pipelines ",
+ "description": "Detected errors in the pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 12,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "9ac0f134",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "c0688ffc",
+ "fields": [
+ {
+ "name": "Pipeline Link",
+ "expression": "`Pipeline Link`"
+ },
+ {
+ "name": "Pipeline Run Id",
+ "expression": "`Pipeline Run Id`"
+ },
+ {
+ "name": "Warning Time",
+ "expression": "`Warning Time`"
+ },
+ {
+ "name": "Warning Message",
+ "expression": "`Warning Message`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Link",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100001,
+ "title": "Pipeline Link",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Pipeline Run Id",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100002,
+ "title": "Pipeline Run Id",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 300
+ },
+ {
+ "fieldName": "Warning Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 100005,
+ "title": "Warning Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Warning Message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 100006,
+ "title": "Warning Message",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 100000,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Run",
+ "type": "string",
+ "displayAs": "string",
+ "order": 100003,
+ "title": "Pipeline Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Run Start Time",
+ "type": "datetime",
+ "displayAs": "datetime",
+ "order": 100004,
+ "title": "Pipeline Run Start Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Warnings in Pipelines",
+ "description": "Detected warnings in the pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 18,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "5527b0d7",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "62d03d54",
+ "fields": [
+ {
+ "name": "Pipeline",
+ "expression": "`Pipeline`"
+ },
+ {
+ "name": "Latest Pipeline Run",
+ "expression": "`Latest Pipeline Run`"
+ },
+ {
+ "name": "Latest Status",
+ "expression": "`Latest Status`"
+ },
+ {
+ "name": "Latest error code",
+ "expression": "`Latest error code`"
+ },
+ {
+ "name": "Latest Create time",
+ "expression": "`Latest Create time`"
+ },
+ {
+ "name": "Latest End time",
+ "expression": "`Latest End time`"
+ },
+ {
+ "name": "Latest run errors",
+ "expression": "`Latest run errors`"
+ },
+ {
+ "name": "Latest run warnings",
+ "expression": "`Latest run warnings`"
+ },
+ {
+ "name": "Latest error message",
+ "expression": "`Latest error message`"
+ },
+ {
+ "name": "Latest Total Seconds",
+ "expression": "`Latest Total Seconds`"
+ },
+ {
+ "name": "Last Updated",
+ "expression": "`Last Updated`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 0,
+ "title": "Pipeline",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Latest Pipeline Run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 2,
+ "title": "Latest Pipeline Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 300
+ },
+ {
+ "fieldName": "Latest Status",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 3,
+ "title": "Latest Status",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 150
+ },
+ {
+ "fieldName": "Latest error code",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 4,
+ "title": "Latest error code",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest Create time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 5,
+ "title": "Latest Create time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest End time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 6,
+ "title": "Latest End time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest run errors",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 9,
+ "title": "Latest run errors",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest run warnings",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 10,
+ "title": "Latest run warnings",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest error message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 11,
+ "title": "Latest error message",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest Total Seconds",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 12,
+ "title": "Latest Total Seconds",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Last Updated",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 13,
+ "title": "Last Updated",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 1,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Latest Status String",
+ "type": "string",
+ "displayAs": "string",
+ "order": 7,
+ "title": "Latest Status String",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Latest error log message",
+ "type": "string",
+ "displayAs": "string",
+ "order": 8,
+ "title": "Latest error log message",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Latest Runs by Pipeline ",
+ "description": "The status of the latest run for each pipeline"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 6,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "pipeline-status-breakdown",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "62d03d54",
+ "fields": [
+ {
+ "name": "count(Latest Status String)",
+ "expression": "COUNT(`Latest Status String`)"
+ },
+ {
+ "name": "Latest Status String",
+ "expression": "`Latest Status String`"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "pie",
+ "encodings": {
+ "angle": {
+ "fieldName": "count(Latest Status String)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Latest Status String",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Pipeline Status Breakdown",
+ "description": "See \"Latest Runs by Pipeline\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 0,
+ "width": 2,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "number-of-errors-in-pipelines",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "82c4e3b5",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)",
+ "style": {
+ "rules": [
+ {
+ "condition": {
+ "operand": {
+ "type": "data-value",
+ "value": "0"
+ },
+ "operator": ">"
+ },
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 4
+ }
+ }
+ ]
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Number of Errors in Pipelines",
+ "description": "See \"Errors in Pipelines\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 0,
+ "width": 2,
+ "height": 3
+ }
+ },
+ {
+ "widget": {
+ "name": "number-of-warnings-in-pipelines",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "c0688ffc",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)",
+ "style": {
+ "rules": [
+ {
+ "condition": {
+ "operand": {
+ "type": "data-value",
+ "value": "0"
+ },
+ "operator": ">"
+ },
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 2
+ }
+ }
+ ]
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Number of Warnings in Pipelines",
+ "description": "See \"Warnings in Pipelines\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 3,
+ "width": 2,
+ "height": 3
+ }
+ },
+ {
+ "widget": {
+ "name": "mv_w5",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "flow_outcomes",
+ "fields": [
+ {
+ "name": "status",
+ "expression": "`status`"
+ },
+ {
+ "name": "flow_runs",
+ "expression": "`flow_runs`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "y": {
+ "fieldName": "status",
+ "scale": {
+ "type": "categorical",
+ "sort": {
+ "by": "x-reversed"
+ }
+ },
+ "axis": {
+ "hideTitle": true
+ },
+ "displayName": "status"
+ },
+ "x": {
+ "fieldName": "flow_runs",
+ "scale": {
+ "type": "quantitative"
+ },
+ "axis": {
+ "hideTitle": true
+ },
+ "displayName": "flow_runs",
+ "format": {
+ "type": "number-plain",
+ "abbreviation": "compact",
+ "decimalPlaces": {
+ "type": "max",
+ "places": 1
+ }
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Flow outcomes"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 30,
+ "width": 6,
+ "height": 6
+ }
+ }
+ ],
+ "pageType": "PAGE_TYPE_CANVAS"
+ },
+ {
+ "name": "0f6e96ec",
+ "displayName": "Pipelines Metrics",
+ "layout": [
+ {
+ "widget": {
+ "name": "6e698544",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "2faba057",
+ "fields": [
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "hourly(Hour)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Hour`)"
+ },
+ {
+ "name": "sum(Number of Errors)",
+ "expression": "SUM(`Number of Errors`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "line",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Hour)",
+ "scale": {
+ "type": "temporal"
+ },
+ "displayName": "Hour"
+ },
+ "y": {
+ "fieldName": "sum(Number of Errors)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Pipeline Name",
+ "scale": {
+ "type": "categorical"
+ },
+ "displayName": "Pipeline Name"
+ },
+ "label": {
+ "show": false
+ }
+ },
+ "annotations": [],
+ "frame": {
+ "showDescription": true,
+ "showTitle": true,
+ "title": "Hourly error rate",
+ "description": "The number of errors per hour across selected pipelines"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 2,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "c972afff",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "82c4e3b5",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ },
+ {
+ "name": "Error Code",
+ "expression": "`Error Code`"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "pie",
+ "encodings": {
+ "angle": {
+ "fieldName": "count(*)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Error Code",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "label": {
+ "show": true
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Distribution of errors codes",
+ "description": "Relative share of different types of errors across pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 8,
+ "width": 6,
+ "height": 7
+ }
+ },
+ {
+ "widget": {
+ "name": "dd54ef0b",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "19d796af",
+ "fields": [
+ {
+ "name": "Status String",
+ "expression": "`Status String`"
+ },
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "minutely(Create time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
+ },
+ {
+ "name": "sum(Total seconds)",
+ "expression": "SUM(`Total seconds`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Create time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Total seconds)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Status String",
+ "scale": {
+ "type": "categorical",
+ "mappings": [
+ {
+ "value": "COMPLETED",
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 3
+ }
+ },
+ {
+ "value": "FAILED",
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 4
+ }
+ },
+ {
+ "value": "WAITING_FOR_RESOURCES",
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 5
+ }
+ }
+ ]
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "Pipeline Name"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Total time of pipeline runs",
+ "description": "Total execution time and final status for recent pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 25,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "c1f6e5a8",
+ "multilineTextboxSpec": {
+ "lines": [
+ "# Errors"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 0,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "f7a042ed",
+ "multilineTextboxSpec": {
+ "lines": [
+ "# Pipeline Durations"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 23,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "6b06b493",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "19d796af",
+ "fields": [
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "minutely(Create time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
+ },
+ {
+ "name": "sum(Starting seconds)",
+ "expression": "SUM(`Starting seconds`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Create time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Starting seconds)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Pipeline Name",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "Pipeline Name"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Starting time of pipeline runs",
+ "description": "Captures the time waiting for resources to start the pipeline"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 31,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "77547bb3",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "19d796af",
+ "fields": [
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "minutely(Create time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
+ },
+ {
+ "name": "sum(Initialization seconds)",
+ "expression": "SUM(`Initialization seconds`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Create time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Initialization seconds)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Pipeline Name",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "Pipeline Name"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Initialization time of pipeline runs",
+ "description": "Captures the time intializing the pipeline"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 37,
+ "width": 6,
+ "height": 7
+ }
+ },
+ {
+ "widget": {
+ "name": "530d0177",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "19d796af",
+ "fields": [
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "minutely(Create time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Create time`)"
+ },
+ {
+ "name": "sum(Running seconds)",
+ "expression": "SUM(`Running seconds`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Create time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Running seconds)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Pipeline Name",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "Pipeline Name"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Execution time of pipeline runs",
+ "description": "Captures the time evaluating the data flow graph of the pipeline"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 44,
+ "width": 6,
+ "height": 7
+ }
+ },
+ {
+ "widget": {
+ "name": "945b778a",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "2660d018",
+ "fields": [
+ {
+ "name": "Pipeline Name",
+ "expression": "`Pipeline Name`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Total Changes)",
+ "expression": "SUM(`Total Changes`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "line",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Total Changes)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Pipeline Name",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly rows changed (by pipeline)",
+ "description": "Rows appended, upserted, and deleted per hour, by pipeline"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 17,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "bc0b65a1",
+ "multilineTextboxSpec": {
+ "lines": [
+ "# Throughput and Rates"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 15,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "mv_w6",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "hygiene",
+ "fields": [
+ {
+ "name": "item",
+ "expression": "`item`"
+ },
+ {
+ "name": "events",
+ "expression": "`events`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "y": {
+ "fieldName": "item",
+ "scale": {
+ "type": "categorical",
+ "sort": {
+ "by": "x-reversed"
+ }
+ },
+ "axis": {
+ "hideTitle": true
+ },
+ "displayName": "item"
+ },
+ "x": {
+ "fieldName": "events",
+ "scale": {
+ "type": "quantitative"
+ },
+ "axis": {
+ "hideTitle": true
+ },
+ "displayName": "events",
+ "format": {
+ "type": "number-plain",
+ "abbreviation": "compact",
+ "decimalPlaces": {
+ "type": "max",
+ "places": 1
+ }
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Deprecated / unsupported / advisory (by feature)"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 51,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "mv_w15",
+ "queries": [
+ {
+ "name": "src",
+ "query": {
+ "datasetName": "util_groupby",
+ "fields": [
+ {
+ "name": "opt",
+ "expression": "`opt`"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "p_dd_util",
+ "query": {
+ "datasetName": "dd_util",
+ "parameters": [
+ {
+ "name": "param_util_groupby",
+ "keyword": "param_util_groupby"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "filter-single-select",
+ "encodings": {
+ "fields": [
+ {
+ "fieldName": "opt",
+ "displayName": "opt",
+ "queryName": "src"
+ },
+ {
+ "parameterName": "param_util_groupby",
+ "queryName": "p_dd_util"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Group by cluster"
+ },
+ "selection": {
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "all clusters"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 57,
+ "width": 2,
+ "height": 1
+ }
+ },
+ {
+ "widget": {
+ "name": "mv_w16",
+ "queries": [
+ {
+ "name": "src",
+ "query": {
+ "datasetName": "util_grain",
+ "fields": [
+ {
+ "name": "grain",
+ "expression": "`grain`"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "p_dd_util",
+ "query": {
+ "datasetName": "dd_util",
+ "parameters": [
+ {
+ "name": "param_util_grain",
+ "keyword": "param_util_grain"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "filter-single-select",
+ "encodings": {
+ "fields": [
+ {
+ "fieldName": "grain",
+ "displayName": "grain",
+ "queryName": "src"
+ },
+ {
+ "parameterName": "param_util_grain",
+ "queryName": "p_dd_util"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Time granularity"
+ },
+ "selection": {
+ "defaultSelection": {
+ "values": {
+ "dataType": "STRING",
+ "values": [
+ {
+ "value": "hour"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 57,
+ "width": 2,
+ "height": 1
+ }
+ },
+ {
+ "widget": {
+ "name": "mv_w17",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "dd_util",
+ "fields": [
+ {
+ "name": "ts",
+ "expression": "`ts`"
+ },
+ {
+ "name": "slot_utilization_pct",
+ "expression": "`slot_utilization_pct`"
+ },
+ {
+ "name": "cluster",
+ "expression": "`cluster`"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "line",
+ "encodings": {
+ "x": {
+ "fieldName": "ts",
+ "scale": {
+ "type": "temporal"
+ },
+ "format": {
+ "type": "date-time",
+ "date": "locale-short-month",
+ "time": "none"
+ },
+ "axis": {
+ "hideTitle": true
+ },
+ "displayName": "ts"
+ },
+ "y": {
+ "fieldName": "slot_utilization_pct",
+ "scale": {
+ "type": "quantitative"
+ },
+ "axis": {
+ "hideTitle": true
+ },
+ "displayName": "slot_utilization_pct",
+ "format": {
+ "type": "number-plain",
+ "abbreviation": "compact",
+ "decimalPlaces": {
+ "type": "max",
+ "places": 1
+ }
+ }
+ },
+ "color": {
+ "fieldName": "cluster",
+ "scale": {
+ "type": "categorical"
+ },
+ "legend": {
+ "position": "bottom",
+ "hideTitle": true
+ },
+ "displayName": "cluster"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "title": "Cluster utilization %"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 58,
+ "width": 6,
+ "height": 6
+ }
+ }
+ ],
+ "pageType": "PAGE_TYPE_CANVAS"
+ },
+ {
+ "name": "6701599f",
+ "displayName": "Tables Health",
+ "layout": [
+ {
+ "widget": {
+ "name": "057ead9b",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "248b74c6",
+ "fields": [
+ {
+ "name": "Pipeline Link",
+ "expression": "`Pipeline Link`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "Latest Run",
+ "expression": "`Latest Run`"
+ },
+ {
+ "name": "Latest State",
+ "expression": "`Latest State`"
+ },
+ {
+ "name": "Latest Changes Time",
+ "expression": "`Latest Changes Time`"
+ },
+ {
+ "name": "Seconds Since Latest Change",
+ "expression": "`Seconds Since Latest Change`"
+ },
+ {
+ "name": "Latest Schema",
+ "expression": "`Latest Schema`"
+ },
+ {
+ "name": "Time of Latest Error",
+ "expression": "`Time of Latest Error`"
+ },
+ {
+ "name": "Code of Latest Error",
+ "expression": "`Code of Latest Error`"
+ },
+ {
+ "name": "Message of Latest Error",
+ "expression": "`Message of Latest Error`"
+ },
+ {
+ "name": "Log Message of Latest Error",
+ "expression": "`Log Message of Latest Error`"
+ },
+ {
+ "name": "Latest Pipeline Run with Error",
+ "expression": "`Latest Pipeline Run with Error`"
+ },
+ {
+ "name": "Last Updated",
+ "expression": "`Last Updated`"
+ },
+ {
+ "name": "Seconds Since Latest Status Update",
+ "expression": "`Seconds Since Latest Status Update`"
+ },
+ {
+ "name": "Last Data Refresh Time",
+ "expression": "`Last Data Refresh Time`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Link",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 1,
+ "title": "Pipeline Link",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Target Table",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 2,
+ "title": "Target Table",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest Run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 3,
+ "title": "Latest Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 300
+ },
+ {
+ "fieldName": "Latest State",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 4,
+ "title": "Latest State",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 125
+ },
+ {
+ "fieldName": "Latest Changes Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 5,
+ "title": "Latest Changes Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Seconds Since Latest Change",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 6,
+ "title": "Seconds Since Latest Change",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest Schema",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "complex",
+ "displayAs": "json",
+ "visible": true,
+ "order": 7,
+ "title": "Latest Schema",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Time of Latest Error",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 8,
+ "title": "Time of Latest Error",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Code of Latest Error",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 9,
+ "title": "Code of Latest Error",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Message of Latest Error",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 10,
+ "title": "Message of Latest Error",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Log Message of Latest Error",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 11,
+ "title": "Log Message of Latest Error",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest Pipeline Run with Error",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 12,
+ "title": "Latest Pipeline Run with Error",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 325
+ },
+ {
+ "fieldName": "Last Updated",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 13,
+ "title": "Last Updated",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Seconds Since Latest Status Update",
+ "numberFormat": "0",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "integer",
+ "displayAs": "number",
+ "visible": true,
+ "order": 14,
+ "title": "Seconds Since Latest Status Update",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Last Data Refresh Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 15,
+ "title": "Last Data Refresh Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 0,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Table Status",
+ "description": "Latest Status of Monitored Tables"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 6,
+ "width": 6,
+ "height": 10
+ }
+ },
+ {
+ "widget": {
+ "name": "cc2c8376",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "7103e200",
+ "fields": [
+ {
+ "name": "Pipeline Link",
+ "expression": "`Pipeline Link`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "Pipeline Run",
+ "expression": "`Pipeline Run`"
+ },
+ {
+ "name": "Table State",
+ "expression": "`Table State`"
+ },
+ {
+ "name": "Error Code",
+ "expression": "`Error Code`"
+ },
+ {
+ "name": "Error Message",
+ "expression": "`Error Message`"
+ },
+ {
+ "name": "Error Message in Log",
+ "expression": "`Error Message in Log`"
+ },
+ {
+ "name": "Error Time",
+ "expression": "`Error Time`"
+ },
+ {
+ "name": "Latest Schema",
+ "expression": "`Latest Schema`"
+ },
+ {
+ "name": "Pipeline Run Create Time",
+ "expression": "`Pipeline Run Create Time`"
+ },
+ {
+ "name": "Pipeline Run End Time",
+ "expression": "`Pipeline Run End Time`"
+ },
+ {
+ "name": "Pipeline Run State",
+ "expression": "`Pipeline Run State`"
+ },
+ {
+ "name": "Last Updated",
+ "expression": "`Last Updated`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Link",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 1,
+ "title": "Pipeline Link",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Target Table",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 2,
+ "title": "Target Table",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Pipeline Run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 3,
+ "title": "Pipeline Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "description": "",
+ "defaultColumnWidth": 325
+ },
+ {
+ "fieldName": "Table State",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 5,
+ "title": "Table State",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 125
+ },
+ {
+ "fieldName": "Error Code",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 6,
+ "title": "Error Code",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 7,
+ "title": "Error Message",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Message in Log",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 8,
+ "title": "Error Message in Log",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 9,
+ "title": "Error Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Latest Schema",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "complex",
+ "displayAs": "json",
+ "visible": true,
+ "order": 10,
+ "title": "Latest Schema",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Pipeline Run Create Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 11,
+ "title": "Pipeline Run Create Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Pipeline Run End Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 12,
+ "title": "Pipeline Run End Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Pipeline Run State",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 13,
+ "title": "Pipeline Run State",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 150
+ },
+ {
+ "fieldName": "Last Updated",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 14,
+ "title": "Last Updated",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 0,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Tables Status per Pipeline Run",
+ "description": "Table statuses for recent pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 46,
+ "width": 6,
+ "height": 9
+ }
+ },
+ {
+ "widget": {
+ "name": "0ec2e19e",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "fields": [
+ {
+ "name": "Pipeline Link",
+ "expression": "`Pipeline Link`"
+ },
+ {
+ "name": "Pipeline Run",
+ "expression": "`Pipeline Run`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "Error Code",
+ "expression": "`Error Code`"
+ },
+ {
+ "name": "Error Message",
+ "expression": "`Error Message`"
+ },
+ {
+ "name": "Error Log Message",
+ "expression": "`Error Log Message`"
+ },
+ {
+ "name": "Error Details",
+ "expression": "`Error Details`"
+ },
+ {
+ "name": "Error Time",
+ "expression": "`Error Time`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Link",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 1,
+ "title": "Pipeline Link",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Pipeline Run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 3,
+ "title": "Pipeline Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 325
+ },
+ {
+ "fieldName": "Target Table",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 4,
+ "title": "Target Table",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Code",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 5,
+ "title": "Error Code",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 6,
+ "title": "Error Message",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Log Message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 7,
+ "title": "Error Log Message",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Details",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "complex",
+ "displayAs": "json",
+ "visible": true,
+ "order": 8,
+ "title": "Error Details",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Error Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 9,
+ "title": "Error Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 0,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Run Id",
+ "type": "string",
+ "displayAs": "string",
+ "order": 2,
+ "title": "Pipeline Run Id",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Start Time",
+ "type": "datetime",
+ "displayAs": "datetime",
+ "order": 10,
+ "title": "Pipeline Start Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Table Errors",
+ "description": "Table error details in recent pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 16,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "be0d5fae",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "8897fc81",
+ "fields": [
+ {
+ "name": "Pipeline Link",
+ "expression": "`Pipeline Link`"
+ },
+ {
+ "name": "Pipeline Run",
+ "expression": "`Pipeline Run`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "Warning Log Message",
+ "expression": "`Warning Log Message`"
+ },
+ {
+ "name": "Warning Time",
+ "expression": "`Warning Time`"
+ }
+ ],
+ "disaggregated": true
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "table",
+ "encodings": {
+ "columns": [
+ {
+ "fieldName": "Pipeline Link",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 1,
+ "title": "Pipeline Link",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 350
+ },
+ {
+ "fieldName": "Pipeline Run",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 3,
+ "title": "Pipeline Run",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": true,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false,
+ "defaultColumnWidth": 325
+ },
+ {
+ "fieldName": "Target Table",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 4,
+ "title": "Target Table",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Warning Log Message",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "string",
+ "displayAs": "string",
+ "visible": true,
+ "order": 5,
+ "title": "Warning Log Message",
+ "allowSearch": true,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "fieldName": "Warning Time",
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "type": "datetime",
+ "displayAs": "datetime",
+ "visible": true,
+ "order": 6,
+ "title": "Warning Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ]
+ },
+ "invisibleColumns": [
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Name",
+ "type": "string",
+ "displayAs": "string",
+ "order": 0,
+ "title": "Pipeline Name",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Run Id",
+ "type": "string",
+ "displayAs": "string",
+ "order": 2,
+ "title": "Pipeline Run Id",
+ "allowSearch": false,
+ "alignContent": "left",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ },
+ {
+ "dateTimeFormat": "DD/MM/YYYY HH:mm:ss.SSS",
+ "booleanValues": [
+ "false",
+ "true"
+ ],
+ "imageUrlTemplate": "{{ @ }}",
+ "imageTitleTemplate": "{{ @ }}",
+ "imageWidth": "",
+ "imageHeight": "",
+ "linkUrlTemplate": "{{ @ }}",
+ "linkTextTemplate": "{{ @ }}",
+ "linkTitleTemplate": "{{ @ }}",
+ "linkOpenInNewTab": true,
+ "name": "Pipeline Start Time",
+ "type": "datetime",
+ "displayAs": "datetime",
+ "order": 7,
+ "title": "Pipeline Start Time",
+ "allowSearch": false,
+ "alignContent": "right",
+ "allowHTML": false,
+ "highlightLinks": false,
+ "useMonospaceFont": false,
+ "preserveWhitespace": false
+ }
+ ],
+ "allowHTMLByDefault": false,
+ "itemsPerPage": 25,
+ "paginationSize": "default",
+ "condensed": true,
+ "withRowNumber": false,
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Table Warnings",
+ "description": "Table warnings details in recent pipeline runs"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 40,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "expectation-failures",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "f786fd7e",
+ "fields": [
+ {
+ "name": "secondly(Check Time)",
+ "expression": "DATE_TRUNC(\"SECOND\", `Check Time`)"
+ },
+ {
+ "name": "sum(Rows Failed)",
+ "expression": "SUM(`Rows Failed`)"
+ },
+ {
+ "name": "max(Failure Percentage)",
+ "expression": "MAX(`Failure Percentage`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 1,
+ "widgetType": "combo",
+ "encodings": {
+ "x": {
+ "fieldName": "secondly(Check Time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "label": {
+ "show": false
+ },
+ "y": {
+ "primary": {
+ "fields": [
+ {
+ "fieldName": "sum(Rows Failed)"
+ }
+ ],
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "secondary": {
+ "fields": [
+ {
+ "fieldName": "max(Failure Percentage)"
+ }
+ ],
+ "scale": {
+ "type": "quantitative"
+ }
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Expectation Failures",
+ "description": "Expectation failures across all selected tables and expectations"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 22,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "break-down-by-expectation",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "f786fd7e",
+ "fields": [
+ {
+ "name": "sum(Rows Failed)",
+ "expression": "SUM(`Rows Failed`)"
+ },
+ {
+ "name": "Expectation Name",
+ "expression": "`Expectation Name`"
+ },
+ {
+ "name": "sum(Rows Passed)",
+ "expression": "SUM(`Rows Passed`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "pie",
+ "encodings": {
+ "angle": {
+ "fieldName": "sum(Rows Failed)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Expectation Name",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "sum(Rows Passed)"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Expectation failure break down by expectation",
+ "description": "The share of failed rows by expectation name"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 28,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "803d2c81",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "f786fd7e",
+ "fields": [
+ {
+ "name": "sum(Rows Failed)",
+ "expression": "SUM(`Rows Failed`)"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "sum(Rows Passed)",
+ "expression": "SUM(`Rows Passed`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "pie",
+ "encodings": {
+ "angle": {
+ "fieldName": "sum(Rows Failed)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "sum(Rows Passed)"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Expectation failure break down by table",
+ "description": "The share of failed rows by table"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 34,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "table-statuses-breakdown",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "248b74c6",
+ "fields": [
+ {
+ "name": "count(Latest State String)",
+ "expression": "COUNT(`Latest State String`)"
+ },
+ {
+ "name": "Latest State String",
+ "expression": "`Latest State String`"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "pie",
+ "encodings": {
+ "angle": {
+ "fieldName": "count(Latest State String)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Latest State String",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Table Statuses Breakdown",
+ "description": "See \"Table Status\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 0,
+ "width": 2,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "table-errors",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)",
+ "style": {
+ "rules": [
+ {
+ "condition": {
+ "operand": {
+ "type": "data-value",
+ "value": "0"
+ },
+ "operator": ">"
+ },
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 4
+ }
+ }
+ ]
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Number of Table Errors",
+ "description": "See \"Table Errors\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 0,
+ "width": 1,
+ "height": 3
+ }
+ },
+ {
+ "widget": {
+ "name": "number-table-warnings",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "8897fc81",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "counter",
+ "encodings": {
+ "value": {
+ "fieldName": "count(*)",
+ "style": {
+ "rules": [
+ {
+ "condition": {
+ "operand": {
+ "type": "data-value",
+ "value": "0"
+ },
+ "operator": ">"
+ },
+ "color": {
+ "themeColorType": "visualizationColors",
+ "position": 2
+ }
+ }
+ ]
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Number of Table Warnings",
+ "description": "See \"Table Warnings\" below"
+ }
+ }
+ },
+ "position": {
+ "x": 2,
+ "y": 3,
+ "width": 1,
+ "height": 3
+ }
+ }
+ ],
+ "pageType": "PAGE_TYPE_CANVAS"
+ },
+ {
+ "name": "f0c5d746",
+ "displayName": "Global Filters",
+ "layout": [
+ {
+ "widget": {
+ "name": "5315d004",
+ "queries": [
+ {
+ "name": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_pipeline_id",
+ "query": {
+ "datasetName": "6740610b",
+ "fields": [
+ {
+ "name": "pipeline_id",
+ "expression": "`pipeline_id`"
+ },
+ {
+ "name": "pipeline_id_associativity",
+ "expression": "COUNT_IF(`associative_filter_predicate_group`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_id",
+ "query": {
+ "datasetName": "0ac0bb93",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_id",
+ "query": {
+ "datasetName": "8db826d7",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_id",
+ "query": {
+ "datasetName": "b18ee129",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b54b96f2b00fb18b522_pipeline_id",
+ "query": {
+ "datasetName": "5e3eb18f",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_id",
+ "query": {
+ "datasetName": "19d796af",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_id",
+ "query": {
+ "datasetName": "c91619f8",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_id",
+ "query": {
+ "datasetName": "82c4e3b5",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_id",
+ "query": {
+ "datasetName": "c0688ffc",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_id",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_id",
+ "query": {
+ "datasetName": "8897fc81",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_id",
+ "query": {
+ "datasetName": "2faba057",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_id",
+ "query": {
+ "datasetName": "62d03d54",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_id",
+ "query": {
+ "datasetName": "2660d018",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_id",
+ "query": {
+ "datasetName": "248b74c6",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_id",
+ "query": {
+ "datasetName": "7103e200",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_id",
+ "query": {
+ "datasetName": "b72dbc41",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_id",
+ "query": {
+ "datasetName": "e9270b22",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_id",
+ "query": {
+ "datasetName": "f786fd7e",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_id",
+ "query": {
+ "datasetName": "bc1ad993",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_id",
+ "query": {
+ "datasetName": "a189e440",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_flow_outcomes_pipeline_id",
+ "query": {
+ "datasetName": "flow_outcomes",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_hygiene_pipeline_id",
+ "query": {
+ "datasetName": "hygiene",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_dd_util_pipeline_id",
+ "query": {
+ "datasetName": "dd_util",
+ "parameters": [
+ {
+ "name": "pipeline_id",
+ "keyword": "pipeline_id"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "filter-single-select",
+ "encodings": {
+ "fields": [
+ {
+ "fieldName": "pipeline_id",
+ "queryName": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b54b96f2b00fb18b522_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_moved_flow_outcomes_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_moved_hygiene_pipeline_id"
+ },
+ {
+ "parameterName": "pipeline_id",
+ "queryName": "parameter_moved_dd_util_pipeline_id"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Pipelines",
+ "description": "Select a pipeline to observe with selected tags"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 2,
+ "width": 1,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "38094f88",
+ "queries": [
+ {
+ "name": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1087b64f359a100390be_table_name",
+ "query": {
+ "datasetName": "8db826d7",
+ "fields": [
+ {
+ "name": "table_name",
+ "expression": "`table_name`"
+ },
+ {
+ "name": "table_name_associativity",
+ "expression": "COUNT_IF(`associative_filter_predicate_group`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1091ba6a3afac7047a0f_table_name",
+ "query": {
+ "datasetName": "b18ee129",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_table_name",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_table_name",
+ "query": {
+ "datasetName": "8897fc81",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_table_name",
+ "query": {
+ "datasetName": "2660d018",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_table_name",
+ "query": {
+ "datasetName": "248b74c6",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_table_name",
+ "query": {
+ "datasetName": "7103e200",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_table_name",
+ "query": {
+ "datasetName": "b72dbc41",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_table_name",
+ "query": {
+ "datasetName": "e9270b22",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_table_name",
+ "query": {
+ "datasetName": "f786fd7e",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_table_name",
+ "query": {
+ "datasetName": "bc1ad993",
+ "parameters": [
+ {
+ "name": "table_name",
+ "keyword": "table_name"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "filter-single-select",
+ "encodings": {
+ "fields": [
+ {
+ "fieldName": "table_name",
+ "queryName": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1087b64f359a100390be_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1091ba6a3afac7047a0f_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_table_name"
+ },
+ {
+ "parameterName": "table_name",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_table_name"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Tables",
+ "description": "Select a table to observe"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 4,
+ "width": 1,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "cc47542d",
+ "queries": [
+ {
+ "name": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1049bab36f0d89f9f773_event_timestamp",
+ "query": {
+ "datasetName": "5e3eb18f",
+ "fields": [
+ {
+ "name": "event_timestamp",
+ "expression": "`event_timestamp`"
+ },
+ {
+ "name": "event_timestamp_associativity",
+ "expression": "COUNT_IF(`associative_filter_predicate_group`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b109daa983b86d2ea0e85_event_range",
+ "query": {
+ "datasetName": "19d796af",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549a1f9682759c5da45cd926_event_range",
+ "query": {
+ "datasetName": "82c4e3b5",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b107b8ce250ca5a5ac326_event_range",
+ "query": {
+ "datasetName": "c0688ffc",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_event_range",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_event_range",
+ "query": {
+ "datasetName": "8897fc81",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b103181194647c214b1aa_event_range",
+ "query": {
+ "datasetName": "2faba057",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10b9badc5ee6b91b8d89_event_range",
+ "query": {
+ "datasetName": "62d03d54",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_event_range",
+ "query": {
+ "datasetName": "2660d018",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_event_range",
+ "query": {
+ "datasetName": "248b74c6",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_event_range",
+ "query": {
+ "datasetName": "7103e200",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_event_range",
+ "query": {
+ "datasetName": "b72dbc41",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_event_range",
+ "query": {
+ "datasetName": "e9270b22",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_event_range",
+ "query": {
+ "datasetName": "f786fd7e",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_event_range",
+ "query": {
+ "datasetName": "bc1ad993",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_flow_outcomes_event_range",
+ "query": {
+ "datasetName": "flow_outcomes",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_hygiene_event_range",
+ "query": {
+ "datasetName": "hygiene",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_dd_util_event_range",
+ "query": {
+ "datasetName": "dd_util",
+ "parameters": [
+ {
+ "name": "event_range",
+ "keyword": "event_range"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "filter-date-range-picker",
+ "encodings": {
+ "fields": [
+ {
+ "fieldName": "event_timestamp",
+ "queryName": "dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1049bab36f0d89f9f773_event_timestamp"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b109daa983b86d2ea0e85_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549a1f9682759c5da45cd926_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b107b8ce250ca5a5ac326_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10ab970ed31098c78e69_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b1024aea58a5922819fe1_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b103181194647c214b1aa_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10b9badc5ee6b91b8d89_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b100c8f076556b0de8fb5_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b105597260198f0433892_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b106e85f1f1d9342869c3_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e549b10c4a4c246ef94cd279b_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b68e926b1b3daa57707d853060ae_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b6904a0e12d39620b3d5d0055c4c_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_dashboards/01f0b68e549a192ea15560791af33ce6/datasets/01f0b690d5ab1755bf99123705eecdfe_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_moved_flow_outcomes_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_moved_hygiene_event_range"
+ },
+ {
+ "parameterName": "event_range",
+ "queryName": "parameter_moved_dd_util_event_range"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Date/Time Range",
+ "description": "Select a event log data/time range to observe"
+ },
+ "selection": {
+ "defaultSelection": {
+ "range": {
+ "dataType": "DATETIME",
+ "min": {
+ "value": "now-24h/h"
+ },
+ "max": {
+ "value": "now/h"
+ }
+ }
+ }
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 6,
+ "width": 1,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "pipeline-tags",
+ "queries": [
+ {
+ "name": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b299faa569d93cdfe85_tag_value",
+ "query": {
+ "datasetName": "5ef35e45",
+ "fields": [
+ {
+ "name": "tag_value",
+ "expression": "`tag_value`"
+ },
+ {
+ "name": "tag_value_associativity",
+ "expression": "COUNT_IF(`associative_filter_predicate_group`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_tag_value",
+ "query": {
+ "datasetName": "6740610b",
+ "parameters": [
+ {
+ "name": "tag_value",
+ "keyword": "tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_tag_value",
+ "query": {
+ "datasetName": "0ac0bb93",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_tag_value",
+ "query": {
+ "datasetName": "8db826d7",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_tag_value",
+ "query": {
+ "datasetName": "b18ee129",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_tag_value",
+ "query": {
+ "datasetName": "82c4e3b5",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_tag_value",
+ "query": {
+ "datasetName": "c0688ffc",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_tag_value",
+ "query": {
+ "datasetName": "19d796af",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_tag_value",
+ "query": {
+ "datasetName": "62d03d54",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_tag_value",
+ "query": {
+ "datasetName": "c91619f8",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_tag_value",
+ "query": {
+ "datasetName": "2faba057",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_tag_value",
+ "query": {
+ "datasetName": "2660d018",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_tag_value",
+ "query": {
+ "datasetName": "8897fc81",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_tag_value",
+ "query": {
+ "datasetName": "248b74c6",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_tag_value",
+ "query": {
+ "datasetName": "7103e200",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_tag_value",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_tag_value",
+ "query": {
+ "datasetName": "b72dbc41",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_tag_value",
+ "query": {
+ "datasetName": "e9270b22",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_tag_value",
+ "query": {
+ "datasetName": "f786fd7e",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_tag_value",
+ "query": {
+ "datasetName": "bc1ad993",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_tag_value",
+ "query": {
+ "datasetName": "a189e440",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_flow_outcomes_pipeline_tag_value",
+ "query": {
+ "datasetName": "flow_outcomes",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_hygiene_pipeline_tag_value",
+ "query": {
+ "datasetName": "hygiene",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ },
+ {
+ "name": "parameter_moved_dd_util_pipeline_tag_value",
+ "query": {
+ "datasetName": "dd_util",
+ "parameters": [
+ {
+ "name": "pipeline_tag_value",
+ "keyword": "pipeline_tag_value"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 2,
+ "widgetType": "filter-single-select",
+ "encodings": {
+ "fields": [
+ {
+ "fieldName": "tag_value",
+ "queryName": "dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b299faa569d93cdfe85_tag_value"
+ },
+ {
+ "parameterName": "tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b32acc049781d0cf011_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b74b28865962df14f44_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b9aa9c13d1f77991e37_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461ba4b22cc5d22af91321_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461a9bbe5fbf4995bdc44f_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b8d9e292d0d2dc67dd6_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bb2a2c9b3b4e6334bc3_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bd6911ce40b60eaf9a5_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b0aad0f649404f8cb25_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b488f6ffac402178c47_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b1ab6fe6f2600b36759_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b3cb83f2bd346127021_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b67bbebc173149c5e91_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461b7facb56e496d7acee2_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bbdb4e1f9e9952a2684_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461be59b3be5a31cd1c7c2_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bf3ba5f4569c02f5f10_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461bff8c74816d8fb2d26d_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6bf6b461c0ca20f659ce33312ee_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_dashboards/01f0d6bf6b46137ca4dacc873d919029/datasets/01f0d6c03f63139e8465a3bb9d4b4785_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_moved_flow_outcomes_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_moved_hygiene_pipeline_tag_value"
+ },
+ {
+ "parameterName": "pipeline_tag_value",
+ "queryName": "parameter_moved_dd_util_pipeline_tag_value"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Pipeline Tags",
+ "description": "Filter pipelines by tag values"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 0,
+ "width": 1,
+ "height": 2
+ }
+ }
+ ],
+ "pageType": "PAGE_TYPE_GLOBAL_FILTERS"
+ },
+ {
+ "name": "75df3911",
+ "displayName": "Tables Metrics",
+ "layout": [
+ {
+ "widget": {
+ "name": "db12c0c1",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "minutely(Error Time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Error Time`)"
+ },
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Error Time)",
+ "scale": {
+ "type": "temporal"
+ },
+ "displayName": "Error Time"
+ },
+ "y": {
+ "fieldName": "count(*)",
+ "scale": {
+ "type": "quantitative"
+ },
+ "displayName": "Count of Records"
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ },
+ "displayName": "Target Table"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Table Errors",
+ "description": "Detected errors while processing tables"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 2,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "841d82eb",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "8897fc81",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "minutely(Warning Time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Warning Time`)"
+ },
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Warning Time)",
+ "scale": {
+ "type": "categorical"
+ },
+ "displayName": "Warning Time"
+ },
+ "y": {
+ "fieldName": "count(*)",
+ "scale": {
+ "type": "quantitative"
+ },
+ "displayName": "Count of Records"
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ },
+ "displayName": "Target Table"
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Table Warnings",
+ "description": "Detected warnings while processing tables"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 23,
+ "width": 6,
+ "height": 5
+ }
+ },
+ {
+ "widget": {
+ "name": "6c9cec76",
+ "multilineTextboxSpec": {
+ "lines": [
+ "# Errors and Warnings"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 0,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "e968324c",
+ "multilineTextboxSpec": {
+ "lines": [
+ "# Rates and Throughput"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 46,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "edb32903",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "2660d018",
+ "fields": [
+ {
+ "name": "Flow",
+ "expression": "`Flow`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Appended Rows)",
+ "expression": "SUM(`Appended Rows`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Appended Rows)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Flow",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Output Rows Throughput",
+ "description": "Number of output rows to the Flows per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 48,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "c4b2aec5",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "3b8b7d86",
+ "fields": [
+ {
+ "name": "count(*)",
+ "expression": "COUNT(`*`)"
+ },
+ {
+ "name": "Error Code",
+ "expression": "`Error Code`"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "pie",
+ "encodings": {
+ "angle": {
+ "fieldName": "count(*)",
+ "scale": {
+ "type": "quantitative"
+ },
+ "displayName": "Number of occurrences"
+ },
+ "color": {
+ "fieldName": "Error Code",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "label": {
+ "show": true
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Error Frequency",
+ "description": "Distribution of errors by error codes"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 8,
+ "width": 6,
+ "height": 8
+ }
+ },
+ {
+ "widget": {
+ "name": "115fa69e",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "2660d018",
+ "fields": [
+ {
+ "name": "Flow",
+ "expression": "`Flow`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Appended Bytes)",
+ "expression": "SUM(`Appended Bytes`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Appended Bytes)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Flow",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Appended Bytes Throughput",
+ "description": "Number of appended Bytes to the Flows per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 54,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "afdff795",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "2660d018",
+ "fields": [
+ {
+ "name": "Flow",
+ "expression": "`Flow`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Upserted Rows)",
+ "expression": "SUM(`Upserted Rows`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Upserted Rows)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Flow",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Upserted Rows Throughput",
+ "description": "Number of Upserted Rows (apply_changes flows) to the Flows per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 60,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "05a978f3",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "2660d018",
+ "fields": [
+ {
+ "name": "Flow",
+ "expression": "`Flow`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Deleted Rows)",
+ "expression": "SUM(`Deleted Rows`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Deleted Rows)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Flow",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Deleted Rows Throughput",
+ "description": "Number of Deleted Rows (apply_changes flows) to the Flows per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 66,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "86b20f22",
+ "multilineTextboxSpec": {
+ "lines": [
+ "# Latencies"
+ ]
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 72,
+ "width": 6,
+ "height": 2
+ }
+ },
+ {
+ "widget": {
+ "name": "20b6da4a",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "b72dbc41",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Backlog Rows)",
+ "expression": "SUM(`Backlog Rows`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Backlog Rows)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Backlogged Rows",
+ "description": "Number of backlogged rows to the Target Tables per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 74,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "2828faa0",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "b72dbc41",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Backlog Bytes)",
+ "expression": "SUM(`Backlog Bytes`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Backlog Bytes)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Backlogged Bytes",
+ "description": "Number of backlogged bytes to the Target Tables per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 80,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "bbe94522",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "b72dbc41",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "sum(Backlog Files)",
+ "expression": "SUM(`Backlog Files`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Backlog Files)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Backlogged Files",
+ "description": "Number of backlogged files to the Target Tables per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 86,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "9d57fd5f",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "b72dbc41",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "hourly(Event Time)",
+ "expression": "DATE_TRUNC(\"HOUR\", `Event Time`)"
+ },
+ {
+ "name": "max(Backlog Seconds)",
+ "expression": "MAX(`Backlog Seconds`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "hourly(Event Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "max(Backlog Seconds)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Hourly Backlog Seconds",
+ "description": "Max of backlog seconds to the Target Tables per Hour"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 92,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "dropped-rows-due-to-expectations-by-table",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "bc1ad993",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "minutely(Event Time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Event Time`)"
+ },
+ {
+ "name": "sum(Expectations Dropped Rows)",
+ "expression": "SUM(`Expectations Dropped Rows`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Event Time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Expectations Dropped Rows)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ }
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Dropped rows",
+ "description": "Number of dropped rows due to expectation failures by table"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 16,
+ "width": 6,
+ "height": 7
+ }
+ },
+ {
+ "widget": {
+ "name": "hourly-expectations-failures-by-expectation",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "e9270b22",
+ "fields": [
+ {
+ "name": "Expectation Name",
+ "expression": "`Expectation Name`"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "minutely(Check Time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Check Time`)"
+ },
+ {
+ "name": "sum(Rows Failed)",
+ "expression": "SUM(`Rows Failed`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Check Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Rows Failed)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Expectation Name",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "Target Table"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Expectations failures by expectation",
+ "description": "Number of rows that failed expectations"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 28,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "548dd03c",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "e9270b22",
+ "fields": [
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "Expectation Name",
+ "expression": "`Expectation Name`"
+ },
+ {
+ "name": "minutely(Check Time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Check Time`)"
+ },
+ {
+ "name": "sum(Rows Failed)",
+ "expression": "SUM(`Rows Failed`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Check Time)",
+ "scale": {
+ "type": "temporal"
+ }
+ },
+ "y": {
+ "fieldName": "sum(Rows Failed)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Target Table",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "Expectation Name"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Expectations failures by table",
+ "description": "Number of rows that failed expectations"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 40,
+ "width": 6,
+ "height": 6
+ }
+ },
+ {
+ "widget": {
+ "name": "a8b05c55",
+ "queries": [
+ {
+ "name": "main_query",
+ "query": {
+ "datasetName": "e9270b22",
+ "fields": [
+ {
+ "name": "Expectation Name",
+ "expression": "`Expectation Name`"
+ },
+ {
+ "name": "sum(Rows Failed)",
+ "expression": "SUM(`Rows Failed`)"
+ },
+ {
+ "name": "sum(Rows Passed)",
+ "expression": "SUM(`Rows Passed`)"
+ },
+ {
+ "name": "Target Table",
+ "expression": "`Target Table`"
+ },
+ {
+ "name": "minutely(Check Time)",
+ "expression": "DATE_TRUNC(\"MINUTE\", `Check Time`)"
+ },
+ {
+ "name": "avg(Failure Percentage)",
+ "expression": "AVG(`Failure Percentage`)"
+ }
+ ],
+ "disaggregated": false
+ }
+ }
+ ],
+ "spec": {
+ "version": 3,
+ "widgetType": "bar",
+ "encodings": {
+ "x": {
+ "fieldName": "minutely(Check Time)",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "y": {
+ "fieldName": "avg(Failure Percentage)",
+ "scale": {
+ "type": "quantitative"
+ }
+ },
+ "color": {
+ "fieldName": "Expectation Name",
+ "scale": {
+ "type": "categorical"
+ }
+ },
+ "extra": [
+ {
+ "fieldName": "sum(Rows Failed)"
+ },
+ {
+ "fieldName": "sum(Rows Passed)"
+ },
+ {
+ "fieldName": "Target Table"
+ }
+ ]
+ },
+ "frame": {
+ "showTitle": true,
+ "showDescription": true,
+ "title": "Expectations failure percentage by expectation",
+ "description": "Percentage of rows that failed expectations"
+ }
+ }
+ },
+ "position": {
+ "x": 0,
+ "y": 34,
+ "width": 6,
+ "height": 6
+ }
+ }
+ ],
+ "pageType": "PAGE_TYPE_CANVAS"
+ }
+ ],
+ "uiSettings": {
+ "theme": {
+ "widgetHeaderAlignment": "ALIGNMENT_UNSPECIFIED"
+ },
+ "applyModeEnabled": false
+ }
+}
\ No newline at end of file