From cb5f7af65748ff24b0504c6f260699a7e1e59557 Mon Sep 17 00:00:00 2001 From: yiman-chen Date: Fri, 2 Jun 2023 17:09:31 +0200 Subject: [PATCH 1/5] make some changes --- .tinyb | 12 +++ datasources/AI_powered_Demo_Data.datasource | 11 +++ datasources/avg_trip_cli_mv.datasource | 9 ++ datasources/bq2tb.datasource | 13 +++ datasources/bq2tb_sheet.datasource | 12 +++ datasources/bq_conector.datasource | 10 ++ datasources/company_id_100040.datasource | 11 +++ datasources/events_single_quotes.datasource | 16 ++++ datasources/events_test_ym.datasource | 11 +++ datasources/mockingbird_test.datasource | 9 ++ datasources/raw_json_example.datasource | 9 ++ datasources/result_api_copy.datasource | 9 ++ datasources/table_barra_n5.datasource | 9 ++ datasources/test2.datasource | 9 ++ datasources/test2_pipe_mv.datasource | 13 +++ .../yellow_tripdata_2018_01.datasource | 23 +++++ datasources/ym_exercise_ui_mv.datasource | 10 ++ pipes/api_copy.pipe | 11 +++ pipes/endpoint_exporter.pipe | 28 ++++++ pipes/exercise_mv.pipe | 20 ++++ pipes/logs_checks.pipe | 10 ++ pipes/onBording_mv.pipe | 17 ++++ pipes/params.pipe | 14 +++ pipes/pip_variable.pipe | 32 +++++++ pipes/pipe_cli_4.pipe | 6 ++ pipes/scheduled_api.pipe | 6 ++ pipes/table_barra_n2_pipe_3126.pipe | 13 +++ pipes/test2_pipe_1364.pipe | 22 +++++ pipes/test_idx.pipe | 6 ++ pipes/test_ym.pipe | 83 ++++++++++++++++ pipes/tripdata.pipe | 41 ++++++++ pipes/untitled_pipe_6685.pipe | 96 +++++++++++++++++++ pipes/yellow_tripdata_2018_01_PD_pipe.pipe | 27 ++++++ ...ellow_tripdata_2018_01_RMT2_pipe_2294.pipe | 6 ++ ...yellow_tripdata_2018_01_RMT_pipe_7557.pipe | 36 +++++++ 35 files changed, 670 insertions(+) create mode 100644 .tinyb create mode 100644 datasources/AI_powered_Demo_Data.datasource create mode 100644 datasources/avg_trip_cli_mv.datasource create mode 100644 datasources/bq2tb.datasource create mode 100644 datasources/bq2tb_sheet.datasource create mode 100644 datasources/bq_conector.datasource create mode 100644 datasources/company_id_100040.datasource create mode 100644 datasources/events_single_quotes.datasource create mode 100644 datasources/events_test_ym.datasource create mode 100644 datasources/mockingbird_test.datasource create mode 100644 datasources/raw_json_example.datasource create mode 100644 datasources/result_api_copy.datasource create mode 100644 datasources/table_barra_n5.datasource create mode 100644 datasources/test2.datasource create mode 100644 datasources/test2_pipe_mv.datasource create mode 100644 datasources/yellow_tripdata_2018_01.datasource create mode 100644 datasources/ym_exercise_ui_mv.datasource create mode 100644 pipes/api_copy.pipe create mode 100644 pipes/endpoint_exporter.pipe create mode 100644 pipes/exercise_mv.pipe create mode 100644 pipes/logs_checks.pipe create mode 100644 pipes/onBording_mv.pipe create mode 100644 pipes/params.pipe create mode 100644 pipes/pip_variable.pipe create mode 100644 pipes/pipe_cli_4.pipe create mode 100644 pipes/scheduled_api.pipe create mode 100644 pipes/table_barra_n2_pipe_3126.pipe create mode 100644 pipes/test2_pipe_1364.pipe create mode 100644 pipes/test_idx.pipe create mode 100644 pipes/test_ym.pipe create mode 100644 pipes/tripdata.pipe create mode 100644 pipes/untitled_pipe_6685.pipe create mode 100644 pipes/yellow_tripdata_2018_01_PD_pipe.pipe create mode 100644 pipes/yellow_tripdata_2018_01_RMT2_pipe_2294.pipe create mode 100644 pipes/yellow_tripdata_2018_01_RMT_pipe_7557.pipe diff --git a/.tinyb b/.tinyb new file mode 100644 index 0000000..33d4181 --- /dev/null +++ b/.tinyb @@ -0,0 +1,12 @@ +{ + "host": "https://api.tinybird.co", + "id": "60ba23df-3071-4d85-af65-8b17ba0cb0ce", + "name": "dev", + "scope": "user", + "token": "p.eyJ1IjogIjYwYmEyM2RmLTMwNzEtNGQ4NS1hZjY1LThiMTdiYTBjYjBjZSIsICJpZCI6ICJhYjRhOGU0YS01NzQyLTRkNDgtOGRmYi1mNDg5YWRjOTQwOWMifQ.gfEFuPfHZwZ4tq5rgX2WUJSmU7uDh8zA6gP6020IAsE", + "tokens": { + "https://api.tinybird.co": "p.eyJ1IjogIjYwYmEyM2RmLTMwNzEtNGQ4NS1hZjY1LThiMTdiYTBjYjBjZSIsICJpZCI6ICJhYjRhOGU0YS01NzQyLTRkNDgtOGRmYi1mNDg5YWRjOTQwOWMifQ.gfEFuPfHZwZ4tq5rgX2WUJSmU7uDh8zA6gP6020IAsE" + }, + "user_email": "yiman.chen@tinybird.co", + "user_id": "6e343f09-dc92-4e80-b0a6-5472925afb0e" +} \ No newline at end of file diff --git a/datasources/AI_powered_Demo_Data.datasource b/datasources/AI_powered_Demo_Data.datasource new file mode 100644 index 0000000..aef265a --- /dev/null +++ b/datasources/AI_powered_Demo_Data.datasource @@ -0,0 +1,11 @@ +DESCRIPTION > + create a datasource with 3 columns + +SCHEMA > + `column1` String `json:$.column1`, + `column2` DateTime `json:$.column2`, + `column3` Float32 `json:$.column3` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(column2)" +ENGINE_SORTING_KEY "column2, column1" diff --git a/datasources/avg_trip_cli_mv.datasource b/datasources/avg_trip_cli_mv.datasource new file mode 100644 index 0000000..3305c18 --- /dev/null +++ b/datasources/avg_trip_cli_mv.datasource @@ -0,0 +1,9 @@ + +SCHEMA > + `date` Date, + `count` AggregateFunction(count, Float32), + `avg_distance` AggregateFunction(avg, Float32) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYear(date)" +ENGINE_SORTING_KEY "date" diff --git a/datasources/bq2tb.datasource b/datasources/bq2tb.datasource new file mode 100644 index 0000000..1589a8a --- /dev/null +++ b/datasources/bq2tb.datasource @@ -0,0 +1,13 @@ +DESCRIPTION > + create this datasource to ingest data from bq to tb + +SCHEMA > + `Postcode` Nullable(String), + `Town_Area` Nullable(String), + `Region` Nullable(String), + `Population` Nullable(Int64), + `Date` Date + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(Date)" +ENGINE_SORTING_KEY "Date" diff --git a/datasources/bq2tb_sheet.datasource b/datasources/bq2tb_sheet.datasource new file mode 100644 index 0000000..493b6f4 --- /dev/null +++ b/datasources/bq2tb_sheet.datasource @@ -0,0 +1,12 @@ +DESCRIPTION > + create this datasource to ingest data from bq to tb + +SCHEMA > + `a` Nullable(Int8), + `b` Nullable(Int8), + `c` Nullable(Int8), + `timestamp` Date + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(timestamp)" +ENGINE_SORTING_KEY "timestamp" diff --git a/datasources/bq_conector.datasource b/datasources/bq_conector.datasource new file mode 100644 index 0000000..ef60619 --- /dev/null +++ b/datasources/bq_conector.datasource @@ -0,0 +1,10 @@ + +SCHEMA > + `a` Nullable(Int64), + `b` Nullable(Int64), + `c` Nullable(Int64), + `insertion_date` DateTime DEFAULT now() + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(insertion_date)" +ENGINE_SORTING_KEY "insertion_date" diff --git a/datasources/company_id_100040.datasource b/datasources/company_id_100040.datasource new file mode 100644 index 0000000..7e3e730 --- /dev/null +++ b/datasources/company_id_100040.datasource @@ -0,0 +1,11 @@ + +SCHEMA > + `publication_id` Int64, + `publication_success` Int16, + `published_at` DateTime, + `correlation_id` Nullable(String), + `schedule_entry_id` Int64 + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(published_at)" +ENGINE_SORTING_KEY "published_at, publication_id, publication_success, schedule_entry_id" diff --git a/datasources/events_single_quotes.datasource b/datasources/events_single_quotes.datasource new file mode 100644 index 0000000..f967c85 --- /dev/null +++ b/datasources/events_single_quotes.datasource @@ -0,0 +1,16 @@ + +SCHEMA > + `arr` Array(Int16) `json:$.arr[:]`, + `hey` String `json:$.hey`, + `lets` Int16 `json:$.lets`, + `obj__this` String `json:$.obj__this`, + `obj__val` Array(Int16) `json:$.obj__val[:]`, + `objar__a` Array(String) `json:$.objar__a[:]`, + `objar__b` Array(Nullable(Int16)) `json:$.objar__b[:]`, + `objar__darr` Array(Nullable(String)) `json:$.objar__darr[:]`, + `objar__dstr` Array(Nullable(String)) `json:$.objar__dstr[:]`, + `objar__f__g` Array(Nullable(Int16)) `json:$.objar__f__g[:]`, + `objar__new` Array(Nullable(String)) `json:$.objar__new[:]` + +ENGINE "MergeTree" +ENGINE_SORTING_KEY "arr, hey, lets, obj__this" diff --git a/datasources/events_test_ym.datasource b/datasources/events_test_ym.datasource new file mode 100644 index 0000000..760903b --- /dev/null +++ b/datasources/events_test_ym.datasource @@ -0,0 +1,11 @@ + +SCHEMA > + `Customer` String `json:$.Customer`, + `Product` String `json:$.Product`, + `Qtr_4` Nullable(String) `json:$.['Qtr 4']`, + `Qtr_2` Nullable(String) `json:$.['Qtr 2']`, + `Qtr_3` Nullable(String) `json:$.['Qtr 3']`, + `Qtr_1` Nullable(String) `json:$.['Qtr 1']` + +ENGINE "MergeTree" +ENGINE_SORTING_KEY "Customer, Product" diff --git a/datasources/mockingbird_test.datasource b/datasources/mockingbird_test.datasource new file mode 100644 index 0000000..efff0c6 --- /dev/null +++ b/datasources/mockingbird_test.datasource @@ -0,0 +1,9 @@ + +SCHEMA > + `b` String `json:$.b`, + `date` DateTime `json:$.date`, + `userid` Int16 `json:$.userid` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(date)" +ENGINE_SORTING_KEY "date, b, userid" diff --git a/datasources/raw_json_example.datasource b/datasources/raw_json_example.datasource new file mode 100644 index 0000000..df6fce3 --- /dev/null +++ b/datasources/raw_json_example.datasource @@ -0,0 +1,9 @@ +DESCRIPTION > + HFI with raw json + +SCHEMA > + `userid` Int32 `json:$.userid`, + `raw_event` String `json:$` + +ENGINE "MergeTree" +ENGINE_SORTING_KEY "userid, raw_event" diff --git a/datasources/result_api_copy.datasource b/datasources/result_api_copy.datasource new file mode 100644 index 0000000..b1ec72b --- /dev/null +++ b/datasources/result_api_copy.datasource @@ -0,0 +1,9 @@ + +SCHEMA > + `column_00` Int16, + `column_01` Nullable(String), + `column_02` DateTime64(3) + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(column_02)" +ENGINE_SORTING_KEY "column_02, column_00" diff --git a/datasources/table_barra_n5.datasource b/datasources/table_barra_n5.datasource new file mode 100644 index 0000000..b1ec72b --- /dev/null +++ b/datasources/table_barra_n5.datasource @@ -0,0 +1,9 @@ + +SCHEMA > + `column_00` Int16, + `column_01` Nullable(String), + `column_02` DateTime64(3) + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(column_02)" +ENGINE_SORTING_KEY "column_02, column_00" diff --git a/datasources/test2.datasource b/datasources/test2.datasource new file mode 100644 index 0000000..d59d643 --- /dev/null +++ b/datasources/test2.datasource @@ -0,0 +1,9 @@ + +SCHEMA > + `userid` Int16 `json:$.userid`, + `chanel` String `json:$.chanel`, + `date` DateTime `json:$.date` + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(date)" +ENGINE_SORTING_KEY "userid, date" diff --git a/datasources/test2_pipe_mv.datasource b/datasources/test2_pipe_mv.datasource new file mode 100644 index 0000000..12c7741 --- /dev/null +++ b/datasources/test2_pipe_mv.datasource @@ -0,0 +1,13 @@ +# Data Source created from Pipe 'test2_pipe_1364' + +SCHEMA > + `userid` Int16, + `chanel` String, + `month` Date, + `num_reg` AggregateFunction(count, DateTime), + `as_uniq_reg` AggregateFunction(uniqExact, DateTime) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYear(month)" +ENGINE_SORTING_KEY "userid, chanel, month" +ENGINE_TTL "month + toIntervalDay(1450)" diff --git a/datasources/yellow_tripdata_2018_01.datasource b/datasources/yellow_tripdata_2018_01.datasource new file mode 100644 index 0000000..75a5fb4 --- /dev/null +++ b/datasources/yellow_tripdata_2018_01.datasource @@ -0,0 +1,23 @@ + +SCHEMA > + `vendorid` Int16, + `tpep_pickup_datetime` DateTime, + `tpep_dropoff_datetime` DateTime, + `passenger_count` Int16, + `trip_distance` Float32, + `ratecodeid` Int16, + `store_and_fwd_flag` String, + `pulocationid` Int32, + `dolocationid` Int32, + `payment_type` Int16, + `fare_amount` Float32, + `extra` Float32, + `mta_tax` Float32, + `tip_amount` Float32, + `tolls_amount` Float32, + `improvement_surcharge` Float32, + `total_amount` Float32 + +ENGINE "MergeTree" +ENGINE_PARTITION_KEY "toYear(tpep_pickup_datetime)" +ENGINE_SORTING_KEY "trip_distance, tpep_pickup_datetime" diff --git a/datasources/ym_exercise_ui_mv.datasource b/datasources/ym_exercise_ui_mv.datasource new file mode 100644 index 0000000..0d46264 --- /dev/null +++ b/datasources/ym_exercise_ui_mv.datasource @@ -0,0 +1,10 @@ +# Data Source created from Pipe 'exercise_mv' + +SCHEMA > + `date` Date, + `count` AggregateFunction(count, Float32), + `avg_distance` AggregateFunction(avg, Float32) + +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date" diff --git a/pipes/api_copy.pipe b/pipes/api_copy.pipe new file mode 100644 index 0000000..c22e59b --- /dev/null +++ b/pipes/api_copy.pipe @@ -0,0 +1,11 @@ +TOKEN "api_copy_endpoint_read_6707" READ + +NODE test_copy_api +DESCRIPTION > + target_datasource: result_copy_api + +SQL > + + SELECT * from table_barra_n5 + + diff --git a/pipes/endpoint_exporter.pipe b/pipes/endpoint_exporter.pipe new file mode 100644 index 0000000..2a9ef2d --- /dev/null +++ b/pipes/endpoint_exporter.pipe @@ -0,0 +1,28 @@ +TOKEN "endpoint_exporter_endpoint_read_5455" READ + +NODE str_names +SQL > + + select 100024 as company_id, 'en_name,en_short_synopsis,en_medium_synopsis,en_long_synopsis,sv_name,sv_short_synopsis,sv_medium_synopsis,sv_long_synopsis,de_name,de_short_synopsis,de_medium_synopsis,de_long_synopsis,he_name,he_short_synopsis,he_medium_synopsis,he_long_synopsis,fr_name,fr_short_synopsis,fr_medium_synopsis,fr_long_synopsis,es_name,es_short_synopsis,es_medium_synopsis,es_long_synopsis,pt_name,pt_short_synopsis,pt_medium_synopsis,pt_long_synopsis,ja_name,ja_short_synopsis,jp_name,jp_short_synopsis,jp_medium_synopsis,jp_long_synopsis,ch_name,ch_short_synopsis,ch_medium_synopsis,ch_long_synopsis,ko_name,ko_short_synopsis,ko_medium_synopsis,ko_long_synopsis,release_year,release_category' AS str_names + + + +NODE exporter_with_filename +DESCRIPTION > + CRON > */2 * * * * + PRENODE > str_names + BUCKET > gs://bebanjo + FILENAME >test_filename + +SQL > + + select * from params + + + +NODE endpoint_exporter_2 +SQL > + + SELECT 'en_name', 'en_short_synopsis', 'en_medium_synopsis', 'en_long_synopsis', 'sv_name', 'sv_short_synopsis', 'sv_medium_synopsis', 'sv_long_synopsis', 'de_name', 'de_short_synopsis', 'de_medium_synopsis', 'de_long_synopsis', 'he_name', 'he_short_synopsis', 'he_medium_synopsis', 'he_long_synopsis', 'fr_name', 'fr_short_synopsis', 'fr_medium_synopsis', 'fr_long_synopsis', 'es_name', 'es_short_synopsis', 'es_medium_synopsis', 'es_long_synopsis', 'pt_name', 'pt_short_synopsis', 'pt_medium_synopsis', 'pt_long_synopsis', 'ja_name', 'ja_short_synopsis', 'jp_name', 'jp_short_synopsis', 'jp_medium_synopsis', 'jp_long_synopsis', 'ch_name', 'ch_short_synopsis', 'ch_medium_synopsis', 'ch_long_synopsis', 'ko_name', 'ko_short_synopsis', 'ko_medium_synopsis', 'ko_long_synopsis', 'release_year', 'release_category' + + diff --git a/pipes/exercise_mv.pipe b/pipes/exercise_mv.pipe new file mode 100644 index 0000000..890c4e0 --- /dev/null +++ b/pipes/exercise_mv.pipe @@ -0,0 +1,20 @@ +DESCRIPTION > + my materialised view exercise from onboarding. + + +NODE test_0_mv +SQL > + + SELECT + toDate(tpep_pickup_datetime) AS date, + countState(trip_distance) AS count, + avgState(trip_distance) AS avg_distance + FROM yellow_tripdata_2018_01 + GROUP BY date + +TYPE materialized +DATASOURCE ym_exercise_ui_mv +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYYYYMM(date)" +ENGINE_SORTING_KEY "date" + diff --git a/pipes/logs_checks.pipe b/pipes/logs_checks.pipe new file mode 100644 index 0000000..1d071b5 --- /dev/null +++ b/pipes/logs_checks.pipe @@ -0,0 +1,10 @@ +NODE checks_0 +SQL > + + select * from tinybird.block_log + where datasource_id = 't_2b6be2b7cd4a412f9b261f92bec32f6c' + and timestamp > now() - INTERVAL 5 hour + + + + diff --git a/pipes/onBording_mv.pipe b/pipes/onBording_mv.pipe new file mode 100644 index 0000000..5911d9c --- /dev/null +++ b/pipes/onBording_mv.pipe @@ -0,0 +1,17 @@ +NODE avg_per_day_cli_mv +DESCRIPTION > + count, calculate avg of trip distance per day + +SQL > + + SELECT + toDate(tpep_pickup_datetime) AS date, + countState(trip_distance) AS count, + avgState(trip_distance) AS avg_distance + FROM yellow_tripdata_2018_01 + GROUP BY date + +TYPE materialized +DATASOURCE avg_trip_cli_mv + + diff --git a/pipes/params.pipe b/pipes/params.pipe new file mode 100644 index 0000000..2e5a2a6 --- /dev/null +++ b/pipes/params.pipe @@ -0,0 +1,14 @@ +TOKEN "params_endpoint_read_7685" READ + +NODE param_1 +SQL > + + % + select + {% for last, x in enumerate_with_last(split_to_array (String(str_names,'col1,col2')) ) %} + {{x}} + {% if not last %},{% end %} + {% end %} + -- from node_str_names + + diff --git a/pipes/pip_variable.pipe b/pipes/pip_variable.pipe new file mode 100644 index 0000000..41ee1de --- /dev/null +++ b/pipes/pip_variable.pipe @@ -0,0 +1,32 @@ +TOKEN "pip_variable_endpoint_read_4893" READ + +NODE ep_ui +SQL > + + % + SELECT + toDate(tpep_pickup_datetime) as date, + avg(trip_distance/(dateDiff(second, tpep_pickup_datetime, tpep_dropoff_datetime)/3600)) as average + FROM csv_ui + where passenger_count = {{Int16(passenger_count, 4, description="number of passengers", required=True)}} + and tpep_dropoff_datetime > tpep_pickup_datetime + --and tpep_dropoff_datetime is not null + --and tpep_pickup_datetime is not null + and trip_distance <> 0 + group by date + ORDER BY date ASC + + + +NODE pip_variable_1 +SQL > + + % + SELECT + toDate(tpep_pickup_datetime) as date, + avg(trip_distance/(dateDiff(second, tpep_pickup_datetime, tpep_dropoff_datetime)/3600)) as average + FROM csv_ui + where passenger_count = {{Int16(passenger_count, 4, description="number of passengers", required=True)}} + group by date + + diff --git a/pipes/pipe_cli_4.pipe b/pipes/pipe_cli_4.pipe new file mode 100644 index 0000000..6459b62 --- /dev/null +++ b/pipes/pipe_cli_4.pipe @@ -0,0 +1,6 @@ +NODE pipe_cli_4_0 +SQL > + + % SELECT toDate(tpep_pickup_datetime) as date, avg(trip_distance/(dateDiff(second, tpep_pickup_datetime, tpep_dropoff_datetime)/3600)) as average FROM csv_ui where passenger_count = {{Int16(passenger_count, 4, description="number of passengers", required=True)}} group by date + + diff --git a/pipes/scheduled_api.pipe b/pipes/scheduled_api.pipe new file mode 100644 index 0000000..f131d61 --- /dev/null +++ b/pipes/scheduled_api.pipe @@ -0,0 +1,6 @@ +NODE endpoint +SQL > + + select 1 as column_00, 'pipe_pushed' as column_01, now() as column_02 + + diff --git a/pipes/table_barra_n2_pipe_3126.pipe b/pipes/table_barra_n2_pipe_3126.pipe new file mode 100644 index 0000000..82d5b5e --- /dev/null +++ b/pipes/table_barra_n2_pipe_3126.pipe @@ -0,0 +1,13 @@ +NODE table_barra_n2_pipe_3126_0 +SQL > + + SELECT * FROM result_copy_api + + + +NODE table_barra_n2_pipe_3126_1 +SQL > + + SELECT replaceAll(column_01, '\n',' ') FROM result_copy_api + + diff --git a/pipes/test2_pipe_1364.pipe b/pipes/test2_pipe_1364.pipe new file mode 100644 index 0000000..ed7ce0b --- /dev/null +++ b/pipes/test2_pipe_1364.pipe @@ -0,0 +1,22 @@ +NODE test2_pipe_1364_0 +SQL > + + SELECT + userid, + chanel, + toStartOfMonth(date) AS month, + countState(date) AS num_reg, + uniqExactState(date) AS as_uniq_reg + FROM test2 + GROUP BY + userid, + chanel, + month + +TYPE materialized +DATASOURCE test2_pipe_mv +ENGINE "AggregatingMergeTree" +ENGINE_PARTITION_KEY "toYear(month)" +ENGINE_SORTING_KEY "userid, chanel, month" +ENGINE_TTL "month + toIntervalDay(1450)" + diff --git a/pipes/test_idx.pipe b/pipes/test_idx.pipe new file mode 100644 index 0000000..dbac972 --- /dev/null +++ b/pipes/test_idx.pipe @@ -0,0 +1,6 @@ +NODE test_idx_0 +SQL > + + select toMonday(now()) as date + + diff --git a/pipes/test_ym.pipe b/pipes/test_ym.pipe new file mode 100644 index 0000000..5efbe61 --- /dev/null +++ b/pipes/test_ym.pipe @@ -0,0 +1,83 @@ +TOKEN "test_ym_endpoint_read_7795" READ + +NODE test_endpoint_exporter +SQL > + + % + SELECT * FROM events_test_ym + WHERE Product IN('product,4','product 2') + limit 10 + + + +NODE regex +SQL > + + select replaceRegexpAll('my test + 2', '\r|\n', ' ') as collection_name + + + +NODE test_ym_2 +SQL > + + INSERT INTO events_test_ym (*) VALUES('A','B',NULL,NULL,NULL,NULL) + + + + +NODE test_ym_3 +SQL > + + SELECT has([1, 2, NULL], NULL) + + + +NODE test_ym_4 +SQL > + + SELECT + toDate(tpep_pickup_datetime) AS date, + countState(trip_distance) AS count, + avgState(trip_distance) AS avg_distance + FROM yellow_tripdata_2018_01 + GROUP BY date + + + +NODE test_ym_5 +SQL > + + SELECT + date, + countMerge(count) AS count, + avgMerge(avg_distance) AS avg_distance + FROM avg_trip_cli_mv + GROUP BY date + order by date desc + + + +NODE test_ym_6 +SQL > + + --SELECT parseDateTimeBestEffort('2022-12-07 T00:00:00+00:00') + SELECT parseDateTimeBestEffort('2022-12-07 T00:00:00+00:00') + JSONExtractString ({"shift":"morning","service_date":"2022-12-07T00:00:00+00:00"},'service_date') as x + + + +NODE test_ym_7 +SQL > + + SELECT toDateTime('2022-12-07 T00:00:00+00:00') + + + +NODE test_ym_8 +SQL > + + SET format_csv_null_representation = 'My NULL'; + SELECT * FROM events_test_ym FORMAT CSV; + + diff --git a/pipes/tripdata.pipe b/pipes/tripdata.pipe new file mode 100644 index 0000000..2616bfc --- /dev/null +++ b/pipes/tripdata.pipe @@ -0,0 +1,41 @@ +NODE tripdata_0 +SQL > + + SELECT * FROM yellow_tripdata_2018_01 where CAST(tpep_pickup_datetime AS DATE) = '2009-01-01' + + + +NODE tripdata_1 +SQL > + + SELECT * FROM yellow_tripdata_2018_01 where trip_distance between '1' and '2' + + + +NODE tripdata_4 +SQL > + + SELECT count (distinct CAST(tpep_pickup_datetime AS DATE)) FROM yellow_tripdata_2018_01 + + + +NODE tripdata_5 +SQL > + + SELECT count (distinct trip_distance ) FROM yellow_tripdata_2018_01 + + + +NODE tripdata_6 +SQL > + + SELECT * FROM yellow_tripdata_2018_01 where tpep_pickup_datetime between '2009-01-01 00:00:00' and '2009-01-01 23:59:59' + + + +NODE tripdata_7 +SQL > + + SELECT count(distinct tpep_pickup_datetime) FROM yellow_tripdata_2018_01 where tpep_pickup_datetime between '2009-01-01 00:00:00' and '2009-01-01 23:59:59' + + diff --git a/pipes/untitled_pipe_6685.pipe b/pipes/untitled_pipe_6685.pipe new file mode 100644 index 0000000..d4d7821 --- /dev/null +++ b/pipes/untitled_pipe_6685.pipe @@ -0,0 +1,96 @@ +NODE untitled_pipe_6685_0 +SQL > + + % + select {{split_to_array(String('/\',/something'), separator=String(separator, ','))}} + + + +NODE untitled_pipe_6685_1 +SQL > + + SELECT replaceAll([`/'`, '/something'] + + + +NODE untitled_pipe_6685_2 +SQL > + + % + select '1' {{split_to_array(code, '1,2,3')}} + + + +NODE untitled_pipe_6685_3 +SQL > + + % + select {{split_to_array(count_unique_vals_columns, 'section_id,city')}} + + + +NODE node +SQL > + + SELECT arrayJoin(['a','b','c']) as path + + + +NODE untitled_pipe_6685_5 +SQL > + + % + SELECT * from node + where path in + {{split_to_array(replaceAll((String(param, "/',/something"), separator=','),'`','"')}} + + + + +NODE untitled_pipe_6685_6 +SQL > + + % + SELECT + {{split_to_array(String(param, 'hi, how are you|fine thanks'), separator='|')}} + + + +NODE untitled_pipe_6685_7 +SQL > + + % + select {{sql_unescape("/',/something","'")}} + + + +NODE path +SQL > + + % + select {{String("/',/something")}} + + + +NODE untitled_pipe_6685_9 +SQL > + + SELECT '/\',/something' + FROM untitled_pipe_6685_10 + + + +NODE untitled_pipe_6685_10 +SQL > + + SELECT * FROM path + + + +NODE untitled_pipe_6685_11 +SQL > + + % + select {{sql_unescape( "testing'", ",")}} + + diff --git a/pipes/yellow_tripdata_2018_01_PD_pipe.pipe b/pipes/yellow_tripdata_2018_01_PD_pipe.pipe new file mode 100644 index 0000000..197f181 --- /dev/null +++ b/pipes/yellow_tripdata_2018_01_PD_pipe.pipe @@ -0,0 +1,27 @@ +NODE yellow_tripdata_2018_01_PD_pipe_2497_0 +SQL > + + SELECT * FROM yellow_tripdata_2018_01_PD where CAST(tpep_dropoff_datetime as DATE ) = '2003-01-01' + + + +NODE yellow_tripdata_2018_01_PD_pipe_1 +SQL > + + SELECT * FROM yellow_tripdata_2018_01_PD where CAST(tpep_pickup_datetime as DATE ) = '2003-01-01' and CAST(tpep_dropoff_datetime as DATE ) = '2003-01-01' + + + +NODE yellow_tripdata_2018_01_PD_pipe_2 +SQL > + + SELECT distinct count(tpep_pickup_datetime) FROM yellow_tripdata_2018_01 + + + +NODE yellow_tripdata_2018_01_PD_pipe_3 +SQL > + + SELECT * FROM yellow_tripdata_2018_01_PD where CAST(tpep_pickup_datetime as DATE ) = '2003-01-01' + + diff --git a/pipes/yellow_tripdata_2018_01_RMT2_pipe_2294.pipe b/pipes/yellow_tripdata_2018_01_RMT2_pipe_2294.pipe new file mode 100644 index 0000000..ec39829 --- /dev/null +++ b/pipes/yellow_tripdata_2018_01_RMT2_pipe_2294.pipe @@ -0,0 +1,6 @@ +NODE yellow_tripdata_2018_01_RMT2_pipe_2294_0 +SQL > + + SELECT * FROM yellow_tripdata_2018_01_RMT2 + + diff --git a/pipes/yellow_tripdata_2018_01_RMT_pipe_7557.pipe b/pipes/yellow_tripdata_2018_01_RMT_pipe_7557.pipe new file mode 100644 index 0000000..4fbe6ea --- /dev/null +++ b/pipes/yellow_tripdata_2018_01_RMT_pipe_7557.pipe @@ -0,0 +1,36 @@ +NODE yellow_tripdata_2018_01_RMT_pipe_7557_0 +SQL > + + SELECT * FROM tinybird.datasources_ops_log + WHERE + datasource_id = 't_20fe960c11f74061bc1c73b040b83d46' + AND timestamp > now() - INTERVAL 1 day + ORDER BY timestamp DESC + + + +NODE yellow_tripdata_2018_01_RMT_pipe_7557_1 +SQL > + + SELECT count()FROM yellow_tripdata_2018_01_RMT + + + +NODE yellow_tripdata_2018_01_RMT_pipe_7557_2 +SQL > + + SELECT distinct passenger_count FROM yellow_tripdata_2018_01_RMT + + + +NODE yellow_tripdata_2018_01_RMT_pipe_7557_3 +SQL > + + SELECT + toDate(tpep_pickup_datetime) AS date, + countState(trip_distance) AS count, + avgState(trip_distance) AS avg_distance + FROM yellow_tripdata_2018_01 + GROUP BY date + + From 92815d5f7b0d72462222d2b4a91765c82ffad5b8 Mon Sep 17 00:00:00 2001 From: yiman-chen Date: Fri, 2 Jun 2023 17:14:08 +0200 Subject: [PATCH 2/5] add ci/cd --- .github/workflows/cd.yaml | 14 ++++++++++++++ .github/workflows/ci.yaml | 17 +++++++++++++++++ .gitignore | 5 +++++ 3 files changed, 36 insertions(+) create mode 100644 .github/workflows/cd.yaml create mode 100644 .github/workflows/ci.yaml create mode 100644 .gitignore diff --git a/.github/workflows/cd.yaml b/.github/workflows/cd.yaml new file mode 100644 index 0000000..0893b24 --- /dev/null +++ b/.github/workflows/cd.yaml @@ -0,0 +1,14 @@ +name: Tinybird - CD Workflow + +on: + workflow_dispatch: + push: + branches: + - main + +jobs: + push_changes: + uses: tinybirdco/ci/.github/workflows/cd.yml@main + secrets: + admin_token: ${{ secrets.ADMIN_TOKEN }} + tb_host: https://api.tinybird.co \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..4430fe3 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,17 @@ +name: Tinybird - CI Workflow + +on: + workflow_dispatch: + pull_request: + branches: + - main + types: [opened, reopened, labeled, unlabeled, synchronize] + +concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} + +jobs: + ci_branching: + uses: tinybirdco/ci/.github/workflows/ci.yml@main + secrets: + admin_token: ${{ secrets.ADMIN_TOKEN }} + tb_host: https://api.tinybird.co \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7666a97 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.diff_tmp +.tinyb +*.swp +*.swo +.e \ No newline at end of file From 205ffdf27b9c756a57ccacea1d123cb33746c0a7 Mon Sep 17 00:00:00 2001 From: yiman-chen Date: Fri, 2 Jun 2023 17:24:19 +0200 Subject: [PATCH 3/5] delete a pipe --- pipes/untitled_pipe_6685.pipe | 96 ----------------------------------- 1 file changed, 96 deletions(-) delete mode 100644 pipes/untitled_pipe_6685.pipe diff --git a/pipes/untitled_pipe_6685.pipe b/pipes/untitled_pipe_6685.pipe deleted file mode 100644 index d4d7821..0000000 --- a/pipes/untitled_pipe_6685.pipe +++ /dev/null @@ -1,96 +0,0 @@ -NODE untitled_pipe_6685_0 -SQL > - - % - select {{split_to_array(String('/\',/something'), separator=String(separator, ','))}} - - - -NODE untitled_pipe_6685_1 -SQL > - - SELECT replaceAll([`/'`, '/something'] - - - -NODE untitled_pipe_6685_2 -SQL > - - % - select '1' {{split_to_array(code, '1,2,3')}} - - - -NODE untitled_pipe_6685_3 -SQL > - - % - select {{split_to_array(count_unique_vals_columns, 'section_id,city')}} - - - -NODE node -SQL > - - SELECT arrayJoin(['a','b','c']) as path - - - -NODE untitled_pipe_6685_5 -SQL > - - % - SELECT * from node - where path in - {{split_to_array(replaceAll((String(param, "/',/something"), separator=','),'`','"')}} - - - - -NODE untitled_pipe_6685_6 -SQL > - - % - SELECT - {{split_to_array(String(param, 'hi, how are you|fine thanks'), separator='|')}} - - - -NODE untitled_pipe_6685_7 -SQL > - - % - select {{sql_unescape("/',/something","'")}} - - - -NODE path -SQL > - - % - select {{String("/',/something")}} - - - -NODE untitled_pipe_6685_9 -SQL > - - SELECT '/\',/something' - FROM untitled_pipe_6685_10 - - - -NODE untitled_pipe_6685_10 -SQL > - - SELECT * FROM path - - - -NODE untitled_pipe_6685_11 -SQL > - - % - select {{sql_unescape( "testing'", ",")}} - - From 96b78f7074c96a4463c9a468b3adb5af61635578 Mon Sep 17 00:00:00 2001 From: yiman-chen Date: Fri, 2 Jun 2023 17:36:16 +0200 Subject: [PATCH 4/5] rename pipe --- pipes/{test2_pipe_1364.pipe => mv_test.pipe} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pipes/{test2_pipe_1364.pipe => mv_test.pipe} (100%) diff --git a/pipes/test2_pipe_1364.pipe b/pipes/mv_test.pipe similarity index 100% rename from pipes/test2_pipe_1364.pipe rename to pipes/mv_test.pipe From 5d6c6cda167f766c198152810bbab497727ec27f Mon Sep 17 00:00:00 2001 From: yiman-chen Date: Fri, 2 Jun 2023 17:40:41 +0200 Subject: [PATCH 5/5] delete a node from test_ym --- pipes/events_test_ym.datasource | 11 +++++++++++ pipes/test_ym.pipe | 32 -------------------------------- 2 files changed, 11 insertions(+), 32 deletions(-) create mode 100644 pipes/events_test_ym.datasource diff --git a/pipes/events_test_ym.datasource b/pipes/events_test_ym.datasource new file mode 100644 index 0000000..760903b --- /dev/null +++ b/pipes/events_test_ym.datasource @@ -0,0 +1,11 @@ + +SCHEMA > + `Customer` String `json:$.Customer`, + `Product` String `json:$.Product`, + `Qtr_4` Nullable(String) `json:$.['Qtr 4']`, + `Qtr_2` Nullable(String) `json:$.['Qtr 2']`, + `Qtr_3` Nullable(String) `json:$.['Qtr 3']`, + `Qtr_1` Nullable(String) `json:$.['Qtr 1']` + +ENGINE "MergeTree" +ENGINE_SORTING_KEY "Customer, Product" diff --git a/pipes/test_ym.pipe b/pipes/test_ym.pipe index 5efbe61..1f59d2a 100644 --- a/pipes/test_ym.pipe +++ b/pipes/test_ym.pipe @@ -18,14 +18,6 @@ SQL > -NODE test_ym_2 -SQL > - - INSERT INTO events_test_ym (*) VALUES('A','B',NULL,NULL,NULL,NULL) - - - - NODE test_ym_3 SQL > @@ -57,27 +49,3 @@ SQL > order by date desc - -NODE test_ym_6 -SQL > - - --SELECT parseDateTimeBestEffort('2022-12-07 T00:00:00+00:00') - SELECT parseDateTimeBestEffort('2022-12-07 T00:00:00+00:00') - JSONExtractString ({"shift":"morning","service_date":"2022-12-07T00:00:00+00:00"},'service_date') as x - - - -NODE test_ym_7 -SQL > - - SELECT toDateTime('2022-12-07 T00:00:00+00:00') - - - -NODE test_ym_8 -SQL > - - SET format_csv_null_representation = 'My NULL'; - SELECT * FROM events_test_ym FORMAT CSV; - -