diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 60c117f437d32..f7c3cdfcf1906 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -117,6 +117,7 @@ body: - amazon - apache-beam - apache-cassandra + - apache-datafusion - apache-drill - apache-druid - apache-flink diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 83b4595d9396c..683232bd19960 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -36,6 +36,9 @@ labelPRBasedOnFilePath: provider:apache-cassandra: - providers/apache/cassandra/** + provider:apache-datafusion: + - providers/apache/datafusion/** + provider:apache-drill: - providers/apache/drill/** diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index eb0acd31df1cb..12f6017dff964 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -194,6 +194,8 @@ custom bash/python providers). +---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-cassandra | ``pip install 'apache-airflow[apache-cassandra]'`` | Cassandra related operators & hooks | +---------------------+-----------------------------------------------------+------------------------------------------------+ +| apache-datafusion | ``pip install 'apache-airflow[apache-datafusion]'`` | Apache DataFusion provider package | ++---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-drill | ``pip install 'apache-airflow[apache-drill]'`` | Drill related operators & hooks | +---------------------+-----------------------------------------------------+------------------------------------------------+ | apache-druid | ``pip install 'apache-airflow[apache-druid]'`` | Druid related operators & hooks | diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index 172f0f6576eb2..18f1414705298 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -241,17 +241,17 @@ Usage:                                                                                                                 breeze build-docs                                                                                                      [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak |   -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache-airflow-providers | apache.cassandra | apache.datafusion | apache.drill | apache.druid | apache.flink |         +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          +databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | +facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index a734b81b0c766..bc6f0459f86df 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -de29604dc4d7031b9a26a1e0350d7806 +3b60fc61bfa1058da1123aa6f146cc85 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 88ba1696907ae..576d6767a036b 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -150,17 +150,17 @@ Usage:                                                                                                                 breeze release-management add-back-references                                                                          [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak |   -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache-airflow-providers | apache.cassandra | apache.datafusion | apache.drill | apache.druid | apache.flink |         +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          +databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | +facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index 9d5b0b1d2fef4..74ffbfa405a86 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -4b44b7c52911adf1c1bdfb87e10e873b +98f2cbb52affbfea2d87e28881f47f65 diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg index e7b78f7153ee1..8b64db6d7b73a 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.svg @@ -162,17 +162,17 @@ Usage:                                                                                                                 breeze release-management classify-provider-changes                                                                    -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.datafusion | apache.drill | apache.druid +| apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin |            +apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |               +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Classify each provider's unreleased changes with hard-coded, high-confidence rules, flagging ambiguous commits as  'needs_llm' for an agent/skill to assess. Outputs JSON - a deterministic alternative to the random '--non-interactive' diff --git a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt index 226f5e3c0feab..766cb36a693c9 100644 --- a/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt +++ b/dev/breeze/doc/images/output_release-management_classify-provider-changes.txt @@ -1 +1 @@ -57466e5cb9470b0d4398724fda2a1825 +d6f78774069055cce998b7523a4af756 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 1ce32a4d94152..e44ea00224def 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -151,17 +151,17 @@ Usage:                                                                                                                 breeze release-management generate-issue-content-providers                                                             -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.datafusion | apache.drill | apache.druid +| apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin |            +apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |               +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index 5231c239ad495..0e67eb0e94f11 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -e2d5118e3339b53ce6e87dc6be6ae053 +8bce09eb48da1b7c1762bc4ef6556bab diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index 40358ab62d9ca..a9b1bcd313b33 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -1,4 +1,4 @@ - + │--provider-id     Provider_id to generate metadata for. If not specified, metadata for all packages will be       │ │generated. This is debug-only option. When this option is specified, the metadata produced for  │ │the distribution will not be written to the file, but printed via console. (airbyte | akeyless |│ -│alibaba | amazon | apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink |│ -│apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | │ -│apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb |│ -│asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai│ -│| common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |│ -│dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ -│github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | │ -│keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql │ -│| neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | │ -│papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | │ -│segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | │ -│tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ -│--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ -│running metadata generation to only this version of the provider. (TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│alibaba | amazon | apache.beam | apache.cassandra | apache.datafusion | apache.drill | │ +│apache.druid | apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | │ +│apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark | │ +│apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | │ +│cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging |│ +│common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | │ +│elasticsearch | exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http │ +│| imap | influxdb | informatica | jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql │ +│| microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | │ +│openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | │ +│postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | │ +│singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ +│--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ +│running metadata generation to only this version of the provider. (TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 088e7af08dc1f..40055d5ba7336 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -93f9b27ce19f188a7d7b70cfcabf8c2b +c89eaff49ab3074ea6a9d3235d472b95 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index 3baea7f764abe..d397cb0b913c7 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-distributions                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.datafusion | apache.drill | apache.druid +| apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin |            +apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |               +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare sdist/whl distributions of Airflow Providers. Each provider directory is wiped with `git clean -fdx`  (preserving .venv, .idea, .vscode) before build to keep in-tree generated files out of the artifact. See dev/breeze  diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index a4104f3c3a4d2..15ec9b7b781ec 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -3581abc65c55d4b8a096005c312df783 +a936d98a919fde245162ac6dfa1d2fd8 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 571cc1a3c80e1..ab346a7f4582e 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -210,17 +210,17 @@ Usage:                                                                                                                 breeze release-management prepare-provider-documentation                                                               -[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.drill | apache.druid | apache.flink |     -apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  -apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | -cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          -databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp -| git | github | google | grpc | hashicorp | http | imap | influxdb | informatica | jdbc | jenkins | keycloak |        -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb |          -zendesk]...                                                                                                            +[OPTIONS] [airbyte | akeyless | alibaba | amazon | apache.cassandra | apache.datafusion | apache.drill | apache.druid +| apache.flink | apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin |            +apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana |               +atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | +common.messaging | common.sql | databricks | datadog | dbt.cloud | dingding | discord | docker | edge3 | elasticsearch +| exasol | fab | facebook | ftp | git | github | google | grpc | hashicorp | http | imap | influxdb | informatica |    +jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |     +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata | trino | vertica | vespa | weaviate |      +yandex | ydb | zendesk]...                                                                                             Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index 9ada882dbcbf1..613aa94dd37aa 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -0a45b898de7677eed8aa19c7e4f07345 +fe23cecf7c7e8773f3f7822f2cd601cb diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index b01450b3c1f6b..26838afbbba82 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -189,17 +189,17 @@ Usage:                                                                                                                 breeze release-management publish-docs                                                                                 [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak |   -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache-airflow-providers | apache.cassandra | apache.datafusion | apache.drill | apache.druid | apache.flink |         +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          +databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | +facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index ca7d9b2eebe9f..d384a4959afb4 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -30b0621c66436e215e0d34ec9bfd407b +98c8722ad7c25e8fe4224c96b8724039 diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index 71bce1509344e..ce70e32759d52 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -183,11 +183,11 @@ │--python-versions Comma separate list of Python versions to update sbom from (defaults to all historical python   │ │versions) (3.6 | 3.7 | 3.8 | 3.9 | 3.10 | 3.11 | 3.12 | 3.13 | 3.14)│ │--provider-id     Provider id to generate the requirements for (airbyte | akeyless | alibaba | amazon | │ -│apache.beam | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | │ -│apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | │ -│apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | │ -│atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai | │ -│common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud | │ +│apache.beam | apache.cassandra | apache.datafusion | apache.drill | apache.druid | apache.flink │ +│| apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | │ +│apache.livy | apache.pig | apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb |│ +│asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes | cohere | common.ai│ +│| common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |│ │dingding | discord | docker | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | │ │github | google | grpc | hashicorp | http | ibm.mq | imap | influxdb | informatica | jdbc | │ │jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | │ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index ff491c1b8cf35..dbce543fe39b2 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -4e0a8dab83f1d257bce1cb90b778b009 +aa7444b5a26f060a76a2a67486e4764e diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 48033b138d1af..4ad533d7bda84 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -204,17 +204,17 @@ Usage:                                                                                                                 breeze workflow-run publish-docs                                                                                       [OPTIONS] [airbyte | akeyless | alibaba | all-providers | amazon | apache-airflow | apache-airflow-ctl |               -apache-airflow-providers | apache.cassandra | apache.drill | apache.druid | apache.flink | apache.hdfs | apache.hive | -apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig | apache.pinot | apache.spark -| apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | cloudant | cncf.kubernetes -| cohere | common.ai | common.compat | common.io | common.messaging | common.sql | databricks | datadog | dbt.cloud |  -dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | facebook | ftp | git | github |    -google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica | jdbc | jenkins | keycloak |   -microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai |         -openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres |     -presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake |    -sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb -| zendesk]...                                                                                                          +apache-airflow-providers | apache.cassandra | apache.datafusion | apache.drill | apache.druid | apache.flink |         +apache.hdfs | apache.hive | apache.iceberg | apache.impala | apache.kafka | apache.kylin | apache.livy | apache.pig |  +apache.pinot | apache.spark | apache.tinkerpop | apprise | arangodb | asana | atlassian.jira | celery | clickhousedb | +cloudant | cncf.kubernetes | cohere | common.ai | common.compat | common.io | common.messaging | common.sql |          +databricks | datadog | dbt.cloud | dingding | discord | docker | docker-stack | edge3 | elasticsearch | exasol | fab | +facebook | ftp | git | github | google | grpc | hashicorp | helm-chart | http | ibm.mq | imap | influxdb | informatica +| jdbc | jenkins | keycloak | microsoft.azure | microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql |   +neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector |   +pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp | singularity | slack | +smtp | snowflake | sqlite | ssh | standard | tableau | task-sdk | telegram | teradata | trino | vertica | vespa |      +weaviate | yandex | ydb | zendesk]...                                                                                  Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index bcdbd541efa08..3da710c69af5e 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -01f2b301d8965951b49e2c8cd114700a +40be646969ce15d25d3cd6e1691d8148 diff --git a/dev/breeze/tests/test_packages.py b/dev/breeze/tests/test_packages.py index 2a11a4ef29d2a..c9075f89efe2b 100644 --- a/dev/breeze/tests/test_packages.py +++ b/dev/breeze/tests/test_packages.py @@ -35,6 +35,7 @@ get_dist_package_name_prefix, get_long_package_name, get_min_airflow_version, + get_not_ready_provider_ids, get_pip_package_name, get_provider_details, get_provider_info_dict, @@ -116,6 +117,11 @@ def test_get_removed_providers(): assert get_removed_provider_ids() == [] +def test_get_not_ready_provider_ids(): + # Modify it every time we add/release the first version of a provider + assert get_not_ready_provider_ids() == ["ibm.mq"] + + def test_get_suspended_provider_ids(): # Modify it every time we suspend/resume provider assert get_suspended_provider_ids() == ["apache.beam"] diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 5c85d4e020897..96daf7fe09d11 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -145,6 +145,17 @@ "excluded-platforms": [], "state": "ready" }, + "apache.datafusion": { + "deps": [ + "apache-airflow>=2.11.0" + ], + "devel-deps": [], + "plugins": [], + "cross-providers-deps": [], + "excluded-python-versions": [], + "excluded-platforms": [], + "state": "ready" + }, "apache.drill": { "deps": [ "apache-airflow-providers-common-compat>=1.8.0", diff --git a/generated/provider_dependencies.json.sha256sum b/generated/provider_dependencies.json.sha256sum index 943fd0fc93e4c..12ba97ee48b04 100644 --- a/generated/provider_dependencies.json.sha256sum +++ b/generated/provider_dependencies.json.sha256sum @@ -1 +1 @@ -93831555f2a141e481c81c147142aeb860c34ea860163ca130d045e5ecd0a83b +6c72fa1143f4112f9670fb7b77e8e4323d5629e48d654be24a404cba5ee734cd diff --git a/providers/apache/datafusion/.gitignore b/providers/apache/datafusion/.gitignore new file mode 100644 index 0000000000000..bff2d7629604d --- /dev/null +++ b/providers/apache/datafusion/.gitignore @@ -0,0 +1 @@ +*.iml diff --git a/providers/apache/datafusion/LICENSE b/providers/apache/datafusion/LICENSE new file mode 100644 index 0000000000000..11069edd79019 --- /dev/null +++ b/providers/apache/datafusion/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/providers/apache/datafusion/NOTICE b/providers/apache/datafusion/NOTICE new file mode 100644 index 0000000000000..a51bd9390d030 --- /dev/null +++ b/providers/apache/datafusion/NOTICE @@ -0,0 +1,5 @@ +Apache Airflow +Copyright 2016-2026 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/providers/apache/datafusion/README.rst b/providers/apache/datafusion/README.rst new file mode 100644 index 0000000000000..d581f0725b12e --- /dev/null +++ b/providers/apache/datafusion/README.rst @@ -0,0 +1,60 @@ + +.. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + +.. http://www.apache.org/licenses/LICENSE-2.0 + +.. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +.. IF YOU WANT TO MODIFY TEMPLATE FOR THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + ``PROVIDER_README_TEMPLATE.rst.jinja2`` IN the ``dev/breeze/src/airflow_breeze/templates`` DIRECTORY + +Package ``apache-airflow-providers-apache-datafusion`` + +Release: ``0.1.0`` + + +`Apache DataFusion `__ + + +Provider package +---------------- + +This is a provider package for ``apache.datafusion`` provider. All classes for this provider package +are in ``airflow.providers.apache.datafusion`` python package. + +You can find package information and changelog for the provider +in the `documentation `_. + +Installation +------------ + +You can install this package on top of an existing Airflow installation (see ``Requirements`` below +for the minimum Airflow version supported) via +``pip install apache-airflow-providers-apache-datafusion`` + +The package supports the following python versions: 3.10,3.11,3.12,3.13,3.14 + +Requirements +------------ + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=2.11.0`` +================== ================== + +The changelog for the provider package can be found in the +`changelog `_. diff --git a/providers/apache/datafusion/docs/.latest-doc-only-change.txt b/providers/apache/datafusion/docs/.latest-doc-only-change.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/providers/apache/datafusion/docs/changelog.rst b/providers/apache/datafusion/docs/changelog.rst new file mode 100644 index 0000000000000..4c26805ad3920 --- /dev/null +++ b/providers/apache/datafusion/docs/changelog.rst @@ -0,0 +1,31 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. NOTE TO CONTRIBUTORS: + Please, only add notes to the Changelog just below the "Changelog" header when there are some breaking changes + and you want to add an explanation to the users on how they are supposed to deal with them. + The changelog is updated and maintained semi-automatically by release manager. + +``apache-airflow-providers-apache-datafusion`` + +Changelog +--------- + +0.1.0 +..... + +Initial not-ready skeleton for the provider. diff --git a/providers/apache/datafusion/docs/commits.rst b/providers/apache/datafusion/docs/commits.rst new file mode 100644 index 0000000000000..1557536f4b3e1 --- /dev/null +++ b/providers/apache/datafusion/docs/commits.rst @@ -0,0 +1,35 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + + .. NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + + .. IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE + `PROVIDER_COMMITS_TEMPLATE.rst.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + .. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN! + +Package apache-airflow-providers-apache-datafusion +------------------------------------------------------ + +`Apache DataFusion `__ + + +This is detailed commit list of changes for versions provider package: ``apache.datafusion``. +For high-level changelog, see :doc:`package information including changelog `. + +.. airflow-providers-commits:: diff --git a/providers/apache/datafusion/docs/conf.py b/providers/apache/datafusion/docs/conf.py new file mode 100644 index 0000000000000..90354bb308fe6 --- /dev/null +++ b/providers/apache/datafusion/docs/conf.py @@ -0,0 +1,27 @@ +# Disable Flake8 because of all the sphinx imports +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Configuration of Providers docs building.""" + +from __future__ import annotations + +import os + +os.environ["AIRFLOW_PACKAGE_NAME"] = "apache-airflow-providers-apache-datafusion" + +from docs.provider_conf import * # noqa: F403 diff --git a/providers/apache/datafusion/docs/index.rst b/providers/apache/datafusion/docs/index.rst new file mode 100644 index 0000000000000..93bdc6d80c63d --- /dev/null +++ b/providers/apache/datafusion/docs/index.rst @@ -0,0 +1,95 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +``apache-airflow-providers-apache-datafusion`` +============================================== + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Basics + + Home + Changelog + Security + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: References + + Python API <_api/airflow/providers/apache/datafusion/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: System tests + + System Tests <_api/tests/system/apache/datafusion/index> + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Resources + + PyPI Repository + Installing from sources + +.. THE REMAINDER OF THE FILE IS AUTOMATICALLY GENERATED. IT WILL BE OVERWRITTEN AT RELEASE TIME! + + +.. toctree:: + :hidden: + :maxdepth: 1 + :caption: Commits + + Detailed list of commits + + +apache-airflow-providers-apache-datafusion package +------------------------------------------------------ + +`Apache DataFusion `__ + + +Release: 0.1.0 + +Provider package +---------------- + +This package is for the ``apache.datafusion`` provider. +All classes for this package are included in the ``airflow.providers.apache.datafusion`` python package. + +Installation +------------ + +You can install this package on top of an existing Airflow installation via +``pip install apache-airflow-providers-apache-datafusion``. +For the minimum Airflow version supported, see ``Requirements`` below. + +Requirements +------------ + +The minimum Apache Airflow version supported by this provider distribution is ``2.11.0``. + +================== ================== +PIP package Version required +================== ================== +``apache-airflow`` ``>=2.11.0`` +================== ================== diff --git a/providers/apache/datafusion/docs/installing-providers-from-sources.rst b/providers/apache/datafusion/docs/installing-providers-from-sources.rst new file mode 100644 index 0000000000000..fdbb17d017579 --- /dev/null +++ b/providers/apache/datafusion/docs/installing-providers-from-sources.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/installing-providers-from-sources.rst diff --git a/providers/apache/datafusion/docs/security.rst b/providers/apache/datafusion/docs/security.rst new file mode 100644 index 0000000000000..351ff007ebf2f --- /dev/null +++ b/providers/apache/datafusion/docs/security.rst @@ -0,0 +1,18 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +.. include:: /../../../../devel-common/src/sphinx_exts/includes/security.rst diff --git a/providers/apache/datafusion/provider.yaml b/providers/apache/datafusion/provider.yaml new file mode 100644 index 0000000000000..ca01290677019 --- /dev/null +++ b/providers/apache/datafusion/provider.yaml @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +--- +package-name: apache-airflow-providers-apache-datafusion +name: Apache DataFusion +description: | + `Apache DataFusion `__ + +state: ready +lifecycle: incubation +source-date-epoch: 1775799686 +# Note that those versions are maintained by release manager - do not update them manually +# with the exception of case where other provider in sources has >= new provider version. +# In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have +# to be done in the same PR +versions: + - 0.1.0 diff --git a/providers/apache/datafusion/pyproject.toml b/providers/apache/datafusion/pyproject.toml new file mode 100644 index 0000000000000..21dd250e6cf4c --- /dev/null +++ b/providers/apache/datafusion/pyproject.toml @@ -0,0 +1,123 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! + +# IF YOU WANT TO MODIFY THIS FILE EXCEPT DEPENDENCIES, YOU SHOULD MODIFY THE TEMPLATE +# `pyproject_TEMPLATE.toml.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +[build-system] +requires = ["flit_core==3.12.0"] +build-backend = "flit_core.buildapi" + +[project] +name = "apache-airflow-providers-apache-datafusion" +version = "0.1.0" +description = "Provider package apache-airflow-providers-apache-datafusion for Apache Airflow" +readme = "README.rst" +license = "Apache-2.0" +license-files = ['LICENSE', 'NOTICE'] +authors = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +maintainers = [ + {name="Apache Software Foundation", email="dev@airflow.apache.org"}, +] +keywords = [ "airflow-provider", "apache.datafusion", "airflow", "integration" ] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Environment :: Web Environment", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "Framework :: Apache Airflow", + "Framework :: Apache Airflow :: Provider", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Topic :: System :: Monitoring", +] +requires-python = ">=3.10" + +# The dependencies should be modified in place in the generated file. +# Any change in the dependencies is preserved when the file is regenerated +# Make sure to run ``prek update-providers-dependencies --all-files`` +# After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` +dependencies = [ + "apache-airflow>=2.11.0", +] + +[dependency-groups] +dev = [ + "apache-airflow", + "apache-airflow-task-sdk", + "apache-airflow-devel-common", + # Additional devel dependencies (do not remove this line and add extra development dependencies) +] + +# To build docs: +# +# uv run --group docs build-docs +# +# To enable auto-refreshing build with server: +# +# uv run --group docs build-docs --autobuild +# +# To see more options: +# +# uv run --group docs build-docs --help +# +docs = [ + "apache-airflow-devel-common[docs]" +] + +[tool.uv.sources] +# These names must match the names as defined in the pyproject.toml of the workspace items, +# *not* the workspace folder paths +apache-airflow = {workspace = true} +apache-airflow-devel-common = {workspace = true} +apache-airflow-task-sdk = {workspace = true} +apache-airflow-providers-common-sql = {workspace = true} +apache-airflow-providers-standard = {workspace = true} + +[project.urls] +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-datafusion/0.1.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-datafusion/0.1.0/changelog.html" +"Bug Tracker" = "https://github.com/apache/airflow/issues" +"Source Code" = "https://github.com/apache/airflow" +"Slack Chat" = "https://s.apache.org/airflow-slack" +"Mastodon" = "https://fosstodon.org/@airflow" +"YouTube" = "https://www.youtube.com/channel/UCSXwxpWZQ7XZ1WL3wqevChA/" + +[project.entry-points."apache_airflow_provider"] +provider_info = "airflow.providers.apache.datafusion.get_provider_info:get_provider_info" + +[tool.flit.module] +name = "airflow.providers.apache.datafusion" + +# Explicit sdist contents so the build does not rely on VCS information +# (flit 4.0 makes --no-use-vcs the default — see https://github.com/pypa/flit/pull/782). +[tool.flit.sdist] +include = [ + "docs/", + "provider.yaml", + "src/airflow/__init__.py", + "src/airflow/providers/__init__.py", + "src/airflow/providers/apache/__init__.py", + "tests/", +] diff --git a/providers/apache/datafusion/src/airflow/__init__.py b/providers/apache/datafusion/src/airflow/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/src/airflow/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/src/airflow/providers/__init__.py b/providers/apache/datafusion/src/airflow/providers/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/src/airflow/providers/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/src/airflow/providers/apache/__init__.py b/providers/apache/datafusion/src/airflow/providers/apache/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/src/airflow/providers/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/src/airflow/providers/apache/datafusion/__init__.py b/providers/apache/datafusion/src/airflow/providers/apache/datafusion/__init__.py new file mode 100644 index 0000000000000..ea2fd8464ebaf --- /dev/null +++ b/providers/apache/datafusion/src/airflow/providers/apache/datafusion/__init__.py @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE +# OVERWRITTEN WHEN PREPARING DOCUMENTATION FOR THE PACKAGES. +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `PROVIDER__INIT__PY_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY +# +from __future__ import annotations + +import packaging.version + +from airflow import __version__ as airflow_version + +__all__ = ["__version__"] + +__version__ = "0.1.0" + +if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( + "2.11.0" +): + raise RuntimeError( + f"The package `apache-airflow-providers-apache-datafusion:{__version__}` needs Apache Airflow 2.11.0+" + ) diff --git a/providers/apache/datafusion/src/airflow/providers/apache/datafusion/get_provider_info.py b/providers/apache/datafusion/src/airflow/providers/apache/datafusion/get_provider_info.py new file mode 100644 index 0000000000000..6c993cf6a2d5f --- /dev/null +++ b/providers/apache/datafusion/src/airflow/providers/apache/datafusion/get_provider_info.py @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# NOTE! THIS FILE IS AUTOMATICALLY GENERATED AND WILL BE OVERWRITTEN! +# +# IF YOU WANT TO MODIFY THIS FILE, YOU SHOULD MODIFY THE TEMPLATE +# `get_provider_info_TEMPLATE.py.jinja2` IN the `dev/breeze/src/airflow_breeze/templates` DIRECTORY + + +def get_provider_info(): + return { + "package-name": "apache-airflow-providers-apache-datafusion", + "name": "Apache DataFusion", + "description": "`Apache DataFusion `__\n", + } diff --git a/providers/apache/datafusion/tests/conftest.py b/providers/apache/datafusion/tests/conftest.py new file mode 100644 index 0000000000000..f56ccce0a3f69 --- /dev/null +++ b/providers/apache/datafusion/tests/conftest.py @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +pytest_plugins = "tests_common.pytest_plugin" diff --git a/providers/apache/datafusion/tests/system/__init__.py b/providers/apache/datafusion/tests/system/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/tests/system/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/tests/system/apache/__init__.py b/providers/apache/datafusion/tests/system/apache/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/tests/system/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/tests/system/apache/datafusion/__init__.py b/providers/apache/datafusion/tests/system/apache/datafusion/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/datafusion/tests/system/apache/datafusion/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/datafusion/tests/unit/__init__.py b/providers/apache/datafusion/tests/unit/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/tests/unit/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/tests/unit/apache/__init__.py b/providers/apache/datafusion/tests/unit/apache/__init__.py new file mode 100644 index 0000000000000..5966d6b1d5261 --- /dev/null +++ b/providers/apache/datafusion/tests/unit/apache/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +__path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/providers/apache/datafusion/tests/unit/apache/datafusion/__init__.py b/providers/apache/datafusion/tests/unit/apache/datafusion/__init__.py new file mode 100644 index 0000000000000..13a83393a9124 --- /dev/null +++ b/providers/apache/datafusion/tests/unit/apache/datafusion/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/providers/apache/datafusion/tests/unit/apache/datafusion/test_example.py b/providers/apache/datafusion/tests/unit/apache/datafusion/test_example.py new file mode 100644 index 0000000000000..d9ff6deefcf17 --- /dev/null +++ b/providers/apache/datafusion/tests/unit/apache/datafusion/test_example.py @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from __future__ import annotations + +from airflow.providers.apache.datafusion.get_provider_info import get_provider_info + + +def test_get_provider_info(): + provider_info = get_provider_info() + + assert provider_info["package-name"] == "apache-airflow-providers-apache-datafusion" + assert provider_info["name"] == "Apache DataFusion" diff --git a/pyproject.toml b/pyproject.toml index bae4c06e3b5ec..345d9f8c17558 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -113,6 +113,9 @@ apache-airflow = "airflow.__main__:main" "apache.cassandra" = [ "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"" ] +"apache.datafusion" = [ + "apache-airflow-providers-apache-datafusion>=0.1.0" +] "apache.drill" = [ "apache-airflow-providers-apache-drill>=2.8.1" ] @@ -412,6 +415,7 @@ apache-airflow = "airflow.__main__:main" "apache-airflow-providers-alibaba>=3.0.0", "apache-airflow-providers-amazon>=9.0.0", "apache-airflow-providers-apache-cassandra>=3.7.0; python_version !=\"3.14\"", + "apache-airflow-providers-apache-datafusion>=0.1.0", "apache-airflow-providers-apache-drill>=2.8.1", "apache-airflow-providers-apache-druid>=3.12.0", "apache-airflow-providers-apache-flink>=1.6.0", @@ -1092,6 +1096,8 @@ mypy_path = [ "$MYPY_CONFIG_FILE_DIR/providers/amazon/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/src", "$MYPY_CONFIG_FILE_DIR/providers/apache/cassandra/tests", + "$MYPY_CONFIG_FILE_DIR/providers/apache/datafusion/src", + "$MYPY_CONFIG_FILE_DIR/providers/apache/datafusion/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/drill/src", "$MYPY_CONFIG_FILE_DIR/providers/apache/drill/tests", "$MYPY_CONFIG_FILE_DIR/providers/apache/druid/src", @@ -1428,6 +1434,7 @@ apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false apache-airflow-providers-apache-cassandra = false +apache-airflow-providers-apache-datafusion = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false apache-airflow-providers-apache-flink = false @@ -1576,6 +1583,7 @@ apache-airflow-providers-akeyless = false apache-airflow-providers-alibaba = false apache-airflow-providers-amazon = false apache-airflow-providers-apache-cassandra = false +apache-airflow-providers-apache-datafusion = false apache-airflow-providers-apache-drill = false apache-airflow-providers-apache-druid = false apache-airflow-providers-apache-flink = false @@ -1738,6 +1746,7 @@ apache-airflow-providers-akeyless = { workspace = true } apache-airflow-providers-alibaba = { workspace = true } apache-airflow-providers-amazon = { workspace = true } apache-airflow-providers-apache-cassandra = { workspace = true } +apache-airflow-providers-apache-datafusion = { workspace = true } apache-airflow-providers-apache-drill = { workspace = true } apache-airflow-providers-apache-druid = { workspace = true } apache-airflow-providers-apache-flink = { workspace = true } @@ -1877,6 +1886,7 @@ members = [ "providers/alibaba", "providers/amazon", "providers/apache/cassandra", + "providers/apache/datafusion", "providers/apache/drill", "providers/apache/druid", "providers/apache/flink", diff --git a/scripts/ci/docker-compose/remove-sources.yml b/scripts/ci/docker-compose/remove-sources.yml index f10ae1472e9d6..c96fbf65cdc1e 100644 --- a/scripts/ci/docker-compose/remove-sources.yml +++ b/scripts/ci/docker-compose/remove-sources.yml @@ -32,6 +32,7 @@ services: - ../../../empty:/opt/airflow/providers/amazon/src - ../../../empty:/opt/airflow/providers/apache/beam/src - ../../../empty:/opt/airflow/providers/apache/cassandra/src + - ../../../empty:/opt/airflow/providers/apache/datafusion/src - ../../../empty:/opt/airflow/providers/apache/drill/src - ../../../empty:/opt/airflow/providers/apache/druid/src - ../../../empty:/opt/airflow/providers/apache/flink/src diff --git a/scripts/ci/docker-compose/tests-sources.yml b/scripts/ci/docker-compose/tests-sources.yml index d33deafc970fa..40b98391ac364 100644 --- a/scripts/ci/docker-compose/tests-sources.yml +++ b/scripts/ci/docker-compose/tests-sources.yml @@ -45,6 +45,7 @@ services: - ../../../providers/amazon/tests:/opt/airflow/providers/amazon/tests - ../../../providers/apache/beam/tests:/opt/airflow/providers/apache/beam/tests - ../../../providers/apache/cassandra/tests:/opt/airflow/providers/apache/cassandra/tests + - ../../../providers/apache/datafusion/tests:/opt/airflow/providers/apache/datafusion/tests - ../../../providers/apache/drill/tests:/opt/airflow/providers/apache/drill/tests - ../../../providers/apache/druid/tests:/opt/airflow/providers/apache/druid/tests - ../../../providers/apache/flink/tests:/opt/airflow/providers/apache/flink/tests diff --git a/uv.lock b/uv.lock index 8d5f31fa71947..7a8b8e366af71 100644 --- a/uv.lock +++ b/uv.lock @@ -122,6 +122,7 @@ apache-airflow-providers-apache-iceberg = false apache-airflow-breeze = false apache-airflow-providers-opsgenie = false apache-airflow-providers-apache-livy = false +apache-airflow-providers-apache-datafusion = false apache-airflow-core = false apache-airflow-providers-hashicorp = false apache-airflow-providers-pagerduty = false @@ -177,6 +178,7 @@ members = [ "apache-airflow-providers-alibaba", "apache-airflow-providers-amazon", "apache-airflow-providers-apache-cassandra", + "apache-airflow-providers-apache-datafusion", "apache-airflow-providers-apache-drill", "apache-airflow-providers-apache-druid", "apache-airflow-providers-apache-flink", @@ -997,6 +999,7 @@ all = [ { name = "apache-airflow-providers-alibaba" }, { name = "apache-airflow-providers-amazon", extra = ["aiobotocore", "python3-saml", "s3fs"] }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*'" }, + { name = "apache-airflow-providers-apache-datafusion" }, { name = "apache-airflow-providers-apache-drill" }, { name = "apache-airflow-providers-apache-druid" }, { name = "apache-airflow-providers-apache-flink" }, @@ -1120,6 +1123,9 @@ apache-atlas = [ apache-cassandra = [ { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*'" }, ] +apache-datafusion = [ + { name = "apache-airflow-providers-apache-datafusion" }, +] apache-drill = [ { name = "apache-airflow-providers-apache-drill" }, ] @@ -1564,6 +1570,8 @@ requires-dist = [ { name = "apache-airflow-providers-amazon", extras = ["s3fs"], marker = "extra == 's3fs'", editable = "providers/amazon" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'all'", editable = "providers/apache/cassandra" }, { name = "apache-airflow-providers-apache-cassandra", marker = "python_full_version != '3.14.*' and extra == 'apache-cassandra'", editable = "providers/apache/cassandra" }, + { name = "apache-airflow-providers-apache-datafusion", marker = "extra == 'all'", editable = "providers/apache/datafusion" }, + { name = "apache-airflow-providers-apache-datafusion", marker = "extra == 'apache-datafusion'", editable = "providers/apache/datafusion" }, { name = "apache-airflow-providers-apache-drill", marker = "extra == 'all'", editable = "providers/apache/drill" }, { name = "apache-airflow-providers-apache-drill", marker = "extra == 'apache-drill'", editable = "providers/apache/drill" }, { name = "apache-airflow-providers-apache-druid", marker = "extra == 'all'", editable = "providers/apache/druid" }, @@ -1773,7 +1781,7 @@ requires-dist = [ { name = "sentry-sdk", marker = "extra == 'sentry'", specifier = ">=2.30.0" }, { name = "uv", marker = "extra == 'uv'", specifier = ">=0.11.19" }, ] -provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "apache-cassandra", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] +provides-extras = ["all-core", "async", "graphviz", "gunicorn", "kerberos", "memray", "otel", "statsd", "all-task-sdk", "airbyte", "akeyless", "alibaba", "amazon", "apache-cassandra", "apache-datafusion", "apache-drill", "apache-druid", "apache-flink", "apache-hdfs", "apache-hive", "apache-iceberg", "apache-impala", "apache-kafka", "apache-kylin", "apache-livy", "apache-pig", "apache-pinot", "apache-spark", "apache-tinkerpop", "apprise", "arangodb", "asana", "atlassian-jira", "celery", "clickhousedb", "cloudant", "cncf-kubernetes", "cohere", "common-ai", "common-compat", "common-io", "common-messaging", "common-sql", "databricks", "datadog", "dbt-cloud", "dingding", "discord", "docker", "edge3", "elasticsearch", "exasol", "fab", "facebook", "ftp", "git", "github", "google", "grpc", "hashicorp", "http", "ibm-mq", "imap", "influxdb", "informatica", "jdbc", "jenkins", "keycloak", "microsoft-azure", "microsoft-mssql", "microsoft-psrp", "microsoft-winrm", "mongo", "mysql", "neo4j", "odbc", "openai", "openfaas", "openlineage", "opensearch", "opsgenie", "oracle", "pagerduty", "papermill", "pgvector", "pinecone", "postgres", "presto", "qdrant", "redis", "salesforce", "samba", "segment", "sendgrid", "sftp", "singularity", "slack", "smtp", "snowflake", "sqlite", "ssh", "standard", "tableau", "telegram", "teradata", "trino", "vertica", "vespa", "weaviate", "yandex", "ydb", "zendesk", "all", "aiobotocore", "apache-atlas", "apache-webhdfs", "amazon-aws-auth", "cloudpickle", "github-enterprise", "google-auth", "ldap", "pandas", "polars", "rabbitmq", "sentry", "s3fs", "uv"] [package.metadata.requires-dev] ci-image = [ @@ -3255,6 +3263,35 @@ dev = [ ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] +[[package]] +name = "apache-airflow-providers-apache-datafusion" +version = "0.1.0" +source = { editable = "providers/apache/datafusion" } +dependencies = [ + { name = "apache-airflow" }, +] + +[package.dev-dependencies] +dev = [ + { name = "apache-airflow" }, + { name = "apache-airflow-devel-common" }, + { name = "apache-airflow-task-sdk" }, +] +docs = [ + { name = "apache-airflow-devel-common", extra = ["docs"] }, +] + +[package.metadata] +requires-dist = [{ name = "apache-airflow", editable = "." }] + +[package.metadata.requires-dev] +dev = [ + { name = "apache-airflow", editable = "." }, + { name = "apache-airflow-devel-common", editable = "devel-common" }, + { name = "apache-airflow-task-sdk", editable = "task-sdk" }, +] +docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] + [[package]] name = "apache-airflow-providers-apache-drill" version = "3.3.3"