pyproject.toml

[build-system]
requires = [ "setuptools>=61.2", "setuptools-git-versioning>=3.0,<4"]
build-backend = "setuptools.build_meta"

[tool.setuptools-git-versioning]
enabled = true
version_file = "onetl/VERSION"
count_commits_from_version_file = true
dev_template = "{tag}.dev{env:GITHUB_RUN_ID:{ccount}}+git.{sha}"
dirty_template = "{tag}.dev{env:GITHUB_RUN_ID:{ccount}}+git.{sha}.dirty"

[project]
name = "onetl"
description = "One ETL tool to rule them all"
license = {text = "Apache-2.0"}
authors = [{name= "MWS Data Bridge", email = "onetools@mts.ru"}]
classifiers = [
    "Development Status :: 4 - Beta",
    "Framework :: Pydantic",
    "Framework :: Pydantic :: 2",
    "Intended Audience :: Developers",
    "Operating System :: OS Independent",
    "Programming Language :: Python :: 3 :: Only",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Programming Language :: Python :: 3.13",
    "Programming Language :: Python :: 3.14",
    "Topic :: Software Development :: Libraries",
    "Topic :: Software Development :: Libraries :: Java Libraries",
    "Topic :: Software Development :: Libraries :: Python Modules",
    "Topic :: System :: Distributed Computing",
    "Typing :: Typed",
]
keywords = ["Spark", "ETL", "JDBC", "HWM"]
requires-python = ">=3.10"
dependencies = [
    "etl-entities>=2.6.1,!=3.0.0,!=3.0.1,<4",
    "evacuator>=1.0,<1.1",
    "frozendict; python_version < '3.15'",
    "humanize",
    "ordered-set",
    "platformdirs",
    "psutil",
    "pydantic>=2.12,<3",
    "pyyaml",
    # https://github.com/python/typing_extensions/pull/188
    "typing-extensions>=4.6.2; python_version < '3.13'",
]
dynamic = ["version"]

[project.readme]
file = "README.rst"
content-type = "text/x-rst"

[tool.setuptools.packages.find]
include = ["onetl*"]
exclude = ["mddocs", "tests"]

[tool.setuptools.package-data]
"*" = ["py.typed", "VERSION"]

[project.urls]  # Optional
"Homepage" = "https://github.com/MTSWebServices/onetl"
"Documentation" = "https://onetl.readthedocs.io/"
"Source" = "https://github.com/MTSWebServices/onetl"
"CI/CD" = "https://github.com/MTSWebServices/onetl/actions"
"Tracker" = "https://github.com/MTSWebServices/onetl/issues"

[project.entry-points."tricoder_package_spy.register"]
onetl = "onetl"


[project.optional-dependencies]
# pyspark package is heavy, make it optional dependency
spark = ["pyspark"]

# same as hdfs[kerberos], but without including hdfs itself
kerberos = ["requests-kerberos>=0.7.0"]

# https://todo.sr.ht/~sschwarzer/ftputil/150
ftp = ["ftputil>=5.0.4"]
ftps = ["ftputil>=5.0.4"]
hdfs = ["hdfs>2"]
# https://github.com/minio/minio-py/pull/1530
s3 = ["minio<8"]
samba = ["pysmb"]
sftp = ["paramiko"]
webdav = ["webdavclient3"]

# all file dependencies from above
files = [
    "ftputil>=5.0.4",
    "hdfs>2",
    "minio<8",
    "pysmb",
    "paramiko",
    "webdavclient3",
]

# spark + kerberos + files
all = [
    "pyspark",
    "ftputil>=5.0.4",
    "hdfs>2",
    "minio<8",
    "pysmb",
    "paramiko",
    "webdavclient3",
    "requests-kerberos>=0.7.0",
]


[dependency-groups]
test-clickhouse = ["clickhouse-driver"]
test-greenplum = ["psycopg2-binary"]
test-kafka = ["confluent-kafka"]
test-mongodb = ["pymongo<4.0"]
test-mssql = ["pymssql"]
test-mysql = ["pymysql[rsa]"]
test-oracle = ["cx_Oracle"]
test-postgres = ["psycopg2-binary"]
test-etl-entities-2 = ["etl-entities<3"]
test-etl-entities-3 = ["etl-entities>=3.0.2,<4"]
test-samba = ["pysmb"]
"test-spark-3.2" = [
    "pyspark~=3.2.4",
    "numpy>=1.16,<1.24",
    "pandas>=1.0,<2",
    "pyarrow>=1.0",
    "sqlalchemy<2.0",
    "setuptools",
]
"test-spark-3.3" = [
    "pyspark~=3.3.4",
    "numpy>=1.16,<1.24",
    "pandas>=1.0,<2",
    "pyarrow>=1.0",
    "sqlalchemy<2.0",
    "setuptools",
]
"test-spark-3.4" = [
    "pyspark~=3.4.4",
    "numpy>=1.16",
    "pandas>=1.0",
    "pyarrow>=1.0",
    "sqlalchemy",
    "setuptools",
]
"test-spark-3.5" = [
    "pyspark~=3.5.8",
    "numpy>=1.16",
    "pandas>=1.0",
    "pyarrow>=4.0",
    "sqlalchemy",
    "setuptools",
]
"test-spark-4.0" = [
    "pyspark~=4.0.4",
    "numpy>=1.24",
    "pandas>=2.0",
    "pyarrow>=11.0",
    "sqlalchemy",
    "setuptools",
]
"test-spark-4.1" = [
    "pyspark~=4.1.3",
    # Upgrading numpy leads to error:
    # .venv/lib/python3.12/site-packages/numpy/__init__.pyi:737: error: Type statement is only supported in Python 3.12 and greater
    "numpy>=1.24,<2.5",
    "pandas>=2.2",
    "pyarrow>=15.0",
    "sqlalchemy",
    "setuptools",
]
"test-spark-4.2" = [
    "pyspark~=4.2.0",
    # Upgrading numpy leads to error:
    # .venv/lib/python3.12/site-packages/numpy/__init__.pyi:737: error: Type statement is only supported in Python 3.12 and greater
    "numpy>=1.24,<2.5",
    "pandas>=2.2",
    "pyarrow>=18.0",
    "sqlalchemy",
    "setuptools",
]
test = [
    "coverage",
    "omegaconf",
    "pytest<8",
    "pytest-lazy-fixture",
    "pytest-mock",
    "pytest-rerunfailures",
    "pytest-xdist[psutil]",
    "requests",
    "responses",
    "filelock",
]
dev = [
    "mypy~=2.3.0",
    "prek>=0.4.12,<0.6.0",
    "ruff~=0.16.1",
    "types-Deprecated~=1.3.1",
    "types-PyYAML~=6.0.12",
    "types-requests~=2.33.0",
]
docs = [
    "mkdocs[i18n]~=1.6.1",
    "mkdocs-material~=9.7.1",
    "mkdocs-autorefs~=1.4.3",
    "mkdocs-literate-nav~=0.6.3",
    "mkdocstrings-python~=2.0.5",
    "griffe-inherited-docstrings~=1.1.3",
    "griffe-fieldz~=0.5.0",
    "mkdocs-git-revision-date-localized-plugin>=1.5.3,<1.7.0",
    "pymdown-extensions>=11,<13",
    "mkdocs-towncrier~=0.1.5",
    "towncrier>=25.8,<26.10",
    "setuptools<82",
]

[tool.uv]
default-groups = []
conflicts = [
    [
        { group = "test-spark-3.2" },
        { group = "test-spark-3.3" },
        { group = "test-spark-3.4" },
        { group = "test-spark-3.5" },
        { group = "test-spark-4.0" },
        { group = "test-spark-4.1" },
        { group = "test-spark-4.2" },
    ],
    [
        { group = "test-etl-entities-2" },
        { group = "test-etl-entities-3" },
    ]
]

[tool.uv.dependency-groups]
dev = {requires-python = ">=3.12"}
docs = {requires-python = ">=3.11"}

[tool.uv.extra-build-dependencies]
# fails on missing pkg_resources
cx-oracle = ["setuptools<82"]

[tool.ruff]
target-version = "py310"
line-length = 120
extend-exclude = ["mddocs/", "Makefile"]

[tool.ruff.lint]
select = ["ALL"]
ignore = [
    "A002",
    "ANN",
    "ARG",
    "COM812",
    "D",
    "FIX002",
    "N805",
    "N815",
    "PLC0415",
    "PYI051",
    "TC001",
    "TC002",
    "TC003",
    "TD",
    "CPY001",
    "E501",
]

[tool.ruff.lint.per-file-ignores]
"onetl/connection/file_connection/*" = ["ERA001"]

"tests/*" = [
    "S",
    "A",
    "PLC0415",
    "TC",
    "ICN001",
    "PT",
    "PLR2004",
    "PLR0913",
    "SLF001",
    "FBT",
    "SIM117",
    "SIM108",
    "PLR0915",
    "DTZ",
    "B017",
    "PERF401",
    "BLE001",
    "PERF203",
    "FURB171",
    "PLR0917",
    "ISC004",
]
"tests/tests_unit/test_hooks/*" = ["C901"]
"tests/libs/*" = ["INP001"]
"mddocs/scripts/rst2md.py" = ["T201", "INP001", "C901", "PLR0915", "PLR0912"]
"mddocs/scripts/generate_anchors.py" = ["T201", "INP001"]

[tool.mypy]
python_version = "3.10"
plugins = ["pydantic.mypy"]
strict_optional = true
ignore_missing_imports = true
follow_imports = "silent"
show_error_codes = true

[[tool.mypy.overrides]]
module=["paramiko.*"]
follow_untyped_imports = true

[tool.codespell]
ignore-words-list = ["INOUT", "inout", "thirdparty"]

[tool.coverage.run]
source_pkgs = ["onetl"]
branch = true
parallel = true
relative_files = true
omit = [
  "tests/*",
]
data_file = "reports/.coverage"

[tool.coverage.report]
precision = 2
exclude_also = [
  "def __repr__",
  "raise AssertionError",
  "raise NotImplementedError",
  "class .*\bProtocol:",
  "@(abc\\.)?abstractmethod",
  "\\.\\.\\.",
  "def downgrade\\(\\)",
]
partial_also = [
  "if .*TYPE_CHECKING:",
  "if .*isEnabledFor.*:",
  "if .*debug.*:",
  "if .*sys.version_info.*:",
  "if pyspark_version",
  "if spark_version",
  "if __name__ == .__main__.:",
  "except .*ImportError",
]

[tool.towncrier]
name = "onETL"
package = "onetl"
filename = "mddocs/docs/changelog/RELEASE_TEMPLATE.md"
start_string = "<!-- towncrier release notes start -->\n"
underlines = ["", "", ""]
directory = "mddocs/docs/changelog/next_release/"
title_format = "# {version} ({project_date}) {{ #DBR-onetl-changelog-{version} }}"
issue_format = "[#{issue}](https://github.com/MTSWebServices/onetl/issues/{issue})"

[[tool.towncrier.type]]
directory = "breaking"
name = "Breaking Changes"
showcontent = true

[[tool.towncrier.type]]
directory = "significant"
name = "Significant Changes"
showcontent = true

[[tool.towncrier.type]]
directory = "feature"
name = "Features"
showcontent = true

[[tool.towncrier.type]]
directory = "improvement"
name = "Improvements"
showcontent = true

[[tool.towncrier.type]]
directory = "bugfix"
name = "Bug Fixes"
showcontent = true

[[tool.towncrier.type]]
directory = "dependency"
name = "Dependencies"
showcontent = true

[[tool.towncrier.type]]
directory = "doc"
name = "Doc only Changes"
showcontent = true

[[tool.towncrier.type]]
directory = "misc"
name = "Misc"
showcontent = true