From 778c8093de58d79e4848145a6b0bca153713dca7 Mon Sep 17 00:00:00 2001 From: Zhang Yanpo Date: Wed, 7 Jan 2026 23:35:44 +0800 Subject: [PATCH] refactor: migrate from setup.py to pyproject.toml Modernize build configuration to PEP 621 compliant pyproject.toml and update tooling to use ruff for linting/formatting and MkDocs for documentation. Changes: - Replace `setup.py` with `pyproject.toml` - Update `__init__.py` to use `importlib.metadata` for version - Switch from Sphinx to MkDocs with mkdocstrings - Replace flake8 with ruff, fix lint errors (E712, E721, E731) - Update GitHub Actions to v5 - Remove obsolete _building scripts and Sphinx config - Add test dependencies to workflow (k3proc, k3daemonize, websocket-client) --- .github/workflows/python-package.yml | 88 +----- .github/workflows/python-publish.yml | 44 ++- .gitignore | 1 + .readthedocs.yaml | 16 ++ __init__.py | 17 +- _building/.gitignore | 129 --------- _building/Makefile | 3 - _building/README.md | 35 +-- _building/README.md.j2 | 38 --- _building/__init__.py | 129 --------- _building/build_readme.py | 85 ------ _building/build_setup.py | 190 ------------- _building/building-requirements.txt | 7 - _building/common.mk | 23 +- _building/install.sh | 12 - _building/populate.py | 23 +- _building/requirements.txt | 1 - docs/Makefile | 20 -- docs/index.md | 32 +++ docs/make.bat | 35 --- docs/source/conf.py | 28 -- docs/source/index.rst | 47 --- mkdocs.yml | 30 ++ pyproject.toml | 63 ++++ requirements.txt | 12 - setup.py | 23 -- synopsis.py | 7 +- test/test_jobs/test_job_echo.py | 5 +- test/test_jobs/test_job_loop_10.py | 3 +- test/test_jobs/test_job_normal.py | 3 +- test/test_jobs/test_job_progress_key.py | 3 +- test/test_jobs/test_job_worker_exception.py | 1 - test/test_k3wsjobd.py | 300 ++++++++++---------- test/wsjobd_server.py | 9 +- wsjobd.py | 200 ++++++------- 35 files changed, 480 insertions(+), 1182 deletions(-) create mode 100644 .readthedocs.yaml delete mode 100644 _building/.gitignore delete mode 100644 _building/Makefile delete mode 100644 _building/README.md.j2 delete mode 100644 _building/__init__.py delete mode 100644 _building/build_readme.py delete mode 100644 _building/build_setup.py delete mode 100644 _building/building-requirements.txt delete mode 100755 _building/install.sh delete mode 100644 _building/requirements.txt delete mode 100644 docs/Makefile create mode 100644 docs/index.md delete mode 100644 docs/make.bat delete mode 100644 docs/source/conf.py delete mode 100644 docs/source/index.rst create mode 100644 mkdocs.yml create mode 100644 pyproject.toml delete mode 100644 requirements.txt delete mode 100644 setup.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 57ad3e7..24a9427 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,6 +1,3 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: Unit test on: @@ -9,116 +6,61 @@ on: jobs: ut: - runs-on: ${{ matrix.os }} strategy: matrix: - # Github action runner update ubuntu to 22.04, which does not have - # python-3.6 in it: https://github.com/actions/setup-python/issues/544#issuecomment-1320295576 - # To fix it: use os: [ubuntu-20.04] instead os: [ubuntu-latest] python-version: [3.9, "3.10", 3.11, 3.12] steps: - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi - - name: Install npm dependencies - run: | - if [ -f package.json ]; then npm install; fi - - # manually add module binary path to github ci - echo "add node module path: $GITHUB_WORKSPACE/node_modules/.bin/" - echo "$GITHUB_WORKSPACE/node_modules/.bin/" >> $GITHUB_PATH - - - name: Install apt dependencies - run: | - if [ -f packages.txt ]; then cat packages.txt | xargs sudo apt-get install; fi + pip install pytest k3ut k3proc k3daemonize websocket-client + pip install -e . - name: Test with pytest - env: - # interactive command such as k3handy.cmdtty to run git, git complains - # if no TERM set: - # out: - (press RETURN) - # err: WARNING: terminal is not fully functional - # And waiting for a RETURN to press for ever - TERM: xterm run: | - cp setup.py .. - cd .. - python setup.py install - cd - - - if [ -f sudo_test ]; then - sudo env "PATH=$PATH" pytest -v - else - pytest -v - fi - - - uses: actions/upload-artifact@v4 - if: failure() - with: - path: test/ + pytest -v build_doc: - runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - python-version: [3.9, "3.10", 3.11, 3.12] + python-version: ["3.12"] steps: - uses: actions/checkout@v5 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi - - name: Test building doc run: | - pip install -r _building/building-requirements.txt - make -C docs html + pip install -e . + pip install mkdocs mkdocs-material "mkdocstrings[python]" + mkdocs build lint: - runs-on: ${{ matrix.os }} strategy: matrix: os: [ubuntu-latest] - python-version: [3.9, "3.10", 3.11, 3.12] + python-version: ["3.12"] steps: - uses: actions/checkout@v5 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - if [ -f test-requirements.txt ]; then pip install -r test-requirements.txt; fi - - - name: Lint with flake8 + - name: Lint with ruff run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + pip install ruff + ruff check . + ruff format --check . diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 3fd68f1..af931b2 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,6 +1,3 @@ -# This workflows will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - name: Upload Python Package on: @@ -10,27 +7,26 @@ on: jobs: deploy: - runs-on: ubuntu-latest steps: - - uses: actions/checkout@v5 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install setuptools wheel twine - - name: Build and publish - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - run: | - cp setup.py .. - cd .. - python setup.py sdist bdist_wheel - pip install dist/*.tar.gz - python -c 'import '${GITHUB_REPOSITORY#*/} - twine upload dist/* + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build twine + + - name: Build package + run: python -m build + + - name: Publish to PyPI + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} + run: twine upload dist/* diff --git a/.gitignore b/.gitignore index 0569721..94d8aab 100644 --- a/.gitignore +++ b/.gitignore @@ -131,3 +131,4 @@ dmypy.json .pyre/ .claude/ +site/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..3e517df --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,16 @@ +version: 2 + +build: + os: ubuntu-22.04 + tools: + python: "3.11" + +mkdocs: + configuration: mkdocs.yml + +python: + install: + - method: pip + path: . + extra_requirements: + - docs diff --git a/__init__.py b/__init__.py index 31fea81..243fff2 100644 --- a/__init__.py +++ b/__init__.py @@ -4,31 +4,30 @@ """ -__version__ = "0.1.0" -__name__ = "k3wsjobd" +from importlib.metadata import version + +__version__ = version("k3wsjobd") from .wsjobd import ( JobdWebSocketApplication, Job, run, - SystemOverloadError, JobError, InvalidMessageError, InvalidProgressError, LoadingError, - JobNotInSessionError + JobNotInSessionError, ) __all__ = [ - 'JobdWebSocketApplication', - 'Job', - 'run', - + "JobdWebSocketApplication", + "Job", + "run", "SystemOverloadError", "JobError", "InvalidMessageError", "InvalidProgressError", "LoadingError", - "JobNotInSessionError" + "JobNotInSessionError", ] diff --git a/_building/.gitignore b/_building/.gitignore deleted file mode 100644 index b6e4761..0000000 --- a/_building/.gitignore +++ /dev/null @@ -1,129 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -pip-wheel-metadata/ -share/python-wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -.python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ -.dmypy.json -dmypy.json - -# Pyre type checker -.pyre/ diff --git a/_building/Makefile b/_building/Makefile deleted file mode 100644 index e5b5931..0000000 --- a/_building/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -# make readme name -readme: - cd .. && python _building/build_readme.py diff --git a/_building/README.md b/_building/README.md index 50205e1..0ee458e 100644 --- a/_building/README.md +++ b/_building/README.md @@ -1,23 +1,24 @@ -# building -building toolkit for pykit3 repos +# _building -This repo should be included in a package, e.g.: +Shared build configuration for pykit3 packages. -``` -vcs/pykit3/k3handy/ -▸ .git/ -▸ .github/ -▸ __pycache__/ -▾ _building/ <-- this repo - ... -``` +## Commands -# Publish python package: +All commands use the `pk3` package: -- `make build_setup_py` does the following steps: - - Builds the `setup.py` and commit it. - - Add a git tag with the name of `"v" + __init__.__ver__`. +```bash +make test # Run tests with pytest +make lint # Format and lint with ruff +make cov # Generate coverage report +make doc # Build documentation with mkdocs +make readme # Generate README.md from docstrings +make release # Create git tag from version in pyproject.toml +make publish # Build and upload to PyPI +``` -- Then `git push` the tag, github Action in the `.github/workflows/python-pubish.yml` will publish a package to `pypi`. +## Release Process - The action spec is copied from template repo: `github.com/pykit3/tmpl`. +1. Update version in `pyproject.toml` +2. Run `make release` to create git tag +3. Run `git push --tags` to trigger GitHub Actions +4. GitHub Actions automatically publishes to PyPI diff --git a/_building/README.md.j2 b/_building/README.md.j2 deleted file mode 100644 index d9baf2c..0000000 --- a/_building/README.md.j2 +++ /dev/null @@ -1,38 +0,0 @@ -# {{ name }} - -[![Action-CI](https://github.com/pykit3/{{ name }}/actions/workflows/python-package.yml/badge.svg)](https://github.com/pykit3/{{ name }}/actions/workflows/python-package.yml) -[![Build Status](https://travis-ci.com/pykit3/{{ name }}.svg?branch=master)](https://travis-ci.com/pykit3/{{ name }}) -[![Documentation Status](https://readthedocs.org/projects/{{ name }}/badge/?version=stable)](https://{{ name }}.readthedocs.io/en/stable/?badge=stable) -[![Package](https://img.shields.io/pypi/pyversions/{{ name }})](https://pypi.org/project/{{ name }}) - -{{ description }} - -{{ name }} is a component of [pykit3] project: a python3 toolkit set. - -{{ package_doc }} - - -# Install - -``` -pip install {{ name }} -``` - -# Synopsis - -```python -{{ synopsis }} -``` - -# Author - -Zhang Yanpo (张炎泼) - -# Copyright and License - -The MIT License (MIT) - -Copyright (c) 2015 Zhang Yanpo (张炎泼) - - -[pykit3]: https://github.com/pykit3 diff --git a/_building/__init__.py b/_building/__init__.py deleted file mode 100644 index 1055c67..0000000 --- a/_building/__init__.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import importlib.util -import sys - -# sys.path.insert(0, os.path.abspath('..')) -# sys.path.insert(0, os.path.abspath('../..')) -# sys.path.insert(0, os.path.abspath('../../..')) - -# __title__ = 'requests' -# __description__ = 'Python HTTP for Humans.' -# __url__ = 'https://requests.readthedocs.io' -# __version__ = '2.23.0' - -__author__ = "Zhang Yanpo" -__author_email__ = "drdr.xp@gmail.com" -__license__ = "MIT" -__copyright__ = "Copyright 2020 Zhang Yanpo" - - -# Configuration file for the Sphinx documentation builder. -# -# This file only contains a selection of the most common options. For a full -# list see the documentation: -# https://www.sphinx-doc.org/en/master/usage/configuration.html - - -# Add any Sphinx extension module names here, as strings. They can be -# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom -# ones. -extensions = [ - "sphinx.ext.autodoc", - "sphinx.ext.napoleon", - # "sphinx.ext.intersphinx", - # "sphinx.ext.todo", - # "sphinx.ext.viewcode", -] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ["_templates"] - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -# This pattern also affects html_static_path and html_extra_path. -exclude_patterns = [] - - -master_doc = "index" - - -# -- Options for HTML output ------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -# -html_theme = "alabaster" - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -# html_static_path = ['_static'] -html_static_path = [] - - -def load_parent_package(): - """ - Load the parent directory as a package module. - - Returns: - tuple: (package_name, package_module) - """ - import os - - parent_dir = os.path.dirname(os.path.dirname(__file__)) - if parent_dir not in sys.path: - sys.path.insert(0, parent_dir) - - # Read the __init__.py file to get the package name - init_file = os.path.join(parent_dir, "__init__.py") - package_name = None - - with open(init_file, "r") as f: - for line in f: - if line.strip().startswith("__name__"): - # Extract package name from __name__ = "package_name" - package_name = line.split("=")[1].strip().strip("\"'") - break - - if not package_name: - # Fallback: use directory name - package_name = os.path.basename(parent_dir) - - # Load the module with proper package context using importlib - spec = importlib.util.spec_from_file_location(package_name, init_file) - pkg = importlib.util.module_from_spec(spec) - sys.modules[package_name] = pkg # Add to sys.modules so relative imports work - spec.loader.exec_module(pkg) - - return package_name, pkg - - -def sphinx_confs(): - """ - Load repo dir as a package - - `readthedocs` use branch name as dir! - Thus the following does not work:: - - import pk3proc - """ - - print("sys.path:", sys.path) - - package_name, pkg = load_parent_package() - - return ( - pkg.__name__, - pkg, - pkg.__version__, - __author__, - __copyright__, - extensions, - templates_path, - exclude_patterns, - master_doc, - html_theme, - html_static_path, - ) diff --git a/_building/build_readme.py b/_building/build_readme.py deleted file mode 100644 index d211e29..0000000 --- a/_building/build_readme.py +++ /dev/null @@ -1,85 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -import doctest -import os -import sys - -import jinja2 -import yaml - -from __init__ import load_parent_package - -# xxx/_building/build_readme.py -this_base = os.path.dirname(__file__) - -j2vars = {} - -# let it be able to find indirectly dependent package locally -# e.g.: `k3fs` depends on `k3confloader` -sys.path.insert(0, os.path.abspath("..")) - -# load package name from __init__.py -package_name, pkg = load_parent_package() -j2vars["name"] = package_name - - -def get_gh_config(): - with open(".github/settings.yml", "r") as f: - cont = f.read() - - cfg = yaml.safe_load(cont) - tags = cfg["repository"]["topics"].split(",") - tags = [x.strip() for x in tags] - cfg["repository"]["topics"] = tags - return cfg - - -cfg = get_gh_config() -j2vars["description"] = cfg["repository"]["description"] - - -def get_examples(pkg): - doc = pkg.__doc__ - parser = doctest.DocTestParser() - es = parser.get_examples(doc) - rst = [] - for e in es: - rst.append(">>> " + e.source.strip()) - rst.append(e.want.strip()) - - rst = "\n".join(rst) - - for fn in ( - "synopsis.txt", - "synopsis.py", - ): - try: - with open(fn, "r") as f: - rst += "\n" + f.read() - - except FileNotFoundError: - pass - - return rst - - -j2vars["synopsis"] = get_examples(pkg) -j2vars["package_doc"] = pkg.__doc__ - - -def render_j2(tmpl_path, tmpl_vars, output_path): - template_loader = jinja2.FileSystemLoader(searchpath="./") - template_env = jinja2.Environment( - loader=template_loader, undefined=jinja2.StrictUndefined - ) - template = template_env.get_template(tmpl_path) - - txt = template.render(tmpl_vars) - - with open(output_path, "w") as f: - f.write(txt) - - -if __name__ == "__main__": - render_j2("_building/README.md.j2", j2vars, "README.md") diff --git a/_building/build_setup.py b/_building/build_setup.py deleted file mode 100644 index ca2ec6c..0000000 --- a/_building/build_setup.py +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -""" -build steup.py for this package. -""" - -import ast -import subprocess -import sys -from string import Template - -import requirements -import yaml - -if hasattr(sys, "getfilesystemencoding"): - defenc = sys.getfilesystemencoding() -if defenc is None: - defenc = sys.getdefaultencoding() - - -def parse_assignment(filename, var_name): - """Parse a Python file and extract the value of a variable assignment using AST.""" - with open(filename, "r") as f: - content = f.read() - - tree = ast.parse(content) - - for node in ast.walk(tree): - if isinstance(node, ast.Assign): - # Check if this assignment is to our target variable - for target in node.targets: - if isinstance(target, ast.Name) and target.id == var_name: - # Extract the literal value - if isinstance(node.value, ast.Constant): # Python 3.8+ - return node.value.value - elif isinstance(node.value, ast.Str): # Python < 3.8 - return node.value.s - elif isinstance(node.value, ast.Num): # Python < 3.8 - return node.value.n - - return None - - -def get_name(): - name = parse_assignment("__init__.py", "__name__") - return name if name is not None else "k3git" # fallback - - -name = get_name() - - -def get_ver(): - version = parse_assignment("__init__.py", "__version__") - if version is None: - raise ValueError("Could not find __version__ in __init__.py") - return version - - -def get_gh_config(): - with open(".github/settings.yml", "r") as f: - cont = f.read() - - cfg = yaml.safe_load(cont) - tags = cfg["repository"]["topics"].split(",") - tags = [x.strip() for x in tags] - cfg["repository"]["topics"] = tags - return cfg - - -def get_travis(): - try: - with open(".travis.yml", "r") as f: - cont = f.read() - except OSError: - return None - - cfg = yaml.safe_load(cont) - return cfg - - -def get_compatible(): - # https://pypi.org/classifiers/ - - rst = [] - t = get_travis() - if t is None: - return ["Programming Language :: Python :: 3"] - - for v in t["python"]: - if v.startswith("pypy"): - v = "Implementation :: PyPy" - rst.append("Programming Language :: Python :: {}".format(v)) - - return rst - - -def get_req(): - try: - with open("requirements.txt", "r") as f: - req = list(requirements.parse(f)) - except OSError: - req = [] - - # req.name, req.specs, req.extras - # Django [('>=', '1.11'), ('<', '1.12')] - # six [('==', '1.10.0')] - req = [x.name + ",".join([a + b for a, b in x.specs]) for x in req] - - return req - - -cfg = get_gh_config() - -ver = get_ver() -description = cfg["repository"]["description"] -long_description = open("README.md").read() -req = get_req() -prog = get_compatible() - - -tmpl = """# DO NOT EDIT!!! built with `python _building/build_setup.py` -import setuptools -setuptools.setup( - name="${name}", - packages=["${name}"], - version="$ver", - license='MIT', - description=$description, - long_description=$long_description, - long_description_content_type="text/markdown", - author='Zhang Yanpo', - author_email='drdr.xp@gmail.com', - url='https://github.com/pykit3/$name', - keywords=$topics, - python_requires='>=3.0', - - install_requires=$req, - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries', - ] + $prog, -) -""" - -s = Template(tmpl) -rst = s.substitute( - name=name, - ver=ver, - description=repr(description), - long_description=repr(long_description), - topics=repr(cfg["repository"]["topics"]), - req=repr(req), - prog=repr(prog), -) -with open("setup.py", "w") as f: - f.write(rst) - - -sb = subprocess.Popen( - ["git", "add", "setup.py"], - encoding=defenc, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -out, err = sb.communicate() -if sb.returncode != 0: - raise Exception("failure to add: ", out, err) - -sb = subprocess.Popen( - ["git", "commit", "setup.py", "-m", "release: v" + ver], - encoding=defenc, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -out, err = sb.communicate() -if sb.returncode != 0: - raise Exception("failure to commit new release: " + ver, out, err) - - -sb = subprocess.Popen( - ["git", "tag", "v" + ver], - encoding=defenc, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, -) -out, err = sb.communicate() -if sb.returncode != 0: - raise Exception("failure to add tag: " + ver, out, err) diff --git a/_building/building-requirements.txt b/_building/building-requirements.txt deleted file mode 100644 index bd303df..0000000 --- a/_building/building-requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -# requirements for building doc, setup.py etc - -semantic_version>=2.9,<3.0 -jinja2>=3.0,<4.0 -PyYAML>=6.0,<7.0 - -sphinx>=4.3,<6.0 diff --git a/_building/common.mk b/_building/common.mk index dbac102..571715a 100644 --- a/_building/common.mk +++ b/_building/common.mk @@ -1,14 +1,19 @@ all: test lint readme doc -.PHONY: test lint +.PHONY: test lint cov sudo_test: - sudo env "PATH=$$PATH" UT_DEBUG=0 PYTHONPATH="$$(cd ..; pwd)" python -m unittest discover -c --failfast -s . + sudo env "PATH=$$PATH" UT_DEBUG=0 pytest -v test: - env "PATH=$$PATH" UT_DEBUG=0 PYTHONPATH="$$(cd ..; pwd)" python -m unittest discover -c --failfast -s . + env UT_DEBUG=0 pytest -v + +cov: + coverage run --source=. -m pytest + coverage html + open htmlcov/index.html doc: - make -C docs html + mkdocs build lint: # ruff format: fast Python code formatter (Black-compatible) @@ -21,13 +26,13 @@ static_check: uvx mypy . --ignore-missing-imports readme: - python _building/build_readme.py + pk3 readme -build_setup_py: - PYTHONPATH="$$(cd ..; pwd)" python _building/build_setup.py +release: + pk3 tag publish: - ./_building/publish.sh + pk3 publish install: - ./_building/install.sh + pip install -e . diff --git a/_building/install.sh b/_building/install.sh deleted file mode 100755 index e82727c..0000000 --- a/_building/install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/sh - - -pwd="$(pwd)" -name="${pwd##*/}" -pip uninstall -y $name - -cp setup.py .. -( -cd .. -python setup.py install -) diff --git a/_building/populate.py b/_building/populate.py index 77deb12..d0a8731 100755 --- a/_building/populate.py +++ b/_building/populate.py @@ -15,22 +15,22 @@ def pjoin(*args): def cp(fn): - - cur = os.path.abspath('.') + cur = os.path.abspath(".") name = os.path.split(cur)[1] - t = '_building/tmpl/' - relfn = fn[len(t):] + t = "_building/tmpl/" + relfn = fn[len(t) :] base = os.path.split(fn)[0] if not os.path.exists(base): os.makedirs(base) src = fn - dst = re.sub(r'xxnamexx', name, relfn) + dst = re.sub(r"xxnamexx", name, relfn) - vs = {'name': name, - 'nameBig': name[0].upper() + name[1:], - } + vs = { + "name": name, + "nameBig": name[0].upper() + name[1:], + } print("populate ", src, " to ", dst) render_j2(src, vs, dst) @@ -39,14 +39,13 @@ def cp(fn): def render_j2(tmpl_path, tmpl_vars, output_path): - template_loader = jinja2.FileSystemLoader(searchpath='./') - template_env = jinja2.Environment(loader=template_loader, - undefined=jinja2.StrictUndefined) + template_loader = jinja2.FileSystemLoader(searchpath="./") + template_env = jinja2.Environment(loader=template_loader, undefined=jinja2.StrictUndefined) template = template_env.get_template(tmpl_path) txt = template.render(tmpl_vars) - with open(output_path, 'w') as f: + with open(output_path, "w") as f: f.write(txt) diff --git a/_building/requirements.txt b/_building/requirements.txt deleted file mode 100644 index 49fe098..0000000 --- a/_building/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -setuptools diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index a4de0bf..0000000 --- a/docs/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line, and also -# from the environment for the first two. -SPHINXOPTS ?= -W -SPHINXBUILD ?= sphinx-build -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..49e09f6 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,32 @@ +# k3wsjobd + +[![Action-CI](https://github.com/pykit3/k3wsjobd/actions/workflows/python-package.yml/badge.svg)](https://github.com/pykit3/k3wsjobd/actions/workflows/python-package.yml) +[![Documentation Status](https://readthedocs.org/projects/k3wsjobd/badge/?version=stable)](https://k3wsjobd.readthedocs.io/en/stable/?badge=stable) +[![Package](https://img.shields.io/pypi/pyversions/k3wsjobd)](https://pypi.org/project/k3wsjobd) + +Gevent-based WebSocket server for async job processing. Receives job descriptions from clients, runs them asynchronously, and reports progress back periodically. + +k3wsjobd is a component of [pykit3](https://github.com/pykit3) project: a python3 toolkit set. + +## Installation + +```bash +pip install k3wsjobd +``` + +## Quick Start + +```python +import k3wsjobd + +# Start WebSocket job server +k3wsjobd.run(ip='127.0.0.1', port=33445) +``` + +## API Reference + +::: k3wsjobd + +## License + +The MIT License (MIT) - Copyright (c) 2015 Zhang Yanpo (张炎泼) diff --git a/docs/make.bat b/docs/make.bat deleted file mode 100644 index 6247f7e..0000000 --- a/docs/make.bat +++ /dev/null @@ -1,35 +0,0 @@ -@ECHO OFF - -pushd %~dp0 - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set SOURCEDIR=source -set BUILDDIR=build - -if "%1" == "" goto help - -%SPHINXBUILD% >NUL 2>NUL -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - exit /b 1 -) - -%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% -goto end - -:help -%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% - -:end -popd diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index e436b3e..0000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,28 +0,0 @@ -import os -import sys - -sys.path.insert(0, os.path.abspath("../..")) - -# In order to find indirect dependency -sys.path.insert(0, os.path.abspath("../../..")) - -# use a try to force not to reorder sys.path and import. -try: - import _building -except Exception as e: - raise e - - -( - project, - pkg, - release, - author, - copyright, - extensions, - templates_path, - exclude_patterns, - master_doc, - html_theme, - html_static_path, -) = _building.sphinx_confs() diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index 5ae3177..0000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. k3wsjobd documentation master file, created by - sphinx-quickstart on Thu May 14 16:58:55 2020. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -k3wsjobd -============ - -.. automodule:: k3wsjobd - -Documentation for the Code -************************** - -Exceptions ----------- - -.. autoexception:: SystemOverloadError - -.. autoexception:: JobError - -.. autoexception:: InvalidMessageError - -.. autoexception:: InvalidProgressError - -.. autoexception:: LoadingError - -.. autoexception:: JobNotInSessionError - -Classes ----------- - -.. autoclass:: JobdWebSocketApplication - -.. autoclass:: Job - - -Functions ---------- - -.. autofunction:: run - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 0000000..0a5e793 --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,30 @@ +site_name: k3wsjobd +site_description: DESCRIPTION +site_url: https://k3wsjobd.readthedocs.io +repo_url: https://github.com/pykit3/k3wsjobd +repo_name: pykit3/k3wsjobd + +theme: + name: material + palette: + primary: blue + accent: blue + +plugins: + - search + - mkdocstrings: + handlers: + python: + paths: [.] + options: + show_source: true + show_root_heading: true + heading_level: 2 + +nav: + - Home: index.md + +markdown_extensions: + - admonition + - pymdownx.highlight + - pymdownx.superfences diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..d735e75 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,63 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "k3wsjobd" +version = "0.1.1" +description = "Gevent-based WebSocket server for async job processing" +readme = "README.md" +license = {text = "MIT"} +requires-python = ">=3.9" +authors = [ + { name = "Zhang Yanpo", email = "drdr.xp@gmail.com" } +] +keywords = ["websocket", "gevent", "job", "async", "server"] +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Topic :: Software Development :: Libraries", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +dependencies = [ + "gevent-websocket", + "psutil", + "k3thread", + "k3utfjson", + "k3jobq", +] + +[project.urls] +Homepage = "https://github.com/pykit3/k3wsjobd" +Documentation = "https://k3wsjobd.readthedocs.io" + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "ruff", + "coverage", +] +publish = [ + "build", + "twine", + "pk3", +] +docs = [ + "mkdocs>=1.5", + "mkdocs-material>=9.0", + "mkdocstrings[python]>=0.24", +] + +[tool.setuptools] +packages = ["k3wsjobd"] + +[tool.setuptools.package-dir] +k3wsjobd = "." + +[tool.ruff] +line-length = 120 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 993bc41..0000000 --- a/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ --r _building/requirements.txt - - -k3ut>=0.1.15,<0.2 -k3utfjson>=0.1.1,<0.2 -k3thread>=0.1.0,<0.2 -k3proc>=0.2.13,<0.3.0 -k3jobq>=0.1.2,<0.2 -psutil>=5.8.0 -gevent-websocket>=0.10.1 -websocket-client>=1.2.0 -k3daemonize>=0.1.0,<0.2 \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index fe1a38d..0000000 --- a/setup.py +++ /dev/null @@ -1,23 +0,0 @@ -# DO NOT EDIT!!! built with `python _building/build_setup.py` -import setuptools -setuptools.setup( - name="k3wsjobd", - packages=["k3wsjobd"], - version="0.1.0", - license='MIT', - description='This module is a gevent based websocket server. When the server receives a job description from a client, it runs that job asynchronously in a thread, and reports the progress back to the client periodically.', - long_description='# k3wsjobd\n\n[![Action-CI](https://github.com/pykit3/k3wsjobd/actions/workflows/python-package.yml/badge.svg)](https://github.com/pykit3/k3wsjobd/actions/workflows/python-package.yml)\n[![Build Status](https://travis-ci.com/pykit3/k3wsjobd.svg?branch=master)](https://travis-ci.com/pykit3/k3wsjobd)\n[![Documentation Status](https://readthedocs.org/projects/k3wsjobd/badge/?version=stable)](https://k3wsjobd.readthedocs.io/en/stable/?badge=stable)\n[![Package](https://img.shields.io/pypi/pyversions/k3wsjobd)](https://pypi.org/project/k3wsjobd)\n\nThis module is a gevent based websocket server. When the server receives a job description from a client, it runs that job asynchronously in a thread, and reports the progress back to the client periodically.\n\nk3wsjobd is a component of [pykit3] project: a python3 toolkit set.\n\n\nThis module is a gevent based websocket server. When the server receives a job description from a client,\nit runs that job asynchronously in a thread, and reports the progress back to the client periodically.\n\n\n\n\n# Install\n\n```\npip install k3wsjobd\n```\n\n# Synopsis\n\n```python\n\nfrom geventwebsocket import Resource, WebSocketServer\nimport k3wsjobd\nfrom k3wsjobd import logging\n\n\ndef run():\n k3wsjobd.run(ip=\'127.0.0.1\', port=33445)\n\n\nif __name__ == "__main__":\n logger = logging.getLogger()\n logger.setLevel(logging.INFO)\n\n file_handler = logging.FileHandler(\'wsjobd.log\')\n formatter = logging.Formatter(\'[%(asctime)s, %(levelname)s] %(message)s\')\n file_handler.setFormatter(formatter)\n logger.addHandler(file_handler)\n run()\n\n```\n\n# Author\n\nZhang Yanpo (张炎泼) \n\n# Copyright and License\n\nThe MIT License (MIT)\n\nCopyright (c) 2015 Zhang Yanpo (张炎泼) \n\n\n[pykit3]: https://github.com/pykit3', - long_description_content_type="text/markdown", - author='Zhang Yanpo', - author_email='drdr.xp@gmail.com', - url='https://github.com/pykit3/k3wsjobd', - keywords=['python', 'thread'], - python_requires='>=3.0', - - install_requires=['k3ut<0.2,>=0.1.15', 'k3utfjson>=0.1.1,<0.2', 'k3thread<0.2,>=0.1.0', 'k3proc<0.3.0,>=0.2.13', 'k3jobq>=0.1.2,<0.2', 'psutil>=5.8.0', 'gevent-websocket>=0.10.1', 'websocket-client>=1.2.0', 'k3daemonize<0.2,>=0.1.0'], - classifiers=[ - 'Development Status :: 4 - Beta', - 'Intended Audience :: Developers', - 'Topic :: Software Development :: Libraries', - ] + ['Programming Language :: Python :: 3'], -) diff --git a/synopsis.py b/synopsis.py index 6adf881..c9c2f8c 100644 --- a/synopsis.py +++ b/synopsis.py @@ -1,18 +1,17 @@ -from geventwebsocket import Resource, WebSocketServer import k3wsjobd from k3wsjobd import logging def run(): - k3wsjobd.run(ip='127.0.0.1', port=33445) + k3wsjobd.run(ip="127.0.0.1", port=33445) if __name__ == "__main__": logger = logging.getLogger() logger.setLevel(logging.INFO) - file_handler = logging.FileHandler('wsjobd.log') - formatter = logging.Formatter('[%(asctime)s, %(levelname)s] %(message)s') + file_handler = logging.FileHandler("wsjobd.log") + formatter = logging.Formatter("[%(asctime)s, %(levelname)s] %(message)s") file_handler.setFormatter(formatter) logger.addHandler(file_handler) run() diff --git a/test/test_jobs/test_job_echo.py b/test/test_jobs/test_job_echo.py index 2765845..c169996 100644 --- a/test/test_jobs/test_job_echo.py +++ b/test/test_jobs/test_job_echo.py @@ -8,8 +8,7 @@ def run(job): - data = job.data - data['result'] = data.get('echo') - time.sleep(data.get('sleep_time', 10)) + data["result"] = data.get("echo") + time.sleep(data.get("sleep_time", 10)) diff --git a/test/test_jobs/test_job_loop_10.py b/test/test_jobs/test_job_loop_10.py index 0d6e9e1..2ec606f 100644 --- a/test/test_jobs/test_job_loop_10.py +++ b/test/test_jobs/test_job_loop_10.py @@ -8,9 +8,8 @@ def run(job): - data = job.data for i in range(10): - data['n'] = i + data["n"] = i time.sleep(1) diff --git a/test/test_jobs/test_job_normal.py b/test/test_jobs/test_job_normal.py index 0c7c04f..1c8d2d6 100644 --- a/test/test_jobs/test_job_normal.py +++ b/test/test_jobs/test_job_normal.py @@ -7,7 +7,6 @@ def run(job): - data = job.data - data['result'] = 'foo' + data["result"] = "foo" diff --git a/test/test_jobs/test_job_progress_key.py b/test/test_jobs/test_job_progress_key.py index 727cfe0..04ce615 100644 --- a/test/test_jobs/test_job_progress_key.py +++ b/test/test_jobs/test_job_progress_key.py @@ -7,7 +7,6 @@ def run(job): - data = job.data - data['foo'] = '80%' + data["foo"] = "80%" diff --git a/test/test_jobs/test_job_worker_exception.py b/test/test_jobs/test_job_worker_exception.py index 9a49a68..54754a6 100644 --- a/test/test_jobs/test_job_worker_exception.py +++ b/test/test_jobs/test_job_worker_exception.py @@ -8,7 +8,6 @@ def run(job): - a = [] logger.info(a[0]) diff --git a/test/test_k3wsjobd.py b/test/test_k3wsjobd.py index 99451f7..1c66d35 100644 --- a/test/test_k3wsjobd.py +++ b/test/test_k3wsjobd.py @@ -22,26 +22,27 @@ def subproc(script, env=None): if env is None: - env = dict(PYTHONPATH=this_base + '/../..', ) + env = dict( + PYTHONPATH=this_base + "/../..", + ) return k3proc.shell_script(script, env=env) class TestWsjobd(unittest.TestCase): - @classmethod def _clean(cls): try: - subproc('python {b}/wsjobd_server.py stop'.format(b=this_base)) + subproc("python {b}/wsjobd_server.py stop".format(b=this_base)) except Exception as e: - dd('failed to stop wsjobd server: ' + repr(e)) + dd("failed to stop wsjobd server: " + repr(e)) time.sleep(0.1) @classmethod def setUpClass(cls): cls._clean() - subproc('python {b}/wsjobd_server.py start'.format(b=this_base)) + subproc("python {b}/wsjobd_server.py start".format(b=this_base)) time.sleep(1) @classmethod @@ -50,7 +51,7 @@ def tearDownClass(cls): def _create_client(self): ws = websocket.WebSocket() - ws.connect('ws://127.0.0.1:%d' % PORT) + ws.connect("ws://127.0.0.1:%d" % PORT) ws.timeout = 6 return ws @@ -61,57 +62,56 @@ def tearDown(self): self.ws.close() def get_random_ident(self): - return 'random_ident_%d' % random.randint(10000, 99999) + return "random_ident_%d" % random.randint(10000, 99999) def _wait_for_result(self, ws): # wait for test_job_echo.run to fillin resp['result'] for _ in range(3): resp = k3utfjson.load(ws.recv()) - if 'result' in resp: + if "result" in resp: break time.sleep(0.1) return resp def test_invalid_jobdesc(self): cases = ( - ('foo', 'not json'), - (k3utfjson.dump('foo'), 'not dict'), - (k3utfjson.dump({}), 'no func'), - (k3utfjson.dump({'func': 'foo'}), 'no ident'), - (k3utfjson.dump({'ident': 'bar'}), 'no func'), - (k3utfjson.dump({'ident': 'bar', 'func': {}}), 'invalid func'), - (k3utfjson.dump({'ident': 44, 'func': 'foo'}), 'invalid ident'), - (k3utfjson.dump({'ident': 'foo', 'func': 'foo', 'jobs_dir': {}}), - 'invalid jobs_dir'), + ("foo", "not json"), + (k3utfjson.dump("foo"), "not dict"), + (k3utfjson.dump({}), "no func"), + (k3utfjson.dump({"func": "foo"}), "no ident"), + (k3utfjson.dump({"ident": "bar"}), "no func"), + (k3utfjson.dump({"ident": "bar", "func": {}}), "invalid func"), + (k3utfjson.dump({"ident": 44, "func": "foo"}), "invalid ident"), + (k3utfjson.dump({"ident": "foo", "func": "foo", "jobs_dir": {}}), "invalid jobs_dir"), ) for msg, desc in cases: ws = self._create_client() ws.send(msg) resp = k3utfjson.load(ws.recv()) - self.assertIn('err', resp, desc) + self.assertIn("err", resp, desc) ws.close() def test_normal_job(self): job_desc = { - 'func': 'test_job_normal.run', - 'ident': self.get_random_ident(), - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_normal.run", + "ident": self.get_random_ident(), + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = self._wait_for_result(self.ws) dd(resp) - self.assertEqual('foo', resp['result'], 'test get result') + self.assertEqual("foo", resp["result"], "test get result") def test_report_interval(self): job_desc = { - 'func': 'test_job_loop_10.run', - 'ident': self.get_random_ident(), - 'progress': { - 'interval': 0.5, + "func": "test_job_loop_10.run", + "ident": self.get_random_ident(), + "progress": { + "interval": 0.5, }, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) @@ -124,20 +124,20 @@ def test_report_interval(self): actual_interval = second_report_time - first_report_time - diff = actual_interval - job_desc['progress']['interval'] + diff = actual_interval - job_desc["progress"]["interval"] # tolerate 0.1 second of difference self.assertLess(diff, 0.1) def test_invalid_progress_key(self): job_desc = { - 'func': 'test_job_progress_key.run', - 'ident': self.get_random_ident(), - 'progress': { - 'key': 'inexistent', + "func": "test_job_progress_key.run", + "ident": self.get_random_ident(), + "progress": { + "key": "inexistent", }, - 'report_system_load': True, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "report_system_load": True, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) @@ -149,16 +149,16 @@ def test_invalid_progress_key(self): def test_progress_key(self): job_desc = { - 'func': 'test_job_progress_key.run', - 'ident': self.get_random_ident(), - 'progress': { - 'key': 'foo', + "func": "test_job_progress_key.run", + "ident": self.get_random_ident(), + "progress": { + "key": "foo", }, # progress_sender may try to read data["foo"] before job starts. # need to preset a value. "foo": "0%", - 'report_system_load': True, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "report_system_load": True, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) @@ -167,33 +167,33 @@ def test_progress_key(self): dd(repr(resp)) resp = k3utfjson.load(resp) # progress may be set by job runner or job desc - self.assertIn(resp, ('80%', '0%')) + self.assertIn(resp, ("80%", "0%")) def test_check_system_load(self): job_desc = { - 'func': 'test_job_normal.run', - 'ident': self.get_random_ident(), - 'check_load': { - 'mem_low_threshold': 100 * 1024 ** 3, - 'cpu_low_threshold': 0, + "func": "test_job_normal.run", + "ident": self.get_random_ident(), + "check_load": { + "mem_low_threshold": 100 * 1024**3, + "cpu_low_threshold": 0, }, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = self.ws.recv() resp = k3utfjson.load(resp) - self.assertEqual('SystemOverloadError', resp['err']) + self.assertEqual("SystemOverloadError", resp["err"]) job_desc = { - 'func': 'test_job_normal.run', - 'ident': self.get_random_ident(), - 'check_load': { - 'cpu_low_threshold': 100.1, - 'mem_low_threshold': 0, + "func": "test_job_normal.run", + "ident": self.get_random_ident(), + "check_load": { + "cpu_low_threshold": 100.1, + "mem_low_threshold": 0, }, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "jobs_dir": "k3wsjobd/test/test_jobs", } ws2 = self._create_client() @@ -202,35 +202,35 @@ def test_check_system_load(self): resp = ws2.recv() resp = k3utfjson.load(resp) ws2.close() - self.assertEqual('SystemOverloadError', resp['err']) + self.assertEqual("SystemOverloadError", resp["err"]) def test_max_client_number(self): job_desc = { - 'func': 'test_job_loop_10.run', - 'ident': self.get_random_ident(), - 'check_load': { - 'max_client_number': 1, - 'cpu_low_threshold': 0, - 'mem_low_threshold': 0, + "func": "test_job_loop_10.run", + "ident": self.get_random_ident(), + "check_load": { + "max_client_number": 1, + "cpu_low_threshold": 0, + "mem_low_threshold": 0, }, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = self.ws.recv() resp = k3utfjson.load(resp) - self.assertNotIn('err', resp) + self.assertNotIn("err", resp) job_desc = { - 'func': 'test_job_loop_10.run', - 'ident': self.get_random_ident(), - 'check_load': { - 'max_client_number': 1, - 'cpu_low_threshold': 0, - 'mem_low_threshold': 0, + "func": "test_job_loop_10.run", + "ident": self.get_random_ident(), + "check_load": { + "max_client_number": 1, + "cpu_low_threshold": 0, + "mem_low_threshold": 0, }, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "jobs_dir": "k3wsjobd/test/test_jobs", } ws2 = self._create_client() @@ -238,56 +238,56 @@ def test_max_client_number(self): resp = ws2.recv() resp = k3utfjson.load(resp) - self.assertEqual('SystemOverloadError', resp['err']) + self.assertEqual("SystemOverloadError", resp["err"]) ws2.close() def test_module_not_exists(self): job_desc = { - 'func': 'foo.bar', - 'ident': self.get_random_ident(), - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "foo.bar", + "ident": self.get_random_ident(), + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = k3utfjson.load(self.ws.recv()) - self.assertIn('err', resp) + self.assertIn("err", resp) def test_report_system_load(self): job_desc = { - 'func': 'test_job_normal.run', - 'ident': self.get_random_ident(), - 'report_system_load': True, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_normal.run", + "ident": self.get_random_ident(), + "report_system_load": True, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = k3utfjson.load(self.ws.recv()) - self.assertIn('mem_available', resp['system_load']) - self.assertIn('cpu_idle_percent', resp['system_load']) - self.assertIn('client_number', resp['system_load']) + self.assertIn("mem_available", resp["system_load"]) + self.assertIn("cpu_idle_percent", resp["system_load"]) + self.assertIn("client_number", resp["system_load"]) def test_same_ident_same_job(self): ident = self.get_random_ident() job_desc = { - 'func': 'test_job_echo.run', - 'ident': ident, - 'echo': 'foo', - 'sleep_time': 10, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.run", + "ident": ident, + "echo": "foo", + "sleep_time": 10, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = k3utfjson.load(self.ws.recv()) - self.assertEqual('foo', resp['result']) + self.assertEqual("foo", resp["result"]) job_desc = { - 'func': 'test_job_echo.run', - 'ident': ident, - 'echo': 'bar', - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.run", + "ident": ident, + "echo": "bar", + "jobs_dir": "k3wsjobd/test/test_jobs", } ws = self._create_client() @@ -297,31 +297,31 @@ def test_same_ident_same_job(self): ws.close() # not bar, because the ident is same as the first job, # if job exists, it will not create a new one - self.assertEqual('foo', resp['result']) - self.assertEqual('foo', resp['echo']) + self.assertEqual("foo", resp["result"]) + self.assertEqual("foo", resp["echo"]) def test_same_ident_different_job(self): ident = self.get_random_ident() job_desc = { - 'func': 'test_job_echo.run', - 'ident': ident, - 'echo': 'foo', - 'sleep_time': 0.1, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.run", + "ident": ident, + "echo": "foo", + "sleep_time": 0.1, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = self._wait_for_result(self.ws) - self.assertEqual('foo', resp['result']) + self.assertEqual("foo", resp["result"]) time.sleep(0.2) job_desc = { - 'func': 'test_job_echo.run', - 'ident': ident, - 'echo': 'bar', - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.run", + "ident": ident, + "echo": "bar", + "jobs_dir": "k3wsjobd/test/test_jobs", } ws2 = self._create_client() @@ -330,17 +330,17 @@ def test_same_ident_different_job(self): resp = self._wait_for_result(ws2) ws2.close() # old job with the same ident has exit, it will create a new one - self.assertEqual('bar', resp['result']) - self.assertEqual('bar', resp['echo']) + self.assertEqual("bar", resp["result"]) + self.assertEqual("bar", resp["echo"]) def test_client_close(self): ident = self.get_random_ident() job_desc = { - 'func': 'test_job_echo.run', - 'ident': ident, - 'echo': 'foo', - 'time_sleep': 10, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.run", + "ident": ident, + "echo": "foo", + "time_sleep": 10, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) @@ -348,80 +348,76 @@ def test_client_close(self): self.ws = self._create_client() job_desc = { - 'func': 'test_job_echo.run', - 'ident': ident, - 'echo': 'bar', - 'time_sleep': 10, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.run", + "ident": ident, + "echo": "bar", + "time_sleep": 10, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = self.ws.recv() resp = k3utfjson.load(resp) - self.assertEqual('foo', resp['result']) + self.assertEqual("foo", resp["result"]) def test_invalid_cpu_sample_interval(self): job_desc = { - 'func': 'test_job_normal.run', - 'ident': self.get_random_ident(), - 'cpu_sample_interval': 'foo', - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_normal.run", + "ident": self.get_random_ident(), + "cpu_sample_interval": "foo", + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = k3utfjson.load(self.ws.recv()) - self.assertEqual('InvalidMessageError', resp['err']) + self.assertEqual("InvalidMessageError", resp["err"]) def test_invalid_check_load_args(self): cases = ( - {'check_load': {'mem_low_threshold': 'foo', - 'cpu_low_threshold': 0}}, - {'check_load': {'cpu_low_threshold': None, - 'mem_low_threshold': 0}}, - {'check_load': {'max_client_number': {}, - 'cpu_low_threshold': 0, - 'mem_low_threshold': 0}}, + {"check_load": {"mem_low_threshold": "foo", "cpu_low_threshold": 0}}, + {"check_load": {"cpu_low_threshold": None, "mem_low_threshold": 0}}, + {"check_load": {"max_client_number": {}, "cpu_low_threshold": 0, "mem_low_threshold": 0}}, ) job_desc = { - 'func': 'test_job_normal.run', - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_normal.run", + "jobs_dir": "k3wsjobd/test/test_jobs", } for case in cases: case.update(job_desc) - case['ident'] = self.get_random_ident() + case["ident"] = self.get_random_ident() ws = self._create_client() ws.send(k3utfjson.dump(case)) resp = k3utfjson.load(ws.recv()) ws.close() - self.assertEqual('InvalidMessageError', resp['err']) + self.assertEqual("InvalidMessageError", resp["err"]) def test_func_not_exists(self): ident = self.get_random_ident() job_desc = { - 'func': 'test_job_echo.func_not_exists', - 'ident': ident, - 'jobs_dir': 'k3wsjobd/test/test_jobs', + "func": "test_job_echo.func_not_exists", + "ident": ident, + "jobs_dir": "k3wsjobd/test/test_jobs", } self.ws.send(k3utfjson.dump(job_desc)) resp = self.ws.recv() resp = k3utfjson.load(resp) - self.assertEqual('LoadingError', resp['err']) + self.assertEqual("LoadingError", resp["err"]) def test_worker_exception(self): ident = self.get_random_ident() job_desc = { - 'func': 'test_job_worker_exception.run', - 'ident': ident, - 'jobs_dir': 'k3wsjobd/test/test_jobs', - 'progress': { - 'interval': 0.1, + "func": "test_job_worker_exception.run", + "ident": ident, + "jobs_dir": "k3wsjobd/test/test_jobs", + "progress": { + "interval": 0.1, }, } @@ -429,7 +425,7 @@ def test_worker_exception(self): for i in range(10): resp = self.ws.recv() - self.assertNotIn('err', resp) + self.assertNotIn("err", resp) with self.assertRaises(Exception): for i in range(3): @@ -442,20 +438,20 @@ def f(self): time.sleep(0.2) return - Job('channel', {'ident': 'a'}, f) - joba = Job.sessions['a'] + Job("channel", {"ident": "a"}, f) + joba = Job.sessions["a"] self.assertEqual(1, len(Job.sessions)) - Job('channel', {'ident': 'a'}, f) - joba1 = Job.sessions['a'] + Job("channel", {"ident": "a"}, f) + joba1 = Job.sessions["a"] # joba already exists self.assertEqual(joba1, joba) self.assertEqual(1, len(Job.sessions)) time.sleep(0.15) - Job('channel', {'ident': 'b'}, f) + Job("channel", {"ident": "b"}, f) self.assertEqual(2, len(Job.sessions)) @@ -466,11 +462,11 @@ def f(self): self.assertEqual(0, len(Job.sessions)) # test use same ident after first one exit - Job('channel', {'ident': 'a'}, f) - joba1 = Job.sessions['a'] + Job("channel", {"ident": "a"}, f) + joba1 = Job.sessions["a"] time.sleep(0.25) - Job('channel', {'ident': 'a'}, f) - joba2 = Job.sessions['a'] + Job("channel", {"ident": "a"}, f) + joba2 = Job.sessions["a"] self.assertNotEqual(joba1, joba2) diff --git a/test/wsjobd_server.py b/test/wsjobd_server.py index f3c7dbd..5db99d8 100644 --- a/test/wsjobd_server.py +++ b/test/wsjobd_server.py @@ -7,17 +7,18 @@ def run(): - k3wsjobd.run(ip='127.0.0.1', port=PORT, jobq_thread_count=20) + k3wsjobd.run(ip="127.0.0.1", port=PORT, jobq_thread_count=20) if __name__ == "__main__": logger = logging.getLogger() logger.setLevel(logging.INFO) - file_handler = logging.FileHandler('/tmp/wsjobd.log') - formatter = logging.Formatter('[%(asctime)s, %(levelname)s] %(message)s') + file_handler = logging.FileHandler("/tmp/wsjobd.log") + formatter = logging.Formatter("[%(asctime)s, %(levelname)s] %(message)s") file_handler.setFormatter(formatter) logger.addHandler(file_handler) import k3daemonize - k3daemonize.daemonize_cli(run, '/tmp/wsjod_server.pid') + + k3daemonize.daemonize_cli(run, "/tmp/wsjod_server.pid") diff --git a/wsjobd.py b/wsjobd.py index 844273b..384af1e 100644 --- a/wsjobd.py +++ b/wsjobd.py @@ -17,10 +17,10 @@ logger = logging.getLogger(__name__) -MEM_AVAILABLE = 'mem_available' -CPU_IDLE_PERCENT = 'cpu_idle_percent' -CLIENT_NUMBER = 'client_number' -JOBS_DIR = 'jobs' +MEM_AVAILABLE = "mem_available" +CPU_IDLE_PERCENT = "cpu_idle_percent" +CLIENT_NUMBER = "client_number" +JOBS_DIR = "jobs" # - `mem_low_threshold` # set the min size of available memory the system should have, if not satified, return error. the default is 500M @@ -33,20 +33,20 @@ CHECK_LOAD_PARAMS = { - 'mem_low_threshold': { - 'load_name': MEM_AVAILABLE, - 'default': 500 * 1024 ** 2, # 500M - 'greater': True, + "mem_low_threshold": { + "load_name": MEM_AVAILABLE, + "default": 500 * 1024**2, # 500M + "greater": True, }, - 'cpu_low_threshold': { - 'load_name': CPU_IDLE_PERCENT, - 'default': 3, # 3% - 'greater': True, + "cpu_low_threshold": { + "load_name": CPU_IDLE_PERCENT, + "default": 3, # 3% + "greater": True, }, - 'max_client_number': { - 'load_name': CLIENT_NUMBER, - 'default': 1000, - 'greater': False, + "max_client_number": { + "load_name": CLIENT_NUMBER, + "default": 1000, + "greater": False, }, } @@ -97,7 +97,7 @@ def __init__(self, channel, msg, func): :param func: required. the function of job, it contain module name and function name, seperated by a dot, the module shoud in the `jobs` directory. """ - self.ident = msg['ident'] + self.ident = msg["ident"] self.channel = channel self.data = msg self.worker = func @@ -106,47 +106,45 @@ def __init__(self, channel, msg, func): self.progress_available = threading.Event() if self.ident in self.sessions: - logger.info('job: %s already exists, created by chennel %s' % - (self.ident, repr(self.sessions[self.ident].channel))) + logger.info( + "job: %s already exists, created by chennel %s" % (self.ident, repr(self.sessions[self.ident].channel)) + ) return else: self.sessions[self.ident] = self - logger.info(('inserted job: %s to sessions by channel %s, ' + - 'there are %d jobs in sessions now') % - (self.ident, repr(self.channel), len(self.sessions))) + logger.info( + ("inserted job: %s to sessions by channel %s, " + "there are %d jobs in sessions now") + % (self.ident, repr(self.channel), len(self.sessions)) + ) - self.thread = k3thread.start(target=self.work, args=(), - daemon=True) + self.thread = k3thread.start(target=self.work, args=(), daemon=True) def work(self): - - logger.info("job %s started, the data is: %s" % - (self.ident, self.data)) + logger.info("job %s started, the data is: %s" % (self.ident, self.data)) try: self.worker(self) except Exception as e: - logger.exception('job %s got exception: %s' % - (self.ident, repr(e))) + logger.exception("job %s got exception: %s" % (self.ident, repr(e))) self.err = e finally: - logger.info('job %s ended' % self.ident) + logger.info("job %s ended" % self.ident) self.close() def close(self): - with self.lock: del self.sessions[self.ident] - logger.info(('removed job: %s from sessions, there are %d ' + - 'jobs in sessions now') % - (self.ident, len(self.sessions))) + logger.info( + ("removed job: %s from sessions, there are %d " + "jobs in sessions now") + % (self.ident, len(self.sessions)) + ) def get_or_create_job(channel, msg, func): with Job.lock: Job(channel, msg, func) - job = Job.sessions.get(msg['ident']) + job = Job.sessions.get(msg["ident"]) return job @@ -160,18 +158,17 @@ def progress_sender(job, channel, interval=5, stat=None): while True: # if thread died due to some reason, still send 10 stats if not job.thread.is_alive(): - logger.info('job %s died: %s' % (job.ident, repr(job.err))) + logger.info("job %s died: %s" % (job.ident, repr(job.err))) if i == 0: channel.ws.close() break i -= 1 - logger.info('jod %s on channel %s send progress: %s' % - (job.ident, repr(channel), repr(stat(data)))) + logger.info("jod %s on channel %s send progress: %s" % (job.ident, repr(channel), repr(stat(data)))) to_send = stat(data) - if channel.report_system_load and type(to_send) == type({}): - to_send['system_load'] = channel.get_system_load() + if channel.report_system_load and isinstance(to_send, dict): + to_send["system_load"] = channel.get_system_load() channel.ws.send(k3utfjson.dump(to_send)) @@ -179,15 +176,15 @@ def progress_sender(job, channel, interval=5, stat=None): job.progress_available.clear() except WebSocketError as e: - if channel.ws.closed == True: - logger.info('the client has closed the connection') + if channel.ws.closed: + logger.info("the client has closed the connection") else: - logger.exception(('got websocket error when sending progress on' + - ' channel %s: %s') % (repr(channel), repr(e))) + logger.exception( + ("got websocket error when sending progress on" + " channel %s: %s") % (repr(channel), repr(e)) + ) except Exception as e: - logger.exception('got exception when sending progress on channel %s: %s' - % (repr(channel), repr(e))) + logger.exception("got exception when sending progress on channel %s: %s" % (repr(channel), repr(e))) channel.ws.close() @@ -195,53 +192,47 @@ class JobdWebSocketApplication(WebSocketApplication): jobq_mgr = None def on_open(self): - logger.info('on open, the channel is: ' + repr(self)) + logger.info("on open, the channel is: " + repr(self)) self.ignore_message = False def _parse_request(self, message): try: try: msg = k3utfjson.load(message) - except Exception as e: - raise InvalidMessageError( - 'message is not a vaild json string: %s' % message) + except Exception: + raise InvalidMessageError("message is not a vaild json string: %s" % message) self._check_msg(msg) - self.report_system_load = msg.get('report_system_load') == True - self.cpu_sample_interval = msg.get('cpu_sample_interval', 0.02) + self.report_system_load = msg.get("report_system_load") is True + self.cpu_sample_interval = msg.get("cpu_sample_interval", 0.02) if not isinstance(self.cpu_sample_interval, (int, float)): - raise InvalidMessageError( - 'cpu_sample_interval is not a number') + raise InvalidMessageError("cpu_sample_interval is not a number") - check_load = msg.get('check_load') - if type(check_load) == type({}): + check_load = msg.get("check_load") + if isinstance(check_load, dict): self._check_system_load(check_load) - self.jobs_dir = msg.get('jobs_dir', JOBS_DIR) + self.jobs_dir = msg.get("jobs_dir", JOBS_DIR) self._setup_response(msg) return except SystemOverloadError as e: - logger.info('system overload on chennel %s, %s' - % (repr(self), repr(e))) + logger.info("system overload on chennel %s, %s" % (repr(self), repr(e))) self._send_err_and_close(e) except JobError as e: - logger.info('error on channel %s while handling message, %s' - % (repr(self), repr(e))) + logger.info("error on channel %s while handling message, %s" % (repr(self), repr(e))) self._send_err_and_close(e) except Exception as e: - logger.exception(('exception on channel %s while handling ' + - 'message, %s') % (repr(self), repr(e))) + logger.exception(("exception on channel %s while handling " + "message, %s") % (repr(self), repr(e))) self._send_err_and_close(e) def on_message(self, message): - logger.info('on message, the channel is: %s, the message is: %s' % - (repr(self), message)) + logger.info("on message, the channel is: %s, the message is: %s" % (repr(self), message)) if self.ignore_message: return @@ -253,19 +244,17 @@ def on_message(self, message): def _send_err_and_close(self, err): try: err_msg = { - 'err': err.__class__.__name__, - 'val': err.args, + "err": err.__class__.__name__, + "val": err.args, } self.ws.send(k3utfjson.dump(err_msg)) except Exception as e: - logger.error(('error on channel %s while sending back error ' - + 'message, %s') % (repr(self), repr(e))) + logger.error(("error on channel %s while sending back error " + "message, %s") % (repr(self), repr(e))) def get_system_load(self): return { MEM_AVAILABLE: psutil.virtual_memory().available, - CPU_IDLE_PERCENT: psutil.cpu_times_percent( - self.cpu_sample_interval).idle, + CPU_IDLE_PERCENT: psutil.cpu_times_percent(self.cpu_sample_interval).idle, CLIENT_NUMBER: len(self.protocol.server.clients), } @@ -273,89 +262,83 @@ def _check_system_load(self, check_load): system_load = self.get_system_load() for param_name, param_attr in CHECK_LOAD_PARAMS.items(): - param_value = check_load.get(param_name, param_attr['default']) + param_value = check_load.get(param_name, param_attr["default"]) if not isinstance(param_value, (int, float)): - raise InvalidMessageError('%s is not a number' % param_name) + raise InvalidMessageError("%s is not a number" % param_name) - load_name = param_attr['load_name'] + load_name = param_attr["load_name"] diff = system_load[load_name] - param_value - if not param_attr['greater']: + if not param_attr["greater"]: diff = 0 - diff if diff < 0: raise SystemOverloadError( - '%s: %d is %s than: %d' % - (load_name, system_load[load_name], - param_attr['greater'] and 'less' or 'greater', - param_value)) + "%s: %d is %s than: %d" + % (load_name, system_load[load_name], param_attr["greater"] and "less" or "greater", param_value) + ) def _check_msg(self, msg): - if type(msg) != type({}): + if not isinstance(msg, dict): raise InvalidMessageError("message is not dictionary") - if 'ident' not in msg: + if "ident" not in msg: raise InvalidMessageError("'ident' is not in message") - if 'func' not in msg: + if "func" not in msg: raise InvalidMessageError("'func' is not in message") def _setup_response(self, msg): - func = self._get_func_by_name(msg) channel = self job = get_or_create_job(channel, msg, func) if job is None: - raise JobNotInSessionError( - 'job not in sessions: ' + repr(Job.sessions)) + raise JobNotInSessionError("job not in sessions: " + repr(Job.sessions)) - progress = msg.get('progress', {}) + progress = msg.get("progress", {}) if progress in (None, False): return - if type(progress) != type({}): - raise InvalidProgressError( - 'the progress in message is not a dictionary') + if not isinstance(progress, dict): + raise InvalidProgressError("the progress in message is not a dictionary") - interval = progress.get('interval', 5) - progress_key = progress.get('key') + interval = progress.get("interval", 5) + progress_key = progress.get("key") - if progress_key is None: - lam = lambda r: r - else: - lam = lambda r: r.get(progress_key) + def get_progress(r): + if progress_key is None: + return r + return r.get(progress_key) - k3thread.start(target=progress_sender, - args=(job, channel, interval, lam), - daemon=True) + k3thread.start(target=progress_sender, args=(job, channel, interval, get_progress), daemon=True) def _get_func_by_name(self, msg): - mod_func = self.jobs_dir.split('/') + msg['func'].split('.') - mod_path = '.'.join(mod_func[:-1]) + mod_func = self.jobs_dir.split("/") + msg["func"].split(".") + mod_path = ".".join(mod_func[:-1]) func_name = mod_func[-1] try: mod = __import__(mod_path) except (ImportError, SyntaxError) as e: - raise LoadingError('failed to import %s: %s' % (mod_path, repr(e))) + raise LoadingError("failed to import %s: %s" % (mod_path, repr(e))) - for mod_name in mod_path.split('.')[1:]: + for mod_name in mod_path.split(".")[1:]: mod = getattr(mod, mod_name) - logger.info('mod imported from: ' + repr(mod.__file__)) + logger.info("mod imported from: " + repr(mod.__file__)) try: func = getattr(mod, func_name) - except AttributeError as e: + except AttributeError: raise LoadingError("function not found: " + repr(func_name)) return func def on_close(self, reason): - logger.info('on close, the channel is: ' + repr(self)) + logger.info("on close, the channel is: " + repr(self)) def _parse_request(args): @@ -363,11 +346,10 @@ def _parse_request(args): app._parse_request(msg) -def run(ip='127.0.0.1', port=63482, jobq_thread_count=10): - JobdWebSocketApplication.jobq_mgr = k3jobq.JobManager( - [(_parse_request, jobq_thread_count)]) +def run(ip="127.0.0.1", port=63482, jobq_thread_count=10): + JobdWebSocketApplication.jobq_mgr = k3jobq.JobManager([(_parse_request, jobq_thread_count)]) WebSocketServer( (ip, port), - Resource(OrderedDict({'/': JobdWebSocketApplication})), + Resource(OrderedDict({"/": JobdWebSocketApplication})), ).serve_forever()