📁 Load CSV File
+ + +Pi Directory Files:
+ + +Or Upload from Browser:
++ to upload path data +
diff --git a/.github/workflows/python-package-conda.yml b/.github/workflows/python-package-conda.yml deleted file mode 100644 index fd03481ab0..0000000000 --- a/.github/workflows/python-package-conda.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: Python package and test - -on: - push: - pull_request: - -jobs: - checkout-test: - name: Checkout and test - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["macos-latest", "ubuntu-latest"] - fail-fast: false - defaults: - run: - shell: bash -l {0} - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Create python 3.11 conda env - uses: conda-incubator/setup-miniconda@v3 - with: - python-version: 3.11 - mamba-version: "*" - activate-environment: donkey - auto-activate-base: false - channels: default, conda-forge, pytorch - channel-priority: true - - name: Conda info and list - run: | - conda info - conda list - - name: Install donkey - run: | - pip install -e .[pc,dev] - pip list - - name: Run tests - run: pytest diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml new file mode 100644 index 0000000000..d2c9f9575a --- /dev/null +++ b/.github/workflows/python-package.yml @@ -0,0 +1,26 @@ +name: Python package and test + +on: + push: + paths-ignore: + - 'docs/**' + pull_request: + +jobs: + checkout-test: + name: Checkout and test + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [macos-latest, ubuntu-latest] + python-version: ['3.12'] + fail-fast: false + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + - name: Create virtual environment + run: uv venv --python ${{ matrix.python-version }} + - name: Install donkey + run: uv pip install -e ".[pc,dev]" + - name: Run tests + run: uv run --no-sync pytest diff --git a/.gitignore b/.gitignore index 5e08b188ba..d0d3c9ac9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ env/* +.venv data/* dist/* diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index b9013ea14d..0000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,3 +0,0 @@ -include donkeycar/templates/* -include scripts -recursive-include donkeycar/parts/web_controller/templates/ * diff --git a/Makefile b/Makefile index ae84f8859b..e34593dad9 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,6 @@ - tests: - pytest + uv run pytest package: - python setup.py sdist - + uv build diff --git a/PYTHON312_MIGRATION.md b/PYTHON312_MIGRATION.md new file mode 100644 index 0000000000..c136bf0b28 --- /dev/null +++ b/PYTHON312_MIGRATION.md @@ -0,0 +1,239 @@ +# Python 3.12/3.13 Migration + +Donkeycar moved from Python 3.11 + conda to Python 3.12+ + uv. This document +covers every technical decision, all code changes, the new install workflow, +and what the documentation team needs to update. + +--- + +## Python version per platform + +| Platform | Python | Reason | +|---|---|---| +| Mac (`[macos]`) | **3.12** | `tensorflow-metal==1.2.0` has no Python 3.13 wheel; TF 2.19 SavedModel export broken on 3.13 | +| PC (`[pc]`) | **3.12** | TF 2.19 SavedModel export broken on 3.13 | +| Raspberry Pi (`[pi]`) | **3.13** | `libcamera`/`picamera2` are Debian system packages for Python 3.13; no pip-installable alternative | + +**Why not 3.13 for Mac/PC:** TF's internal `inspect.getattr_static` behavior +changed in 3.13, breaking all SavedModel export paths (`model.export()`, +`tf.saved_model.save()`). The `_DictWrapper` workaround via +`tf.function + from_concrete_functions` produces TFLite with no +`serving_default` signature, requiring a tensor-API fallback in the `TfLite` +interpreter. This is too invasive for the current release cycle. + +**Why 3.13 on Pi:** `python3-libcamera` and `python3-picamera2` are Debian +packages installed for the system Python (3.13 on Trixie). The venv must use +the system Python with `--system-site-packages` to access them. A Python 3.12 +venv cannot reach these packages even with `--system-site-packages` because +Debian installs them under `/usr/lib/python3/dist-packages`, not under the +custom Python 3.12 tree at `/usr/local`. + +--- + +## Why uv (not conda) + +- Single cross-platform tool: same workflow on Pi, Mac, and PC. +- Persistent named venv at a fixed path — `donkey` commands work from any + directory once the venv is activated in the shell profile. +- `uv pip install` is a drop-in for pip; editable installs (`-e`) work + unchanged. +- On Pi, the Debian system Python 3.13 is used directly (not uv's bundled + CPython) because camera libraries are Debian system packages that must be + visible to the venv via `--system-site-packages`. + +--- + +## TensorFlow version decision + +| Platform | TF version | Notes | +|---|---|---| +| Mac (`macos` extra) | `2.19.*` | Highest version compatible with `tensorflow-metal==1.2.0` | +| PC (`pc` extra) | `2.19.*` | Kept in sync with Mac for model format compatibility | +| Pi | — | Pi uses `ai-edge-litert` (TFLite), not TF | + +**`tensorflow-metal` compatibility wall:** TF 2.20 changed the internal +`_pywrap_tensorflow_internal.so` rpath, breaking `libmetal_plugin.dylib`. +TF 2.18 and 2.19 both work with `tensorflow-metal==1.2.0` on **Python 3.12 +only**. TF 2.20 and 2.21 do not. Check back when Apple releases +tensorflow-metal 1.3+. + +**`tensorflow-metal==1.1.0`** (the previous version) has no Python 3.12 wheel. +`1.2.0` is the first release with a `cp312` wheel. There is no `cp313` wheel. + +**Inference performance:** TFLite on Pi 5 (aarch64, XNNPACK): **~282 fps** +for a `KerasLinear` 160×120×3 model. No regression vs TF 2.21 on the same +model. + +--- + +## Key dependency changes + +| Dependency | Before | After | Reason | +|---|---|---|---| +| Python (Mac/PC) | 3.11 | 3.12 | Stability, TF 2.19 support | +| Python (Pi) | 3.11 | 3.13 | System Python required for Debian camera packages | +| TF (pc/mac) | `2.15.*` | `2.19.*` | Latest compatible with metal | +| `tflite-runtime` | present | **removed** | Dead project (last release Python 3.11) | +| `ai-edge-litert` | absent | `>=2.1.4` | Google's official TFLite successor, drop-in API | +| `tensorflow-metal` | `1.1.0` | `1.2.0` | First release with Python 3.12 wheel | +| `RPi.GPIO` | present | **removed** | No wheels past Python 3.9 | +| `gpiozero` | absent | present | Supports Python 3.12+, covers same hardware | +| `torch` | `2.1.*` | `2.6.*` | First series with Python 3.12+ aarch64 wheels | +| `picamera2` (pi extra) | present | **removed** | Debian system package only; install via `apt` | + +--- + +## Codebase changes + +### Package metadata (`pyproject.toml`) + +`setup.cfg` and `MANIFEST.in` were deleted. All metadata now lives in +`pyproject.toml`: + +- `[project]` — name, dynamic version, authors, license, classifiers, + `requires-python = ">=3.12.0,<3.14"`, core dependencies +- `[project.optional-dependencies]` — `pi`, `nano`, `pc`, `macos`, `dev`, + `torch` extras +- `[project.scripts]` — `donkey` entry point +- `[tool.setuptools.dynamic]` — `version = {attr = "donkeycar.__version__"}` +- `[tool.setuptools.packages.find]` — `namespaces = true` +- `[tool.setuptools.package-data]` — covers `*.html/ini/txt/kv` plus + `donkeycar/management/tub_web/static/**/*` (was in MANIFEST.in) + +### CI (`.github/workflows/python-package.yml`) + +Replaced `python-package-conda.yml`. New workflow: + +```yaml +- uses: astral-sh/setup-uv@v5 + with: + python-version: '3.12' +- run: uv pip install -e ".[pc,dev]" +- run: uv run pytest +``` + +### TF internal API imports + +`tensorflow.python.keras.*` was removed in TF 2.16+. All occurrences replaced +with `tensorflow.keras.*` or wrapped in `try/except ImportError`: + +- `donkeycar/parts/interpreter.py` +- `donkeycar/parts/keras.py` +- `donkeycar/pipeline/training.py` +- `donkeycar/parts/keras_2.py` +- `donkeycar/management/makemovie.py` +- `donkeycar/management/base.py` + +### Keras 3.x breaking changes (TF 2.16+) + +TF 2.16 switched from bundled Keras 2 to standalone Keras 3. The `tf-keras` +package restores the Keras 2 API at `tensorflow.keras.*`: + +- `from keras.backend import concatenate` → + `from tensorflow.keras.layers import concatenate` +- `workers=1, use_multiprocessing=False` removed from `model.fit()` (gone in + Keras 3) +- `model.input_names` removed in Keras 3 → replaced with + `[inp.name for inp in model.inputs]` +- Default model save format: `savedmodel` → `keras` +- `model_prefix_map` in `pipeline/database.py` updated for `.keras` extension + +### TFLite on Pi (`ai-edge-litert`) + +`tflite-runtime` is dead (last release: TF 2.14 / Python 3.11 max). Replaced +by `ai-edge-litert`, which is Google's official successor with a drop-in API. +No inference code changes required — only the import and package name changed. + +The `TfLite` interpreter in `interpreter.py` was updated with a tensor-API +fallback for models without `serving_default` signatures (needed when +converting from Keras 3 via `from_concrete_functions`). + +--- + +## Pi: Python 3.13 (system) on Debian Trixie + +Pi 5 running Debian 13 (Trixie) ships Python 3.13 as the system default. +Camera support requires `libcamera` and `picamera2`, which are Debian packages +installed for that system Python — they are not available on PyPI. The venv +must therefore use the system Python 3.13 with `--system-site-packages`: + +```zsh +sudo apt install python3-libcamera python3-picamera2 +uv venv ~/env --python 3.13 --system-site-packages +``` + +Using `--python 3.12` (custom-built at `/usr/local/bin/python3.12`) will not +work: its `--system-site-packages` only includes +`/usr/local/lib/python3.12/site-packages`, not Debian's +`/usr/lib/python3/dist-packages` where `libcamera` and `picamera2` live. + +--- + +## Install workflow (uv) + +### First-time setup — all platforms + +```zsh +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +### Raspberry Pi + +```zsh +sudo apt install python3-libcamera python3-picamera2 +uv venv ~/env --python 3.13 --system-site-packages +echo 'source ~/env/bin/activate' >> ~/.zshrc +source ~/env/bin/activate + +# User install (PyPI): +uv pip install donkeycar[pi] + +# Developer install (git clone): +uv pip install -e ".[pi,dev]" +``` + +### Mac + +```zsh +uv venv ~/.venvs/donkeycar --python 3.12 +echo 'source ~/.venvs/donkeycar/bin/activate' >> ~/.zshrc +source ~/.venvs/donkeycar/bin/activate + +# User install (PyPI): +uv pip install donkeycar[macos] + +# Developer install (git clone): +uv pip install -e ".[macos,dev]" +``` + +### PC (Linux / Windows) + +```zsh +uv venv ~/.venvs/donkeycar --python 3.12 +echo 'source ~/.venvs/donkeycar/bin/activate' >> ~/.zshrc +source ~/.venvs/donkeycar/bin/activate + +# User install (PyPI): +uv pip install donkeycar[pc] + +# Developer install (git clone): +uv pip install -e ".[pc,dev]" +``` + +--- + +## Documentation updates required + +The following pages on docs.donkeycar.com need updating before this branch +is merged to main: + +| Page | What to change | +|---|---| +| Install — Raspberry Pi | Replace conda/pip steps with uv workflow above; note Pi uses TFLite via `ai-edge-litert`, not full TF | +| Install — Mac | Replace conda steps with uv + `[macos]` extra; note TF 2.19 + Metal GPU | +| Install — PC / Linux | Replace conda steps with uv + `[pc]` extra | +| Software requirements | Update Python version from 3.11 to 3.12 (Mac/PC) or 3.13 (Pi); remove conda prerequisite; add uv install step | +| Upgrade guide | Add section: "Upgrading from conda to uv" — remove old env, install uv, create new venv | +| Training / model formats | Note default save format is now `.keras` (was `.savedmodel`) | +| Pi inference | Update package name from `tflite-runtime` to `ai-edge-litert`; confirm API is identical | +| CI badge in README | Already updated to `python-package.yml` | diff --git a/README.md b/README.md index eb7fba3810..2c7fcfa7b1 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Donkeycar: a python self driving library - +   @@ -15,7 +15,7 @@  -Donkeycar is minimalist and modular self driving library for Python. It is developed for hobbyists and students with a focus on allowing fast experimentation and easy community contributions. It is being actively used at the high school and university level for learning and research. It offers a [rich graphical interface](https://docs.donkeycar.com/utility/ui/) and includes a [simulator](https://docs.donkeycar.com/guide/deep_learning/simulator/) so you can experiment with self-driving even before you build a robot. +Donkeycar is a minimalist and modular self driving library for Python. It is developed for hobbyists and students with a focus on allowing fast experimentation and easy community contributions. It is being actively used at the high school and university level for learning and research. It offers a [rich graphical interface](https://docs.donkeycar.com/utility/ui/) and includes a [simulator](https://docs.donkeycar.com/guide/deep_learning/simulator/) so you can experiment with self-driving even before you build a robot. #### Quick Links * [Donkeycar Updates & Examples](http://donkeycar.com) @@ -79,5 +79,84 @@ V.add(tub, inputs=['image'], outputs=['num_records']) V.start(rate_hz=10) ``` +## Installation + +Donkeycar uses [uv](https://docs.astral.sh/uv/) for environment management. +Install uv first: + +```zsh +curl -LsSf https://astral.sh/uv/install.sh | sh +``` + +Then create a persistent virtual environment and activate it on login by +adding the `source` line to your shell profile. + +### Raspberry Pi + +Camera support (`picamera2`, `libcamera`) requires system packages that are +not on PyPI. Install them first: + +```zsh +sudo apt install python3-libcamera python3-picamera2 +``` + +Then create the venv using the system Python (3.13 on Raspberry Pi OS Trixie) +with `--system-site-packages` so the venv can see the camera libraries: + +```zsh +uv venv ~/env --python 3.13 --system-site-packages +echo 'source ~/env/bin/activate' >> ~/.zshrc +source ~/env/bin/activate +``` + +User install (from PyPI): +```zsh +uv pip install donkeycar[pi] +``` + +Developer install (from a git clone): +```zsh +uv pip install -e ".[pi,dev]" +``` + +### Mac + +```zsh +uv venv ~/.venvs/donkeycar --python 3.12 +echo 'source ~/.venvs/donkeycar/bin/activate' >> ~/.zshrc +source ~/.venvs/donkeycar/bin/activate +``` + +User install (from PyPI): +```zsh +uv pip install donkeycar[macos] +``` + +Developer install (from a git clone): +```zsh +uv pip install -e ".[macos,dev]" +``` + +### PC (Linux/Windows) + +```zsh +uv venv ~/.venvs/donkeycar --python 3.12 +echo 'source ~/.venvs/donkeycar/bin/activate' >> ~/.zshrc +source ~/.venvs/donkeycar/bin/activate +``` + +User install (from PyPI): +```zsh +uv pip install donkeycar[pc] +``` + +Developer install (from a git clone): +```zsh +uv pip install -e ".[pc,dev]" +``` + +Once the venv is activated in your shell profile the `donkey` command is +available from any directory without needing `uv run`. + See [home page](http://donkeycar.com), [docs](http://docs.donkeycar.com) or join the [Discord server](http://www.donkeycar.com/community.html) to learn more. diff --git a/donkeycar/__init__.py b/donkeycar/__init__.py index 00cc209727..63619dc568 100644 --- a/donkeycar/__init__.py +++ b/donkeycar/__init__.py @@ -3,7 +3,7 @@ from pyfiglet import Figlet import logging -__version__ = '5.2.dev2' +__version__ = '5.4.dev1' logging.basicConfig(level=os.environ.get('LOGLEVEL', 'INFO').upper()) @@ -13,8 +13,9 @@ print(f.renderText('Donkey Car')) print(f'using donkey v{__version__} ...') -if sys.version_info.major < 3 or sys.version_info.minor < 11: - msg = f'Donkey Requires Python 3.11 or greater. You are using {sys.version}' +if sys.version_info.major < 3 or sys.version_info.minor < 12: + msg = (f'Donkey requires Python 3.12+ (PC/Mac) and 3.13+ (Raspberry Pi). ' + f'You are using {sys.version}') raise ValueError(msg) # The default recursion limits in CPython are too small. diff --git a/donkeycar/gym/__init__.py b/donkeycar/gym/__init__.py deleted file mode 100755 index e69de29bb2..0000000000 diff --git a/donkeycar/gym/gym_real.py b/donkeycar/gym/gym_real.py deleted file mode 100755 index eaf66904d9..0000000000 --- a/donkeycar/gym/gym_real.py +++ /dev/null @@ -1,89 +0,0 @@ -''' -file: gym_real.py -author: Tawn Kramer -date: 2019-01-24 -desc: Control a real donkey robot via the gym interface -''' -import os -import time - -import gym -import numpy as np -from gym import error, spaces, utils - -from .remote_controller import DonkeyRemoteContoller - - -class DonkeyRealEnv(gym.Env): - """ - OpenAI Gym Environment for a real Donkey - """ - - metadata = { - "render.modes": ["human", "rgb_array"], - } - - ACTION_NAMES = ["steer", "throttle"] - STEER_LIMIT_LEFT = -1.0 - STEER_LIMIT_RIGHT = 1.0 - THROTTLE_MIN = 0.0 - THROTTLE_MAX = 5.0 - VAL_PER_PIXEL = 255 - - def __init__(self, time_step=0.05, frame_skip=2): - - print("starting DonkeyGym env") - - try: - donkey_name = str(os.environ['DONKEY_NAME']) - except: - donkey_name = 'my_robot1234' - print("No DONKEY_NAME environment var. Using default:", donkey_name) - - try: - mqtt_broker = str(os.environ['DONKEY_MQTT_BROKER']) - except: - mqtt_broker = "iot.eclipse.org" - print("No DONKEY_MQTT_BROKER environment var. Using default:", mqtt_broker) - - # start controller - self.controller = DonkeyRemoteContoller(donkey_name=donkey_name, mqtt_broker=mqtt_broker) - - # steering and throttle - self.action_space = spaces.Box(low=np.array([self.STEER_LIMIT_LEFT, self.THROTTLE_MIN]), - high=np.array([self.STEER_LIMIT_RIGHT, self.THROTTLE_MAX]), dtype=np.float32 ) - - # camera sensor data - self.observation_space = spaces.Box(0, self.VAL_PER_PIXEL, self.controller.get_sensor_size(), dtype=np.uint8) - - # Frame Skipping - self.frame_skip = frame_skip - - # wait until loaded - self.controller.wait_until_connected() - - - def close(self): - self.controller.quit() - - def step(self, action): - for i in range(self.frame_skip): - self.controller.take_action(action) - time.sleep(0.05) - observation = self.controller.observe() - reward, done, info = 0.1, False, None - return observation, reward, done, info - - def reset(self): - observation = self.controller.observe() - reward, done, info = 0.1, False, None - return observation - - def render(self, mode="human", close=False): - if close: - self.controller.quit() - - return self.controller.observe() - - def is_game_over(self): - return False diff --git a/donkeycar/gym/remote_controller.py b/donkeycar/gym/remote_controller.py deleted file mode 100755 index 8954d823fa..0000000000 --- a/donkeycar/gym/remote_controller.py +++ /dev/null @@ -1,42 +0,0 @@ -''' -file: remote_controller.py -author: Tawn Kramer -date: 2019-01-24 -desc: Control a remote donkey robot over network -''' - -import time - -from donkeycar.parts.network import MQTTValueSub, MQTTValuePub -from donkeycar.parts.image import JpgToImgArr - -class DonkeyRemoteContoller: - def __init__(self, donkey_name, mqtt_broker, sensor_size=(120, 160, 3)): - self.camera_sub = MQTTValueSub("donkey/%s/camera" % donkey_name, broker=mqtt_broker) - self.controller_pub = MQTTValuePub("donkey/%s/controls" % donkey_name, broker=mqtt_broker) - self.jpgToImg = JpgToImgArr() - self.sensor_size = sensor_size - - def get_sensor_size(self): - return self.sensor_size - - def wait_until_connected(self): - pass - - def take_action(self, action): - self.controller_pub.run(action) - - def quit(self): - self.camera_sub.shutdown() - self.controller_pub.shutdown() - - def get_original_image(self): - return self.img - - def observe(self): - jpg = self.camera_sub.run() - self.img = self.jpgToImg.run(jpg) - return self.img - - - diff --git a/donkeycar/management/base.py b/donkeycar/management/base.py index e906aa4a7e..3c03864e10 100644 --- a/donkeycar/management/base.py +++ b/donkeycar/management/base.py @@ -375,7 +375,7 @@ def get_activations(self, image_path, model_path, cfg): returns activations/features ''' - from tensorflow.python.keras.models import load_model, Model + from tensorflow.keras.models import load_model, Model model_path = os.path.expanduser(model_path) image_path = os.path.expanduser(image_path) diff --git a/donkeycar/management/makemovie.py b/donkeycar/management/makemovie.py index df6106f552..3687cc98a7 100755 --- a/donkeycar/management/makemovie.py +++ b/donkeycar/management/makemovie.py @@ -1,12 +1,13 @@ import tempfile +import logging -from tensorflow.python.keras import activations -from tensorflow.python.keras import backend as K -from tensorflow.python.keras.models import load_model +logging.getLogger('tensorflow').setLevel(logging.WARNING) import tensorflow as tf -import cv2 +from tensorflow.keras import activations +from tensorflow.keras import backend as K +from tensorflow.keras.models import load_model from matplotlib import cm - +import cv2 import donkeycar as dk from donkeycar.parts.tub_v2 import Tub @@ -90,18 +91,15 @@ def run(self, args, parser): if args.type is None and args.model is not None: args.type = self.cfg.DEFAULT_MODEL_TYPE - print("Model type not provided. Using default model type from " - "config file") + print("Model type not provided. Using default model type from config file") if args.salient: if args.model is None: - print("ERR>> salient visualization requires a model. Pass " - "with the --model arg.") + print("ERR>> salient visualization requires a model. Pass with the --model arg.") parser.print_help() if args.type not in ['linear', 'categorical']: - print(f"Model type {args.type} is not supported. Only linear " - f"or categorical is supported for salient visualization") + print("Model type {} is not supported. Only linear or categorical is supported for salient visualization".format(args.type)) parser.print_help() return diff --git a/donkeycar/management/ui/car_screen.kv b/donkeycar/management/ui/car_screen.kv index 3586670144..1a9e57c430 100644 --- a/donkeycar/management/ui/car_screen.kv +++ b/donkeycar/management/ui/car_screen.kv @@ -91,8 +91,8 @@ id: btn_h5 text: 'Sync h5' RoundedToggleButton: - id: btn_savedmodel - text: 'Sync savedmodel' + id: btn_keras + text: 'Sync keras' RoundedToggleButton: id: btn_tflite text: 'Sync tflite' diff --git a/donkeycar/management/ui/car_screen.py b/donkeycar/management/ui/car_screen.py index 4676f6a564..95948429c7 100644 --- a/donkeycar/management/ui/car_screen.py +++ b/donkeycar/management/ui/car_screen.py @@ -66,14 +66,14 @@ def send_pilot(self): # add trailing '/' src = os.path.join(self.config.MODELS_PATH, '') # check if any sync buttons are pressed and update path accordingly - buttons = ['h5', 'savedmodel', 'tflite', 'trt'] + buttons = ['h5', 'keras', 'tflite', 'trt'] select = [btn for btn in buttons if self.ids[f'btn_{btn}'].state == 'down'] - # build filter: for example this rsyncs all .tfilte and .trt models + # build filter: for example this rsyncs all .tflite and .trt models # --include=*.trt/*** --include=*.tflite --exclude=* filter = ['--include=database.json'] for ext in select: - if ext in ['savedmodel', 'trt']: + if ext == 'trt': ext += '/***' filter.append(f'--include=*.{ext}') # if nothing selected, sync all diff --git a/donkeycar/management/ui/pilot_screen.kv b/donkeycar/management/ui/pilot_screen.kv index d04532c10f..dede108359 100644 --- a/donkeycar/management/ui/pilot_screen.kv +++ b/donkeycar/management/ui/pilot_screen.kv @@ -72,7 +72,7 @@ value: 1000 MyLabel: valign: 'center' - text: f'Selected records 0 to {int(slider.value)}' + text: 'Selected records 0 to {}'.format(int(slider.value)) RoundedButton: text: 'Tub plot' on_release: diff --git a/donkeycar/management/ui/pilot_screen.py b/donkeycar/management/ui/pilot_screen.py index bd4bca64a7..8ed155c3ee 100644 --- a/donkeycar/management/ui/pilot_screen.py +++ b/donkeycar/management/ui/pilot_screen.py @@ -18,7 +18,7 @@ from donkeycar.utils import get_model_by_type -ALL_FILTERS = ['*.h5', '*.tflite', '*.savedmodel', '*.trt'] +ALL_FILTERS = ['*.h5', '*.keras', '*.tflite', '*.trt'] class PilotLoader(BoxLayout, FileChooserBase): @@ -66,9 +66,9 @@ def on_model_type(self, obj, model_type): if 'tflite' in self.model_type: self.filters = ['*.tflite'] elif 'tensorrt' in self.model_type: - self.filters = ['*.trt', '*.savedmodel'] + self.filters = ['*.trt'] else: - self.filters = ['*.h5', '*.savedmodel'] + self.filters = ['*.h5', '*.keras'] except Exception as e: status(f'Error: {e}') diff --git a/donkeycar/management/ui/train_screen.py b/donkeycar/management/ui/train_screen.py index 5bdce73e4b..288696376c 100644 --- a/donkeycar/management/ui/train_screen.py +++ b/donkeycar/management/ui/train_screen.py @@ -2,6 +2,7 @@ import os from threading import Thread import json +import traceback import pandas as pd from kivy import Logger @@ -121,7 +122,7 @@ def build_widgets(self, labels): class TransferSelector(BoxLayout, FileChooserBase): """ Class to select transfer model""" - filters = ['*.h5', '*.savedmodel'] + filters = ['*.h5', '*.keras'] class ConfigViewerPopup(Popup): @@ -195,21 +196,26 @@ class TrainScreen(AppScreen): train_checker = False def train_call(self, *args): + try: + import tensorflow as tf + tf.keras.backend.clear_session() + except Exception: + pass tub_path = get_app_screen('tub').ids.tub_loader.tub.base_path transfer = self.ids.transfer_spinner.text model_type = self.ids.train_spinner.text + keras = os.path.join(self.config.MODELS_PATH, transfer + '.keras') h5 = os.path.join(self.config.MODELS_PATH, transfer + '.h5') - sm = os.path.join(self.config.MODELS_PATH, transfer + '.savedmodel') if transfer == 'Choose transfer model': transfer_model = None - elif os.path.exists(sm): - transfer_model = str(sm) + elif os.path.exists(keras): + transfer_model = str(keras) elif os.path.exists(h5): transfer_model = str(h5) else: transfer_model = None - status(f'Could find neither {sm} nor {h5} - training without ' + status(f'Could find neither {keras} nor {h5} - training without ' f'transfer') try: history = train(self.config, tub_paths=tub_path, @@ -217,7 +223,7 @@ def train_call(self, *args): transfer=transfer_model, comment=self.ids.comment.text) except Exception as e: - Logger.error(e) + Logger.error(f'Training error: {e}\n{traceback.format_exc()}') status(f'Training failed see console') def train(self): diff --git a/donkeycar/parts/camera.py b/donkeycar/parts/camera.py index 1377089c7c..228ea0acb4 100644 --- a/donkeycar/parts/camera.py +++ b/donkeycar/parts/camera.py @@ -35,7 +35,7 @@ def __init__(self, image_w=160, image_h=120, image_d=3, self.camera = Picamera2() config = self.camera.create_preview_configuration( config_dict, transform=transform) - self.camera.align_configuration(config) + # self.camera.align_configuration(config) # this created issues with the libcamera2 library on the Pi5, which automatically changes the resolution to 128x120 in an attempt to align with native sensor resolution self.camera.configure(config) # try min / max frame rate as 0.1 / 1 ms (it will be slower though) self.camera.set_controls({"FrameDurationLimits": (100, 1000)}) diff --git a/donkeycar/parts/datastore_v2.py b/donkeycar/parts/datastore_v2.py index c1ae561205..e0a0f40ed4 100644 --- a/donkeycar/parts/datastore_v2.py +++ b/donkeycar/parts/datastore_v2.py @@ -454,6 +454,9 @@ def __init__(self, manifest): self.current_catalog_index = 0 self.current_catalog = None + def __iter__(self): + return self + def __next__(self): while True: if not self.has_catalogs: diff --git a/donkeycar/parts/dgym.py b/donkeycar/parts/dgym.py index fb067d60ad..f782043a48 100644 --- a/donkeycar/parts/dgym.py +++ b/donkeycar/parts/dgym.py @@ -1,7 +1,15 @@ import os import time -import gym -import gym_donkeycar + +try: + import gymnasium as gym + import gym_donkeycar + +except ImportError: + raise ImportError( + "You need to install gymnasium and gym-donkeycar to use the DonkeyGymEnv." + "Please follow the instructions at https://docs.donkeycar.com/guide/deep_learning/simulator/" + ) def is_exe(fpath): @@ -26,7 +34,7 @@ def __init__(self, sim_path, host="127.0.0.1", port=9091, headless=0, env_name=" conf["guid"] = 0 conf["frame_skip"] = 1 self.env = gym.make(env_name, conf=conf) - self.frame = self.env.reset() + self.frame, _ = self.env.reset() self.action = [0.0, 0.0, 0.0] self.running = True self.info = {'pos': (0., 0., 0.), @@ -64,10 +72,10 @@ def delay_buffer(self, frame, info): def update(self): while self.running: if self.delay > 0.0: - current_frame, _, _, current_info = self.env.step(self.action) + current_frame, _, _, _, current_info = self.env.step(self.action) self.delay_buffer(current_frame, current_info) else: - self.frame, _, _, self.info = self.env.step(self.action) + self.frame, _, _, _, self.info = self.env.step(self.action) def run_threaded(self, steering, throttle, brake=None): if steering is None or throttle is None: diff --git a/donkeycar/parts/gps_imu_fusion.py b/donkeycar/parts/gps_imu_fusion.py new file mode 100644 index 0000000000..2c0060c575 --- /dev/null +++ b/donkeycar/parts/gps_imu_fusion.py @@ -0,0 +1,161 @@ +import math +import time +import numpy as np +import logging + +logger = logging.getLogger(__name__) + + +class EKFFusion: + """ + Extended Kalman Filter for fusing GPS (UTM) and IMU (Quaternions/Accel). + Includes an active Accelerometer Bias state to eliminate dead-reckoning drift. + """ + + def __init__(self, debug=False): + self.debug = debug + # State Vector [x, y, yaw, velocity, accel_bias] + self.x = np.zeros((5, 1)) + + # Covariance Matrix (Our uncertainty about the state) + self.P = np.eye(5) * 1.0 + + # Process Noise (Q) - How much do we distrust our predictions? + self.Q = np.diag([ + 0.05, # variance in x + 0.05, # variance in y + 0.05, # variance in yaw + 0.05, # variance in velocity + 0.0001 # variance in accel_bias (Very small! Bias changes very slowly) + ]) + + # Measurement Noise (R) - How much do we distrust the GPS? + self.R = np.diag([ + 10.0, # GPS X variance in meters + 10.0 # GPS Y variance in meters + ]) + + # Measurement Matrix (H) - Maps state to measurement [x, y] + self.H = np.array([ + [1.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0, 0.0] + ]) + + # Low-Pass Filter (EMA) Setup + self.alpha = 0.10 # Lower = smoother data but more lag. Higher = more jagged data but faster. + self.smoothed_x = None + self.smoothed_y = None + + self.last_time = time.time() + self.last_gps = (0.0, 0.0) + self.initialized = False + + def _quat_to_yaw(self, qi, qj, qk, qr): + siny_cosp = 2 * (qr * qk + qi * qj) + cosy_cosp = 1 - 2 * (qj * qj + qk * qk) + return math.atan2(siny_cosp, cosy_cosp) + + def run(self, gps, accel, gyro, quat): + gps_x, gps_y = gps + ax, ay, az = accel + gx, gy, gz = gyro + qi, qj, qk, qr = quat + + # Graceful fallback if sensors are still booting up + if any(v is None for v in [gps_x, gps_y, ax, qi]): + return 0.0, 0.0, 0.0 + + current_time = time.time() + dt = current_time - self.last_time + self.last_time = current_time + + yaw = self._quat_to_yaw(qi, qj, qk, qr) + + # --------------------------------------------------------- + # 1. INITIALIZATION + # --------------------------------------------------------- + if not self.initialized: + if gps_x != 0.0 and gps_y != 0.0: + self.x[0, 0] = gps_x + self.x[1, 0] = gps_y + self.x[2, 0] = yaw + self.x[3, 0] = 0.0 + self.x[4, 0] = 0.0 # Assume 0 bias to start + self.last_gps = (gps_x, gps_y) + self.initialized = True + return gps_x, gps_y, yaw + + # --------------------------------------------------------- + # 2. PREDICTION STEP + # --------------------------------------------------------- + if abs(ay) < 0.2: + ay = 0.0 + + # Calculate true acceleration by subtracting our estimated bias + true_ax = ay - self.x[4, 0] + + # 2. Velocity Decay (Artificial Friction) + # Multiplier slowly bleeds off fake velocity generated by GPS noise + friction_factor = 0.95 + v_pred = (self.x[3, 0] * friction_factor) + (true_ax * dt) + + # Non-linear state transition + self.x[0, 0] = self.x[0, 0] + v_pred * math.cos(self.x[2, 0]) * dt + self.x[1, 0] = self.x[1, 0] + v_pred * math.sin(self.x[2, 0]) * dt + self.x[2, 0] = yaw + self.x[3, 0] = v_pred + # self.x[4, 0] remains unchanged during prediction (constant bias model) + + # Calculate the Jacobian (F) matrix + F = np.eye(5) + F[0, 2] = -v_pred * math.sin(self.x[2, 0]) * dt + F[0, 3] = math.cos(self.x[2, 0]) * dt + F[1, 2] = v_pred * math.cos(self.x[2, 0]) * dt + F[1, 3] = math.sin(self.x[2, 0]) * dt + F[3, 4] = -dt # This links the bias variable to velocity error + + # Update Covariance + self.P = F @ self.P @ F.T + self.Q + + # --------------------------------------------------------- + # 3. UPDATE STEP (Pull position back to reality using GPS) + # --------------------------------------------------------- + if gps_x != 0.0 and gps_y != 0.0 and (gps_x, gps_y) != self.last_gps: + z = np.array([[gps_x], [gps_y]]) + + # Calculate Kalman Gain + S = self.H @ self.P @ self.H.T + self.R + K = self.P @ self.H.T @ np.linalg.inv(S) + + # Residual (Difference between GPS and our Prediction) + y = z - (self.H @ self.x) + + # Correct the state and covariance (this naturally adjusts self.x[4, 0]!) + self.x = self.x + (K @ y) + self.P = (np.eye(5) - (K @ self.H)) @ self.P + + self.last_gps = (gps_x, gps_y) + + # --------------------------------------------------------- + # 4. OUTPUT + # --------------------------------------------------------- + raw_x = float(self.x[0, 0]) + raw_y = float(self.x[1, 0]) + yaw = float(self.x[2, 0]) + + # Initialize the EMA filter on the very first loop + if self.smoothed_x is None or self.smoothed_y is None: + self.smoothed_x = raw_x + self.smoothed_y = raw_y + else: + # Apply the Exponential Moving Average formula to X and Y + self.smoothed_x = (self.alpha * raw_x) + ((1.0 - self.alpha) * self.smoothed_x) + self.smoothed_y = (self.alpha * raw_y) + ((1.0 - self.alpha) * self.smoothed_y) + + if self.debug: + logger.info(f"EKF FUSED: X={self.smoothed_x:.4f}, Y={self.smoothed_y:.4f}, Yaw={yaw:.4f}") + + return self.smoothed_x, self.smoothed_y, yaw + + def shutdown(self): + pass \ No newline at end of file diff --git a/donkeycar/parts/imu.py b/donkeycar/parts/imu.py index 4690fc5e46..2e2ee84d87 100755 --- a/donkeycar/parts/imu.py +++ b/donkeycar/parts/imu.py @@ -1,15 +1,17 @@ #!/usr/bin/env python3 import time + SENSOR_MPU6050 = 'mpu6050' SENSOR_MPU9250 = 'mpu9250' DLP_SETTING_DISABLED = 0 CONFIG_REGISTER = 0x1A + class IMU: ''' Installation: - + - MPU6050 sudo apt install python3-smbus or @@ -20,10 +22,10 @@ class IMU: sudo python setup.py install pip install mpu6050-raspberrypi - + - MPU9250 pip install mpu9250-jmdev - + ''' def __init__(self, addr=0x68, poll_delay=0.0166, sensor=SENSOR_MPU6050, dlp_setting=DLP_SETTING_DISABLED): @@ -31,10 +33,10 @@ def __init__(self, addr=0x68, poll_delay=0.0166, sensor=SENSOR_MPU6050, dlp_sett if self.sensortype == SENSOR_MPU6050: from mpu6050 import mpu6050 as MPU6050 self.sensor = MPU6050(addr) - - if(dlp_setting > 0): + + if (dlp_setting > 0): self.sensor.bus.write_byte_data(self.sensor.address, CONFIG_REGISTER, dlp_setting) - + else: from mpu9250_jmdev.registers import AK8963_ADDRESS, GFS_1000, AFS_4G, AK8963_BIT_16, AK8963_MODE_C100HZ from mpu9250_jmdev.mpu_9250 import MPU9250 @@ -48,17 +50,18 @@ def __init__(self, addr=0x68, poll_delay=0.0166, sensor=SENSOR_MPU6050, dlp_sett afs=AFS_4G, mfs=AK8963_BIT_16, mode=AK8963_MODE_C100HZ) - - if(dlp_setting > 0): + + if (dlp_setting > 0): self.sensor.writeSlave(CONFIG_REGISTER, dlp_setting) self.sensor.calibrateMPU6500() self.sensor.configure() - - self.accel = { 'x' : 0., 'y' : 0., 'z' : 0. } - self.gyro = { 'x' : 0., 'y' : 0., 'z' : 0. } - self.mag = {'x': 0., 'y': 0., 'z': 0.} - self.temp = 0. + # self.accel = {'x': 0., 'y': 0., 'z': 0.} + # self.gyro = {'x': 0., 'y': 0., 'z': 0.} + # self.quat = {'i': 0., 'j': 0., 'k': 0., 'real': 0.} + self.accel = (0., 0., 0.) + self.gyro = (0., 0., 0.) + self.quat = (0., 0., 0., 0.) self.poll_delay = poll_delay self.on = True @@ -66,7 +69,7 @@ def update(self): while self.on: self.poll() time.sleep(self.poll_delay) - + def poll(self): try: if self.sensortype == SENSOR_MPU6050: @@ -74,19 +77,91 @@ def poll(self): else: from mpu9250_jmdev.registers import GRAVITY ret = self.sensor.getAllData() - self.accel = { 'x' : ret[1] * GRAVITY, 'y' : ret[2] * GRAVITY, 'z' : ret[3] * GRAVITY } - self.gyro = { 'x' : ret[4], 'y' : ret[5], 'z' : ret[6] } - self.mag = { 'x' : ret[13], 'y' : ret[14], 'z' : ret[15] } + self.accel = (ret[1] * GRAVITY, ret[2] * GRAVITY, ret[3] * GRAVITY) + self.gyro = (ret[4], ret[5], ret[6]) + self.mag = (ret[13], ret[14], ret[15]) self.temp = ret[16] except: print('failed to read imu!!') - + def run_threaded(self): - return self.accel['x'], self.accel['y'], self.accel['z'], self.gyro['x'], self.gyro['y'], self.gyro['z'], self.temp + return self.accel, self.gyro, self.temp def run(self): self.poll() - return self.accel['x'], self.accel['y'], self.accel['z'], self.gyro['x'], self.gyro['y'], self.gyro['z'], self.temp + return self.accel, self.gyro, self.temp + + def shutdown(self): + self.on = False + + +class Bno08xIMU: + """ + Installation: + pip install adafruit-circuitpython-bno08x + """ + + def __init__(self, poll_delay=0.0166, addr=0x4A): + import board + import busio + from adafruit_bno08x.i2c import BNO08X_I2C + from adafruit_bno08x import BNO_REPORT_ACCELEROMETER, BNO_REPORT_GYROSCOPE, BNO_REPORT_ROTATION_VECTOR + + # Initialize I2C bus + self.i2c = busio.I2C(board.SCL, board.SDA) + self.sensor = BNO08X_I2C(self.i2c, address=addr) + + # Explicitly enable the streams needed for navigation and fusion + self.sensor.enable_feature(BNO_REPORT_ACCELEROMETER) + self.sensor.enable_feature(BNO_REPORT_GYROSCOPE) + self.sensor.enable_feature(BNO_REPORT_ROTATION_VECTOR) + + # self.accel = {'x': 0., 'y': 0., 'z': 0.} + # self.gyro = {'x': 0., 'y': 0., 'z': 0.} + # self.quat = {'i': 0., 'j': 0., 'k': 0., 'real': 0.} + self.accel = (0., 0., 0.) + self.gyro = (0., 0., 0.) + self.quat = (0., 0., 0., 0.) + self.poll_delay = poll_delay + self.on = True + + def update(self): + import time + while self.on: + self.poll() + time.sleep(self.poll_delay) + + def poll(self): + try: + ax, ay, az = self.sensor.acceleration + self.accel = (ax, ay, az) + + gx, gy, gz = self.sensor.gyro + self.gyro = (gx, gy, gz) + + # Quaternions are ideal for avoiding gimbal lock during fusion + qi, qj, qk, qr = self.sensor.quaternion + self.quat = (qi, qj, qk, qr) + except Exception as e: + print(f"Failed to read BNO08x: {e}") + + # Instead of returning flat scalars: + # return self.accel['x'], self.accel['y'], self.accel['z'], self.gyro['x'] ... + + def run_threaded(self): + accel = self.accel # (ax, ay, az) + gyro = self.gyro # (gx, gy, gz) + + # If using the BNO08x, include the quaternion tuple as well + if hasattr(self, 'quat'): + quat = self.quat # (qi, qj, qk, qr) + return accel, gyro, quat + + return accel, gyro, self.temp + + def run(self): + self.poll() + return self.run_threaded() def shutdown(self): self.on = False @@ -95,17 +170,21 @@ def shutdown(self): if __name__ == "__main__": iter = 0 import sys - sensor_type = SENSOR_MPU6050 + + sensor_type = SENSOR_MPU6050 dlp_setting = DLP_SETTING_DISABLED if len(sys.argv) > 1: sensor_type = sys.argv[1] if len(sys.argv) > 2: dlp_setting = int(sys.argv[2]) - p = IMU(sensor=sensor_type) + if sensor_type.lower() == 'bno08x': + p = Bno08xIMU() + else: + p = IMU(sensor=sensor_type) + while iter < 100: data = p.run() print(data) time.sleep(0.1) iter += 1 - diff --git a/donkeycar/parts/interpreter.py b/donkeycar/parts/interpreter.py index 8af44e7480..2c10e7fc00 100755 --- a/donkeycar/parts/interpreter.py +++ b/donkeycar/parts/interpreter.py @@ -1,18 +1,50 @@ +import importlib.metadata import os +import sys from abc import ABC, abstractmethod import logging import numpy as np -from typing import Union, Sequence, List +from typing import Union, Sequence -import tensorflow as tf -from tensorflow import keras -from tensorflow.python.saved_model import tag_constants, signature_constants -from tensorflow.python.compiler.tensorrt import trt_convert as trt +try: + import tensorflow as tf + from tensorflow import keras + logging.getLogger('tensorflow').setLevel(logging.WARNING) + try: + import tensorflow.compiler.tf2tensorrt.wrap_py_utils as trt + except ImportError: + trt = None +except ImportError: + tf = None + keras = None + trt = None logger = logging.getLogger(__name__) +def _is_metal_installed() -> bool: + if sys.platform != 'darwin': + return False + try: + importlib.metadata.version('tensorflow-metal') + return True + except importlib.metadata.PackageNotFoundError: + return False + +def get_tflite_interpreter(): + try: + from ai_edge_litert.interpreter import Interpreter + return Interpreter + except ImportError: + pass + if tf is not None: + return tf.lite.Interpreter + raise ImportError('No TFLite interpreter found. Install ai-edge-litert.') + + def has_trt_support(): + if trt is None: + return False try: converter = trt.TrtGraphConverterV2() return True @@ -29,7 +61,19 @@ def keras_model_to_tflite(in_filename, out_filename, data_gen=None): def keras_to_tflite(model, out_filename, data_gen=None): - converter = tf.lite.TFLiteConverter.from_keras_model(model) + # from_keras_model is broken with Keras 3.x; use tf.function + + # from_concrete_functions which bypasses the problematic Keras export path + input_sig = [tf.TensorSpec(shape=(1,) + tuple(inp.shape[1:]), + dtype=tf.float32, name=inp.name) + for inp in model.inputs] + if len(input_sig) == 1: + tf_func = tf.function(model, input_signature=input_sig) + else: + tf_func = tf.function(lambda *args: model(list(args)), + input_signature=input_sig) + concrete = tf_func.get_concrete_function() + converter = tf.lite.TFLiteConverter.from_concrete_functions( + [concrete], tf_func) converter.target_spec.supported_ops = [tf.lite.OpsSet.TFLITE_BUILTINS, tf.lite.OpsSet.SELECT_TF_OPS] converter.allow_custom_ops = True @@ -91,14 +135,14 @@ def set_model(self, pilot: 'KerasPilot') -> None: """ Some interpreters will need the model""" pass - def set_optimizer(self, optimizer: tf.keras.optimizers.Optimizer) -> None: + def set_optimizer(self, optimizer) -> None: pass def compile(self, **kwargs): raise NotImplementedError('Requires implementation') @abstractmethod - def get_input_shape(self, input_name) -> tf.TensorShape: + def get_input_shape(self, input_name): pass def predict(self, img_arr: np.ndarray, *other_arr: np.ndarray) \ @@ -115,7 +159,7 @@ def predict(self, img_arr: np.ndarray, *other_arr: np.ndarray) \ def predict_from_dict(self, input_dict) -> Sequence[Union[float, np.ndarray]]: pass - def summary(self) -> str: + def summary(self) -> None: pass def __str__(self) -> str: @@ -127,7 +171,7 @@ class KerasInterpreter(Interpreter): def __init__(self): super().__init__() - self.model: tf.keras.Model = None + self.model = None def set_model(self, pilot: 'KerasPilot') -> None: self.model = pilot.create_model() @@ -141,22 +185,38 @@ def set_model(self, pilot: 'KerasPilot') -> None: if type(output_shape) is not list: output_shape = [output_shape] - self.input_keys = self.model.input_names + self.input_keys = [inp.name for inp in self.model.inputs] self.output_keys = self.model.output_names self.shapes = (dict(zip(self.input_keys, input_shape)), dict(zip(self.output_keys, output_shape))) - def set_optimizer(self, optimizer: tf.keras.optimizers.Optimizer) -> None: + def set_optimizer(self, optimizer) -> None: self.model.optimizer = optimizer - def get_input_shape(self, input_name) -> tf.TensorShape: + def get_input_shape(self, input_name): assert self.model, 'Model not set' return self.shapes[0][input_name] def compile(self, **kwargs): assert self.model, 'Model not set' + kwargs.setdefault('jit_compile', False) + if _is_metal_installed(): + # Safe production fallback: the current compiled Metal training + # path is unsafe across the covered optimizers on the real + # workload, so force eager execution until regression tests + # prove a compiled path is both correct and faster. + kwargs.setdefault('run_eagerly', True) self.model.compile(**kwargs) + def fit(self, x, steps_per_epoch, batch_size, callbacks, + validation_data, validation_steps, epochs, verbose): + return self.model.fit( + x=x, steps_per_epoch=steps_per_epoch, + batch_size=batch_size, callbacks=callbacks, + validation_data=validation_data, + validation_steps=validation_steps, + epochs=epochs, verbose=verbose) + def predict_from_dict(self, input_dict): for k, v in input_dict.items(): input_dict[k] = self.expand_and_convert(v) @@ -172,16 +232,20 @@ def predict_from_dict(self, input_dict): return outputs.numpy().squeeze(axis=0) def load(self, model_path: str) -> None: - logger.info(f'Loading model {model_path}') self.model = keras.models.load_model(model_path, compile=False) + # composite model output names can be lost when exporting to SavedModel + # for TRT; overwrite them from the pilot's declared output_keys + logger.info(f'Loading model {model_path} and overwriting model output ' + f'names {self.model.output_names} with {self.output_keys}') + self.model.output_names = self.output_keys def load_weights(self, model_path: str, by_name: bool = True) -> \ None: assert self.model, 'Model not set' self.model.load_weights(model_path, by_name=by_name) - def summary(self) -> str: - return self.model.summary() + def summary(self) -> None: + self.model.summary(expand_nested=True, show_trainable=True) @staticmethod def expand_and_convert(arr): @@ -243,9 +307,6 @@ def load(self, model_path: str) -> None: logger.info(self.model) self.model.eval() - def summary(self) -> str: - return self.model - class TfLite(Interpreter): """ @@ -258,16 +319,39 @@ def __init__(self): self.runner = None self.signatures = None + @staticmethod + def _normalise_tensor_name(raw): + return raw.removeprefix('serving_default_').removesuffix(':0') + def load(self, model_path): assert os.path.splitext(model_path)[1] == '.tflite', \ 'TFlitePilot should load only .tflite files' logger.info(f'Loading model {model_path}') - # Load TFLite model and extract input and output keys - self.interpreter = tf.lite.Interpreter(model_path=model_path) + TfliteInterpreter = get_tflite_interpreter() + self.interpreter = TfliteInterpreter(model_path=model_path) self.signatures = self.interpreter.get_signature_list() + if not self.signatures: + return self._load_via_tensor_api() self.runner = self.interpreter.get_signature_runner() - self.input_keys = self.signatures['serving_default']['inputs'] - self.output_keys = self.signatures['serving_default']['outputs'] + self.input_keys = list( + self.signatures['serving_default']['inputs']) + self.output_keys = list( + self.signatures['serving_default']['outputs']) + + def _load_via_tensor_api(self): + logger.info( + 'No TFLite signatures found; using tensor API fallback') + self.runner = None + self.interpreter.allocate_tensors() + in_details = self.interpreter.get_input_details() + out_details = sorted(self.interpreter.get_output_details(), + key=lambda d: d['index']) + self.input_keys = [ + self._normalise_tensor_name(d['name']) for d in in_details] + self._input_index_map = { + k: d['index'] for k, d in zip(self.input_keys, in_details)} + self._output_indices = [d['index'] for d in out_details] + self.output_keys = [str(i) for i in range(len(out_details))] def compile(self, **kwargs): pass @@ -275,17 +359,35 @@ def compile(self, **kwargs): def predict_from_dict(self, input_dict): for k, v in input_dict.items(): input_dict[k] = self.expand_and_convert(v) + if self.runner is not None: + return self._predict_via_runner(input_dict) + return self._predict_via_tensor_api(input_dict) + + def _predict_via_runner(self, input_dict): outputs = self.runner(**input_dict) ret = list(outputs[k][0] for k in self.output_keys) return ret if len(ret) > 1 else ret[0] + def _predict_via_tensor_api(self, input_dict): + for k, v in input_dict.items(): + self.interpreter.set_tensor(self._input_index_map[k], v) + self.interpreter.invoke() + ret = [self.interpreter.get_tensor(i)[0] + for i in self._output_indices] + return ret if len(ret) > 1 else ret[0] + def get_input_shape(self, input_name): - assert self.interpreter is not None, "Need to load tflite model first" + assert self.interpreter is not None, \ + "Need to load tflite model first" details = self.interpreter.get_input_details() - for detail in details: - if detail['name'] == f"serving_default_{input_name}:0": - return detail['shape'] - raise RuntimeError(f'{input_name} not found in TFlite model') + match = next( + (d for d in details + if self._normalise_tensor_name(d['name']) == input_name), + None) + if match is None: + raise RuntimeError( + f'{input_name} not found in TFlite model') + return match['shape'] @staticmethod def expand_and_convert(arr): @@ -312,7 +414,7 @@ def set_model(self, pilot: 'KerasPilot') -> None: # state as the trt model hasn't been loaded yet self.pilot = pilot - def get_input_shape(self, input_name) -> tf.TensorShape: + def get_input_shape(self, input_name): assert self.graph_func, "Requires loadin the tensorrt model first" return self.graph_func.structured_input_signature[1][input_name].shape @@ -323,23 +425,13 @@ def load(self, model_path: str) -> None: logger.info(f'Loading TensorRT model {model_path}') assert self.pilot, "Need to set pilot first" try: - ext = os.path.splitext(model_path)[1] - if ext == '.savedmodel': - # first load tf model format to extract input and output keys - model = tf.keras.models.load_model(model_path, compile=False) - self.input_keys = model.input_names - self.output_keys = model.output_names - converter \ - = trt.TrtGraphConverterV2(input_saved_model_dir=model_path) - self.graph_func = converter.convert() - else: - trt_model_loaded = tf.saved_model.load( - model_path, tags=[tag_constants.SERVING]) - self.graph_func = trt_model_loaded.signatures[ - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY] - inputs, outputs = self.pilot.output_shapes() - self.input_keys = list(inputs.keys()) - self.output_keys = list(outputs.keys()) + trt_model_loaded = tf.saved_model.load( + model_path, tags=[tf.saved_model.SERVING]) + self.graph_func = trt_model_loaded.signatures[ + tf.saved_model.DEFAULT_SERVING_SIGNATURE_DEF_KEY] + inputs, outputs = self.pilot.output_shapes() + self.input_keys = list(inputs.keys()) + self.output_keys = list(outputs.keys()) logger.info(f'Finished loading TensorRT model.') except Exception as e: logger.error(f'Could not load TensorRT model because: {e}') @@ -347,12 +439,12 @@ def load(self, model_path: str) -> None: def predict_from_dict(self, input_dict): for k, v in input_dict.items(): input_dict[k] = self.expand_and_convert(v) - out_list = self.graph_func(**input_dict) + out_dict = self.graph_func(**input_dict) # Squeeze here because we send a batch of size one, so pick first # element. To return the order of outputs as defined in the model we # need to iterate through the model's output shapes here - outputs = [k.numpy().squeeze(axis=0) for k in out_list] - + outputs = [out_dict[k].numpy().squeeze(axis=0) for k in + self.output_keys] # don't return list if output is 1d return outputs if len(outputs) > 1 else outputs[0] diff --git a/donkeycar/parts/keras.py b/donkeycar/parts/keras.py index 9c89f81a53..e3955d62f1 100644 --- a/donkeycar/parts/keras.py +++ b/donkeycar/parts/keras.py @@ -8,38 +8,56 @@ """ import datetime +from datetime import datetime +from os import path from abc import ABC, abstractmethod from collections import deque import numpy as np from typing import Dict, Tuple, Optional, Union, List, Sequence, Callable, Any -from logging import getLogger - -from tensorflow.python.data.ops.dataset_ops import DatasetV1, DatasetV2 +import logging import donkeycar as dk from donkeycar.utils import normalize_image, linear_bin from donkeycar.pipeline.types import TubRecord -from donkeycar.parts.interpreter import Interpreter, KerasInterpreter +from donkeycar.parts.interpreter import ( + Interpreter, + KerasInterpreter, + _is_metal_installed, +) + +try: + import tensorflow as tf + from tensorflow import keras + from tensorflow.keras.layers import (Dense, Input, Convolution2D, + MaxPooling2D, Activation, Dropout, Flatten, LSTM, BatchNormalization, + Conv3D, MaxPooling3D, Conv2DTranspose) + from tensorflow.keras.layers import TimeDistributed as TD + from tensorflow.keras.layers import concatenate + from tensorflow.keras.models import Model + from tensorflow.keras.callbacks import (EarlyStopping, ModelCheckpoint, + TensorBoard) + logging.getLogger('tensorflow').setLevel(logging.WARNING) +except ImportError: + tf = None + keras = None -import tensorflow as tf -from tensorflow import keras -from tensorflow.keras.layers import (Dense, Input,Convolution2D, - MaxPooling2D, Activation, Dropout, Flatten, LSTM, BatchNormalization, - Conv3D, MaxPooling3D, Conv2DTranspose) +ONE_BYTE_SCALE = 1.0 / 255.0 -from tensorflow.keras.layers import TimeDistributed as TD -from tensorflow.keras.backend import concatenate -from tensorflow.keras.models import Model -from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint -ONE_BYTE_SCALE = 1.0 / 255.0 +def _tshape(shape): + """Shape helper for LiteRT-only Pi inference when TensorFlow is absent.""" + if isinstance(shape, (int, np.integer)): + shape = (int(shape),) + else: + shape = tuple(shape) + return tf.TensorShape(shape) if tf is not None else shape # type of x XY = Union[float, np.ndarray, Tuple[Union[float, np.ndarray], ...]] -logger = getLogger(__name__) +logger = logging.getLogger(__name__) class KerasPilot(ABC): @@ -76,22 +94,29 @@ def create_model(self): def set_optimizer(self, optimizer_type: str, rate: float, decay: float) -> None: + # decay is emulated via ExponentialDecay (Keras 3 removed the + # per-step decay kwarg from optimizer constructors). + lr = keras.optimizers.schedules.ExponentialDecay( + rate, decay_steps=1, decay_rate=1 - decay) if decay else rate if optimizer_type == "adam": - optimizer = keras.optimizers.Adam(lr=rate, decay=decay) + optimizer = keras.optimizers.Adam(learning_rate=lr) elif optimizer_type == "sgd": - optimizer = keras.optimizers.SGD(lr=rate, decay=decay) + optimizer = keras.optimizers.SGD(learning_rate=lr) elif optimizer_type == "rmsprop": - optimizer = keras.optimizers.RMSprop(lr=rate, decay=decay) + optimizer = keras.optimizers.RMSprop(learning_rate=lr) else: raise Exception(f"Unknown optimizer type: {optimizer_type}") self.interpreter.set_optimizer(optimizer) - def get_input_shape(self, input_name) -> tf.TensorShape: + def get_input_shape(self, input_name): return self.interpreter.get_input_shape(input_name) def seq_size(self) -> int: return 0 + def use_lap_pct(self) -> bool: + return False + def run(self, img_arr: np.ndarray, *other_arr: List[float]) \ -> Tuple[Union[float, np.ndarray], ...]: """ @@ -137,22 +162,32 @@ def interpreter_to_output( def train(self, model_path: str, - train_data: Union[DatasetV1, DatasetV2], + train_data: Any, train_steps: int, batch_size: int, - validation_data: Union[DatasetV1, DatasetV2], + validation_data: Any, validation_steps: int, epochs: int, verbose: int = 1, min_delta: float = .0005, patience: int = 5, - show_plot: bool = False) -> tf.keras.callbacks.History: + show_plot: bool = False): """ trains the model """ assert isinstance(self.interpreter, KerasInterpreter) + dev = tf.config.list_physical_devices('GPU') + if not dev: + logger.warning("No GPU found for training") + model = self.interpreter.model self.compile() + if _is_metal_installed() and batch_size < 512: + logger.info( + 'Metal eager training may need BATCH_SIZE >= 512 ' + 'for a clear speedup over CPU. Current BATCH_SIZE=%s.', + batch_size, + ) callbacks = [ EarlyStopping(monitor='val_loss', @@ -163,9 +198,18 @@ def train(self, save_best_only=True, verbose=verbose)] - tic = datetime.datetime.now() + # Create a TensorBoard callback + log_path = path.join(path.dirname(model_path), + "logs", datetime.now().strftime("%Y%m%d-%H%M%S")) + board = TensorBoard( + log_dir=log_path, + histogram_freq=1, + profile_batch=(0, 4)) + callbacks.append(board) + + tic = datetime.now() logger.info('////////// Starting training //////////') - history: tf.keras.callbacks.History = model.fit( + history = self.interpreter.fit( x=train_data, steps_per_epoch=train_steps, batch_size=batch_size, @@ -173,10 +217,8 @@ def train(self, validation_data=validation_data, validation_steps=validation_steps, epochs=epochs, - verbose=verbose, - workers=1, - use_multiprocessing=False) - toc = datetime.datetime.now() + verbose=verbose) + toc = datetime.now() logger.info(f'////////// Finished training in: {toc - tic} //////////') if show_plot: @@ -231,10 +273,17 @@ def x_transform( def y_transform(self, record: Union[TubRecord, List[TubRecord]]) \ -> Dict[str, Union[float, List[float]]]: """ Transforms the record into dictionary for y for training the - model to x,y. All model ouputs layer's names must be matched by + model to x,y. All model outputs layer's names must be matched by dictionary keys. """ - raise NotImplementedError(f'{self} not ready yet for new training ' - f'pipeline') + raise NotImplementedError(f'y_transform for {self} not implemented') + + def w_transform(self, records: Union[TubRecord, List[TubRecord]]) \ + -> Dict[str, Union[float, List[float]]]: + """ Transforms the record into dictionary for weights for training the + model to x,y. All model outputs layer's names must be matched by + dictionary keys, i.e the format of the dictionary here has to match + the output of y_transform. """ + raise NotImplementedError(f'w_transform for {self} not implemented') def output_types(self) -> Tuple[Dict[str, np.typename], ...]: """ Used in tf.data, assume all types are doubles""" @@ -242,13 +291,36 @@ def output_types(self) -> Tuple[Dict[str, np.typename], ...]: types = tuple({k: tf.float64 for k in d} for d in shapes) return types - def output_shapes(self) -> Dict[str, tf.TensorShape]: + def output_shapes(self) -> dict: return {} def __str__(self) -> str: """ For printing model initialisation """ return type(self).__name__ + def get_num_last_layers_to_train(self): + """ Find the canonically named Flatten layer and return number of + layers after that""" + assert isinstance(self.interpreter, KerasInterpreter) and \ + self.interpreter.model, "Wrong interpreter or no model set" + i = 0 + while self.interpreter.model.layers[i].name != 'flattened': + i += 1 + return len(self.interpreter.model.layers) - i - 1 + + def freeze_first_layers(self, num_last_layers_to_train=None): + if num_last_layers_to_train is None: + num_last_layers_to_train = self.get_num_last_layers_to_train() + assert isinstance(self.interpreter, KerasInterpreter), \ + 'Can only freeze layers in Keras model but not in TfLite and others' + num_to_freeze = len(self.interpreter.model.layers) - num_last_layers_to_train + frozen_layers = [] + for i in range(num_to_freeze): + self.interpreter.model.layers[i].trainable = False + frozen_layers.append(self.interpreter.model.layers[i].name) + logger.info(f'Freezing layers {frozen_layers}') + return num_to_freeze + class KerasCategorical(KerasPilot): """ @@ -275,9 +347,10 @@ def create_model(self): return default_categorical(self.input_shape) def compile(self): + # Keras 3.x requires one metric per output for multi-output models self.interpreter.compile( optimizer=self.optimizer, - metrics=['accuracy'], + metrics=['accuracy', 'accuracy'], loss={'angle_out': 'categorical_crossentropy', 'throttle_out': 'categorical_crossentropy'}, loss_weights={'angle_out': 0.5, 'throttle_out': 0.5}) @@ -302,9 +375,9 @@ def y_transform(self, record: Union[TubRecord, List[TubRecord]]) \ def output_shapes(self): # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] - shapes = ({'img_in': tf.TensorShape(img_shape)}, - {'angle_out': tf.TensorShape([15]), - 'throttle_out': tf.TensorShape([20])}) + shapes = ({'img_in': _tshape(img_shape)}, + {'angle_out': _tshape([15]), + 'throttle_out': _tshape([20])}) return shapes def __str__(self) -> str: @@ -346,9 +419,9 @@ def y_transform(self, record: Union[TubRecord, List[TubRecord]]) \ def output_shapes(self): # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] - shapes = ({'img_in': tf.TensorShape(img_shape)}, - {'n_outputs0': tf.TensorShape([]), - 'n_outputs1': tf.TensorShape([])}) + shapes = ({'img_in': _tshape(img_shape)}, + {'n_outputs0': _tshape([]), + 'n_outputs1': _tshape([])}) return shapes @@ -433,16 +506,16 @@ def y_transform(self, records: Union[TubRecord, List[TubRecord]]) \ def output_shapes(self): # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] - shapes = ({'img_in': tf.TensorShape(img_shape), - 'mem_in': tf.TensorShape(2 * self.mem_length)}, - {'n_outputs0': tf.TensorShape([]), - 'n_outputs1': tf.TensorShape([])}) + shapes = ({'img_in': _tshape(img_shape), + 'mem_in': _tshape(2 * self.mem_length)}, + {'n_outputs0': _tshape([]), + 'n_outputs1': _tshape([])}) return shapes def __str__(self) -> str: """ For printing model initialisation """ - return super().__str__() \ - + f'-L:{self.mem_length}-D:{self.mem_depth}' + return super().__str__() + f'-L:{self.mem_length}-D' + \ + f':{self.mem_depth}-SS:{self.mem_start_speed}' class KerasInferred(KerasPilot): @@ -462,17 +535,18 @@ def interpreter_to_output(self, interpreter_out): return steering, dk.utils.throttle(steering) def y_transform(self, record: Union[TubRecord, List[TubRecord]]) \ - -> Dict[str, Union[float, List[float]]]: + -> Union[float, Dict[str, Union[float, List[float]]]]: assert isinstance(record, TubRecord), "TubRecord expected" - angle: float = record.underlying['user/angle'] - return {'n_outputs0': angle} + # Keras 3.x: single-output models require raw value, not dict + return record.underlying['user/angle'] def output_shapes(self): - # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] - shapes = ({'img_in': tf.TensorShape(img_shape)}, - {'n_outputs0': tf.TensorShape([])}) - return shapes + # Keras 3.x: single-output shape is a TensorShape, not a dict + return ({'img_in': _tshape(img_shape)}, _tshape([])) + + def output_types(self): + return ({'img_in': tf.float64}, tf.float64) class KerasIMU(KerasPilot): @@ -527,10 +601,10 @@ def output_shapes(self): # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] # the keys need to match the models input/output layers - shapes = ({'img_in': tf.TensorShape(img_shape), - 'imu_in': tf.TensorShape([self.num_imu_inputs])}, - {'out_0': tf.TensorShape([]), - 'out_1': tf.TensorShape([])}) + shapes = ({'img_in': _tshape(img_shape), + 'imu_in': _tshape([self.num_imu_inputs])}, + {'out_0': _tshape([]), + 'out_1': _tshape([])}) return shapes @@ -566,10 +640,10 @@ def output_shapes(self): # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] # the keys need to match the models input/output layers - shapes = ({'img_in': tf.TensorShape(img_shape), - 'xbehavior_in': tf.TensorShape([self.num_behavior_inputs])}, - {'angle_out': tf.TensorShape([15]), - 'throttle_out': tf.TensorShape([20])}) + shapes = ({'img_in': _tshape(img_shape), + 'xbehavior_in': _tshape([self.num_behavior_inputs])}, + {'angle_out': _tshape([15]), + 'throttle_out': _tshape([20])}) return shapes @@ -590,7 +664,9 @@ def create_model(self): input_shape=self.input_shape) def compile(self): - self.interpreter.compile(optimizer=self.optimizer, metrics=['acc'], + # Keras 3.x requires metrics per output for multi-output models + self.interpreter.compile(optimizer=self.optimizer, + metrics={'zloc': 'accuracy'}, loss='mse') def interpreter_to_output(self, interpreter_out) \ @@ -613,10 +689,10 @@ def output_shapes(self): # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] # the keys need to match the models input/output layers - shapes = ({'img_in': tf.TensorShape(img_shape)}, - {'angle': tf.TensorShape([]), - 'throttle': tf.TensorShape([]), - 'zloc': tf.TensorShape([self.num_locations])}) + shapes = ({'img_in': _tshape(img_shape)}, + {'angle': _tshape([]), + 'throttle': _tshape([]), + 'zloc': _tshape([self.num_locations])}) return shapes @@ -658,12 +734,13 @@ def x_transform( return {'img_in': np.array(img_arrays)} def y_transform(self, records: Union[TubRecord, List[TubRecord]]) \ - -> Dict[str, Union[float, List[float]]]: + -> Union[np.ndarray, Dict[str, Union[float, List[float]]]]: """ Only return the last entry of angle/throttle""" assert isinstance(records, list), 'List[TubRecord] expected' angle = records[-1].underlying['user/angle'] throttle = records[-1].underlying['user/throttle'] - return {'model_outputs': [angle, throttle]} + # Keras 3.x: single-output models require raw array, not dict + return np.array([angle, throttle]) def run(self, img_arr, *other_arr): if img_arr.shape[2] == 3 and self.input_shape[2] == 1: @@ -687,12 +764,13 @@ def interpreter_to_output(self, interpreter_out) \ return steering, throttle def output_shapes(self): - # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] - # the keys need to match the models input/output layers - shapes = ({'img_in': tf.TensorShape(img_shape)}, - {'model_outputs': tf.TensorShape([self.num_outputs])}) - return shapes + # Keras 3.x: single-output shape is a TensorShape, not a dict + return ({'img_in': _tshape(img_shape)}, + _tshape([self.num_outputs])) + + def output_types(self): + return ({'img_in': tf.float64}, tf.float64) def __str__(self) -> str: """ For printing model initialisation """ @@ -735,12 +813,13 @@ def x_transform( return {'img_in': np.array(img_seq)} def y_transform(self, records: Union[TubRecord, List[TubRecord]]) \ - -> Dict[str, Union[float, List[float]]]: + -> Union[np.ndarray, Dict[str, Union[float, List[float]]]]: """ Only return the last entry of angle/throttle""" assert isinstance(records, list), 'List[TubRecord] expected' angle = records[-1].underlying['user/angle'] throttle = records[-1].underlying['user/throttle'] - return {'outputs': [angle, throttle]} + # Keras 3.x: single-output models require raw array, not dict + return np.array([angle, throttle]) def run(self, img_arr, *other_arr): if img_arr.shape[2] == 3 and self.input_shape[2] == 1: @@ -764,12 +843,13 @@ def interpreter_to_output(self, interpreter_out) \ return steering, throttle def output_shapes(self): - # need to cut off None from [None, 120, 160, 3] tensor shape img_shape = self.get_input_shape('img_in')[1:] - # the keys need to match the models input/output layers - shapes = ({'img_in': tf.TensorShape(img_shape)}, - {'outputs': tf.TensorShape([self.num_outputs])}) - return shapes + # Keras 3.x: single-output shape is a TensorShape, not a dict + return ({'img_in': _tshape(img_shape)}, + _tshape([self.num_outputs])) + + def output_types(self): + return ({'img_in': tf.float64}, tf.float64) class KerasLatent(KerasPilot): diff --git a/donkeycar/parts/oak_d.py b/donkeycar/parts/oak_d.py new file mode 100644 index 0000000000..862eeb2d43 --- /dev/null +++ b/donkeycar/parts/oak_d.py @@ -0,0 +1,415 @@ +""" +Author: Brian Henry & Manav Gagvani +File: oak_d.py +Date: February 13 2022, revised July 10, 2025 +Notes: + Based on realsense435i.py by Ed Murphy: https://github.com/autorope/donkeycar/blob/454be3068ea5dfbac226c3be4d84b0a61d1cec84/donkeycar/parts/realsense435i.py + Based on https://github.com/luxonis/depthai-tutorials/blob/d571473911f876b0d4ac52b7ffdc0fb2beae1641/1-hello-world/hello_world.py + + https://docs.luxonis.com/en/latest/pages/tutorials/first_steps/#first-steps-with-depthai + > If you are using a Linux system, in most cases you have to add a new udev rule for our script to be able to access the device correctly. You can add and apply new rules by running + $ echo 'SUBSYSTEM=="usb", ATTRS{idVendor}=="03e7", MODE="0666"' | sudo tee /etc/udev/rules.d/80-movidius.rules + $ sudo udevadm control --reload-rules && sudo udevadm trigger + (or: "RuntimeError: No DepthAI (Oak-D-Lite) device (camera) found!") + + `sudo pip3 install --extra-index-url https://developer.download.nvidia.com/compute/redist/jp/v461 tensorflow` +""" + +import argparse +import string +import time +import sys + +import numpy as np # numpy - manipulate the packet data returned by depthai +import cv2 as cv2 # opencv - display the video stream +import depthai # depthai - access the camera and its data packets +from depthai import Pipeline, DataOutputQueue, ImgFrame, ImgDetections, ImgDetection +from numpy import ndarray +from typing import List + +WIDTH = 640 +HEIGHT = 480 + + +class OakD(object): + """ + Donkeycar part for the Oak-D camera + Intel Movidius based depth sensing camera + https://docs.luxonis.com/projects/hardware/en/latest/pages/DM9095.html + https://www.kickstarter.com/projects/opencv/opencv-ai-kit-oak-depth-camera-4k-cv-edge-object-detection + https://shop.luxonis.com/ + """ + + def __init__( + self, + width=WIDTH, + height=HEIGHT, + enable_rgb=True, + enable_depth=True, + device_id=None, + rgb_output_mode="isp", + rgb_isp_scale_num=1, + rgb_isp_scale_den=6, + rgb_sensor_crop_x=None, + rgb_sensor_crop_y=None, + ): + self.device_id = device_id # "18443010C1E4681200" # serial number of device to use|None to use default|"list" to list devices and exit + self.enable_rgb = enable_rgb + self.enable_depth = enable_depth + self.rgb_output_mode = rgb_output_mode + self.rgb_isp_scale_num = rgb_isp_scale_num + self.rgb_isp_scale_den = rgb_isp_scale_den + self.rgb_sensor_crop_x = rgb_sensor_crop_x + self.rgb_sensor_crop_y = rgb_sensor_crop_y + + self.width = width + self.height = height + + # TODO: Accommodate using device native resolutions to avoid resizing. + self.resize = (width != WIDTH) or (height != HEIGHT) + if self.resize: + print( + f"The output images will be resized from {(WIDTH, HEIGHT)} to {(self.width, self.height)} using OpenCV. Device resolution in use is 640x480." + ) + + self.pipeline = None + if self.enable_depth or self.enable_rgb: + self.pipeline = depthai.Pipeline() + + device_info = self.get_depthai_device_info(device_id) + + if self.enable_depth: + self.setup_depth_camera(WIDTH, HEIGHT) + + if self.enable_rgb: + self.setup_rgb_camera(WIDTH, HEIGHT) + + self.oak_d_device = depthai.Device(self.pipeline, device_info) + + # initialize frame state + self.color_image = None + self.depth_image = None + self.frame_count = 0 + self.start_time = time.time() + self.frame_time = self.start_time + + self.running = True + + # Taken from the demo application. + def get_depthai_device_info(self, device_id: string): + device_infos = depthai.Device.getAllAvailableDevices() + if len(device_infos) == 0: + raise RuntimeError("No DepthAI (Oak-D-Lite) device (camera) found!") + else: + print("Available devices:") + for i, deviceInfo in enumerate(device_infos): + print(f"[{i}] {deviceInfo.getMxId()} [{deviceInfo.state.name}]") + + # Set the deviceId to "list" in order to list the connected devices' ids. + if device_id == "list": + raise SystemExit(0) + elif device_id is not None: + matching_device = next( + filter(lambda info: info.getMxId() == device_id, device_infos), None + ) + if matching_device is None: + raise RuntimeError( + f"No DepthAI device found with id matching {device_id} !" + ) + return matching_device + elif len(device_infos) == 1: + return device_infos[0] + else: + val = input("Which DepthAI Device you want to use: ") + try: + return device_infos[int(val)] + except: + raise ValueError(f"Incorrect value supplied: {val}") + + def setup_depth_camera(self, width, height): + # Set up left and right cameras + mono_left = self.get_mono_camera(self.pipeline, True) + mono_right = self.get_mono_camera(self.pipeline, False) + + # Combine left and right cameras to form a stereo pair + stereo: depthai.node.StereoDepth = self.get_stereo_pair( + self.pipeline, mono_left, mono_right + ) + + # Define and name output depth map + xout_depth = self.pipeline.createXLinkOut() + xout_depth.setStreamName("depth") + + stereo.depth.link(xout_depth.input) + + def setup_rgb_camera(self, width, height): + cam_rgb = self.pipeline.create(depthai.node.ColorCamera) + + # 'video' is always center-cropped to 16:9 (max 4K) before scaling, + # regardless of setResolution(), which clips the sensor's full FOV. + # 'isp' preserves the full sensor FOV, so downscale from there + # instead; setIspScale() can't land on an exact pixel size, so + # _poll() resizes the result down to the exact requested dimensions. + res = depthai.ColorCameraProperties.SensorResolution.THE_13_MP + cam_rgb.setResolution(res) + + xout_rgb = self.pipeline.create(depthai.node.XLinkOut) + xout_rgb.setStreamName("rgb") + + if self.rgb_output_mode == "video": + if self.rgb_sensor_crop_x is not None or self.rgb_sensor_crop_y is not None: + cam_rgb.setSensorCrop( + 0.0 if self.rgb_sensor_crop_x is None else self.rgb_sensor_crop_x, + 0.0 if self.rgb_sensor_crop_y is None else self.rgb_sensor_crop_y, + ) + cam_rgb.setVideoSize(self.width, self.height) + cam_rgb.video.link(xout_rgb.input) + else: + cam_rgb.setIspScale(self.rgb_isp_scale_num, self.rgb_isp_scale_den) + cam_rgb.isp.link(xout_rgb.input) + + def get_mono_camera(self, pipeline: Pipeline, is_left: bool): + # Configure mono camera + mono = pipeline.createMonoCamera() + + # Set camera resolution + mono.setResolution(depthai.MonoCameraProperties.SensorResolution.THE_480_P) + + if is_left: + # Get left camera + mono.setBoardSocket(depthai.CameraBoardSocket.LEFT) + else: + # Get right camera + mono.setBoardSocket(depthai.CameraBoardSocket.RIGHT) + + return mono + + def get_stereo_pair(self, pipeline: Pipeline, mono_left, mono_right): + # Configure the stereo pair for depth estimation + new_stereo = pipeline.createStereoDepth() + # Checks occluded pixels and marks them as invalid + new_stereo.setLeftRightCheck(True) + + # Configure left and right cameras to work as a stereo pair + mono_left.out.link(new_stereo.left) + mono_right.out.link(new_stereo.right) + + return new_stereo + + def get_frame(self, queue: DataOutputQueue): + # Get frame from queue + new_frame: ImgFrame = queue.get() + # Convert to OpenCV format + return new_frame.getCvFrame() + + def _poll(self): + last_time = self.frame_time + self.frame_time = time.time() - self.start_time + self.frame_count += 1 + + # + # convert camera frames to images + # + # RGB and depth queues are initialized independently based on which + # streams are enabled; querying a queue for a disabled stream raises + # a RuntimeError since no XLinkOut was ever wired for it. + if self.enable_rgb: + self.rgb_queue: DataOutputQueue = self.oak_d_device.getOutputQueue( + "rgb", maxSize=1, blocking=False + ) + rgb_frame = self.get_frame(self.rgb_queue) + if rgb_frame.shape[1] != self.width or rgb_frame.shape[0] != self.height: + # setIspScale() lands on an approximate size; resize to the + # exact requested dimensions without re-cropping the FOV. + rgb_frame = cv2.resize( + rgb_frame, (self.width, self.height), interpolation=cv2.INTER_NEAREST + ) + self.color_image = rgb_frame + else: + self.color_image = None # Explicitly set None to prevent AttributeError later + + if self.enable_depth: + self.depth_queue: DataOutputQueue = self.oak_d_device.getOutputQueue( + name="depth", maxSize=1, blocking=False + ) + depth_frame = self.get_frame(self.depth_queue) + self.depth_image = depth_frame + else: + self.depth_image = None # Explicitly set None to prevent AttributeError later + + + if self.resize: + if self.width != WIDTH or self.height != HEIGHT: + self.color_image = ( + cv2.resize( + self.color_image, (self.width, self.height), cv2.INTER_NEAREST + ) + if self.enable_rgb + else None + ) + self.depth_image = ( + cv2.resize( + self.depth_image, (self.width, self.height), cv2.INTER_NEAREST + ) + if self.enable_depth + else None + ) + + def update(self): + """ + When running threaded, update() is called from the background thread + to update the state. run_threaded() is called to return the latest state. + """ + while self.running: + self._poll() + + def run_threaded(self): + """ + Return the latest state read by update(). This will not block. + All 4 states are returned, but may be None if the feature is not enabled when the camera part is constructed. + For gyroscope, x is pitch, y is yaw and z is roll. + :return: (rbg_image: nparray, depth_image: nparray, acceleration: (x:float, y:float, z:float), gyroscope: (x:float, y:float, z:float)) + """ + return self.color_image, self.depth_image + + def run(self): + """ + Read and return frame from camera. This will block while reading the frame. + see run_threaded() for return types. + """ + self._poll() + return self.run_threaded() + + def shutdown(self): + self.running = False + time.sleep(2) # give thread enough time to shutdown + + # done running + self.oak_d_device.close() + + +# +# self test +# +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + parser.add_argument( + "--rgb", default=False, action="store_true", help="Stream RGB camera" + ) + parser.add_argument( + "--depth", default=False, action="store_true", help="Stream depth camera" + ) + parser.add_argument( + "--device_id", + help='Camera id (if more than one camera connected), or "list" to print the connected device ids', + ) + args = parser.parse_args() + + if not (args.rgb or args.depth): + print("Must specify one or more of --rgb, --depth") + parser.print_help() + sys.exit(0) + + show_opencv_window = ( + args.rgb or args.depth + ) # True to show images in opencv window: note that default donkeycar environment is not configured for this. + if show_opencv_window: + import cv2 + + enable_rgb = args.rgb + enable_depth = args.depth + + devices = depthai.Device.getAllAvailableDevices() + + device_id = args.device_id # getMxId + + width = 640 + height = 480 + channels = 3 + + profile_frames = 0 # set to non-zero to calculate the max frame rate using given number of frames + + camera = None + try: + camera = OakD( + width=width, + height=height, + enable_rgb=enable_rgb, + enable_depth=enable_depth, + device_id=device_id, + ) + + frame_count = 0 + start_time = time.time() + frame_time = start_time + while True: + # + # read data from camera + # + color_image, depth_image = camera.run() + + # maintain frame timing + frame_count += 1 + last_time = frame_time + frame_time = time.time() + + # Show images + if show_opencv_window and not profile_frames: + cv2.namedWindow("Oak-D", cv2.WINDOW_AUTOSIZE) + if enable_rgb or enable_depth: + # make sure depth and color images have same number of channels so we can show them together in the window + if 3 == channels: + depth_colormap = ( + cv2.applyColorMap( + cv2.convertScaleAbs(depth_image, alpha=0.03), + cv2.COLORMAP_JET, + ) + if enable_depth + else None + ) + else: + depth_colormap = ( + cv2.cvtColor( + cv2.applyColorMap( + cv2.convertScaleAbs(depth_image, alpha=0.03), + cv2.COLORMAP_JET, + ), + cv2.COLOR_RGB2GRAY, + ) + if enable_depth + else None + ) + + # Stack both images horizontally (i.e. side by side). + images = None + if enable_rgb: + images = ( + np.hstack((color_image, depth_colormap)) + if enable_depth + else color_image + ) + elif enable_depth: + images = depth_colormap + + if images is not None: + cv2.imshow("Oak-D", images) + + # Press esc or 'q' to close the image window + key = cv2.waitKey(1) + if key & 0xFF == ord("q") or key == 27: + cv2.destroyAllWindows() + break + if profile_frames > 0: + if frame_count == profile_frames: + print( + f"Acquired {frame_count} frames in {frame_time - start_time} seconds for {frame_count / (frame_time - start_time)} fps" + ) + + break + else: + time.sleep(0.05) + finally: + if camera is not None: + camera.shutdown() diff --git a/donkeycar/parts/oled.py b/donkeycar/parts/oled.py index 2ce137a462..f8cd768cf0 100644 --- a/donkeycar/parts/oled.py +++ b/donkeycar/parts/oled.py @@ -1,161 +1,200 @@ -# requires the Adafruit ssd1306 library: pip install adafruit-circuitpython-ssd1306 - -import subprocess -import time -from board import SCL, SDA -import busio -from PIL import Image, ImageDraw, ImageFont -import adafruit_ssd1306 - - -class OLEDDisplay(object): - ''' - Manages drawing of text on the OLED display. - ''' - def __init__(self, rotation=0, resolution=1): - # Placeholder - self._EMPTY = '' - # Total number of lines of text - self._SLOT_COUNT = 4 - self.slots = [self._EMPTY] * self._SLOT_COUNT - self.display = None - self.rotation = rotation - if resolution == 2: - self.height = 64 - else: - self.height = 32 - - def init_display(self): - ''' - Initializes the OLED display. - ''' - if self.display is None: - # Create the I2C interface. - i2c = busio.I2C(SCL, SDA) - # Create the SSD1306 OLED class. - # The first two parameters are the pixel width and pixel height. Change these - # to the right size for your display! - self.display = adafruit_ssd1306.SSD1306_I2C(128, self.height, i2c) - self.display.rotation = self.rotation - - - self.display.fill(0) - self.display.show() - - # Create blank image for drawing. - # Make sure to create image with mode '1' for 1-bit color. - self.width = self.display.width - self.image = Image.new("1", (self.width, self.height)) - - # Get drawing object to draw on image. - self.draw = ImageDraw.Draw(self.image) - - # Draw a black filled box to clear the image. - self.draw.rectangle((0, 0, self.width, self.height), outline=0, fill=0) - # Load Fonts - self.font = ImageFont.load_default() - self.clear_display() - - def clear_display(self): - if self.draw is not None: - self.draw.rectangle((0, 0, self.width, self.height), outline=0, fill=0) - - def update_slot(self, index, text): - if index < len(self.slots): - self.slots[index] = text - - def clear_slot(self, index): - if index < len(self.slots): - self.slots[index] = self._EMPTY - - def update(self): - '''Display text''' - x = 0 - top = -2 - self.clear_display() - for i in range(self._SLOT_COUNT): - text = self.slots[i] - if len(text) > 0: - self.draw.text((x, top), text, font=self.font, fill=255) - top += 8 - - # Update - self.display.rotation = self.rotation - self.display.image(self.image) - self.display.show() - - -class OLEDPart(object): - ''' - The part that updates status on the oled display. - ''' - def __init__(self, rotation, resolution, auto_record_on_throttle=False): - self.oled = OLEDDisplay(rotation, resolution) - self.oled.init_display() - self.on = False - if auto_record_on_throttle: - self.recording = 'AUTO' - else: - self.recording = 'NO' - self.num_records = 0 - self.user_mode = None - eth0 = OLEDPart.get_ip_address('eth0') - wlan0 = OLEDPart.get_ip_address('wlan0') - if eth0 is not None: - self.eth0 = 'eth0:%s' % (eth0) - else: - self.eth0 = None - if wlan0 is not None: - self.wlan0 = 'wlan0:%s' % (wlan0) - else: - self.wlan0 = None - - def run(self): - if not self.on: - self.on = True - - def run_threaded(self, recording, num_records, user_mode): - if num_records is not None and num_records > 0: - self.num_records = num_records - - if recording: - self.recording = 'YES (Records = %s)' % (self.num_records) - else: - self.recording = 'NO (Records = %s)' % (self.num_records) - - self.user_mode = 'User Mode (%s)' % (user_mode) - - def update_slots(self): - updates = [self.eth0, self.wlan0, self.recording, self.user_mode] - index = 0 - # Update slots - for update in updates: - if update is not None: - self.oled.update_slot(index, update) - index += 1 - - # Update display - self.oled.update() - - def update(self): - self.on = True - # Run threaded loop by itself - while self.on: - self.update_slots() - - def shutdown(self): - self.oled.clear_display() - self.on = False - - # https://github.com/NVIDIA-AI-IOT/jetbot/blob/master/jetbot/utils/utils.py - - @classmethod - def get_ip_address(cls, interface): - if OLEDPart.get_network_interface_state(interface) == 'down': - return None - cmd = "ifconfig %s | grep -Eo 'inet (addr:)?([0-9]*\.){3}[0-9]*' | grep -Eo '([0-9]*\.){3}[0-9]*' | grep -v '127.0.0.1'" % interface - return subprocess.check_output(cmd, shell=True).decode('ascii')[:-1] - - @classmethod - def get_network_interface_state(cls, interface): - return subprocess.check_output('cat /sys/class/net/%s/operstate' % interface, shell=True).decode('ascii')[:-1] +# requires the Adafruit ssd1306 library: pip install adafruit-circuitpython-ssd1306 + + +import os +import re +import subprocess +import time +from board import SCL, SDA +import busio +from PIL import Image, ImageDraw, ImageFont +import adafruit_ssd1306 + + +class OLEDDisplay(object): + ''' + Manages drawing of text on the OLED display. + ''' + def __init__(self, rotation=0, resolution=1): + # Placeholder + self._EMPTY = '' + # Total number of lines of text + self._SLOT_COUNT = 4 + self.slots = [self._EMPTY] * self._SLOT_COUNT + self.display = None + self.rotation = rotation + if resolution == 2: + self.height = 64 + else: + self.height = 32 + + def init_display(self): + ''' + Initializes the OLED display. + ''' + if self.display is None: + # Create the I2C interface. + i2c = busio.I2C(SCL, SDA) + # Create the SSD1306 OLED class. + # The first two parameters are the pixel width and pixel height. Change these + # to the right size for your display! + self.display = adafruit_ssd1306.SSD1306_I2C(128, self.height, i2c) + self.display.rotation = self.rotation + + self.display.fill(0) + self.display.show() + + # Create blank image for drawing. + # Make sure to create image with mode '1' for 1-bit color. + self.width = self.display.width + self.image = Image.new("1", (self.width, self.height)) + + # Get drawing object to draw on image. + self.draw = ImageDraw.Draw(self.image) + + # Draw a black filled box to clear the image. + self.draw.rectangle((0, 0, self.width, self.height), outline=0, fill=0) + # Load Fonts + self.font = ImageFont.load_default() + self.clear_display() + + def clear_display(self): + if self.draw is not None: + self.draw.rectangle((0, 0, self.width, self.height), outline=0, fill=0) + + def update_slot(self, index, text): + if index < len(self.slots): + self.slots[index] = text + + def clear_slot(self, index): + if index < len(self.slots): + self.slots[index] = self._EMPTY + + def update(self): + '''Display text''' + x = 0 + top = -2 + self.clear_display() + for i in range(self._SLOT_COUNT): + text = self.slots[i] + if len(text) > 0: + self.draw.text((x, top), text, font=self.font, fill=255) + top += 8 + + # Update + self.display.rotation = self.rotation + self.display.image(self.image) + self.display.show() + + +class OLEDPart(object): + ''' + The part that updates status on the oled display. + ''' + def __init__(self, rotation, resolution, auto_record_on_throttle=False): + self.oled = OLEDDisplay(rotation, resolution) + self.oled.init_display() + self.on = False + if auto_record_on_throttle: + self.recording = 'AUTO' + else: + self.recording = 'NO' + self.num_records = 0 + self.user_mode = None + + # Bookworm / systemd often doesn't have "eth0" (predictable interface names). + # Only query interfaces that actually exist to avoid crashing. + eth0 = None + wlan0 = None + + if os.path.exists('/sys/class/net/eth0'): + eth0 = OLEDPart.get_ip_address('eth0') + if os.path.exists('/sys/class/net/wlan0'): + wlan0 = OLEDPart.get_ip_address('wlan0') + + if eth0: + self.eth0 = f'eth0:{eth0}' + else: + self.eth0 = None + + if wlan0: + self.wlan0 = f'wlan0:{wlan0}' + else: + self.wlan0 = None + + def run(self): + if not self.on: + self.on = True + + def run_threaded(self, recording, num_records, user_mode): + if num_records is not None and num_records > 0: + self.num_records = num_records + + if recording: + self.recording = 'YES (Records = %s)' % (self.num_records) + else: + self.recording = 'NO (Records = %s)' % (self.num_records) + + self.user_mode = 'User Mode (%s)' % (user_mode) + + def update_slots(self): + updates = [self.eth0, self.wlan0, self.recording, self.user_mode] + index = 0 + # Update slots + for update in updates: + if update is not None: + self.oled.update_slot(index, update) + index += 1 + + # Update display + self.oled.update() + + def update(self): + self.on = True + # Run threaded loop by itself + while self.on: + self.update_slots() + + def shutdown(self): + self.oled.clear_display() + self.on = False + + @classmethod + def get_ip_address(cls, interface): + # If interface is missing or down, don't crash. + if cls.get_network_interface_state(interface) != 'up': + return None + + # Prefer `ip` (present on Bookworm Lite) over `ifconfig` (often not installed). + try: + out = subprocess.check_output( + ["ip", "-4", "addr", "show", "dev", interface], + stderr=subprocess.DEVNULL, + text=True, + ) + except Exception: + return None + + # Parse e.g. "inet 192.168.86.62/24 ..." + m = re.search(r"\binet\s+([0-9]+(?:\.[0-9]+){3})/", out) + if not m: + return None + ip = m.group(1) + if ip == "127.0.0.1": + return None + return ip + + @classmethod + def get_network_interface_state(cls, interface): + # Return 'down' for missing interfaces instead of throwing. + path = f"/sys/class/net/{interface}/operstate" + try: + with open(path, "r") as f: + state = f.read().strip() + except FileNotFoundError: + return "down" + except Exception: + return "down" + + # Normalize common values: up/down/unknown/dormant... + return state if state else "down" diff --git a/donkeycar/parts/pytorch/ResNet18.py b/donkeycar/parts/pytorch/ResNet18.py index 2990466e2f..37c64a123d 100644 --- a/donkeycar/parts/pytorch/ResNet18.py +++ b/donkeycar/parts/pytorch/ResNet18.py @@ -11,7 +11,7 @@ def load_resnet18(num_classes=2): # Load the pre-trained model (on ImageNet) - model = models.resnet18(pretrained=True) + model = models.resnet18(weights=models.ResNet18_Weights.IMAGENET1K_V1) # Don't allow model feature extraction layers to be modified for layer in model.parameters(): diff --git a/donkeycar/parts/pytorch/torch_train.py b/donkeycar/parts/pytorch/torch_train.py index 7413b00090..c1894f19cf 100644 --- a/donkeycar/parts/pytorch/torch_train.py +++ b/donkeycar/parts/pytorch/torch_train.py @@ -30,10 +30,13 @@ def train(cfg, tub_paths, model_output_path, model_type, checkpoint_path=None): model = get_model_by_type(model_type, cfg, checkpoint_path=checkpoint_path) if torch.cuda.is_available(): print('Using CUDA') - gpus = -1 + accelerator, devices = 'cuda', -1 + elif torch.backends.mps.is_available(): + print('Using MPS (Apple GPU)') + accelerator, devices = 'mps', 1 else: - print('Not using CUDA') - gpus = 0 + print('Using CPU') + accelerator, devices = 'cpu', 1 logger = None if cfg.VERBOSE_TRAIN: @@ -46,8 +49,9 @@ def train(cfg, tub_paths, model_output_path, model_type, checkpoint_path=None): if cfg.PRINT_MODEL_SUMMARY: summarize(model) - trainer = pl.Trainer(accelerator='cpu', logger=logger, - max_epochs=cfg.MAX_EPOCHS, default_root_dir=output_dir) + trainer = pl.Trainer(logger=logger, max_epochs=cfg.MAX_EPOCHS, + default_root_dir=output_dir, + accelerator=accelerator, devices=devices) data_module = TorchTubDataModule(cfg, tub_paths) trainer.fit(model, data_module) diff --git a/donkeycar/parts/robohat.py b/donkeycar/parts/robohat.py index 768787a7fe..935ae01a73 100755 --- a/donkeycar/parts/robohat.py +++ b/donkeycar/parts/robohat.py @@ -180,9 +180,13 @@ class RoboHATDriver: This is developed by Robotics Masters """ - def __init__(self, cfg, debug=False): + def __init__(self, cfg, serial_port=None, debug=False): # Initialise the Robo HAT using the serial port - self.pwm = serial.Serial(cfg.MM1_SERIAL_PORT, 115200, timeout=1) + # Use shared serial port if provided to avoid opening the same port twice + if serial_port is not None: + self.pwm = serial_port + else: + self.pwm = serial.Serial(cfg.MM1_SERIAL_PORT, 115200, timeout=1) self.MAX_FORWARD = cfg.MM1_MAX_FORWARD self.MAX_REVERSE = cfg.MM1_MAX_REVERSE self.STOPPED_PWM = cfg.MM1_STOPPED_PWM diff --git a/donkeycar/pipeline/augmentations.py b/donkeycar/pipeline/augmentations.py index a421d0e4c6..151b245399 100644 --- a/donkeycar/pipeline/augmentations.py +++ b/donkeycar/pipeline/augmentations.py @@ -2,7 +2,8 @@ import logging import albumentations as A from albumentations import GaussianBlur -from albumentations.augmentations.transforms import RandomBrightnessContrast +from albumentations.augmentations import RandomBrightnessContrast + from donkeycar.config import Config @@ -11,14 +12,14 @@ class ImageAugmentation: - def __init__(self, cfg, key, prob=0.5, always_apply=False): + def __init__(self, cfg, key, prob=0.5): aug_list = getattr(cfg, key, []) - augmentations = [ImageAugmentation.create(a, cfg, prob, always_apply) + augmentations = [ImageAugmentation.create(a, cfg, prob) for a in aug_list] self.augmentations = A.Compose(augmentations) @classmethod - def create(cls, aug_type: str, config: Config, prob, always) -> \ + def create(cls, aug_type: str, config: Config, prob) -> \ albumentations.core.transforms_interface.BasicTransform: """ Augmentation factory. Cropping and trapezoidal mask are transformations which should be applied in training, validation @@ -30,13 +31,13 @@ def create(cls, aug_type: str, config: Config, prob, always) -> \ logger.info(f'Creating augmentation {aug_type} {b_limit}') return RandomBrightnessContrast(brightness_limit=b_limit, contrast_limit=b_limit, - p=prob, always_apply=always) + p=prob) elif aug_type == 'BLUR': b_range = getattr(config, 'AUG_BLUR_RANGE', 3) logger.info(f'Creating augmentation {aug_type} {b_range}') return GaussianBlur(sigma_limit=b_range, blur_limit=(13, 13), - p=prob, always_apply=always) + p=prob) # Parts interface def run(self, img_arr): diff --git a/donkeycar/pipeline/database.py b/donkeycar/pipeline/database.py index ef22217910..18c6b7ab5b 100644 --- a/donkeycar/pipeline/database.py +++ b/donkeycar/pipeline/database.py @@ -44,7 +44,7 @@ def generate_model_name(self) -> Tuple[str, int]: this_num = 0 date = time.strftime('%y-%m-%d') ext = 'h5' if getattr(self.cfg, 'SAVE_MODEL_AS_H5', False) \ - else 'savedmodel' + else 'keras' name = f'pilot_{date}_{this_num}.{ext}' return os.path.join(self.cfg.MODELS_PATH, name), this_num @@ -127,7 +127,7 @@ def time_fmt(t): return datetime.fromtimestamp(t).strftime(fmt) def transfer_fmt(model_name): - return model_name.replace('.h5', '').replace('.savedmodel', '') + return model_name.replace('.h5', '').replace('.keras', '') return {'Time': time_fmt, 'Transfer': transfer_fmt} @@ -147,3 +147,23 @@ def pretty_print(self, group_tubs=False): def get_pilot_names(self): return [entry['Name'] for entry in self.entries] + + +def update_config_from_database(cfg, model_path, model_type): + """Load model config overrides and infer model type from database.""" + overwrite = ['TRANSFORMATIONS', 'POST_TRANSFORMATIONS', + 'ROI_CROP_BOTTOM', 'ROI_CROP_LEFT', 'ROI_CROP_RIGHT', + 'ROI_CROP_TOP', 'SEQUENCE_LENGTH'] + model_prefix_map = {'.tflite': 'tflite_', '.trt': 'tensorrt_', + '.keras': '', '.h5': ''} + db = PilotDatabase(cfg) + model_basename, model_ext = os.path.splitext( + os.path.basename(model_path)) + pilot_entry = db.get_entry(model_basename) + if pilot_entry: + logger.info(f'Found {model_basename} in database') + cfg_train_dict = pilot_entry['Config'] + cfg.from_dict(cfg_train_dict, overwrite) + model_type = model_prefix_map[model_ext] + pilot_entry['Type'] + + return model_type diff --git a/donkeycar/pipeline/training.py b/donkeycar/pipeline/training.py index d58276cf63..25c5bb9f61 100644 --- a/donkeycar/pipeline/training.py +++ b/donkeycar/pipeline/training.py @@ -4,8 +4,6 @@ from typing import List, Dict, Union, Tuple import logging -from tensorflow.python.keras.models import load_model - from donkeycar.config import Config from donkeycar.parts.keras import KerasPilot from donkeycar.parts.interpreter import keras_model_to_tflite, \ @@ -16,8 +14,13 @@ from donkeycar.pipeline.augmentations import ImageAugmentation from donkeycar.parts.image_transformations import ImageTransformations from donkeycar.utils import get_model_by_type, normalize_image, train_test_split -import tensorflow as tf import numpy as np +try: + import tensorflow as tf + from tensorflow.keras.models import load_model + logging.getLogger('tensorflow').setLevel(logging.WARNING) +except ImportError: + tf = None logger = logging.getLogger(__name__) @@ -82,7 +85,7 @@ def get_y(record: TubRecord) -> Dict[str, Union[float, np.ndarray]]: y_transform=get_y) return pipeline - def create_tf_data(self) -> tf.data.Dataset: + def create_tf_data(self): """ Assembles the tf data pipeline """ dataset = tf.data.Dataset.from_generator( generator=lambda: self.pipeline, @@ -102,7 +105,7 @@ def get_model_train_details(database: PilotDatabase, model: str = None) \ def train(cfg: Config, tub_paths: str, model: str = None, model_type: str = None, transfer: str = None, comment: str = None) \ - -> tf.keras.callbacks.History: + : """ Train the model """ @@ -113,9 +116,14 @@ def train(cfg: Config, tub_paths: str, model: str = None, get_model_train_details(database, model) base_path, ext = tuple(os.path.splitext(model_path)) + if tf is not None: + from tensorflow import keras as _keras + _keras.backend.clear_session() kl = get_model_by_type(model_type, cfg) if transfer: kl.load(transfer) + if getattr(cfg, 'FREEZE_LAYERS', False): + kl.freeze_first_layers() if cfg.PRINT_MODEL_SUMMARY: kl.interpreter.summary() @@ -131,18 +139,21 @@ def train(cfg: Config, tub_paths: str, model: str = None, dataset.close() # We need augmentation in validation when using crop / trapeze - if 'fastai_' in model_type: from donkeycar.parts.pytorch.torch_data \ import TorchTubDataset, get_default_transform transform = get_default_transform(resize=False) - dataset_train = TorchTubDataset(cfg, training_records, transform=transform) - dataset_validate = TorchTubDataset(cfg, validation_records, transform=transform) + dataset_train = TorchTubDataset(cfg, training_records, + transform=transform) + dataset_validate = TorchTubDataset(cfg, validation_records, + transform=transform) train_size = len(training_records) val_size = len(validation_records) else: - training_pipe = BatchSequence(kl, cfg, training_records, is_train=True) - validation_pipe = BatchSequence(kl, cfg, validation_records, is_train=False) + training_pipe = BatchSequence(kl, cfg, training_records, + is_train=True) + validation_pipe = BatchSequence(kl, cfg, validation_records, + is_train=False) tune = tf.data.experimental.AUTOTUNE dataset_train = training_pipe.create_tf_data().prefetch(tune) dataset_validate = validation_pipe.create_tf_data().prefetch(tune) @@ -175,13 +186,9 @@ def train(cfg: Config, tub_paths: str, model: str = None, keras_model_to_tflite(model_path, tf_lite_model_path) if getattr(cfg, 'CREATE_TENSOR_RT', False): - # convert .h5 model to .savedmodel, only if we are using h5 format - if ext == '.h5': - logger.info(f"Converting from .h5 to .savedmodel first") - model_tmp = load_model(model_path, compile=False) - # save in tensorflow savedmodel format (i.e. directory) - model_tmp.save(f'{base_path}.savedmodel') - # pass savedmodel to the rt converter + logger.info(f"Exporting to .savedmodel for TRT conversion") + model_tmp = load_model(model_path, compile=False) + model_tmp.export(f'{base_path}.savedmodel') saved_model_to_tensor_rt(f'{base_path}.savedmodel', f'{base_path}.trt') database_entry = { diff --git a/donkeycar/templates/cfg_basic.py b/donkeycar/templates/cfg_basic.py index f718e8ae47..eaac3e2abf 100755 --- a/donkeycar/templates/cfg_basic.py +++ b/donkeycar/templates/cfg_basic.py @@ -25,7 +25,7 @@ MAX_LOOPS = None #CAMERA -CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST) +CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|OAKD|MOCK|IMAGE_LIST) IMAGE_W = 160 IMAGE_H = 120 IMAGE_DEPTH = 3 # default RGB=3, make 1 for mono @@ -75,7 +75,7 @@ DEFAULT_MODEL_TYPE = 'linear' #(linear|categorical|rnn|imu|behavior|3d|localizer|latent) CREATE_TF_LITE = True # automatically create tflite model in training CREATE_TENSOR_RT = False # automatically create tensorrt model in training -SAVE_MODEL_AS_H5 = False # if old keras format should be used instead of savedmodel +SAVE_MODEL_AS_H5 = False # if True saves as .h5, otherwise uses .keras format BATCH_SIZE = 128 TRAIN_TEST_SPLIT = 0.8 MAX_EPOCHS = 100 diff --git a/donkeycar/templates/cfg_complete.py b/donkeycar/templates/cfg_complete.py index d468f6b3cd..33760cdaeb 100644 --- a/donkeycar/templates/cfg_complete.py +++ b/donkeycar/templates/cfg_complete.py @@ -9,48 +9,132 @@ import dk cfg = dk.load_config(config_path='~/mycar/config.py') print(cfg.CAMERA_RESOLUTION) - """ - import os -#PATHS +# ============================================================================== +# 1. HARDWARE CONFIGURATION & I/O +# (Camera, Drive Train, Sensors, Pins, Display) +# ============================================================================== + +# PATHS CAR_PATH = PACKAGE_PATH = os.path.dirname(os.path.realpath(__file__)) DATA_PATH = os.path.join(CAR_PATH, 'data') MODELS_PATH = os.path.join(CAR_PATH, 'models') -#VEHICLE -DRIVE_LOOP_HZ = 20 # the vehicle loop will pause if faster than this speed. -MAX_LOOPS = None # the vehicle loop can abort after this many iterations, when given a positive integer. - -#CAMERA -CAMERA_TYPE = "PICAM" # (PICAM|WEBCAM|CVCAM|CSIC|V4L|D435|MOCK|IMAGE_LIST) +# ------------------------------------------------------------------------------ +# CAMERA SETUP +# ------------------------------------------------------------------------------ + +# Select the camera type. +# 'PICAM': Raspberry Pi Camera (CSI) +# 'WEBCAM': USB Camera +# 'CVCAM': OpenCV Camera (often same as WEBCAM) +# 'CSIC': High-speed CSI camera (e.g. Arducam) +# 'D435': Intel Realsense D435 +# 'OAKD': Luxonis OAK-D +# 'MOCK': Simulation/Testing or when using GPS path following +CAMERA_TYPE = "PICAM" + +# The resolution of the input image. Higher resolution needs more processing power. IMAGE_W = 160 IMAGE_H = 120 -IMAGE_DEPTH = 3 # default RGB=3, make 1 for mono -CAMERA_FRAMERATE = DRIVE_LOOP_HZ + +# The depth of the image. 3 for RGB, 1 for Greyscale. +IMAGE_DEPTH = 3 + +# The framerate of the camera. Should generally match DRIVE_LOOP_HZ. +CAMERA_FRAMERATE = 20 + +# Flip the image vertically (useful if camera is mounted upside down). CAMERA_VFLIP = False + +# Flip the image horizontally. CAMERA_HFLIP = False -CAMERA_INDEX = 0 # used for 'WEBCAM' and 'CVCAM' when there is more than one camera connected -# For CSIC camera - If the camera is mounted in a rotated position, changing the below parameter will correct the output frame orientation -CSIC_CAM_GSTREAMER_FLIP_PARM = 0 # (0 => none , 4 => Flip horizontally, 6 => Flip vertically) -BGR2RGB = False # true to convert from BRG format to RGB format; requires opencv -SHOW_PILOT_IMAGE = False # show the image used to do the inference when in autopilot mode -# For IMAGE_LIST camera -# PATH_MASK = "~/mycar/data/tub_1_20-03-12/*.jpg" +# Used for 'WEBCAM' and 'CVCAM' when there is more than one camera connected. +CAMERA_INDEX = 0 -#9865, over rides only if needed, ie. TX2.. -PCA9685_I2C_ADDR = 0x40 #I2C address, use i2cdetect to validate this number -PCA9685_I2C_BUSNUM = None #None will auto detect, which is fine on the pi. But other platforms should specify the bus num. +# CSIC Camera: 0=None, 4=Flip Horizontal, 6=Flip Vertical. +CSIC_CAM_GSTREAMER_FLIP_PARM = 0 -#SSD1306_128_32 -USE_SSD1306_128_32 = False # Enable the SSD_1306 OLED Display -SSD1306_128_32_I2C_ROTATION = 0 # 0 = text is right-side up, 1 = rotated 90 degrees clockwise, 2 = 180 degrees (flipped), 3 = 270 degrees -SSD1306_RESOLUTION = 1 # 1 = 128x32; 2 = 128x64 +# Convert Blue-Green-Red (OpenCV default) to Red-Green-Blue. +BGR2RGB = False + +# Intel Realsense D435 specific settings +REALSENSE_D435_RGB = True # True to capture RGB image +REALSENSE_D435_DEPTH = True # True to capture depth as image array +REALSENSE_D435_IMU = False # True to capture IMU data (D435i only) +REALSENSE_D435_ID = None # Serial number of camera or None for auto-detect + +# OAK-D Camera specific settings +OAKD_RGB = True # True to capture RGB image +OAKD_DEPTH = True # True to capture depth as image array +OAKD_ID = None # Serial number of camera or None for auto-detect +OAKD_RGB_OUTPUT_MODE = "isp" # "isp" preserves full FOV; "video" uses crop-based framing +OAKD_RGB_ISP_SCALE_NUM = 1 # Used when OAKD_RGB_OUTPUT_MODE == "isp" +OAKD_RGB_ISP_SCALE_DEN = 6 # Used when OAKD_RGB_OUTPUT_MODE == "isp" +OAKD_RGB_SENSOR_CROP_X = None # Used when OAKD_RGB_OUTPUT_MODE == "video" +OAKD_RGB_SENSOR_CROP_Y = None # Used when OAKD_RGB_OUTPUT_MODE == "video" + + +# ------------------------------------------------------------------------------ +# I2C & DISPLAY +# ------------------------------------------------------------------------------ + +# I2C address of the PCA9685 servo driver (standard is 0x40). +PCA9685_I2C_ADDR = 0x40 + +# I2C bus number. None will auto-detect (usually 1 on Pi). +PCA9685_I2C_BUSNUM = None + +# Enable the SSD1306 OLED display (small screen on the car). +USE_SSD1306_128_32 = False + +# OLED Rotation: 0 = normal, 1 = 90 deg, 2 = 180 deg, 3 = 270 deg. +SSD1306_128_32_I2C_ROTATION = 0 + +# OLED Resolution: 1 = 128x32, 2 = 128x64. +SSD1306_RESOLUTION = 1 + + +# ------------------------------------------------------------------------------ +# INPUT DEVICES (JOYSTICK / CONTROLLER) +# ------------------------------------------------------------------------------ + +# If True, the joystick is enabled by default without needing '--js' flag. +USE_JOYSTICK_AS_DEFAULT = True + +# The maximum throttle output (0.0 to 1.0) allowed by the joystick. +# Useful for limiting speed for beginners. +JOYSTICK_MAX_THROTTLE = 0.5 + +# Scalar for steering. 1.0 is normal. <1.0 is less sensitive. +JOYSTICK_STEERING_SCALE = 1.0 + +# The "deadzone" where small joystick movements are ignored (0.0 to 1.0). +JOYSTICK_DEADZONE = 0.01 + +# Set to -1.0 to flip forward/backward direction on the joystick. +JOYSTICK_THROTTLE_DIR = -1.0 + +# The linux device file for the joystick. +JOYSTICK_DEVICE_FILE = "/dev/input/js0" + +# The type of controller being used. +# Options: 'ps3', 'ps4', 'xbox', 'nimbus', 'wiiu', 'F710', 'rc3', 'MM1 (use for RC Hat)', 'custom' +CONTROLLER_TYPE = 'xbox' + +# Enable listening for remote joystick control over the network. +USE_NETWORKED_JS = False +NETWORK_JS_SERVER_IP = None + + +# ------------------------------------------------------------------------------ +# DRIVE TRAIN CONFIGURATION +# ------------------------------------------------------------------------------ -# # DRIVE_TRAIN_TYPE # These options specify which chasis and motor setup you are using. # See Actuators documentation https://docs.donkeycar.com/parts/actuators/ @@ -66,55 +150,37 @@ # "DC_TWO_WHEEL_L298N" using HBridge in 3-pin mode to control two drive motors, one of the left and one on the right. # "MOCK" no drive train. This can be used to test other features in a test rig. # "VESC" VESC Motor controller to set servo angle and duty cycle -# (deprecated) "SERVO_HBRIDGE_PWM" use ServoBlaster to output pwm control from the PiZero directly to control steering, -# and HBridge for a drive motor. -# (deprecated) "PIGPIO_PWM" uses Raspberrys internal PWM -# (deprecated) "I2C_SERVO" uses PCA9685 servo controller to control a steering servo and an ESC, as in a standard RC car -# + +# Select the drive train type. This determines how the software talks to the motors. +# "PWM_STEERING_THROTTLE": Standard RC car (Servo + ESC) +# "MM1": RoboHat MM1 or RC Hat +# "SERVO_HBRIDGE_2PIN": Servo for steering, HBridge (2 pin) for motor +# "SERVO_HBRIDGE_3PIN": Servo for steering, HBridge (3 pin) for motor +# "DC_STEER_THROTTLE": DC Motor for steering, DC Motor for drive (L298N) +# "DC_TWO_WHEEL": Differential drive (tank style), 2 Pin HBridge +# "DC_TWO_WHEEL_L298N": Differential drive (tank style), 3 Pin HBridge +# "VESC": VESC Motor Controller DRIVE_TRAIN_TYPE = "PWM_STEERING_THROTTLE" -# -# PWM_STEERING_THROTTLE -# +# Configuration for PWM_STEERING_THROTTLE (Standard RC Car) # Drive train for RC car with a steering servo and ESC. # Uses a PwmPin for steering (servo) and a second PwmPin for throttle (ESC) # Base PWM Frequence is presumed to be 60hz; use PWM_xxxx_SCALE to adjust pulse with for non-standard PWM frequencies -# +# Requires calibration using 'donkey calibrate'. PWM_STEERING_THROTTLE = { - "PWM_STEERING_PIN": "PCA9685.1:40.1", # PWM output pin for steering servo - "PWM_STEERING_SCALE": 1.0, # used to compensate for PWM frequency differents from 60hz; NOT for adjusting steering range - "PWM_STEERING_INVERTED": False, # True if hardware requires an inverted PWM pulse - "PWM_THROTTLE_PIN": "PCA9685.1:40.0", # PWM output pin for ESC - "PWM_THROTTLE_SCALE": 1.0, # used to compensate for PWM frequence differences from 60hz; NOT for increasing/limiting speed - "PWM_THROTTLE_INVERTED": False, # True if hardware requires an inverted PWM pulse - "STEERING_LEFT_PWM": 460, #pwm value for full left steering - "STEERING_RIGHT_PWM": 290, #pwm value for full right steering - "THROTTLE_FORWARD_PWM": 500, #pwm value for max forward throttle - "THROTTLE_STOPPED_PWM": 370, #pwm value for no movement - "THROTTLE_REVERSE_PWM": 220, #pwm value for max reverse throttle + "PWM_STEERING_PIN": "PCA9685.1:40.1", # Pin for steering servo + "PWM_STEERING_SCALE": 1.0, # PWM frequency compensation + "PWM_STEERING_INVERTED": False, # Invert steering direction + "PWM_THROTTLE_PIN": "PCA9685.1:40.0", # Pin for ESC (Throttle) + "PWM_THROTTLE_SCALE": 1.0, # PWM frequency compensation + "PWM_THROTTLE_INVERTED": False, # Invert throttle direction + "STEERING_LEFT_PWM": 460, # Calibrated value: Full Left + "STEERING_RIGHT_PWM": 290, # Calibrated value: Full Right + "THROTTLE_FORWARD_PWM": 500, # Calibrated value: Max Forward + "THROTTLE_STOPPED_PWM": 370, # Calibrated value: Stopped + "THROTTLE_REVERSE_PWM": 220, # Calibrated value: Max Reverse } -# -# I2C_SERVO (deprecated in favor of PWM_STEERING_THROTTLE) -# -STEERING_CHANNEL = 1 #(deprecated) channel on the 9685 pwm board 0-15 -STEERING_LEFT_PWM = 460 #pwm value for full left steering -STEERING_RIGHT_PWM = 290 #pwm value for full right steering -THROTTLE_CHANNEL = 0 #(deprecated) channel on the 9685 pwm board 0-15 -THROTTLE_FORWARD_PWM = 500 #pwm value for max forward throttle -THROTTLE_STOPPED_PWM = 370 #pwm value for no movement -THROTTLE_REVERSE_PWM = 220 #pwm value for max reverse throttle - -# -# PIGPIO_PWM (deprecated in favor of PWM_STEERING_THROTTLE) -# -STEERING_PWM_PIN = 13 #(deprecated) Pin numbering according to Broadcom numbers -STEERING_PWM_FREQ = 50 #Frequency for PWM -STEERING_PWM_INVERTED = False #If PWM needs to be inverted -THROTTLE_PWM_PIN = 18 #(deprecated) Pin numbering according to Broadcom numbers -THROTTLE_PWM_FREQ = 50 #Frequency for PWM -THROTTLE_PWM_INVERTED = False #If PWM needs to be inverted - # # SERVO_HBRIDGE_2PIN # - configures a steering servo and an HBridge in 2pin mode (2 pwm pins) @@ -145,14 +211,15 @@ # - RPI_GPIO, PIGPIO and PCA9685 can be mixed arbitrarily, # although it is discouraged to mix RPI_GPIO and PIGPIO. # +# Configuration for SERVO_HBRIDGE_2PIN SERVO_HBRIDGE_2PIN = { - "FWD_DUTY_PIN": "RPI_GPIO.BOARD.18", # provides forward duty cycle to motor - "BWD_DUTY_PIN": "RPI_GPIO.BOARD.16", # provides reverse duty cycle to motor - "PWM_STEERING_PIN": "RPI_GPIO.BOARD.33", # provides servo pulse to steering servo - "PWM_STEERING_SCALE": 1.0, # used to compensate for PWM frequency differents from 60hz; NOT for adjusting steering range - "PWM_STEERING_INVERTED": False, # True if hardware requires an inverted PWM pulse - "STEERING_LEFT_PWM": 460, # pwm value for full left steering (use `donkey calibrate` to measure value for your car) - "STEERING_RIGHT_PWM": 290, # pwm value for full right steering (use `donkey calibrate` to measure value for your car) + "FWD_DUTY_PIN": "RPI_GPIO.BOARD.18", # Pin for Forward + "BWD_DUTY_PIN": "RPI_GPIO.BOARD.16", # Pin for Reverse + "PWM_STEERING_PIN": "RPI_GPIO.BOARD.33", # Pin for Servo + "PWM_STEERING_SCALE": 1.0, + "PWM_STEERING_INVERTED": False, + "STEERING_LEFT_PWM": 460, + "STEERING_RIGHT_PWM": 290, } # @@ -189,51 +256,18 @@ # - RPI_GPIO, PIGPIO and PCA9685 can be mixed arbitrarily, # although it is discouraged to mix RPI_GPIO and PIGPIO. # +# Configuration for SERVO_HBRIDGE_3PIN SERVO_HBRIDGE_3PIN = { - "FWD_PIN": "RPI_GPIO.BOARD.18", # ttl pin, high enables motor forward - "BWD_PIN": "RPI_GPIO.BOARD.16", # ttl pin, high enables motor reverse - "DUTY_PIN": "RPI_GPIO.BOARD.35", # provides duty cycle to motor - "PWM_STEERING_PIN": "RPI_GPIO.BOARD.33", # provides servo pulse to steering servo - "PWM_STEERING_SCALE": 1.0, # used to compensate for PWM frequency differents from 60hz; NOT for adjusting steering range - "PWM_STEERING_INVERTED": False, # True if hardware requires an inverted PWM pulse - "STEERING_LEFT_PWM": 460, # pwm value for full left steering (use `donkey calibrate` to measure value for your car) - "STEERING_RIGHT_PWM": 290, # pwm value for full right steering (use `donkey calibrate` to measure value for your car) + "FWD_PIN": "RPI_GPIO.BOARD.18", # Enable Forward + "BWD_PIN": "RPI_GPIO.BOARD.16", # Enable Reverse + "DUTY_PIN": "RPI_GPIO.BOARD.35", # Speed Control (PWM) + "PWM_STEERING_PIN": "RPI_GPIO.BOARD.33", + "PWM_STEERING_SCALE": 1.0, + "PWM_STEERING_INVERTED": False, + "STEERING_LEFT_PWM": 460, + "STEERING_RIGHT_PWM": 290, } -# -# DRIVETRAIN_TYPE == "SERVO_HBRIDGE_PWM" (deprecated in favor of SERVO_HBRIDGE_2PIN) -# - configures a steering servo and an HBridge in 2pin mode (2 pwm pins) -# - Uses ServoBlaster library, which is NOT installed by default, so -# you will need to install it to make this work. -# - Servo takes a standard servo PWM pulse between 1 millisecond (fully reverse) -# and 2 milliseconds (full forward) with 1.5ms being neutral. -# - the motor is controlled by two pwm pins, -# one for forward and one for backward (reverse). -# - the pwm pins produce a duty cycle from 0 (completely LOW) -# to 1 (100% completely high), which is proportional to the -# amount of power delivered to the motor. -# - in forward mode, the reverse pwm is 0 duty_cycle, -# in backward mode, the forward pwm is 0 duty cycle. -# - both pwms are 0 duty cycle (LOW) to 'detach' motor and -# and glide to a stop. -# - both pwms are full duty cycle (100% HIGH) to brake -# -HBRIDGE_PIN_FWD = 18 # provides forward duty cycle to motor -HBRIDGE_PIN_BWD = 16 # provides reverse duty cycle to motor -STEERING_CHANNEL = 0 # PCA 9685 channel for steering control -STEERING_LEFT_PWM = 460 # pwm value for full left steering (use `donkey calibrate` to measure value for your car) -STEERING_RIGHT_PWM = 290 # pwm value for full right steering (use `donkey calibrate` to measure value for your car) - -#VESC controller, primarily need to change VESC_SERIAL_PORT and VESC_MAX_SPEED_PERCENT -VESC_MAX_SPEED_PERCENT =.2 # Max speed as a percent of the actual speed -VESC_SERIAL_PORT= "/dev/ttyACM0" # Serial device to use for communication. Can check with ls /dev/tty* -VESC_HAS_SENSOR= True # Whether or not the bldc motor is using a hall effect sensor -VESC_START_HEARTBEAT= True # Whether or not to automatically start the heartbeat thread that will keep commands alive. -VESC_BAUDRATE= 115200 # baudrate for the serial communication. Shouldn't need to change this. -VESC_TIMEOUT= 0.05 # timeout for the serial communication -VESC_STEERING_SCALE= 0.5 # VESC accepts steering inputs from 0 to 1. Joystick is usually -1 to 1. This changes it to -0.5 to 0.5 -VESC_STEERING_OFFSET = 0.5 # VESC accepts steering inputs from 0 to 1. Coupled with above change we move Joystick to 0 to 1 - # # DC_STEER_THROTTLE with one motor as steering, one as drive # - uses L298N type motor controller in two pin wiring @@ -254,14 +288,14 @@ # - RPI_GPIO, PIGPIO and PCA9685 can be mixed arbitrarily, # although it is discouraged to mix RPI_GPIO and PIGPIO. # +# Configuration for DC_STEER_THROTTLE (Motor for steering, Motor for drive) DC_STEER_THROTTLE = { - "LEFT_DUTY_PIN": "RPI_GPIO.BOARD.18", # pwm pin produces duty cycle for steering left - "RIGHT_DUTY_PIN": "RPI_GPIO.BOARD.16", # pwm pin produces duty cycle for steering right - "FWD_DUTY_PIN": "RPI_GPIO.BOARD.15", # pwm pin produces duty cycle for forward drive - "BWD_DUTY_PIN": "RPI_GPIO.BOARD.13", # pwm pin produces duty cycle for reverse drive + "LEFT_DUTY_PIN": "RPI_GPIO.BOARD.18", # Steer Left + "RIGHT_DUTY_PIN": "RPI_GPIO.BOARD.16", # Steer Right + "FWD_DUTY_PIN": "RPI_GPIO.BOARD.15", # Drive Forward + "BWD_DUTY_PIN": "RPI_GPIO.BOARD.13", # Drive Reverse } -# # DC_TWO_WHEEL pin configuration # - configures L298N_HBridge_2pin driver # - two wheels as differential drive, left and right. @@ -290,14 +324,14 @@ # - RPI_GPIO, PIGPIO and PCA9685 can be mixed arbitrarily, # although it is discouraged to mix RPI_GPIO and PIGPIO. # +# Configuration for DC_TWO_WHEEL (Differential/Tank Drive) DC_TWO_WHEEL = { - "LEFT_FWD_DUTY_PIN": "RPI_GPIO.BOARD.18", # pwm pin produces duty cycle for left wheel forward - "LEFT_BWD_DUTY_PIN": "RPI_GPIO.BOARD.16", # pwm pin produces duty cycle for left wheel reverse - "RIGHT_FWD_DUTY_PIN": "RPI_GPIO.BOARD.15", # pwm pin produces duty cycle for right wheel forward - "RIGHT_BWD_DUTY_PIN": "RPI_GPIO.BOARD.13", # pwm pin produces duty cycle for right wheel reverse + "LEFT_FWD_DUTY_PIN": "RPI_GPIO.BOARD.18", + "LEFT_BWD_DUTY_PIN": "RPI_GPIO.BOARD.16", + "RIGHT_FWD_DUTY_PIN": "RPI_GPIO.BOARD.15", + "RIGHT_BWD_DUTY_PIN": "RPI_GPIO.BOARD.13", } -# # DC_TWO_WHEEL_L298N pin configuration # - configures L298N_HBridge_3pin driver # - two wheels as differential drive, left and right. @@ -329,37 +363,95 @@ # - for example "PCA9685.1:40.13" # - RPI_GPIO, PIGPIO and PCA9685 can be mixed arbitrarily, # although it is discouraged to mix RPI_GPIO and PIGPIO. -# +# Configuration for DC_TWO_WHEEL_L298N (Differential Drive 3-pin) DC_TWO_WHEEL_L298N = { - "LEFT_FWD_PIN": "RPI_GPIO.BOARD.16", # TTL output pin enables left wheel forward - "LEFT_BWD_PIN": "RPI_GPIO.BOARD.18", # TTL output pin enables left wheel reverse - "LEFT_EN_DUTY_PIN": "RPI_GPIO.BOARD.22", # PWM pin generates duty cycle for left motor speed - - "RIGHT_FWD_PIN": "RPI_GPIO.BOARD.15", # TTL output pin enables right wheel forward - "RIGHT_BWD_PIN": "RPI_GPIO.BOARD.13", # TTL output pin enables right wheel reverse - "RIGHT_EN_DUTY_PIN": "RPI_GPIO.BOARD.11", # PWM pin generates duty cycle for right wheel speed + "LEFT_FWD_PIN": "RPI_GPIO.BOARD.16", + "LEFT_BWD_PIN": "RPI_GPIO.BOARD.18", + "LEFT_EN_DUTY_PIN": "RPI_GPIO.BOARD.22", + "RIGHT_FWD_PIN": "RPI_GPIO.BOARD.15", + "RIGHT_BWD_PIN": "RPI_GPIO.BOARD.13", + "RIGHT_EN_DUTY_PIN": "RPI_GPIO.BOARD.11", } -#ODOMETRY -HAVE_ODOM = False # Do you have an odometer/encoder -ENCODER_TYPE = 'GPIO' # What kind of encoder? GPIO|Arduino|Astar -MM_PER_TICK = 12.7625 # How much travel with a single tick, in mm. Roll you car a meter and divide total ticks measured by 1,000 -ODOM_PIN = 13 # if using GPIO, which GPIO board mode pin to use as input -ODOM_DEBUG = False # Write out values on vel and distance as it runs +# Configuration for VESC Motor Controller +VESC_MAX_SPEED_PERCENT = .2 +VESC_SERIAL_PORT = "/dev/ttyACM0" +VESC_HAS_SENSOR = True +VESC_START_HEARTBEAT = True +VESC_BAUDRATE = 115200 +VESC_TIMEOUT = 0.05 +VESC_STEERING_SCALE = 0.5 +VESC_STEERING_OFFSET = 0.5 + +# Configuration for RoboHat MM1 and RC Hat +MM1_STEERING_MID = 1500 +MM1_MAX_FORWARD = 2000 +MM1_STOPPED_PWM = 1500 +MM1_MAX_REVERSE = 1000 +MM1_SHOW_STEERING_VALUE = False +MM1_SERIAL_PORT = '/dev/ttyAMA0' + + +# ------------------------------------------------------------------------------ +# SENSORS & ADD-ONS +# ------------------------------------------------------------------------------ + +# ODOMETRY: Set to True if you have an encoder/odometer installed. +HAVE_ODOM = False +ENCODER_TYPE = 'GPIO' # GPIO|Arduino|Astar +MM_PER_TICK = 12.7625 # Calibration: MM travel per encoder tick +ODOM_PIN = 13 # GPIO pin for encoder +ODOM_DEBUG = False -# #LIDAR +# LIDAR: Set to True if you have a LIDAR (RP or YD). USE_LIDAR = False -LIDAR_TYPE = 'RP' #(RP|YD) -LIDAR_LOWER_LIMIT = 90 # angles that will be recorded. Use this to block out obstructed areas on your car, or looking backwards. Note that for the RP A1M8 Lidar, "0" is in the direction of the motor +LIDAR_TYPE = 'RP' # (RP|YD) +LIDAR_LOWER_LIMIT = 90 # Angle limit to ignore (e.g. looking back at car) LIDAR_UPPER_LIMIT = 270 -# TFMINI +# TFMINI: Short range laser radar. HAVE_TFMINI = False -TFMINI_SERIAL_PORT = "/dev/serial0" # tfmini serial port, can be wired up or use usb/serial adapter +TFMINI_SERIAL_PORT = "/dev/serial0" -#TRAINING -# The default AI framework to use. Choose from (tensorflow|pytorch) -DEFAULT_AI_FRAMEWORK = 'tensorflow' +# IMU: Inertial Measurement Unit (e.g. MPU6050). +HAVE_IMU = False +IMU_SENSOR = 'mpu6050' # (mpu6050|mpu9250|bno08x) +IMU_ADDRESS = 0x68 # I2C address +IMU_DLP_CONFIG = 0 # Digital Lowpass Filter (0-6) + +# GPS + IMU Fusion: Use the BNO08x IMU to update position faster +USE_FUSION = False # Activate GPS + IMU Fusion +FUSION_DEBUG = False # Logs the position and yaw from the kalman filter + +# SOMBRERO HAT: Enable if using the Sombrero Hat. +HAVE_SOMBRERO = False + +# LEDS: RGB Status LED configuration. +HAVE_RGB_LED = False +LED_INVERT = False # True for Common Anode +LED_PIN_R = 12 +LED_PIN_G = 10 +LED_PIN_B = 16 +LED_R = 0 +LED_G = 0 +LED_B = 1 + +# Hardware Alert Logic (Blink LED when recording count reached) +REC_COUNT_ALERT = 1000 +REC_COUNT_ALERT_CYC = 15 +REC_COUNT_ALERT_BLINK_RATE = 0.4 +RECORD_ALERT_COLOR_ARR = [ (0, (1, 1, 1)), (3000, (5, 5, 5)), (5000, (5, 2, 0)), (10000, (0, 5, 0)), (15000, (0, 5, 5)), (20000, (0, 0, 5)) ] +MODEL_RELOADED_LED_R = 100 +MODEL_RELOADED_LED_G = 0 +MODEL_RELOADED_LED_B = 0 + + +# ============================================================================== +# 2. AI, MODELS & TRAINING +# (Frameworks, Hyperparams, Transformations, Augmentations) +# ============================================================================== + +# TRAINING FUNDAMENTALS # The DEFAULT_MODEL_TYPE will choose which model will be created at training # time. This chooses between different neural network designs. You can @@ -367,32 +459,91 @@ # python manage.py train and drive commands. # tensorflow models: (linear|categorical|tflite_linear|tensorrt_linear) # pytorch models: (resnet18) +# The AI framework to use (tensorflow|pytorch). +DEFAULT_AI_FRAMEWORK = 'tensorflow' + +# The architecture of the model to use. +# 'linear': Standard regression (predicts steer/throttle floats) +# 'categorical': Classification (bins steer/throttle into categories) +# 'resnet18': Pytorch heavy model DEFAULT_MODEL_TYPE = 'linear' -BATCH_SIZE = 128 #how many records to use when doing one pass of gradient decent. Use a smaller number if your gpu is running out of memory. -TRAIN_TEST_SPLIT = 0.8 #what percent of records to use for training. the remaining used for validation. -MAX_EPOCHS = 100 #how many times to visit all records of your data -SHOW_PLOT = True #would you like to see a pop up display of final loss? -VERBOSE_TRAIN = True #would you like to see a progress bar with text during training? -USE_EARLY_STOP = True #would you like to stop the training if we see it's not improving fit? -EARLY_STOP_PATIENCE = 5 #how many epochs to wait before no improvement -MIN_DELTA = .0005 #early stop will want this much loss change before calling it improved. -PRINT_MODEL_SUMMARY = True #print layers and weights to stdout -OPTIMIZER = None #adam, sgd, rmsprop, etc.. None accepts default -LEARNING_RATE = 0.001 #only used when OPTIMIZER specified -LEARNING_RATE_DECAY = 0.0 #only used when OPTIMIZER specified -SEND_BEST_MODEL_TO_PI = False #change to true to automatically send best model during training -CREATE_TF_LITE = True # automatically create tflite model in training -CREATE_TENSOR_RT = False # automatically create tensorrt model in training -SAVE_MODEL_AS_H5 = False # if old keras format should be used instead of savedmodel -CACHE_POLICY = 'ARRAY' # if images are cached as array in training other options are 'NOCACHE' and 'BINARY' - -PRUNE_CNN = False #This will remove weights from your model. The primary goal is to increase performance. -PRUNE_PERCENT_TARGET = 75 # The desired percentage of pruning. -PRUNE_PERCENT_PER_ITERATION = 20 # Percenge of pruning that is perform per iteration. -PRUNE_VAL_LOSS_DEGRADATION_LIMIT = 0.2 # The max amout of validation loss that is permitted during pruning. -PRUNE_EVAL_PERCENT_OF_DATASET = .05 # percent of dataset used to perform evaluation of model. -# +# Number of training samples per pass. +BATCH_SIZE = 128 + +# Percentage of data used for training vs validation (0.8 = 80% train). +TRAIN_TEST_SPLIT = 0.8 + +# Max training iterations. +MAX_EPOCHS = 100 + +# Show a plot of loss after training. +SHOW_PLOT = True + +# Show text progress bar during training. +VERBOSE_TRAIN = True + +# Stop training early if loss stops improving. +USE_EARLY_STOP = True +EARLY_STOP_PATIENCE = 5 +MIN_DELTA = .0005 + +# Print model summary to console. +PRINT_MODEL_SUMMARY = True + +# Optimizer (None uses default for framework). +OPTIMIZER = None +LEARNING_RATE = 0.001 +LEARNING_RATE_DECAY = 0.0 + +# Store images as 'ARRAY' (faster), 'BINARY', or 'NOCACHE' (saves RAM). +CACHE_POLICY = 'ARRAY' + +# MODEL OPTIMIZATION +# Automatically create TFLite model for faster inference on Pi. +CREATE_TF_LITE = True +CREATE_TENSOR_RT = False +SAVE_MODEL_AS_H5 = False +SEND_BEST_MODEL_TO_PI = False + +# Model Pruning (Remove weights to increase speed). +PRUNE_CNN = False +PRUNE_PERCENT_TARGET = 75 +PRUNE_PERCENT_PER_ITERATION = 20 +PRUNE_VAL_LOSS_DEGRADATION_LIMIT = 0.2 +PRUNE_EVAL_PERCENT_OF_DATASET = .05 + +# MODEL SPECIFIC SETTINGS +#Limits the upper bound of the learned throttle for categorical models. +#For the categorical model, this limits the upper bound of the learned throttle +#it's very IMPORTANT that this value is matched from the training PC config.py and the robot.py +#and ideally wouldn't change once set. +MODEL_CATEGORICAL_MAX_THROTTLE_RANGE = 0.8 + +# Number of images in a sequence for RNN/3D models. +SEQUENCE_LENGTH = 3 + +# Transfer Learning options. +FREEZE_LAYERS = False +NUM_LAST_LAYERS_TO_TRAIN = 7 + + +# ------------------------------------------------------------------------------ +# AUGMENTATIONS (Applied randomly ONLY during training) +# ------------------------------------------------------------------------------ +# List of augmentations to apply. e.g. ['MULTIPLY', 'BLUR'] +AUGMENTATIONS = [] + +# Brightness range for augmentation [-0.2, 0.2]. +AUG_BRIGHTNESS_RANGE = 0.2 + +# Blur range for augmentation (kernel size). +AUG_BLUR_RANGE = (0, 3) + + +# ------------------------------------------------------------------------------ +# TRANSFORMATIONS (Applied during Training AND Inference) +# ------------------------------------------------------------------------------ # Augmentations and Transformations # # - Augmentations are changes to the image that are only applied during @@ -482,17 +633,10 @@ # return self.blur.run(image) # ``` # -AUGMENTATIONS = [] # changes to image only applied in training to create - # more variety in the data. -TRANSFORMATIONS = [] # changes applied _before_ training augmentations, - # such that augmentations are applied to the transformed image, -POST_TRANSFORMATIONS = [] # transformations applied _after_ training augmentations, - # such that changes are applied to the augmented image - -# Settings for brightness and blur, use 'MULTIPLY' and/or 'BLUR' in -# AUGMENTATIONS -AUG_BRIGHTNESS_RANGE = 0.2 # this is interpreted as [-0.2, 0.2] -AUG_BLUR_RANGE = (0, 3) + +# Operations applied to the image before it hits the AI. +TRANSFORMATIONS = [] +POST_TRANSFORMATIONS = [] # "CROP" Transformation # Apply mask to borders of the image @@ -508,10 +652,10 @@ # xxxxxxxxxxxxxxxxxxxxx # bottom # xxxxxxxxxxxxxxxxxxxxx # # # # # # # # # # # # # # -ROI_CROP_TOP = 45 # the number of rows of pixels to ignore on the top of the image -ROI_CROP_BOTTOM = 0 # the number of rows of pixels to ignore on the bottom of the image -ROI_CROP_RIGHT = 0 # the number of rows of pixels to ignore on the right of the image -ROI_CROP_LEFT = 0 # the number of rows of pixels to ignore on the left of the image +ROI_CROP_TOP = 45 +ROI_CROP_BOTTOM = 0 +ROI_CROP_RIGHT = 0 +ROI_CROP_LEFT = 0 # "TRAPEZE" tranformation # Apply mask to borders of image @@ -531,161 +675,56 @@ ROI_TRAPEZE_MIN_Y = 60 ROI_TRAPEZE_MAX_Y = 120 -# "CANNY" Canny Edge Detection tranformation -CANNY_LOW_THRESHOLD = 60 # Canny edge detection low threshold value of intensity gradient -CANNY_HIGH_THRESHOLD = 110 # Canny edge detection high threshold value of intensity gradient -CANNY_APERTURE = 3 # Canny edge detect aperture in pixels, must be odd; choices=[3, 5, 7] - -# "BLUR" transformation (not this is SEPARATE from the blur augmentation) -BLUR_KERNEL = 5 # blur kernel horizontal size in pixels -BLUR_KERNEL_Y = None # blur kernel vertical size in pixels or None for square kernel -BLUR_GAUSSIAN = True # blur is gaussian if True, simple if False - -# "RESIZE" transformation -RESIZE_WIDTH = 160 # horizontal size in pixels -RESIZE_HEIGHT = 120 # vertical size in pixels - -# "SCALE" transformation -SCALE_WIDTH = 1.0 # horizontal scale factor -SCALE_HEIGHT = None # vertical scale factor or None to maintain aspect ratio - -#Model transfer options -#When copying weights during a model transfer operation, should we freeze a certain number of layers -#to the incoming weights and not allow them to change during training? -FREEZE_LAYERS = False #default False will allow all layers to be modified by training -NUM_LAST_LAYERS_TO_TRAIN = 7 #when freezing layers, how many layers from the last should be allowed to train? - -#WEB CONTROL -WEB_CONTROL_PORT = int(os.getenv("WEB_CONTROL_PORT", 8887)) # which port to listen on when making a web controller -WEB_INIT_MODE = "user" # which control mode to start in. one of user|local_angle|local. Setting local will start in ai mode. - -#JOYSTICK -USE_JOYSTICK_AS_DEFAULT = False #when starting the manage.py, when True, will not require a --js option to use the joystick -JOYSTICK_MAX_THROTTLE = 0.5 #this scalar is multiplied with the -1 to 1 throttle value to limit the maximum throttle. This can help if you drop the controller or just don't need the full speed available. -JOYSTICK_STEERING_SCALE = 1.0 #some people want a steering that is less sensitve. This scalar is multiplied with the steering -1 to 1. It can be negative to reverse dir. -AUTO_RECORD_ON_THROTTLE = True #if true, we will record whenever throttle is not zero. if false, you must manually toggle recording with some other trigger. Usually circle button on joystick. -CONTROLLER_TYPE = 'xbox' #(ps3|ps4|xbox|pigpio_rc|nimbus|wiiu|F710|rc3|MM1|custom) custom will run the my_joystick.py controller written by the `donkey createjs` command -USE_NETWORKED_JS = False #should we listen for remote joystick control over the network? -NETWORK_JS_SERVER_IP = None #when listening for network joystick control, which ip is serving this information -JOYSTICK_DEADZONE = 0.01 # when non zero, this is the smallest throttle before recording triggered. -JOYSTICK_THROTTLE_DIR = -1.0 # use -1.0 to flip forward/backward, use 1.0 to use joystick's natural forward/backward -USE_FPV = False # send camera data to FPV webserver -JOYSTICK_DEVICE_FILE = "/dev/input/js0" # this is the unix file use to access the joystick. +# "CANNY" Edge Detection Settings. +CANNY_LOW_THRESHOLD = 60 +CANNY_HIGH_THRESHOLD = 110 +CANNY_APERTURE = 3 -#For the categorical model, this limits the upper bound of the learned throttle -#it's very IMPORTANT that this value is matched from the training PC config.py and the robot.py -#and ideally wouldn't change once set. -MODEL_CATEGORICAL_MAX_THROTTLE_RANGE = 0.8 +# "BLUR" Transformation Settings. +BLUR_KERNEL = 5 +BLUR_KERNEL_Y = None +BLUR_GAUSSIAN = True -#RNN or 3D -SEQUENCE_LENGTH = 3 #some models use a number of images over time. This controls how many. +# "RESIZE" / "SCALE" Settings. +RESIZE_WIDTH = 160 +RESIZE_HEIGHT = 120 +SCALE_WIDTH = 1.0 +SCALE_HEIGHT = None -#IMU -HAVE_IMU = False #when true, this add a Mpu6050 part and records the data. Can be used with a -IMU_SENSOR = 'mpu6050' # (mpu6050|mpu9250) -IMU_ADDRESS = 0x68 # if AD0 pin is pulled high them address is 0x69, otherwise it is 0x68 -IMU_DLP_CONFIG = 0 # Digital Lowpass Filter setting (0:250Hz, 1:184Hz, 2:92Hz, 3:41Hz, 4:20Hz, 5:10Hz, 6:5Hz) -#SOMBRERO -HAVE_SOMBRERO = False #set to true when using the sombrero hat from the Donkeycar store. This will enable pwm on the hat. +# ============================================================================== +# 3. MODES, FEATURES & OPERATION +# (Driving modes, Web/Joystick control, Simulation, Logging) +# ============================================================================== -#PIGPIO RC control -STEERING_RC_GPIO = 26 -THROTTLE_RC_GPIO = 20 -DATA_WIPER_RC_GPIO = 19 -PIGPIO_STEERING_MID = 1500 # Adjust this value if your car cannot run in a straight line -PIGPIO_MAX_FORWARD = 2000 # Max throttle to go fowrward. The bigger the faster -PIGPIO_STOPPED_PWM = 1500 -PIGPIO_MAX_REVERSE = 1000 # Max throttle to go reverse. The smaller the faster -PIGPIO_SHOW_STEERING_VALUE = False -PIGPIO_INVERT = False -PIGPIO_JITTER = 0.025 # threshold below which no signal is reported +# VEHICLE LOOP +# The main loop frequency (Hz). Hardware is updated this many times per second. +DRIVE_LOOP_HZ = 20 +# Max loops to run before quitting (useful for testing, None = infinite). +MAX_LOOPS = None -#ROBOHAT MM1 -MM1_STEERING_MID = 1500 # Adjust this value if your car cannot run in a straight line -MM1_MAX_FORWARD = 2000 # Max throttle to go fowrward. The bigger the faster -MM1_STOPPED_PWM = 1500 -MM1_MAX_REVERSE = 1000 # Max throttle to go reverse. The smaller the faster -MM1_SHOW_STEERING_VALUE = False -# Serial port -# -- Default Pi: '/dev/ttyS0' -# -- Jetson Nano: '/dev/ttyTHS1' -# -- Google coral: '/dev/ttymxc0' -# -- Windows: 'COM3', Arduino: '/dev/ttyACM0' -# -- MacOS/Linux:please use 'ls /dev/tty.*' to find the correct serial port for mm1 -# eg.'/dev/tty.usbmodemXXXXXX' and replace the port accordingly -MM1_SERIAL_PORT = '/dev/ttyS0' # Serial Port for reading and sending MM1 data. - -#LOGGING -HAVE_CONSOLE_LOGGING = True -LOGGING_LEVEL = 'INFO' # (Python logging level) 'NOTSET' / 'DEBUG' / 'INFO' / 'WARNING' / 'ERROR' / 'FATAL' / 'CRITICAL' -LOGGING_FORMAT = '%(message)s' # (Python logging format - https://docs.python.org/3/library/logging.html#formatter-objects - -#TELEMETRY -HAVE_MQTT_TELEMETRY = False -TELEMETRY_DONKEY_NAME = 'my_robot1234' -TELEMETRY_MQTT_TOPIC_TEMPLATE = 'donkey/%s/telemetry' -TELEMETRY_MQTT_JSON_ENABLE = False -TELEMETRY_MQTT_BROKER_HOST = 'broker.hivemq.com' -TELEMETRY_MQTT_BROKER_PORT = 1883 -TELEMETRY_PUBLISH_PERIOD = 1 -TELEMETRY_LOGGING_ENABLE = True -TELEMETRY_LOGGING_LEVEL = 'INFO' # (Python logging level) 'NOTSET' / 'DEBUG' / 'INFO' / 'WARNING' / 'ERROR' / 'FATAL' / 'CRITICAL' -TELEMETRY_LOGGING_FORMAT = '%(message)s' # (Python logging format - https://docs.python.org/3/library/logging.html#formatter-objects -TELEMETRY_DEFAULT_INPUTS = 'pilot/angle,pilot/throttle,recording' -TELEMETRY_DEFAULT_TYPES = 'float,float' +# AUTOMATION & BEHAVIORS -# PERF MONITOR -HAVE_PERFMON = False - -#RECORD OPTIONS -RECORD_DURING_AI = False #normally we do not record during ai mode. Set this to true to get image and steering records for your Ai. Be careful not to use them to train. -AUTO_CREATE_NEW_TUB = False #create a new tub (tub_YY_MM_DD) directory when recording or append records to data directory directly - -#LED -HAVE_RGB_LED = False #do you have an RGB LED like https://www.amazon.com/dp/B07BNRZWNF -LED_INVERT = False #COMMON ANODE? Some RGB LED use common anode. like https://www.amazon.com/Xia-Fly-Tri-Color-Emitting-Diffused/dp/B07MYJQP8B - -#LED board pin number for pwm outputs -#These are physical pinouts. See: https://www.raspberrypi-spy.co.uk/2012/06/simple-guide-to-the-rpi-gpio-header-and-pins/ -LED_PIN_R = 12 -LED_PIN_G = 10 -LED_PIN_B = 16 - -#LED status color, 0-100 -LED_R = 0 -LED_G = 0 -LED_B = 1 +# Show the image the pilot sees (with overlays) in the web UI. +SHOW_PILOT_IMAGE = False -#LED Color for record count indicator -REC_COUNT_ALERT = 1000 #how many records before blinking alert -REC_COUNT_ALERT_CYC = 15 #how many cycles of 1/20 of a second to blink per REC_COUNT_ALERT records -REC_COUNT_ALERT_BLINK_RATE = 0.4 #how fast to blink the led in seconds on/off - -#first number is record count, second tuple is color ( r, g, b) (0-100) -#when record count exceeds that number, the color will be used -RECORD_ALERT_COLOR_ARR = [ (0, (1, 1, 1)), - (3000, (5, 5, 5)), - (5000, (5, 2, 0)), - (10000, (0, 5, 0)), - (15000, (0, 5, 5)), - (20000, (0, 0, 5)), ] - - -#LED status color, 0-100, for model reloaded alert -MODEL_RELOADED_LED_R = 100 -MODEL_RELOADED_LED_G = 0 -MODEL_RELOADED_LED_B = 0 +# Scale all AI throttle output by this multiplier. +AI_THROTTLE_MULT = 1.0 +# "Launch Control": Boost throttle for X seconds at start of autonomous mode. +AI_LAUNCH_DURATION = 0.0 +AI_LAUNCH_THROTTLE = 0.0 +AI_LAUNCH_ENABLE_BUTTON = 'R2' +AI_LAUNCH_KEEP_ENABLED = False #BEHAVIORS #When training the Behavioral Neural Network model, make a list of the behaviors, #Set the TRAIN_BEHAVIORS = True, and use the BEHAVIOR_LED_COLORS to give each behavior a color TRAIN_BEHAVIORS = False BEHAVIOR_LIST = ['Left_Lane', "Right_Lane"] -BEHAVIOR_LED_COLORS = [(0, 10, 0), (10, 0, 0)] #RGB tuples 0-100 per chanel +BEHAVIOR_LED_COLORS = [(0, 10, 0), (10, 0, 0)] #Localizer #The localizer is a neural network that can learn to predict its location on the track. @@ -693,74 +732,101 @@ #to predict the segement of the course, where the course is divided into NUM_LOCATIONS segments. TRAIN_LOCALIZER = False NUM_LOCATIONS = 10 -BUTTON_PRESS_NEW_TUB = False #when enabled, makes it easier to divide our data into one tub per track length if we make a new tub on each X button press. +BUTTON_PRESS_NEW_TUB = False + + +# PATH FOLLOWING (GPS or Odometry based) +PATH_FILENAME = "donkey_path.pkl" +PATH_SCALE = 5.0 +PATH_OFFSET = (0, 0) +PATH_MIN_DIST = 0.3 +PID_P = -10.0 +PID_I = 0.000 +PID_D = -0.2 +PID_THROTTLE = 0.2 +USE_CONSTANT_THROTTLE = False +SAVE_PATH_BTN = "cross" +RESET_ORIGIN_BTN = "triangle" + +# STOP SIGN DETECTOR +STOP_SIGN_DETECTOR = False +STOP_SIGN_MIN_SCORE = 0.2 +STOP_SIGN_SHOW_BOUNDING_BOX = True +STOP_SIGN_MAX_REVERSE_COUNT = 10 +STOP_SIGN_REVERSE_THROTTLE = -0.5 + + +# RECORDING & LOGGING + +# Automatically record data when throttle is > 0 (Standard training data collection). +AUTO_RECORD_ON_THROTTLE = True + +# Record data even when the AI is driving (Careful: don't train on this data!). +RECORD_DURING_AI = False + +# Create a new directory for every session (True) or append to existing (False). +AUTO_CREATE_NEW_TUB = False + +# Console logging settings. +HAVE_CONSOLE_LOGGING = True +LOGGING_LEVEL = 'INFO' +LOGGING_FORMAT = '%(message)s' +HAVE_PERFMON = False +SHOW_FPS = False +FPS_DEBUG_INTERVAL = 10 + + +# TELEMETRY (MQTT) +HAVE_MQTT_TELEMETRY = False +TELEMETRY_DONKEY_NAME = 'my_robot1234' +TELEMETRY_MQTT_TOPIC_TEMPLATE = 'donkey/%s/telemetry' +TELEMETRY_MQTT_JSON_ENABLE = False +TELEMETRY_MQTT_BROKER_HOST = 'broker.hivemq.com' +TELEMETRY_MQTT_BROKER_PORT = 1883 +TELEMETRY_PUBLISH_PERIOD = 1 +TELEMETRY_LOGGING_ENABLE = True +TELEMETRY_LOGGING_LEVEL = 'INFO' +TELEMETRY_LOGGING_FORMAT = '%(message)s' +TELEMETRY_DEFAULT_INPUTS = 'pilot/angle,pilot/throttle,recording' +TELEMETRY_DEFAULT_TYPES = 'float,float' + -#DonkeyGym +# SIMULATION (DONKEY GYM) #Only on Ubuntu linux, you can use the simulator as a virtual donkey and #issue the same python manage.py drive command as usual, but have them control a virtual car. #This enables that, and sets the path to the simualator and the environment. #You will want to download the simulator binary from: https://github.com/tawnkramer/donkey_gym/releases/download/v18.9/DonkeySimLinux.zip #then extract that and modify DONKEY_SIM_PATH. + +# Settings for connecting to the Donkey Gym Unity simulator. DONKEY_GYM = False -DONKEY_SIM_PATH = "path to sim" #"/home/tkramer/projects/sdsandbox/sdsim/build/DonkeySimLinux/donkey_sim.x86_64" when racing on virtual-race-league use "remote", or user "remote" when you want to start the sim manually first. -DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" # ("donkey-generated-track-v0"|"donkey-generated-roads-v0"|"donkey-warehouse-v0"|"donkey-avc-sparkfun-v0") -GYM_CONF = { "body_style" : "donkey", "body_rgb" : (128, 128, 128), "car_name" : "car", "font_size" : 100} # body style(donkey|bare|car01) body rgb 0-255 +DONKEY_SIM_PATH = "path to sim" +DONKEY_GYM_ENV_NAME = "donkey-generated-track-v0" +GYM_CONF = { "body_style" : "donkey", "body_rgb" : (128, 128, 128), "car_name" : "car", "font_size" : 100} GYM_CONF["racer_name"] = "Your Name" GYM_CONF["country"] = "Place" GYM_CONF["bio"] = "I race robots." - -SIM_HOST = "127.0.0.1" # when racing on virtual-race-league use host "trainmydonkey.com" -SIM_ARTIFICIAL_LATENCY = 0 # this is the millisecond latency in controls. Can use useful in emulating the delay when useing a remote server. values of 100 to 400 probably reasonable. - -# Save info from Simulator (pln) +SIM_HOST = "127.0.0.1" +SIM_ARTIFICIAL_LATENCY = 0 SIM_RECORD_LOCATION = False -SIM_RECORD_GYROACCEL= False +SIM_RECORD_GYROACCEL = False SIM_RECORD_VELOCITY = False SIM_RECORD_LIDAR = False +PUB_CAMERA_IMAGES = False # Publish camera over network +USE_FPV = False # send camera data to FPV webserver -#publish camera over network -#This is used to create a tcp service to publish the camera feed -PUB_CAMERA_IMAGES = False - -#When racing, to give the ai a boost, configure these values. -AI_LAUNCH_DURATION = 0.0 # the ai will output throttle for this many seconds -AI_LAUNCH_THROTTLE = 0.0 # the ai will output this throttle value -AI_LAUNCH_ENABLE_BUTTON = 'R2' # this keypress will enable this boost. It must be enabled before each use to prevent accidental trigger. -AI_LAUNCH_KEEP_ENABLED = False # when False ( default) you will need to hit the AI_LAUNCH_ENABLE_BUTTON for each use. This is safest. When this True, is active on each trip into "local" ai mode. - -#Scale the output of the throttle of the ai pilot for all model types. -AI_THROTTLE_MULT = 1.0 # this multiplier will scale every throttle value for all output from NN models - -#Path following -PATH_FILENAME = "donkey_path.pkl" # the path will be saved to this filename -PATH_SCALE = 5.0 # the path display will be scaled by this factor in the web page -PATH_OFFSET = (0, 0) # 255, 255 is the center of the map. This offset controls where the origin is displayed. -PATH_MIN_DIST = 0.3 # after travelling this distance (m), save a path point -PID_P = -10.0 # proportional mult for PID path follower -PID_I = 0.000 # integral mult for PID path follower -PID_D = -0.2 # differential mult for PID path follower -PID_THROTTLE = 0.2 # constant throttle value during path following -USE_CONSTANT_THROTTLE = False # whether or not to use the constant throttle or variable throttle captured during path recording -SAVE_PATH_BTN = "cross" # joystick button to save path -RESET_ORIGIN_BTN = "triangle" # joystick button to press to move car back to origin - -# Intel Realsense D435 and D435i depth sensing camera -REALSENSE_D435_RGB = True # True to capture RGB image -REALSENSE_D435_DEPTH = True # True to capture depth as image array -REALSENSE_D435_IMU = False # True to capture IMU data (D435i only) -REALSENSE_D435_ID = None # serial number of camera or None if you only have one camera (it will autodetect) - -# Stop Sign Detector -STOP_SIGN_DETECTOR = False -STOP_SIGN_MIN_SCORE = 0.2 -STOP_SIGN_SHOW_BOUNDING_BOX = True -STOP_SIGN_MAX_REVERSE_COUNT = 10 # How many times should the car reverse when detected a stop sign, set to 0 to disable reversing -STOP_SIGN_REVERSE_THROTTLE = -0.5 # Throttle during reversing when detected a stop sign -# FPS counter -SHOW_FPS = False -FPS_DEBUG_INTERVAL = 10 # the interval in seconds for printing the frequency info into the shell - -# PI connection +# PI CONNECTION PI_USERNAME = "pi" PI_HOSTNAME = "donkeypi.local" + + +# WEB CONTROL +# The port for the web server (default 8887). +WEB_CONTROL_PORT = int(os.getenv("WEB_CONTROL_PORT", 8887)) + +# Initial mode on startup. +# 'user': Human control +# 'local_angle': AI Steering, Human Throttle +# 'local': AI Steering and Throttle +WEB_INIT_MODE = "user" diff --git a/donkeycar/templates/cfg_path_follow.py b/donkeycar/templates/cfg_path_follow.py index 4ac8a78954..d9859cd4d6 100644 --- a/donkeycar/templates/cfg_path_follow.py +++ b/donkeycar/templates/cfg_path_follow.py @@ -4,7 +4,7 @@ # This file is read by your car application's manage.py script to change the car # performance -# If desired, all config overrides can be specified here. +# If desired, all config overrides can be specified here. # The update operation will not touch this file. # """ @@ -410,11 +410,14 @@ # IMU for imu model -HAVE_IMU = False #when true, this add a Mpu6050 part and records the data. Can be used with a -IMU_SENSOR = 'mpu6050' # (mpu6050|mpu9250) +HAVE_IMU = False # when true, this adds a Mpu6050 part and records the data. Can be used with a +IMU_SENSOR = 'mpu6050' # (mpu6050|mpu9250|bno08x) IMU_ADDRESS = 0x68 # if AD0 pin is pulled high them address is 0x69, otherwise it is 0x68 IMU_DLP_CONFIG = 0 # Digital Lowpass Filter setting (0:250Hz, 1:184Hz, 2:92Hz, 3:41Hz, 4:20Hz, 5:10Hz, 6:5Hz) +# GPS + IMU Fusion: Use the BNO08x IMU to update position faster +USE_FUSION = False # Activate GPS + IMU Fusion +FUSION_DEBUG = False # Logs the position and yaw from the kalman filter # # Input controllers @@ -641,9 +644,15 @@ PATH_SCALE = 10.0 # the path display will be scaled by this factor in the web page PATH_OFFSET = (255, 255) # 255, 255 is the center of the map. This offset controls where the origin is displayed. PATH_MIN_DIST = 0.2 # after travelling this distance (m), save a path point -PATH_SEARCH_LENGTH = None # number of points to search for closest point, None to search entire path +PATH_SEARCH_LENGTH = 10 # number of points to search for closest point, None to search entire path + # ideally this is set to a number that is large enough to find the closest point + # but not so large that it finds a point on a different part of the track. + # For instance, if you are racing on a track with long straightaways and tight turns, + # you may want to set this to a number that is large enough to find the closest point + # on the straightaway but not so large that it finds a point on the straightaway + # when you are in the middle of the turn. PATH_LOOK_AHEAD = 1 # number of points ahead of the closest point to include in cte track -PATH_LOOK_BEHIND = 1 # number of points behind the closest point to include in cte track +PATH_LOOK_BEHIND = 1 # number of points behind the closest point to include in cte track PID_P = -0.5 # proportional mult for PID path follower PID_I = 0.000 # integral mult for PID path follower PID_D = -0.3 # differential mult for PID path follower @@ -671,4 +680,3 @@ # Intel Realsense T265 tracking camera REALSENSE_T265_ID = None # serial number of camera or None if you only have one camera (it will autodetect) WHEEL_ODOM_CALIB = "calibration_odometry.json" - diff --git a/donkeycar/templates/cfg_simulator.py b/donkeycar/templates/cfg_simulator.py index ea498fc5d2..d80df81ef4 100644 --- a/donkeycar/templates/cfg_simulator.py +++ b/donkeycar/templates/cfg_simulator.py @@ -274,6 +274,15 @@ REALSENSE_D435_IMU = False # True to capture IMU data (D435i only) REALSENSE_D435_ID = None # serial number of camera or None if you only have one camera (it will autodetect) +OAKD_RGB = True # True to capture RGB image +OAKD_DEPTH = True # True to capture depth as image array +OAKD_ID = None # serial number of camera or None if you only have one camera (it will autodetect) +OAKD_RGB_OUTPUT_MODE = "isp" # "isp" preserves full FOV; "video" uses crop-based framing +OAKD_RGB_ISP_SCALE_NUM = 1 # Used when OAKD_RGB_OUTPUT_MODE == "isp" +OAKD_RGB_ISP_SCALE_DEN = 6 # Used when OAKD_RGB_OUTPUT_MODE == "isp" +OAKD_RGB_SENSOR_CROP_X = None # Used when OAKD_RGB_OUTPUT_MODE == "video" +OAKD_RGB_SENSOR_CROP_Y = None # Used when OAKD_RGB_OUTPUT_MODE == "video" + # Stop Sign Detector STOP_SIGN_DETECTOR = False STOP_SIGN_MIN_SCORE = 0.2 diff --git a/donkeycar/templates/complete.py b/donkeycar/templates/complete.py index 32c4a7168d..b9b17cfa60 100644 --- a/donkeycar/templates/complete.py +++ b/donkeycar/templates/complete.py @@ -25,7 +25,6 @@ except: pass - import donkeycar as dk from donkeycar.parts.transform import TriggeredCallback, DelayedTrigger from donkeycar.parts.tub_v2 import TubWriter @@ -60,9 +59,9 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, """ logger.info(f'PID: {os.getpid()}') if cfg.DONKEY_GYM: - #the simulator will use cuda and then we usually run out of resources - #if we also try to use cuda. so disable for donkey_gym. - os.environ["CUDA_VISIBLE_DEVICES"]="-1" + # the simulator will use cuda and then we usually run out of resources + # if we also try to use cuda. so disable for donkey_gym. + os.environ["CUDA_VISIBLE_DEVICES"] = "-1" if model_type is None: if cfg.TRAIN_LOCALIZER: @@ -85,32 +84,29 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, if cfg.HAVE_MQTT_TELEMETRY: from donkeycar.parts.telemetry import MqttTelemetry tel = MqttTelemetry(cfg) - + # # if we are using the simulator, set it up # add_simulator(V, cfg) - # # setup encoders, odometry and pose estimation # add_odometry(V, cfg) - # # setup primary camera # add_camera(V, cfg, camera_type) - # add lidar if cfg.USE_LIDAR: from donkeycar.parts.lidar import RPLidar if cfg.LIDAR_TYPE == 'RP': print("adding RP lidar part") - lidar = RPLidar(lower_limit = cfg.LIDAR_LOWER_LIMIT, upper_limit = cfg.LIDAR_UPPER_LIMIT) - V.add(lidar, inputs=[],outputs=['lidar/dist_array'], threaded=True) + lidar = RPLidar(lower_limit=cfg.LIDAR_LOWER_LIMIT, upper_limit=cfg.LIDAR_UPPER_LIMIT) + V.add(lidar, inputs=[], outputs=['lidar/dist_array'], threaded=True) if cfg.LIDAR_TYPE == 'YD': print("YD Lidar not yet supported") @@ -152,7 +148,7 @@ def drive(cfg, model_path=None, use_joystick=False, model_type=None, V.add(Lambda(lambda v: print(f"web/w4 clicked")), inputs=["web/w4"], run_condition="web/w4") V.add(Lambda(lambda v: print(f"web/w5 clicked")), inputs=["web/w5"], run_condition="web/w5") - #this throttle filter will allow one tap back for esc reverse + # this throttle filter will allow one tap back for esc reverse th_filter = ThrottleFilter() V.add(th_filter, inputs=['user/throttle'], outputs=['user/throttle']) @@ -168,7 +164,7 @@ def __init__(self, cfg): self.cfg = cfg def run(self, mode, recording, recording_alert, behavior_state, model_file_changed, track_loc): - #returns a blink rate. 0 for off. -1 for on. positive for rate. + # returns a blink rate. 0 for off. -1 for on. positive for rate. if track_loc is not None: led.set_rgb(*self.cfg.LOC_COLORS[track_loc]) @@ -189,10 +185,10 @@ def run(self, mode, recording, recording_alert, behavior_state, model_file_chang if behavior_state is not None and model_type == 'behavior': r, g, b = self.cfg.BEHAVIOR_LED_COLORS[behavior_state] led.set_rgb(r, g, b) - return -1 #solid on + return -1 # solid on if recording: - return -1 #solid on + return -1 # solid on elif mode == 'user': return 1 elif mode == 'local_angle': @@ -206,7 +202,8 @@ def run(self, mode, recording, recording_alert, behavior_state, model_file_chang led = RGB_LED(cfg.LED_PIN_R, cfg.LED_PIN_G, cfg.LED_PIN_B, cfg.LED_INVERT) led.set_rgb(cfg.LED_R, cfg.LED_G, cfg.LED_B) - V.add(LedConditionLogic(cfg), inputs=['user/mode', 'recording', "records/alert", 'behavior/state', 'modelfile/modified', "pilot/loc"], + V.add(LedConditionLogic(cfg), + inputs=['user/mode', 'recording', "records/alert", 'behavior/state', 'modelfile/modified', "pilot/loc"], outputs=['led/blink_rate']) V.add(led, inputs=['led/blink_rate']) @@ -253,22 +250,24 @@ def run(self, num_records): def show_record_count_status(): rec_tracker_part.last_num_rec_print = 0 rec_tracker_part.force_alert = 1 - if (cfg.CONTROLLER_TYPE != "pigpio_rc") and (cfg.CONTROLLER_TYPE != "MM1"): # these controllers don't use the joystick class + + if (cfg.CONTROLLER_TYPE != "pigpio_rc") and ( + cfg.CONTROLLER_TYPE != "MM1"): # these controllers don't use the joystick class if isinstance(ctr, JoystickController): - ctr.set_button_down_trigger('circle', show_record_count_status) #then we are not using the circle button. hijack that to force a record count indication + ctr.set_button_down_trigger('circle', + show_record_count_status) # then we are not using the circle button. hijack that to force a record count indication else: - + show_record_count_status() - #Sombrero + # Sombrero if cfg.HAVE_SOMBRERO: from donkeycar.parts.sombrero import Sombrero s = Sombrero() - #IMU + # IMU add_imu(V, cfg) - # Use the FPV preview, which will show the cropped image output, or the full frame. if cfg.USE_FPV: V.add(WebFpv(), inputs=['cam/image_array'], threaded=True) @@ -277,14 +276,14 @@ def load_model(kl, model_path): start = time.time() print('loading model', model_path) kl.load(model_path) - print('finished loading in %s sec.' % (str(time.time() - start)) ) + print('finished loading in %s sec.' % (str(time.time() - start))) def load_weights(kl, weights_path): start = time.time() try: print('loading model weights', weights_path) kl.model.load_weights(weights_path) - print('finished loading in %s sec.' % (str(time.time() - start)) ) + print('finished loading in %s sec.' % (str(time.time() - start))) except Exception as e: print(e) print('ERR>> problems loading weights', weights_path) @@ -297,7 +296,7 @@ def load_model_json(kl, json_fnm): with open(json_fnm, 'r') as handle: contents = handle.read() kl.model = keras.models.model_from_json(contents) - print('finished loading json in %s sec.' % (str(time.time() - start)) ) + print('finished loading json in %s sec.' % (str(time.time() - start))) except Exception as e: print(e) print("ERR>> problems loading model json", json_fnm) @@ -315,7 +314,7 @@ def load_model_json(kl, json_fnm): # model_reload_cb = None if '.h5' in model_path or '.trt' in model_path or '.tflite' in \ - model_path or '.savedmodel' in model_path or '.pth' in model_path: + model_path or '.pth' in model_path or '.keras' in model_path: # load the whole model with weigths, etc load_model(kl, model_path) @@ -378,13 +377,14 @@ def reload_weights(filename): assert cfg.HAVE_IMU, 'Missing imu parameter in config' class Vectorizer: - def run(self, *components): - return components + def run(self, accel, gyro): + # accel is (ax, ay, az) and gyro is (gx, gy, gz) + return list(accel) + list(gyro) - V.add(Vectorizer, inputs=['imu/acl_x', 'imu/acl_y', 'imu/acl_z', - 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'], + # Add it to the vehicle loop + V.add(Vectorizer(), + inputs=['imu/accel', 'imu/gyro'], outputs=['imu_array']) - inputs = ['cam/image_array', 'imu_array'] else: inputs = ['cam/image_array'] @@ -426,7 +426,7 @@ def run(self, *components): cfg.STOP_SIGN_REVERSE_THROTTLE), inputs=['cam/image_array', 'pilot/throttle'], outputs=['pilot/throttle', 'cam/image_array']) - V.add(ThrottleFilter(), + V.add(ThrottleFilter(), inputs=['pilot/throttle'], outputs=['pilot/throttle']) @@ -452,12 +452,10 @@ def run(self, *components): 'pilot/angle', 'pilot/throttle'], outputs=['steering', 'throttle']) - if (cfg.CONTROLLER_TYPE != "pigpio_rc") and (cfg.CONTROLLER_TYPE != "MM1"): if isinstance(ctr, JoystickController): ctr.set_button_down_trigger(cfg.AI_LAUNCH_ENABLE_BUTTON, aiLauncher.enable_ai_launch) - # Ai Recording recording_control = ToggleRecording(cfg.AUTO_RECORD_ON_THROTTLE, cfg.RECORD_DURING_AI) V.add(recording_control, inputs=['user/mode', "recording"], outputs=["recording"]) @@ -467,7 +465,6 @@ def run(self, *components): # add_drivetrain(V, cfg) - # # OLED display setup # @@ -482,10 +479,10 @@ def run(self, *components): # if cfg.USE_LIDAR: inputs = ['cam/image_array', 'lidar/dist_array', 'user/angle', 'user/throttle', 'user/mode'] - types = ['image_array', 'nparray','float', 'float', 'str'] + types = ['image_array', 'nparray', 'float', 'float', 'str'] else: - inputs=['cam/image_array','user/angle', 'user/throttle', 'user/mode'] - types=['image_array','float', 'float','str'] + inputs = ['cam/image_array', 'user/angle', 'user/throttle', 'user/mode'] + types = ['image_array', 'float', 'float', 'str'] if cfg.HAVE_ODOM: inputs += ['enc/speed'] @@ -499,27 +496,28 @@ def run(self, *components): inputs += ['cam/depth_array'] types += ['gray16_array'] - if cfg.HAVE_IMU or (cfg.CAMERA_TYPE == "D435" and cfg.REALSENSE_D435_IMU): - inputs += ['imu/acl_x', 'imu/acl_y', 'imu/acl_z', - 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'] + if cfg.CAMERA_TYPE == "OAKD" and cfg.OAKD_DEPTH: + inputs += ['cam/depth_array'] + types += ['gray16_array'] - types +=['float', 'float', 'float', - 'float', 'float', 'float'] + if cfg.HAVE_IMU or (cfg.CAMERA_TYPE == "D435" and cfg.REALSENSE_D435_IMU): + inputs += ['imu/accel', 'imu/gyro'] + types += ['vector', 'vector'] # rbx if cfg.DONKEY_GYM: if cfg.SIM_RECORD_LOCATION: inputs += ['pos/pos_x', 'pos/pos_y', 'pos/pos_z', 'pos/speed', 'pos/cte'] - types += ['float', 'float', 'float', 'float', 'float'] + types += ['float', 'float', 'float', 'float', 'float'] if cfg.SIM_RECORD_GYROACCEL: inputs += ['gyro/gyro_x', 'gyro/gyro_y', 'gyro/gyro_z', 'accel/accel_x', 'accel/accel_y', 'accel/accel_z'] - types += ['float', 'float', 'float', 'float', 'float', 'float'] + types += ['float', 'float', 'float', 'float', 'float', 'float'] if cfg.SIM_RECORD_VELOCITY: inputs += ['vel/vel_x', 'vel/vel_y', 'vel/vel_z'] - types += ['float', 'float', 'float'] + types += ['float', 'float', 'float'] if cfg.SIM_RECORD_LIDAR: inputs += ['lidar/dist_array'] - types += ['nparray'] + types += ['nparray'] if cfg.RECORD_DURING_AI: inputs += ['pilot/angle', 'pilot/throttle'] @@ -556,7 +554,6 @@ def run(self, *components): V.add(ImgArrToJpg(), inputs=['cam/image_array'], outputs=['jpg/bin']) V.add(pub, inputs=['jpg/bin']) - if cfg.DONKEY_GYM: print("You can now go to http://localhost:%d to drive your car." % cfg.WEB_CONTROL_PORT) else: @@ -655,11 +652,11 @@ def run(self, mode, elif mode == 'local_angle': return pilot_steering if pilot_steering else 0.0, user_throttle return (pilot_steering if pilot_steering else 0.0, - pilot_throttle * self.ai_throttle_mult if pilot_throttle else 0.0) + pilot_throttle * self.ai_throttle_mult if pilot_throttle else 0.0) class UserPilotCondition: - def __init__(self, show_pilot_image:bool = False) -> None: + def __init__(self, show_pilot_image: bool = False) -> None: """ :param show_pilot_image:bool True to show pilot image in pilot mode False to show user image in pilot mode @@ -765,7 +762,7 @@ def add_simulator(V, cfg): gym = DonkeyGymEnv(cfg.DONKEY_SIM_PATH, host=cfg.SIM_HOST, env_name=cfg.DONKEY_GYM_ENV_NAME, conf=cfg.GYM_CONF, record_location=cfg.SIM_RECORD_LOCATION, record_gyroaccel=cfg.SIM_RECORD_GYROACCEL, record_velocity=cfg.SIM_RECORD_VELOCITY, record_lidar=cfg.SIM_RECORD_LIDAR, - # record_distance=cfg.SIM_RECORD_DISTANCE, record_orientation=cfg.SIM_RECORD_ORIENTATION, + # record_distance=cfg.SIM_RECORD_DISTANCE, record_orientation=cfg.SIM_RECORD_ORIENTATION, delay=cfg.SIM_ARTIFICIAL_LATENCY) threaded = True inputs = ['steering', 'throttle'] @@ -810,7 +807,8 @@ def get_camera(cfg): framerate=cfg.CAMERA_FRAMERATE, gstreamer_flip=cfg.CSIC_CAM_GSTREAMER_FLIP_PARM) elif cfg.CAMERA_TYPE == "V4L": from donkeycar.parts.camera import V4LCamera - cam = V4LCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, framerate=cfg.CAMERA_FRAMERATE) + cam = V4LCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, + framerate=cfg.CAMERA_FRAMERATE) elif cfg.CAMERA_TYPE == "IMAGE_LIST": from donkeycar.parts.camera import ImageListCamera cam = ImageListCamera(path_mask=cfg.PATH_MASK) @@ -821,7 +819,7 @@ def get_camera(cfg): from donkeycar.parts.camera import MockCamera cam = MockCamera(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH) else: - raise(Exception("Unkown camera type: %s" % cfg.CAMERA_TYPE)) + raise (Exception("Unkown camera type: %s" % cfg.CAMERA_TYPE)) return cam @@ -833,21 +831,21 @@ def add_camera(V, cfg, camera_type): On output this will be modified. :param cfg: the configuration (from myconfig.py) """ - logger.info("cfg.CAMERA_TYPE %s"%cfg.CAMERA_TYPE) + logger.info("cfg.CAMERA_TYPE %s" % cfg.CAMERA_TYPE) if camera_type == "stereo": if cfg.CAMERA_TYPE == "WEBCAM": from donkeycar.parts.camera import Webcam - camA = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam = 0) - camB = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam = 1) + camA = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=0) + camB = Webcam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=1) elif cfg.CAMERA_TYPE == "CVCAM": from donkeycar.parts.cv import CvCam - camA = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam = 0) - camB = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam = 1) + camA = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=0) + camB = CvCam(image_w=cfg.IMAGE_W, image_h=cfg.IMAGE_H, image_d=cfg.IMAGE_DEPTH, iCam=1) else: - raise(Exception("Unsupported camera type: %s" % cfg.CAMERA_TYPE)) + raise (Exception("Unsupported camera type: %s" % cfg.CAMERA_TYPE)) V.add(camA, outputs=['cam/image_array_a'], threaded=True) V.add(camB, outputs=['cam/image_array_b'], threaded=True) @@ -855,7 +853,7 @@ def add_camera(V, cfg, camera_type): from donkeycar.parts.image import StereoPair V.add(StereoPair(), inputs=['cam/image_array_a', 'cam/image_array_b'], - outputs=['cam/image_array']) + outputs=['cam/image_array']) if cfg.BGR2RGB: from donkeycar.parts.cv import ImgBGR2RGB V.add(ImgBGR2RGB(), inputs=["cam/image_array_a"], outputs=["cam/image_array_a"]) @@ -870,9 +868,24 @@ def add_camera(V, cfg, camera_type): device_id=cfg.REALSENSE_D435_ID) V.add(cam, inputs=[], outputs=['cam/image_array', 'cam/depth_array', - 'imu/acl_x', 'imu/acl_y', 'imu/acl_z', - 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'], + 'imu/accel', 'imu/gyro', 'imu/temp'], + threaded=True) + + elif cfg.CAMERA_TYPE == "OAKD": + from donkeycar.parts.oak_d import OakD + cam = OakD( + rgb_output_mode=cfg.OAKD_RGB_OUTPUT_MODE, + rgb_isp_scale_num=cfg.OAKD_RGB_ISP_SCALE_NUM, + rgb_isp_scale_den=cfg.OAKD_RGB_ISP_SCALE_DEN, + rgb_sensor_crop_x=cfg.OAKD_RGB_SENSOR_CROP_X, + rgb_sensor_crop_y=cfg.OAKD_RGB_SENSOR_CROP_Y, + enable_rgb=cfg.OAKD_RGB, + enable_depth=cfg.OAKD_DEPTH, + device_id=cfg.OAKD_ID) + V.add(cam, inputs=[], + outputs=['cam/image_array', 'cam/depth_array'], threaded=True) + else: inputs = [] outputs = ['cam/image_array'] @@ -899,11 +912,11 @@ def add_odometry(V, cfg, threaded=True): poll_delay_secs = 0.01 # pose estimation runs at 100hz kinematics = UnicyclePose(cfg, poll_delay_secs) if cfg.HAVE_ODOM_2 else BicyclePose(cfg, poll_delay_secs) V.add(kinematics, - inputs = ["throttle", "steering", None], - outputs = ['enc/distance', 'enc/speed', 'pos/x', 'pos/y', + inputs=["throttle", "steering", None], + outputs=['enc/distance', 'enc/speed', 'pos/x', 'pos/y', 'pos/angle', 'vel/x', 'vel/y', 'vel/angle', 'nul/timestamp'], - threaded = threaded) + threaded=threaded) # @@ -912,12 +925,17 @@ def add_odometry(V, cfg, threaded=True): def add_imu(V, cfg): imu = None if cfg.HAVE_IMU: - from donkeycar.parts.imu import IMU - - imu = IMU(sensor=cfg.IMU_SENSOR, addr=cfg.IMU_ADDRESS, - dlp_setting=cfg.IMU_DLP_CONFIG) - V.add(imu, outputs=['imu/acl_x', 'imu/acl_y', 'imu/acl_z', - 'imu/gyr_x', 'imu/gyr_y', 'imu/gyr_z'], threaded=True) + if cfg.IMU_SENSOR.lower() == "bno08x": + from donkeycar.parts.imu import Bno08xIMU + imu = Bno08xIMU(addr=cfg.IMU_ADDRESS) + V.add(imu, + outputs=['imu/accel', 'imu/gyro', 'imu/quat'], + threaded=True) + else: + from donkeycar.parts.imu import IMU + imu = IMU(sensor=cfg.IMU_SENSOR, addr=cfg.IMU_ADDRESS, + dlp_setting=cfg.IMU_DLP_CONFIG) + V.add(imu, outputs=['imu/accel', 'imu/gyro', 'imu/temp'], threaded=True) return imu @@ -925,7 +943,6 @@ def add_imu(V, cfg): # Drive train setup # def add_drivetrain(V, cfg): - if (not cfg.DONKEY_GYM) and cfg.DRIVE_TRAIN_TYPE != "MOCK": from donkeycar.parts import actuator, pins from donkeycar.parts.actuator import TwoWheelSteeringThrottle @@ -954,17 +971,17 @@ def add_drivetrain(V, cfg): pwm_scale=dt["PWM_STEERING_SCALE"], pwm_inverted=dt["PWM_STEERING_INVERTED"]) steering = PWMSteering(controller=steering_controller, - left_pulse=dt["STEERING_LEFT_PWM"], - right_pulse=dt["STEERING_RIGHT_PWM"]) + left_pulse=dt["STEERING_LEFT_PWM"], + right_pulse=dt["STEERING_RIGHT_PWM"]) throttle_controller = PulseController( pwm_pin=pins.pwm_pin_by_id(dt["PWM_THROTTLE_PIN"]), pwm_scale=dt["PWM_THROTTLE_SCALE"], pwm_inverted=dt['PWM_THROTTLE_INVERTED']) throttle = PWMThrottle(controller=throttle_controller, - max_pulse=dt['THROTTLE_FORWARD_PWM'], - zero_pulse=dt['THROTTLE_STOPPED_PWM'], - min_pulse=dt['THROTTLE_REVERSE_PWM']) + max_pulse=dt['THROTTLE_FORWARD_PWM'], + zero_pulse=dt['THROTTLE_STOPPED_PWM'], + min_pulse=dt['THROTTLE_REVERSE_PWM']) V.add(steering, inputs=['steering'], threaded=True) V.add(throttle, inputs=['throttle'], threaded=True) @@ -977,14 +994,14 @@ def add_drivetrain(V, cfg): steering_controller = PCA9685(cfg.STEERING_CHANNEL, cfg.PCA9685_I2C_ADDR, busnum=cfg.PCA9685_I2C_BUSNUM) steering = PWMSteering(controller=steering_controller, - left_pulse=cfg.STEERING_LEFT_PWM, - right_pulse=cfg.STEERING_RIGHT_PWM) + left_pulse=cfg.STEERING_LEFT_PWM, + right_pulse=cfg.STEERING_RIGHT_PWM) throttle_controller = PCA9685(cfg.THROTTLE_CHANNEL, cfg.PCA9685_I2C_ADDR, busnum=cfg.PCA9685_I2C_BUSNUM) throttle = PWMThrottle(controller=throttle_controller, - max_pulse=cfg.THROTTLE_FORWARD_PWM, - zero_pulse=cfg.THROTTLE_STOPPED_PWM, - min_pulse=cfg.THROTTLE_REVERSE_PWM) + max_pulse=cfg.THROTTLE_FORWARD_PWM, + zero_pulse=cfg.THROTTLE_STOPPED_PWM, + min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['steering'], threaded=True) V.add(throttle, inputs=['throttle'], threaded=True) @@ -1039,8 +1056,8 @@ def add_drivetrain(V, cfg): pwm_scale=dt['PWM_STEERING_SCALE'], pwm_inverted=dt['PWM_STEERING_INVERTED']) steering = PWMSteering(controller=steering_controller, - left_pulse=dt['STEERING_LEFT_PWM'], - right_pulse=dt['STEERING_RIGHT_PWM']) + left_pulse=dt['STEERING_LEFT_PWM'], + right_pulse=dt['STEERING_RIGHT_PWM']) motor = actuator.L298N_HBridge_2pin( pins.pwm_pin_by_id(dt['FWD_DUTY_PIN']), @@ -1061,8 +1078,8 @@ def add_drivetrain(V, cfg): pwm_scale=dt['PWM_STEERING_SCALE'], pwm_inverted=dt['PWM_STEERING_INVERTED']) steering = PWMSteering(controller=steering_controller, - left_pulse=dt['STEERING_LEFT_PWM'], - right_pulse=dt['STEERING_RIGHT_PWM']) + left_pulse=dt['STEERING_LEFT_PWM'], + right_pulse=dt['STEERING_RIGHT_PWM']) motor = actuator.L298N_HBridge_3pin( pins.output_pin_by_id(dt['FWD_PIN']), @@ -1078,10 +1095,10 @@ def add_drivetrain(V, cfg): # This driver will be removed in a future release # from donkeycar.parts.actuator import ServoBlaster, PWMSteering - steering_controller = ServoBlaster(cfg.STEERING_CHANNEL) #really pin + steering_controller = ServoBlaster(cfg.STEERING_CHANNEL) # really pin # PWM pulse values should be in the range of 100 to 200 - assert(cfg.STEERING_LEFT_PWM <= 200) - assert(cfg.STEERING_RIGHT_PWM <= 200) + assert (cfg.STEERING_LEFT_PWM <= 200) + assert (cfg.STEERING_RIGHT_PWM <= 200) steering = PWMSteering(controller=steering_controller, left_pulse=cfg.STEERING_LEFT_PWM, right_pulse=cfg.STEERING_RIGHT_PWM) @@ -1094,6 +1111,7 @@ def add_drivetrain(V, cfg): elif cfg.DRIVE_TRAIN_TYPE == "MM1": from donkeycar.parts.robohat import RoboHATDriver + # Share serial port with controller to avoid opening the same port twice V.add(RoboHATDriver(cfg), inputs=['steering', 'throttle']) elif cfg.DRIVE_TRAIN_TYPE == "PIGPIO_PWM": @@ -1116,18 +1134,18 @@ def add_drivetrain(V, cfg): min_pulse=cfg.THROTTLE_REVERSE_PWM) V.add(steering, inputs=['steering'], threaded=True) V.add(throttle, inputs=['throttle'], threaded=True) - + elif cfg.DRIVE_TRAIN_TYPE == "VESC": from donkeycar.parts.actuator import VESC logger.info("Creating VESC at port {}".format(cfg.VESC_SERIAL_PORT)) vesc = VESC(cfg.VESC_SERIAL_PORT, - cfg.VESC_MAX_SPEED_PERCENT, - cfg.VESC_HAS_SENSOR, - cfg.VESC_START_HEARTBEAT, - cfg.VESC_BAUDRATE, - cfg.VESC_TIMEOUT, - cfg.VESC_STEERING_SCALE, - cfg.VESC_STEERING_OFFSET + cfg.VESC_MAX_SPEED_PERCENT, + cfg.VESC_HAS_SENSOR, + cfg.VESC_START_HEARTBEAT, + cfg.VESC_BAUDRATE, + cfg.VESC_TIMEOUT, + cfg.VESC_STEERING_SCALE, + cfg.VESC_STEERING_OFFSET ) V.add(vesc, inputs=['steering', 'throttle']) diff --git a/donkeycar/templates/path_follow.py b/donkeycar/templates/path_follow.py index 1d029bbf2a..7275f91da4 100644 --- a/donkeycar/templates/path_follow.py +++ b/donkeycar/templates/path_follow.py @@ -464,20 +464,51 @@ def add_gps(V, cfg): nmea_player = None if cfg.GPS_NMEA_PATH: nmea_writer = CsvLogger(cfg.GPS_NMEA_PATH, separator='\t', field_count=2) - V.add(nmea_writer, inputs=['recording', 'gps/nmea'], outputs=['gps/recorded/nmea']) # only record nmea sentences in user mode + V.add(nmea_writer, inputs=['recording', 'gps/nmea'], + outputs=['gps/recorded/nmea']) # only record nmea sentences in user mode nmea_player = GpsPlayer(nmea_writer) - V.add(nmea_player, inputs=['run_pilot', 'gps/nmea'], outputs=['gps/playing', 'gps/nmea']) # only play nmea sentences in autopilot mode + V.add(nmea_player, inputs=['run_pilot', 'gps/nmea'], + outputs=['gps/playing', 'gps/nmea']) # only play nmea sentences in autopilot mode gps_positions = GpsNmeaPositions(debug=cfg.GPS_DEBUG) V.add(gps_positions, inputs=['gps/nmea'], outputs=['gps/positions']) gps_latest_position = GpsLatestPosition(debug=cfg.GPS_DEBUG) - V.add(gps_latest_position, inputs=['gps/positions'], outputs=['gps/timestamp', 'gps/utm/longitude', 'gps/utm/latitude']) - - # rename gps utm position to pose values - V.add(Pipe(), inputs=['gps/utm/longitude', 'gps/utm/latitude'], outputs=['pos/x', 'pos/y']) - - return nmea_player + V.add(gps_latest_position, inputs=['gps/positions'], + outputs=['gps/timestamp', 'gps/utm/longitude', 'gps/utm/latitude']) + + if cfg.USE_FUSION: + from donkeycar.parts.gps_imu_fusion import EKFFusion + from donkeycar.parts.transform import Lambda + fusion = EKFFusion(debug=cfg.FUSION_DEBUG) + + # Combine the two GPS scalars into a single (x, y) tuple. + # gps/pos -> (easting, northing) in UTM meters (lon=x, lat=y) + V.add(Lambda(lambda x, y: (x, y)), + inputs=['gps/utm/longitude', 'gps/utm/latitude'], + outputs=['gps/pos']) + + # IMU already publishes accel/gyro/quat as tuples. + # MPU-class IMUs have no quaternion, so feed an identity quat. + if cfg.IMU_SENSOR.lower() == "bno08x": + quat_key = 'imu/quat' + else: + quat_key = 'imu/quat_identity' + V.add(Lambda(lambda _a: (0.0, 0.0, 0.0, 1.0)), + inputs=['imu/accel'], + outputs=[quat_key]) + + # Fusion consumes exactly 4 tuple inputs. + V.add(fusion, + inputs=['gps/pos', 'imu/accel', 'imu/gyro', quat_key], + outputs=['pos/x', 'pos/y', 'pos/yaw'], + threaded=False) + return None + else: + # rename gps utm position to pose values + V.add(Pipe(), inputs=['gps/utm/longitude', 'gps/utm/latitude'], outputs=['pos/x', 'pos/y']) + return nmea_player + return None if __name__ == '__main__': args = docopt(__doc__) @@ -491,4 +522,4 @@ def add_gps(V, cfg): if args['drive']: - drive(cfg, use_joystick=args['--js'], camera_type=args['--camera']) + drive(cfg, use_joystick=args['--js'], camera_type=args['--camera']) \ No newline at end of file diff --git a/donkeycar/templates/simulator.py b/donkeycar/templates/simulator.py index 8330c025bf..ef0086b392 100644 --- a/donkeycar/templates/simulator.py +++ b/donkeycar/templates/simulator.py @@ -273,8 +273,8 @@ def load_model_json(kl, json_fnm): model_reload_cb = None - if '.h5' in model_path or '.savedmodel' in model_path or '.uff' in \ - model_path or 'tflite' in model_path or '.pkl' in model_path: + if '.h5' in model_path or '.uff' in model_path or 'tflite' in \ + model_path or '.pkl' in model_path or '.keras' in model_path: #when we have a .h5 extension #load everything from the model file load_model(kl, model_path) diff --git a/donkeycar/tests/pytest.ini b/donkeycar/tests/pytest.ini index 078a34f6c8..221a9f7092 100644 --- a/donkeycar/tests/pytest.ini +++ b/donkeycar/tests/pytest.ini @@ -5,3 +5,4 @@ filterwarnings = log_cli = True log_cli_level = INFO +reruns = 3 diff --git a/donkeycar/tests/test_keras.py b/donkeycar/tests/test_keras.py index b807a190e1..eff24129f4 100644 --- a/donkeycar/tests/test_keras.py +++ b/donkeycar/tests/test_keras.py @@ -4,6 +4,9 @@ import pytest import os +tf = pytest.importorskip('tensorflow', + reason='TensorFlow not installed, skipping keras tests') + from donkeycar.parts.interpreter import keras_to_tflite, \ saved_model_to_tensor_rt, TfLite, TensorRT, has_trt_support from donkeycar.parts.keras import * @@ -19,9 +22,16 @@ def tmp_dir() -> str: shutil.rmtree(tmp_dir) -test_data = [KerasLinear, KerasCategorical, KerasInferred, KerasLSTM, - KerasLocalizer, KerasIMU, Keras3D_CNN, KerasMemory, - KerasBehavioral] +test_data = [ + KerasLinear, KerasCategorical, KerasInferred, + # KerasLSTM uses CudnnRNNV3 which TFLite cannot run without CUDA + pytest.param(KerasLSTM, marks=pytest.mark.xfail( + reason='KerasLSTM uses CudnnRNNV3, not supported in TFLite on macOS', + strict=False)), + KerasLocalizer, KerasIMU, + Keras3D_CNN, + KerasMemory, KerasBehavioral, +] def create_models(keras_pilot, dir): @@ -33,14 +43,13 @@ def create_models(keras_pilot, dir): keras_to_tflite(interpreter.model, tflite_model_path) kl = keras_pilot(interpreter=TfLite()) kl.load(tflite_model_path) - # save model in savedmodel format - savedmodel_path = os.path.join(dir, 'model.savedmodel') - interpreter.model.save(savedmodel_path) + keras_path = os.path.join(dir, 'model.keras') + interpreter.model.save(keras_path) krt = None # load tensorrt only if supported if has_trt_support(): krt = keras_pilot(interpreter=TensorRT()) - krt.load(savedmodel_path) + krt.load(keras_path) return km, kl, krt diff --git a/donkeycar/tests/test_metal_gradients.py b/donkeycar/tests/test_metal_gradients.py new file mode 100644 index 0000000000..da45f77e63 --- /dev/null +++ b/donkeycar/tests/test_metal_gradients.py @@ -0,0 +1,322 @@ +""" +Regression tests for tensorflow-metal training correctness. + +tensorflow-metal 1.2.0 paired with TensorFlow 2.19 / Keras 3 produces +incorrect results in compiled training on the Metal GPU. Eager training +matches CPU for the optimizers covered here, but compiled tf.function +execution is currently unsafe on the training path we care about. +KerasInterpreter therefore forces run_eagerly=True on Metal. + +Tests are skipped on non-macOS platforms or when tensorflow-metal is absent. +""" +import numpy as np +import pytest + +from donkeycar.parts.interpreter import _is_metal_installed + +skip_no_metal = pytest.mark.skipif( + not _is_metal_installed(), + reason='tensorflow-metal not installed or not on macOS', +) + + +def _make_tiny_model(tf, seed=42): + tf.random.set_seed(seed) + ki0 = tf.keras.initializers.GlorotUniform(seed=seed) + ki1 = tf.keras.initializers.GlorotUniform(seed=seed + 1) + return tf.keras.Sequential([ + tf.keras.layers.Dense(4, activation='relu', input_shape=(3,), + kernel_initializer=ki0, + bias_initializer='zeros'), + tf.keras.layers.Dense(1, kernel_initializer=ki1, + bias_initializer='zeros'), + ]) + + +def _fixed_data(): + np.random.seed(42) + x = np.random.randn(8, 3).astype(np.float32) + y = np.random.randn(8, 1).astype(np.float32) + return x, y + + +def _optimizer(tf, name): + if name == 'adam': + return tf.keras.optimizers.Adam(1e-3) + if name == 'sgd': + return tf.keras.optimizers.SGD(1e-3) + if name == 'rmsprop': + return tf.keras.optimizers.RMSprop(1e-3) + raise ValueError(f'unsupported optimizer: {name}') + + +def _cpu_reference_weights(tf, optimizer_name='adam'): + """Return weights after one CPU training step.""" + x, y = _fixed_data() + model = _make_tiny_model(tf) + with tf.device('/CPU:0'): + model.compile(optimizer=_optimizer(tf, optimizer_name), loss='mse') + model.fit(x, y, epochs=1, batch_size=8, verbose=0) + return [w.numpy().copy() for w in model.weights] + + +def _compiled_gradients(tf, model, x_t, y_t): + @tf.function + def grad_fn(): + with tf.GradientTape() as tape: + pred = model(x_t, training=True) + loss = tf.reduce_mean(tf.square(pred - y_t)) + return tape.gradient(loss, model.trainable_variables) + + return grad_fn() + + +@skip_no_metal +@pytest.mark.parametrize('optimizer_name', ['adam', 'sgd', 'rmsprop']) +def test_metal_training_matches_cpu_reference(optimizer_name): + """Step 1: eager Metal training matches CPU for common optimizers.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + x, y = _fixed_data() + cpu_weights = _cpu_reference_weights(tf, optimizer_name) + + model = _make_tiny_model(tf) + model.compile( + optimizer=_optimizer(tf, optimizer_name), + loss='mse', + run_eagerly=True, + ) + model.fit(x, y, epochs=1, batch_size=8, verbose=0) + metal_weights = [w.numpy().copy() for w in model.weights] + + for i, (cpu_w, metal_w) in enumerate(zip(cpu_weights, metal_weights)): + np.testing.assert_allclose( + metal_w, + cpu_w, + atol=1e-5, + rtol=1e-4, + err_msg=( + f'weight[{i}]: Metal eager {optimizer_name} ' + f'diverges from CPU ref' + ), + ) + + +@skip_no_metal +def test_step2a_gpu_gradients_match_cpu(): + """Step 2a: Eager gradient computation on Metal matches CPU.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + x, y = _fixed_data() + x_t = tf.constant(x) + y_t = tf.constant(y) + + model_cpu = _make_tiny_model(tf) + _ = model_cpu(x_t) + model_gpu = _make_tiny_model(tf) + _ = model_gpu(x_t) + for wc, wg in zip(model_cpu.weights, model_gpu.weights): + wg.assign(wc) + + with tf.device('/CPU:0'): + with tf.GradientTape() as tape_cpu: + pred_cpu = model_cpu(x_t, training=True) + loss_cpu = tf.reduce_mean(tf.square(pred_cpu - y_t)) + grads_cpu = tape_cpu.gradient(loss_cpu, model_cpu.trainable_variables) + + with tf.GradientTape() as tape_gpu: + pred_gpu = model_gpu(x_t, training=True) + loss_gpu = tf.reduce_mean(tf.square(pred_gpu - y_t)) + grads_gpu = tape_gpu.gradient(loss_gpu, model_gpu.trainable_variables) + + for i, (gc, gg) in enumerate(zip(grads_cpu, grads_gpu)): + np.testing.assert_allclose( + gg.numpy(), gc.numpy(), atol=1e-5, rtol=1e-4, + err_msg=f'grad[{i}]: GPU gradient differs from CPU reference') + + +@skip_no_metal +def test_step2a_compiled_gpu_gradients_diverge_from_cpu(): + """Compiled tf.function gradients on Metal are currently unsafe.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + x, y = _fixed_data() + x_t = tf.constant(x) + y_t = tf.constant(y) + + model_cpu = _make_tiny_model(tf) + _ = model_cpu(x_t) + model_gpu = _make_tiny_model(tf) + _ = model_gpu(x_t) + for wc, wg in zip(model_cpu.weights, model_gpu.weights): + wg.assign(wc) + + with tf.device('/CPU:0'): + with tf.GradientTape() as tape_cpu: + pred_cpu = model_cpu(x_t, training=True) + loss_cpu = tf.reduce_mean(tf.square(pred_cpu - y_t)) + grads_cpu = tape_cpu.gradient(loss_cpu, model_cpu.trainable_variables) + + grads_gpu = _compiled_gradients(tf, model_gpu, x_t, y_t) + max_diffs = [ + np.max(np.abs(gc.numpy() - gg.numpy())) + for gc, gg in zip(grads_cpu, grads_gpu) + ] + + assert max(max_diffs) > 1e-3, ( + 'Compiled Metal gradients unexpectedly match CPU; re-evaluate the ' + 'Metal workaround before changing production code.') + + +@skip_no_metal +def test_step2b_eager_apply_on_gpu_matches_cpu(): + """Step 2b: eager apply_gradients on GPU matches CPU reference.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + x, y = _fixed_data() + x_t = tf.constant(x) + y_t = tf.constant(y) + + cpu_weights = _cpu_reference_weights(tf) + + model = _make_tiny_model(tf) + _ = model(x_t) + model_ref = _make_tiny_model(tf) + _ = model_ref(x_t) + for wr, wm in zip(model_ref.weights, model.weights): + wm.assign(wr) + + optimizer = tf.keras.optimizers.Adam(1e-3) + with tf.GradientTape() as tape: + pred = model(x_t, training=True) + loss = tf.reduce_mean(tf.square(pred - y_t)) + grads = tape.gradient(loss, model.trainable_variables) + # Eager apply (not inside tf.function) - correct on Metal GPU + optimizer.apply_gradients(zip(grads, model.trainable_variables)) + + for i, (cpu_w, model_w) in enumerate(zip(cpu_weights, model.weights)): + np.testing.assert_allclose( + model_w.numpy(), cpu_w, atol=1e-5, rtol=1e-4, + err_msg=f'weight[{i}]: eager GPU apply diverges from CPU ref') + + +@skip_no_metal +@pytest.mark.parametrize('optimizer_name', ['adam', 'sgd', 'rmsprop']) +def test_keras_interpreter_compile_sets_run_eagerly(optimizer_name): + """Production path: KerasInterpreter uses eager training on Metal.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + x, y = _fixed_data() + cpu_weights = _cpu_reference_weights(tf, optimizer_name) + + from donkeycar.parts.interpreter import KerasInterpreter + + interp = KerasInterpreter() + model = _make_tiny_model(tf) + interp.model = model + interp.input_keys = [] + interp.output_keys = [] + interp.shapes = ({}, {}) + interp.compile(optimizer=_optimizer(tf, optimizer_name), loss='mse') + + assert model.run_eagerly, ( + 'KerasInterpreter.compile() must set run_eagerly=True on Metal') + + dataset = tf.data.Dataset.from_tensor_slices((x, y)).batch(8) + interp.fit(x=dataset, steps_per_epoch=1, batch_size=8, + callbacks=[], validation_data=None, + validation_steps=0, epochs=1, verbose=0) + + for i, (cpu_w, model_w) in enumerate(zip(cpu_weights, model.weights)): + np.testing.assert_allclose( + model_w.numpy(), + cpu_w, + atol=1e-5, + rtol=1e-4, + err_msg=( + f'weight[{i}]: Metal eager {optimizer_name} fit incorrect' + ), + ) + + +@skip_no_metal +@pytest.mark.parametrize('optimizer_name', ['adam', 'sgd', 'rmsprop']) +def test_compiled_metal_training_diverges_for_common_optimizers( + optimizer_name, +): + """Compiled Metal training is unsafe across common optimizers.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + x, y = _fixed_data() + cpu_weights = _cpu_reference_weights(tf, optimizer_name) + + model = _make_tiny_model(tf) + model.compile(optimizer=_optimizer(tf, optimizer_name), loss='mse') + model.fit(x, y, epochs=1, batch_size=8, verbose=0) + metal_weights = [w.numpy().copy() for w in model.weights] + max_diffs = [ + np.max(np.abs(cpu_w - metal_w)) + for cpu_w, metal_w in zip(cpu_weights, metal_weights) + ] + + assert max(max_diffs) > 1e-4, ( + f'Compiled Metal {optimizer_name} unexpectedly matches CPU; ' + 're-evaluate the eager-only Metal workaround before changing ' + 'production code.' + ) + + +@skip_no_metal +def test_keras_interpreter_accepts_dict_inputs(): + """KerasInterpreter must support dict-shaped inputs used by train.""" + import tensorflow as tf + gpus = tf.config.list_physical_devices('GPU') + if not gpus: + pytest.skip('Metal GPU not visible to TensorFlow') + + from donkeycar.parts.interpreter import KerasInterpreter + + x, y = _fixed_data() + inputs = tf.keras.Input(shape=(3,), name='img_in') + hidden = tf.keras.layers.Dense(4, activation='relu')(inputs) + outputs = tf.keras.layers.Dense(1, name='angle_out')(hidden) + model = tf.keras.Model(inputs=inputs, outputs=outputs) + + interp = KerasInterpreter() + interp.model = model + interp.input_keys = ['img_in'] + interp.output_keys = ['angle_out'] + interp.shapes = ({'img_in': (None, 3)}, {'angle_out': (None, 1)}) + interp.compile(optimizer=tf.keras.optimizers.Adam(1e-3), loss='mse') + + dataset = tf.data.Dataset.from_tensor_slices(({'img_in': x}, y)).batch(8) + history = interp.fit( + x=dataset, + steps_per_epoch=1, + batch_size=8, + callbacks=[], + validation_data=None, + validation_steps=0, + epochs=1, + verbose=0, + ) + + assert history.history['loss'], 'expected one successful training step' diff --git a/donkeycar/tests/test_scripts.py b/donkeycar/tests/test_scripts.py index 3001feea76..d12581d512 100755 --- a/donkeycar/tests/test_scripts.py +++ b/donkeycar/tests/test_scripts.py @@ -1,4 +1,5 @@ import os +import shutil import subprocess import tarfile @@ -6,6 +7,14 @@ import pytest +tf_available = pytest.mark.skipif( + not __import__('importlib').util.find_spec('tensorflow'), + reason='TensorFlow not installed' +) + +DONKEY_CLI_AVAILABLE = shutil.which('donkey') is not None + + def is_error(err): for e in err: # Catch error if 'Error' is in the stderr output. @@ -23,14 +32,23 @@ def cardir(tmpdir_factory): return path +@pytest.mark.skipif( + not DONKEY_CLI_AVAILABLE, + reason="donkey CLI not installed in PATH" +) def test_createcar(cardir): cmd = ['donkey', 'createcar', '--path', cardir] out, err, proc_id = utils.run_shell_command(cmd) assert is_error(err) is False +@pytest.mark.skipif( + not DONKEY_CLI_AVAILABLE, + reason="donkey CLI not installed in PATH" +) +@tf_available def test_drivesim(cardir): - cmd = ['donkey', 'createcar', '--path', cardir ,'--template', 'square'] + cmd = ['donkey', 'createcar', '--path', cardir, '--template', 'square'] out, err, proc_id = utils.run_shell_command(cmd, timeout=10) cmd = ['python', 'manage.py', 'drive'] out, err, proc_id = utils.run_shell_command(cmd, cwd=cardir) @@ -42,6 +60,10 @@ def test_drivesim(cardir): raise ValueError(err) +@pytest.mark.skipif( + not DONKEY_CLI_AVAILABLE, + reason="donkey CLI not installed in PATH" +) def test_bad_command_fails(): cmd = ['donkey', 'not a comand'] out, err, proc_id = utils.run_shell_command(cmd) @@ -50,25 +72,29 @@ def test_bad_command_fails(): assert is_error(err) is True +@pytest.mark.skipif( + not DONKEY_CLI_AVAILABLE, + reason="donkey CLI not installed in PATH" +) +@tf_available def test_tubplot(cardir): # create empy KerasLinear model in car directory model_dir = os.path.join(cardir, 'models') os.mkdir(model_dir) - model_path = os.path.join(model_dir, 'model.savedmodel') + model_path = os.path.join(model_dir, 'model.keras') from donkeycar.parts.keras import KerasLinear KerasLinear().interpreter.model.save(model_path) # extract tub.tar.gz into car_dir/tub this_dir = os.path.dirname(os.path.abspath(__file__)) with tarfile.open(os.path.join(this_dir, 'tub', 'tub.tar.gz')) as file: - file.extractall(cardir) - # define the tub dir + file.extractall(cardir, filter='data') tub_dir = os.path.join(cardir, 'tub') # put a dummy config file into the car dir cfg_file = os.path.join(cardir, 'config.py') with open(cfg_file, "w+") as f: - f.writelines(["# config file\n", "IMAGE_H = 120\n", "IMAGE_W = 160\n", - "IMAGE_DEPTH = 3\n", "\n"]) + f.writelines(["# config file\n", "IMAGE_H = 120\n", + "IMAGE_W = 160\n", "IMAGE_DEPTH = 3\n", "\n"]) cmd = ['donkey', 'tubplot', '--tub', tub_dir, '--model', model_path, '--type', 'linear', '--noshow'] @@ -80,4 +106,3 @@ def test_tubplot(cardir): line = pipe.stdout.readline().decode() print(f'List model dir: {os.listdir(model_dir)}') assert os.path.exists(model_path + '_pred.png') - diff --git a/donkeycar/tests/test_telemetry.py b/donkeycar/tests/test_telemetry.py index 02a8f6907a..a2427e2da9 100644 --- a/donkeycar/tests/test_telemetry.py +++ b/donkeycar/tests/test_telemetry.py @@ -2,46 +2,98 @@ # -*- coding: utf-8 -*- import time from unittest import mock -from paho.mqtt.client import Client -from paho.mqtt.enums import CallbackAPIVersion - +from unittest.mock import patch, MagicMock import donkeycar.templates.cfg_complete as cfg from donkeycar.parts.telemetry import MqttTelemetry +import pytest from random import randint -def test_mqtt_telemetry(): - +@patch('donkeycar.parts.telemetry.MQTTClient') +def test_mqtt_telemetry(mock_mqtt_client): + """Test MQTT telemetry functionality with mocked MQTT client""" + + # Setup configuration cfg.TELEMETRY_DEFAULT_INPUTS = 'pilot/angle,pilot/throttle' cfg.TELEMETRY_DONKEY_NAME = 'test{}'.format(randint(0, 1000)) cfg.TELEMETRY_MQTT_JSON_ENABLE = True - # Create receiver - sub = Client(callback_api_version=CallbackAPIVersion.VERSION2, - clean_session=True) - - on_message_mock = mock.Mock() - sub.on_message = on_message_mock - sub.connect(cfg.TELEMETRY_MQTT_BROKER_HOST) - sub.loop_start() - name = "donkey/%s/#" % cfg.TELEMETRY_DONKEY_NAME - sub.subscribe(name) + # Create mock MQTT client + mock_client_instance = MagicMock() + mock_mqtt_client.return_value = mock_client_instance + # Create telemetry instance t = MqttTelemetry(cfg) + + # Verify MQTT client was initialized correctly + mock_mqtt_client.assert_called_once_with(callback_api_version=mock.ANY) + mock_client_instance.connect.assert_called_once_with( + cfg.TELEMETRY_MQTT_BROKER_HOST, + cfg.TELEMETRY_MQTT_BROKER_PORT + ) + mock_client_instance.loop_start.assert_called_once() + + # Test adding step inputs t.add_step_inputs(inputs=['my/voltage'], types=['float']) + expected_inputs = ['pilot/angle', 'pilot/throttle', 'my/voltage'] + expected_types = ['float', 'float', 'float'] + assert t._step_inputs == expected_inputs + assert t._step_types == expected_types + + # Test initial publish (should do nothing as queue is empty) t.publish() + mock_client_instance.publish.assert_not_called() + # Test reporting data timestamp = t.report({'my/speed': 16, 'my/voltage': 12}) + assert isinstance(timestamp, int) + assert t.qsize == 1 + + # Test run method (adds step inputs to queue) t.run(33.3, 22.2, 11.1) assert t.qsize == 2 - time.sleep(1.5) - + # Test publishing with data t.publish() assert t.qsize == 0 + + # Verify publish was called + assert mock_client_instance.publish.called + call_args = mock_client_instance.publish.call_args + topic, payload = call_args[0] + + # Verify topic format + expected_topic = cfg.TELEMETRY_MQTT_TOPIC_TEMPLATE % cfg.TELEMETRY_DONKEY_NAME + assert topic == expected_topic + + # Verify JSON payload structure + import json + payload_data = json.loads(payload) + assert isinstance(payload_data, list) + assert len(payload_data) >= 1 + + # Check that data contains expected keys + data_entry = payload_data[0] + assert 'ts' in data_entry + assert 'values' in data_entry + assert 'my/speed' in data_entry['values'] + assert 'pilot/angle' in data_entry['values'] + assert 'pilot/throttle' in data_entry['values'] + + +def test_mqtt_telemetry_connection_error(): + """Test MQTT telemetry handles connection errors gracefully""" + + cfg.TELEMETRY_DEFAULT_INPUTS = 'pilot/angle,pilot/throttle' + cfg.TELEMETRY_DONKEY_NAME = 'test{}'.format(randint(0, 1000)) + cfg.TELEMETRY_MQTT_JSON_ENABLE = True - time.sleep(0.5) + with patch('donkeycar.parts.telemetry.MQTTClient') as mock_mqtt_client: + # Simulate connection failure + mock_client_instance = MagicMock() + mock_client_instance.connect.side_effect = ConnectionError("Connection failed") + mock_mqtt_client.return_value = mock_client_instance - res = str.encode('[{"ts": %s, "values": {"my/speed": 16, "my/voltage": 11.1, "pilot/angle": 33.3, ' - '"pilot/throttle": 22.2}}]' % timestamp) - assert on_message_mock.call_args_list[0][0][2].payload == res + # Connection error should be raised during initialization + with pytest.raises(ConnectionError): + t = MqttTelemetry(cfg) diff --git a/donkeycar/tests/test_torch.py b/donkeycar/tests/test_torch.py index f6bdd21934..decb56170c 100644 --- a/donkeycar/tests/test_torch.py +++ b/donkeycar/tests/test_torch.py @@ -5,8 +5,18 @@ from collections import namedtuple from donkeycar.config import Config -Data = namedtuple('Data', ['type', 'name', 'convergence', 'pretrained']) +try: + import torch as _torch +except ImportError: + pytest.skip( + 'PyTorch not installed. Install with: uv pip install ' + '--python ~/.venvs/donkeycar/bin/python ' + '"torch==2.11.*" "torchvision==0.26.*" "torchaudio==2.11.*" ' + 'pytorch-lightning fastai', + allow_module_level=True, + ) +Data = namedtuple('Data', ['type', 'name', 'convergence', 'pretrained']) is_jetson = pytest.mark.skipif( platform.machine() == 'aarch64', @@ -39,7 +49,7 @@ def car_dir(tmpdir_factory): # extract tub.tar.gz into temp car_dir/tub this_dir = os.path.dirname(os.path.abspath(__file__)) with tarfile.open(os.path.join(this_dir, 'tub', 'tub.tar.gz')) as file: - file.extractall(dir) + file.extractall(dir, filter='data') return dir @@ -49,8 +59,6 @@ def car_dir(tmpdir_factory): @is_jetson -@pytest.mark.skipif("GITHUB_ACTIONS" in os.environ, - reason='Suppress training test in CI') @pytest.mark.parametrize('data', test_data) def test_train(config: Config, car_dir: str, data: Data) -> None: """ @@ -76,8 +84,6 @@ def pilot_path(name): @is_jetson -@pytest.mark.skipif("GITHUB_ACTIONS" in os.environ, - reason='Suppress training test in CI') @pytest.mark.parametrize('model_type', ['resnet18']) def test_training_pipeline(config: Config, model_type: str, car_dir: str) \ -> None: diff --git a/donkeycar/tests/test_train.py b/donkeycar/tests/test_train.py index 6f08828fb3..d6889f141e 100644 --- a/donkeycar/tests/test_train.py +++ b/donkeycar/tests/test_train.py @@ -1,6 +1,8 @@ from copy import copy import pytest +pytest.importorskip('tensorflow', + reason='TensorFlow not installed, skipping training tests') import tarfile import os import numpy as np @@ -84,7 +86,7 @@ def car_dir(tmpdir_factory, base_config, imu_fields) -> str: # extract tub.tar.gz into car_dir/tub this_dir = os.path.dirname(os.path.abspath(__file__)) with tarfile.open(os.path.join(this_dir, 'tub', 'tub.tar.gz')) as file: - file.extractall(car_dir) + file.extractall(car_dir, filter='data') # now create a second tub with additonal imu data tub_dir = os.path.join(car_dir, 'tub') tub = Tub(base_path=tub_dir) @@ -106,6 +108,8 @@ def car_dir(tmpdir_factory, base_config, imu_fields) -> str: record['localizer/location'] = 3 * count // len(tub) tub_full.write_record(record) count += 1 + tub_full.close() + tub.close() return car_dir @@ -126,7 +130,17 @@ def car_dir(tmpdir_factory, base_config, imu_fields) -> str: d14 = Data(type='fastai_linear', name='linfastai1', convergence=0.6, tf_lite=False, tensor_rt=False) -test_data = [d1, d2, d3, d6, d7, d8, d9, d10, d11, d12, d14] +try: + import fastai as _fastai # noqa: F401 + _skip_fastai = () +except ImportError: + _skip_fastai = (pytest.mark.skip( + reason='fastai not installed. Install with: uv pip install ' + '--python ~/.venvs/donkeycar/bin/python ' + 'pytorch-lightning fastai'),) + +test_data = [d1, d2, d3, d6, d7, d8, d9, d10, d11, d12, + pytest.param(d14, marks=_skip_fastai)] full_tub = ['imu', 'behavior', 'localizer'] @@ -141,7 +155,7 @@ def test_train(config: Config, data: Data) -> None: :return: None """ def pilot_path(name): - pilot_name = f'pilot_{name}.savedmodel' + pilot_name = f'pilot_{name}.keras' return os.path.join(config.MODELS_PATH, pilot_name) cfg = copy(config) @@ -204,31 +218,35 @@ def test_training_pipeline(config: Config, model_type: str, # this takes all batches into one list tf_batch = list(data_train.take(num_whole_batches).as_numpy_iterator()) it = iter(training_records) - for xy_batch in tf_batch: - # extract x and y values from records, asymmetric in x and y b/c x - # requires image manipulations - batch_records = [next(it) for _ in range(config.BATCH_SIZE)] - # if we cache images then the normalisation here would not work, because - # the tf batch might already have taken the images out and written - # the uint8 images into the cache. - records_x = [kl.x_transform(r, normalize_image) for r in batch_records] - records_y = [kl.y_transform(r) for r in batch_records] - # from here all checks are symmetrical between x and y - for batch, o_type, records \ - in zip(xy_batch, kl.output_types(), (records_x, records_y)): - # check batch dictionary have expected keys - assert batch.keys() == o_type.keys(), \ - 'batch keys need to match models output types' - # convert record values into arrays of batch size - values = defaultdict(list) - for r in records: - for k, v in r.items(): - values[k].append(v) - # now convert arrays of floats or numpy arrays into numpy arrays - np_dict = dict() - for k, v in values.items(): - np_dict[k] = np.array(v) - # compare record values with values from tf.data - for k, v in batch.items(): - assert np.isclose(v, np_dict[k]).all() + try: + for xy_batch in tf_batch: + # extract x and y values from records, asymmetric in x and y b/c x + # requires image manipulations + batch_records = [next(it) for _ in range(config.BATCH_SIZE)] + # if we cache images then the normalisation here would not work, + # because the tf batch might already have taken the images out and + # written the uint8 images into the cache. + records_x = [kl.x_transform(r, normalize_image) + for r in batch_records] + records_y = [kl.y_transform(r) for r in batch_records] + # from here all checks are symmetrical between x and y + for batch, o_type, records in zip( + xy_batch, kl.output_types(), (records_x, records_y)): + if not isinstance(batch, dict): + # single-output models return raw arrays (Keras 3.x) + assert np.isclose(batch, np.array(records)).all() + continue + assert batch.keys() == o_type.keys(), \ + 'batch keys need to match models output types' + values = defaultdict(list) + for r in records: + for k, v in r.items(): + values[k].append(v) + np_dict = dict() + for k, v in values.items(): + np_dict[k] = np.array(v) + for k, v in batch.items(): + assert np.isclose(v, np_dict[k]).all() + finally: + dataset.close() diff --git a/donkeycar/tests/test_web_socket.py b/donkeycar/tests/test_web_socket.py index 4bb0c45b85..4270998d8e 100644 --- a/donkeycar/tests/test_web_socket.py +++ b/donkeycar/tests/test_web_socket.py @@ -1,13 +1,10 @@ from tornado import testing import tornado.websocket import tornado.web -import tornado.ioloop import json from unittest.mock import Mock from donkeycar.parts.web_controller.web import WebSocketCalibrateAPI -from time import sleep - -SLEEP = 0.5 +import tornado.gen class WebSocketCalibrateTest(testing.AsyncHTTPTestCase): @@ -24,6 +21,17 @@ def get_app(self): def get_ws_url(self): return "ws://localhost:" + str(self.get_http_port()) + "/" + + async def wait_for_attribute_value(self, obj, attr_name, + expected_value, timeout_seconds=5): + """Poll until an object's attribute equals the expected value or timeout.""" + iterations = int(timeout_seconds / 0.1) + for _ in range(iterations): + if (hasattr(obj, attr_name) + and getattr(obj, attr_name) == expected_value): + return True + await tornado.gen.sleep(0.1) + return False @tornado.testing.gen_test def test_calibrate_servo_esc_1b(self): @@ -31,6 +39,8 @@ def test_calibrate_servo_esc_1b(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.left_pulse = None + mock.right_pulse = None self.app.drive_train = dict() self.app.drive_train['steering'] = mock self.app.drive_train_type = "I2C_SERVO" @@ -38,9 +48,12 @@ def test_calibrate_servo_esc_1b(self): data = {"config": {"STEERING_LEFT_PWM": 444}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train['steering'], 'left_pulse', 444) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train['steering'].left_pulse == 444 - assert isinstance(self.app.drive_train['steering'].right_pulse, Mock) + assert self.app.drive_train['steering'].right_pulse is None @tornado.testing.gen_test def test_calibrate_servo_esc_1a(self): @@ -48,6 +61,8 @@ def test_calibrate_servo_esc_1a(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.left_pulse = None + mock.right_pulse = None self.app.drive_train = dict() self.app.drive_train['steering'] = mock self.app.drive_train_type = "PWM_STEERING_THROTTLE" @@ -55,9 +70,12 @@ def test_calibrate_servo_esc_1a(self): data = {"config": {"STEERING_LEFT_PWM": 444}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train['steering'], 'left_pulse', 444) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train['steering'].left_pulse == 444 - assert isinstance(self.app.drive_train['steering'].right_pulse, Mock) + assert self.app.drive_train['steering'].right_pulse is None @tornado.testing.gen_test def test_calibrate_servo_esc_2b(self): @@ -65,6 +83,8 @@ def test_calibrate_servo_esc_2b(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.left_pulse = None + mock.right_pulse = None self.app.drive_train = dict() self.app.drive_train['steering'] = mock self.app.drive_train_type = "I2C_SERVO" @@ -72,9 +92,12 @@ def test_calibrate_servo_esc_2b(self): data = {"config": {"STEERING_RIGHT_PWM": 555}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train['steering'], 'right_pulse', 555) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train['steering'].right_pulse == 555 - assert isinstance(self.app.drive_train['steering'].left_pulse, Mock) + assert self.app.drive_train['steering'].left_pulse is None @tornado.testing.gen_test def test_calibrate_servo_esc_2a(self): @@ -82,6 +105,8 @@ def test_calibrate_servo_esc_2a(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.left_pulse = None + mock.right_pulse = None self.app.drive_train = dict() self.app.drive_train['steering'] = mock self.app.drive_train_type = "PWM_STEERING_THROTTLE" @@ -89,9 +114,12 @@ def test_calibrate_servo_esc_2a(self): data = {"config": {"STEERING_RIGHT_PWM": 555}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train['steering'], 'right_pulse', 555) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train['steering'].right_pulse == 555 - assert isinstance(self.app.drive_train['steering'].left_pulse, Mock) + assert self.app.drive_train['steering'].left_pulse is None @tornado.testing.gen_test def test_calibrate_servo_esc_3b(self): @@ -99,6 +127,8 @@ def test_calibrate_servo_esc_3b(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.max_pulse = None + mock.min_pulse = None self.app.drive_train = dict() self.app.drive_train['throttle'] = mock self.app.drive_train_type = "I2C_SERVO" @@ -106,9 +136,12 @@ def test_calibrate_servo_esc_3b(self): data = {"config": {"THROTTLE_FORWARD_PWM": 666}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train['throttle'], 'max_pulse', 666) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train['throttle'].max_pulse == 666 - assert isinstance(self.app.drive_train['throttle'].min_pulse, Mock) + assert self.app.drive_train['throttle'].min_pulse is None @tornado.testing.gen_test def test_calibrate_servo_esc_3a(self): @@ -116,6 +149,8 @@ def test_calibrate_servo_esc_3a(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.max_pulse = None + mock.min_pulse = None self.app.drive_train = dict() self.app.drive_train['throttle'] = mock self.app.drive_train_type = "PWM_STEERING_THROTTLE" @@ -123,9 +158,12 @@ def test_calibrate_servo_esc_3a(self): data = {"config": {"THROTTLE_FORWARD_PWM": 666}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train['throttle'], 'max_pulse', 666) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train['throttle'].max_pulse == 666 - assert isinstance(self.app.drive_train['throttle'].min_pulse, Mock) + assert self.app.drive_train['throttle'].min_pulse is None @tornado.testing.gen_test def test_calibrate_mm1(self): @@ -133,10 +171,14 @@ def test_calibrate_mm1(self): # Now we can run a test on the WebSocket. mock = Mock() + mock.STEERING_MID = None self.app.drive_train = mock self.app.drive_train_type = "MM1" data = {"config": {"MM1_STEERING_MID": 1234}} yield ws_client.write_message(json.dumps(data)) yield ws_client.close() - sleep(SLEEP) + + result = yield self.wait_for_attribute_value( + self.app.drive_train, 'STEERING_MID', 1234) + assert result, "WebSocket message not processed within timeout" assert self.app.drive_train.STEERING_MID == 1234 diff --git a/donkeycar/utilities/TrackSpeedPlanner/README.md b/donkeycar/utilities/TrackSpeedPlanner/README.md new file mode 100644 index 0000000000..affbcc4615 --- /dev/null +++ b/donkeycar/utilities/TrackSpeedPlanner/README.md @@ -0,0 +1,216 @@ +# Donkey Car Path Data Visualizer & Editor + +A web-based tool for visualizing and editing Donkey Car path data CSV files. This application provides an intuitive interface to load, visualize, and modify speed values for autonomous vehicle path planning. + +## Features + +### 🎯 **Interactive Path Visualization** +- Canvas-based path display with automatic scaling and centering +- Color-coded speed visualization (red = slow, green = fast) +- Click-to-select path points with visual feedback +- Real-time updates as you modify speed values + +### 📁 **Flexible File Operations** +- **Pi Directory Access**: Browse and load CSV files directly from the server directory +- **Upload from Browser**: Drag-and-drop or click to upload CSV files from your computer +- **Dual Save Options**: + - Save back to Pi directory (for server files) + - Download to local machine (for uploaded files) + +### ⚡ **Speed Editing** +- Individual speed controls for each path point +- Speed range: 0.1 to 1.0 in 0.1 increments +- Auto-scroll to selected point's controls +- Instant visual feedback on canvas + +### 📱 **Responsive Design** +- Works on desktop, tablet, and mobile devices +- Adaptive layout that adjusts to screen size +- Touch-friendly controls for mobile editing + +## Quick Start + +### 1. Start the Server +```bash +# Default port (note: avoid port 5000 on macOS due to AirPlay conflict) +python trackeditor.py --port=8080 + +# With debug mode +python trackeditor.py --port=8080 --debug +``` + +### 2. Access the Application +Open your browser and navigate to: +- **Local**: http://localhost:8080 +- **Network**: http://[your-ip]:8080 + +### 3. Load Path Data +Choose one of these methods: +- **From Pi Directory**: Use the dropdown to select existing CSV files +- **Upload File**: Drag & drop or click the upload area + +### 4. Edit and Save +- Click on path points to select them +- Adjust speed values using the right panel controls +- Save your changes back to the Pi or download locally + +## Server Configuration + +### Command Line Options +```bash +python trackeditor.py --help +``` + +Available options: +- `--port=PORT`: Server port (default: 5000, recommend 8080 on macOS) +- `--debug`: Enable debug mode with detailed logging + +### API Endpoints +- `GET /`: Main application interface +- `GET /api/health`: Server health check +- `GET /api/files`: List CSV files in server directory +- `POST /api/upload`: Upload CSV file from browser +- `POST /api/loadfile`: Load CSV file from server directory +- `POST /api/save`: Save modifications back to server +- `POST /api/export`: Download modified CSV file +- `POST /api/shutdown`: Graceful server shutdown + +## CSV File Format + +The application supports CSV files with the following formats: + +### With Headers (Recommended) +```csv +x,y,speed +10.5,20.3,0.8 +11.2,21.0,0.7 +12.1,22.5,0.9 +``` + +### Alternative Headers +Also supports: `pos_x,pos_y,throttle` and `X,Y,Speed` + +### Without Headers +```csv +10.5,20.3,0.8 +11.2,21.0,0.7 +12.1,22.5,0.9 +``` + +**Notes:** +- Speed values are automatically clamped to 0.1-1.0 range +- Missing speed values default to 0.5 +- Invalid rows are skipped with console warnings + +## File Structure + +``` +TrackSpeedPlanner/ +├── trackeditor.py # Main Tornado web server +├── static/ +│ └── index.html # Complete web interface (HTML/CSS/JS) +├── test_path.csv # Sample CSV file +└── attached_assets/ # Additional CSV files +``` + +## Requirements + +- **Python 3.6+** +- **Tornado web framework** +You should run this out of your donkey environment which has python=3.11 with +tornado installed. The above is only relevant when running out of another +environment. Please see next section for that. + +### Installation +```bash +# Install Tornado if not available +pip install tornado + +# Or with conda +conda install tornado +``` + +## Technical Details + +### Visualization Engine +- **Canvas Size**: 800x600 pixels (responsive) +- **Auto-scaling**: Automatically fits path data to canvas +- **Color Coding**: RGB interpolation based on speed values +- **Selection Radius**: 15-pixel click detection around points + +### Performance +- **Raspberry Pi Zero/1**: Suitable for small to medium paths +- **Raspberry Pi 2/3**: Good performance for typical donkey car paths +- **Raspberry Pi 4+**: Excellent performance with large datasets +- **Desktop/Laptop**: Handles very large path files efficiently + +### Browser Compatibility +- Modern browsers with HTML5 Canvas support +- Chrome, Firefox, Safari, Edge +- Mobile browsers on iOS and Android + +## Troubleshooting + +### Server Won't Start +```bash +# Check Python version +python --version # Should be 3.6+ + +# Install Tornado +pip install tornado + +# Check port availability (especially on macOS) +lsof -i :5000 # If in use, try different port +python trackeditor.py --port=8080 +``` + +### Can't Access from Network +```bash +# Check your IP address +hostname -I # Linux/Pi +ipconfig # Windows +ifconfig # macOS + +# Test locally first +curl http://localhost:8080/api/health +``` + +### File Upload Issues +- Ensure files have `.csv` extension +- Check file format matches expected CSV structure +- Large files may take time to process on slower hardware + +### Port 5000 Conflicts (macOS) +macOS uses port 5000 for AirPlay. Use an alternative port: +```bash +python trackeditor.py --port=8080 +``` + +## Integration with Donkey Car + +This tool is designed to work with Donkey Car path planning: + +1. **Record Paths**: Use Donkey Car to record driving paths +2. **Edit Speeds**: Load and modify speed profiles using this tool +3. **Deploy**: Save edited paths back to your Donkey Car for autonomous driving + +## Development + +### Server Development +- Edit `trackeditor.py` for backend changes +- Supports hot reload in debug mode: `--debug` + +### Frontend Development +- Edit `static/index.html` for UI changes +- Contains all HTML, CSS, and JavaScript in a single file +- Refresh browser to see changes + +### Adding Features +The application uses a RESTful API design. To add new functionality: +1. Create new handler class in `trackeditor.py` +2. Add route to `make_app()` function +3. Implement frontend calls in `static/index.html` + +## License + +This tool is part of the Donkey Car project and follows the same open-source principles. \ No newline at end of file diff --git a/donkeycar/utilities/TrackSpeedPlanner/attached_assets/donkey_path (1)_1749922476174.csv b/donkeycar/utilities/TrackSpeedPlanner/attached_assets/donkey_path (1)_1749922476174.csv new file mode 100644 index 0000000000..cbaed4ccfe --- /dev/null +++ b/donkeycar/utilities/TrackSpeedPlanner/attached_assets/donkey_path (1)_1749922476174.csv @@ -0,0 +1,296 @@ +-0.0030673221917822957, 0.0018730079755187035, 0.10 +-0.007616000366397202, 0.21041828347370028, 0.21 +-0.011573938943911344, 0.4121252675540745, 0.19 +-0.01824282720917836, 0.6764627313241363, 0.21 +-0.03438853187253699, 1.1043250309303403, 0.13 +-0.05552761268336326, 1.365461734123528, 0.18 +-0.07756315887672827, 1.6166332405991852, 0.22 +-0.09831937844865024, 1.8845999259501696, 0.19 +-0.11706995643908158, 2.178405029233545, 0.2 +-0.1370603431132622, 2.4790539648383856, 0.16 +-0.14352418953785673, 2.7864206824451685, 0.2 +-0.1350443068658933, 3.2665225761011243, 0.2 +-0.13057457859395072, 3.9041928029619157, 0.17 +-0.2759307601954788, 4.08069723425433, 0.18 +-0.26287036488065496, 4.413014094810933, 0.19 +-0.3238702082890086, 4.738206502050161, 0.18 +-0.317751364025753, 5.017763447016478, 0.14 +-0.3245541320065968, 5.2854263400658965, 0.16 +-0.31516807473963127, 5.529496371280402, 0.23 +-0.3057356822537258, 5.893055530264974, 0.21 +-0.32409560115775093, 6.175591673702002, 0.21 +-0.20024774060584605, 6.445094198919833, 0.22 +-0.20232785487314686, 6.923808777704835, 0.2 +-0.21382020541932434, 7.269447682891041, 0.19 +-0.21788450243184343, 7.610775483772159, 0.21 +-0.2115321487071924, 7.955708012916148, 0.21 +-0.20915608439827338, 8.310093288309872, 0.19 +-0.18674702144926414, 8.672987627796829, 0.21 +-0.18244739610236138, 9.02458624728024, 0.22 +-0.17332758504198864, 9.560641455464065, 0.2 +-0.17610309325391427, 9.919872448313981, 0.2 +-0.17738666926743463, 10.28278435766697, 0.21 +-0.17360054177697748, 10.650270047597587, 0.2 +-0.1684942006249912, 11.020330023020506, 0.2 +-0.16525320091750473, 11.380987118463963, 0.2 +-0.151132392056752, 11.74247475201264, 0.19 +-0.14113290887326002, 12.096425587311387, 0.19 +-0.1391996091697365, 12.436963462736458, 0.2 +-0.12483004067325965, 12.770561928395182, 0.19 +-0.10620368458330631, 13.108556433580816, 0.19 +-0.09411374182673171, 13.441250945907086, 0.2 +-0.08369441644754261, 13.939435167703778, 0.2 +-0.07446853868896142, 14.278802696149796, 0.2 +-0.05699604615801945, 14.785791465081275, 0.19 +-0.04335269337752834, 15.12936898181215, 0.19 +-0.03141556039918214, 15.463172893971205, 0.19 +-0.01712639193283394, 15.787353243678808, 0.2 +-0.0009073510300368071, 16.109485612250865, 0.21 +0.016964310547336936, 16.435112840030342, 0.22 +0.04258392611518502, 16.774894473608583, 0.21 +0.057979221222922206, 17.133600993081927, 0.17 +0.07151232560863718, 17.483273970428854, 0.16 +0.08968014811398461, 17.805574386846274, 0.15 +0.0993453242117539, 18.101353115867823, 0.14 +0.09989347273949534, 18.374678417574614, 0.16 +0.09441870549926534, 18.626816655974835, 0.16 +0.0819710700889118, 18.860546175390482, 0.16 +0.06464427232276648, 19.07400227803737, 0.15 +0.04883137933211401, 19.274702372029424, 0.17 +0.00900184892816469, 19.57053424604237, 0.18 +-0.056796766992192715, 19.86584907397628, 0.17 +-0.18286227021599188, 20.249033151660115, 0.18 +-0.26627612952142954, 20.438860008493066, 0.22 +-0.3643774958909489, 20.638046366162598, 0.16 +-0.5019444086938165, 20.852528900373727, 0.18 +-0.6410112387384288, 21.04338488727808, 0.18 +-0.7983938870602287, 21.217775813303888, 0.18 +-0.9639892141567543, 21.380601855460554, 0.17 +-1.1309032734134234, 21.52201599860564, 0.17 +-1.3028351159300655, 21.6459281463176, 0.16 +-1.637306509364862, 21.83438921859488, 0.17 +-1.88962384784827, 21.93257800769061, 0.16 +-2.1394269879092462, 22.002119564451277, 0.19 +-2.5065589963342063, 22.0525323394686, 0.19 +-2.7168480378459208, 22.05543560255319, 0.2 +-2.937315894290805, 22.044574580155313, 0.19 +-3.1725628016283736, 22.0133399781771, 0.2 +-3.4235481740906835, 21.96340212924406, 0.19 +-3.826888970565051, 21.82131560239941, 0.2 +-4.0977833410725, 21.68234611535445, 0.19 +-4.365809155919123, 21.499213048722595, 0.19 +-4.610208795464132, 21.270261735655367, 0.19 +-4.916357762471307, 20.860479433089495, 0.17 +-5.079315411276184, 20.55289688752964, 0.17 +-5.207706963701639, 20.245388622395694, 0.18 +-5.317029797064606, 19.928483449853957, 0.18 +-5.396341048006434, 19.61039865994826, 0.19 +-5.448629320191685, 19.283218421507627, 0.18 +-5.479991402826272, 18.955120807979256, 0.19 +-5.491415091499221, 18.62445211224258, 0.19 +-5.484423849266022, 18.28383804159239, 0.19 +-5.482304791279603, 17.771695592440665, 0.2 +-5.484244559134822, 17.43041903525591, 0.19 +-5.50052320380928, 16.911231032572687, 0.19 +-5.514527516148519, 16.563408902846277, 0.18 +-5.535436813312117, 16.22414111159742, 0.2 +-5.561508715094533, 15.888426358345896, 0.18 +-5.587731849460397, 15.554005674552172, 0.19 +-5.619251478870865, 15.226463454309851, 0.18 +-5.656113327422645, 14.900444301310927, 0.17 +-5.6966679152683355, 14.579073733650148, 0.18 +-5.733083840052132, 14.267271320335567, 0.19 +-5.773057037906256, 13.956976734567434, 0.19 +-5.81742249650415, 13.645426867995411, 0.18 +-5.864255073538516, 13.330019732937217, 0.18 +-5.91576976532815, 13.017422806005925, 0.18 +-5.96446127386298, 12.712373756803572, 0.16 +-6.0223973040119745, 12.407957674004138, 0.17 +-6.069540124794003, 12.113237618934363, 0.17 +-6.11194250889821, 11.822540109977126, 0.17 +-6.148421284044161, 11.54139551660046, 0.15 +-6.179633405234199, 11.268886036705226, 0.14 +-6.189783743757289, 11.011340711265802, 0.15 +-6.194082877191249, 10.773875819519162, 0.19 +-6.191221747023519, 10.538750671315938, 0.17 +-6.189203486777842, 10.299939072225243, 0.18 +-6.180239249079023, 10.057374525815248, 0.17 +-6.170320324134082, 9.812585640698671, 0.19 +-6.158895497967023, 9.53509511006996, 0.18 +-6.146881507534999, 9.250581619795412, 0.17 +-6.131272407073993, 8.968992357607931, 0.17 +-6.111252126574982, 8.690874401014298, 0.18 +-6.095234932436142, 8.419069822411984, 0.17 +-6.083907731925137, 8.149152107071131, 0.17 +-6.060171733202878, 7.755021872464567, 0.19 +-6.049237527768128, 7.096905063837767, 0.18 +-6.032837463484611, 6.831930467393249, 0.16 +-6.025043771602213, 6.585312901996076, 0.17 +-6.014293004584033, 6.342917788773775, 0.18 +-6.008058929757681, 6.103700836189091, 0.17 +-5.999913903942797, 5.860219873487949, 0.18 +-5.983473622589372, 5.200763284228742, 0.18 +-5.982503050297964, 4.953280556481332, 0.21 +-5.990941184631083, 4.649180646520108, 0.17 +-5.990375204302836, 4.183222070336342, 0.19 +-6.019994155445602, 3.7171518155373633, 0.17 +-6.027824205812067, 3.4082449730485678, 0.18 +-6.0477328937267885, 3.110152824781835, 0.17 +-6.06179206300294, 2.8129341518506408, 0.17 +-6.077425037568901, 2.521454067900777, 0.17 +-6.1049633483053185, 2.242079977877438, 0.17 +-6.127303571149241, 1.8302442184649408, 0.17 +-6.148689898371231, 1.5491554676555097, 0.18 +-6.168582712823991, 1.2719322978518903, 0.17 +-6.18979416473303, 0.9923195131123066, 0.18 +-6.205935905280057, 0.7172805098816752, 0.14 +-6.220681760751177, 0.4536799010820687, 0.15 +-6.23177575081354, 0.19965160032734275, 0.16 +-6.234200189646799, -0.04632468428462744, 0.17 +-6.226472593843937, -0.28167605539783835, 0.17 +-6.2231546033290215, -0.5203140652738512, 0.18 +-6.2245912424405105, -0.7646366409026086, 0.17 +-6.226796769245993, -1.0229885149747133, 0.11 +-6.228753546136431, -1.3092295587994158, 0.11 +-6.235858633473981, -1.5712332543917, 0.16 +-6.241867145581637, -1.8188410829752684, 0.17 +-6.247505407140125, -2.0610962817445397, 0.17 +-6.26160208293004, -2.305680143646896, 0.16 +-6.271030831092503, -2.554736233782023, 0.16 +-6.276966826932039, -2.7938492889516056, 0.18 +-6.286932715040166, -3.029788340907544, 0.2 +-6.305014460871462, -3.4180211033672094, 0.19 +-6.318147216457874, -3.701765866484493, 0.18 +-6.331692788458895, -3.9958493197336793, 0.18 +-6.345709400018677, -4.2880819295533, 0.2 +-6.364543029514607, -4.745379496365786, 0.2 +-6.376225057931151, -5.068289315328002, 0.2 +-6.386946090089623, -5.3926847986876965, 0.21 +-6.395901210664306, -5.7383338352665305, 0.2 +-6.403380268428009, -6.1011709361337125, 0.2 +-6.416631709085777, -6.474855022039264, 0.21 +-6.434793751104735, -6.853483611252159, 0.22 +-6.448680644913111, -7.054484212305397, 0.21 +-6.467507634719368, -7.453976237680763, 0.21 +-6.477391435939353, -7.661290214397013, 0.21 +-6.486641930707265, -7.870456422679126, 0.2 +-6.495588385616429, -8.082026096526533, 0.2 +-6.524012937385123, -8.503228714689612, 0.22 +-6.5402425257489085, -8.712520023807883, 0.19 +-6.551985369238537, -8.9284982024692, 0.19 +-6.569215367722791, -9.14092057198286, 0.2 +-6.586892755993176, -9.34872206300497, 0.2 +-6.60186992114177, -9.563379854895175, 0.19 +-6.631458728807047, -9.987896815408021, 0.21 +-6.66829125926597, -10.405518687330186, 0.21 +-6.706335340102669, -10.83199497172609, 0.19 +-6.727308536821511, -11.045678162947297, 0.19 +-6.772747574665118, -11.463965290691704, 0.19 +-6.790943300409708, -11.675456004682928, 0.19 +-6.810922395961825, -11.88674680935219, 0.19 +-6.825367045763414, -12.096053357236087, 0.19 +-6.84090342768468, -12.300179475452751, 0.2 +-6.845361111394595, -12.70828955899924, 0.19 +-6.8295608789194375, -12.910467007663101, 0.19 +-6.805793757550418, -13.110865643247962, 0.2 +-6.778116438537836, -13.309450836852193, 0.19 +-6.721929965831805, -13.704227453097701, 0.22 +-6.677289731043857, -14.101860418450087, 0.18 +-6.655346964776982, -14.306860540062189, 0.2 +-6.6208854574360885, -14.699235502164811, 0.19 +-6.602681154443417, -14.904942438937724, 0.2 +-6.586696574522648, -15.104535905178636, 0.19 +-6.572500093141571, -15.304483472369611, 0.19 +-6.559116529591847, -15.504608781542629, 0.18 +-6.5455659635481425, -15.70418146904558, 0.19 +-6.522173349803779, -16.10052494890988, 0.2 +-6.501548283093143, -16.687991101294756, 0.2 +-6.486073686159216, -17.08038865402341, 0.19 +-6.440251832071226, -17.472675912082195, 0.19 +-6.36428901553154, -17.850999934598804, 0.19 +-6.262275043467525, -18.219204135239124, 0.18 +-6.1392165693687275, -18.574475537519902, 0.2 +-5.995439703168813, -18.916996038053185, 0.2 +-5.82680745475227, -19.256404439453036, 0.2 +-5.639526880986523, -19.578427204396576, 0.19 +-5.435757920437027, -19.888955731876194, 0.19 +-5.210173988540191, -20.172337487339973, 0.19 +-4.977268960385118, -20.43454325897619, 0.2 +-4.730871926352847, -20.675071679521352, 0.2 +-4.467291708278935, -20.89875599881634, 0.2 +-4.1789457486593165, -21.091809764504433, 0.2 +-3.869326696614735, -21.245153723284602, 0.2 +-3.532948974694591, -21.36252835020423, 0.19 +-3.1829956967849284, -21.448438162449747, 0.21 +-2.8091336390934885, -21.49004360055551, 0.2 +-2.4208278381847776, -21.47452077968046, 0.2 +-2.0313652566983365, -21.380517796613276, 0.2 +-1.8465069359517656, -21.30194366723299, 0.2 +-1.5211410725605674, -21.074422171339393, 0.2 +-1.3832566144992597, -20.92859201366082, 0.19 +-1.1700898366398178, -20.594474390614778, 0.19 +-1.050913849612698, -20.225942220538855, 0.19 +-1.0023386162356474, -19.85846951743588, 0.2 +-0.9697019049199298, -19.495108449365944, 0.2 +-0.9172337215277366, -19.10883137024939, 0.21 +-0.8606301248073578, -18.73237776197493, 0.21 +-0.7884204352158122, -18.352733087725937, 0.2 +-0.7270369815523736, -17.967640206217766, 0.2 +-0.6831958081456833, -17.584429083857685, 0.19 +-0.6613254428957589, -17.20860238838941, 0.2 +-0.6603923634975217, -16.83315101219341, 0.2 +-0.6465109558193944, -16.461688498500735, 0.21 +-0.6254944946267642, -16.09084093850106, 0.21 +-0.5970987328910269, -15.711006932891905, 0.2 +-0.5658231334527954, -15.335999253671616, 0.2 +-0.53315547487, -14.778906599152833, 0.19 +-0.5095244732801802, -14.234849255532026, 0.21 +-0.5074288263567723, -13.875290488824248, 0.19 +-0.5041894205496646, -13.514818074181676, 0.21 +-0.5201187747879885, -12.98556698486209, 0.21 +-0.5449145335587673, -12.449406960979104, 0.2 +-0.5511244894587435, -12.093286258168519, 0.2 +-0.5525593199417926, -11.729080291930586, 0.2 +-0.5538571049110033, -11.367830416187644, 0.2 +-0.5460127148544416, -11.000194695778191, 0.2 +-0.5372593509964645, -10.640138713642955, 0.2 +-0.5218230555183254, -10.276630812324584, 0.18 +-0.5084112170734443, -9.922155069652945, 0.12 +-0.487563535629306, -9.5900893737562, 0.14 +-0.47043087967904285, -9.294005033560097, 0.18 +-0.44222241727402434, -8.879079768434167, 0.17 +-0.4225362437427975, -8.62604822870344, 0.17 +-0.39984264987288043, -8.381907118484378, 0.17 +-0.3839407638297416, -8.153960082214326, 0.17 +-0.3676176664303057, -7.819639571942389, 0.18 +-0.35717150644632056, -7.603465637657791, 0.18 +-0.3537375403684564, -7.391294858884066, 0.18 +-0.3571330072591081, -7.180727923288941, 0.18 +-0.3589121018303558, -6.970913514494896, 0.17 +-0.3638231527293101, -6.766797524876893, 0.19 +-0.3713273367029615, -6.562290033791214, 0.19 +-0.38785169285256416, -6.235815151128918, 0.19 +-0.3994927635649219, -6.00264657381922, 0.18 +-0.4094317204435356, -5.646124672610313, 0.19 +-0.41835511795943603, -5.398758745286614, 0.19 +-0.420521495048888, -5.139261462260038, 0.19 +-0.4209439161350019, -4.884580806829035, 0.18 +-0.41695042565697804, -4.625874835532159, 0.18 +-0.4130515170400031, -4.378249000757933, 0.18 +-0.40592302032746375, -4.132497558835894, 0.2 +-0.39784771675476804, -3.8898938009515405, 0.18 +-0.3842253560433164, -3.643828428350389, 0.16 +-0.3669228610233404, -3.4038889915682375, 0.16 +-0.34668704052455723, -3.181519421748817, 0.16 +-0.3303708662861027, -2.962071313057095, 0.17 +-0.31298660242464393, -2.750573665369302, 0.16 +-0.30068939056945965, -2.545681159477681, 0.17 +-0.2819177901255898, -2.152652540244162, 0.16 +-0.2668647714308463, -1.8720637680962682, 0.16 +-0.2542174984118901, -1.6071524657309055, 0.17 +-0.23450717522064224, -1.2562394505366683, 0.18 +-0.2238464216934517, -0.976905960123986, 0.17 +-0.20772531983675435, -0.704267649911344, 0.18 +-0.18620157655095682, -0.33102354081347585, 0.16 +-0.1767618251615204, -0.06165247969329357, -0.0 +-0.16035895358072594, 0.16795242298394442, -0.01 diff --git a/donkeycar/utilities/TrackSpeedPlanner/static/index.html b/donkeycar/utilities/TrackSpeedPlanner/static/index.html new file mode 100644 index 0000000000..3ef29ed4b4 --- /dev/null +++ b/donkeycar/utilities/TrackSpeedPlanner/static/index.html @@ -0,0 +1,734 @@ + + +
+ + +