diff --git a/.github/workflows/build-os-bookworm-dx.yml b/.github/workflows/build-os-bookworm-dx.yml index a1d61b26d..d6055fb94 100644 --- a/.github/workflows/build-os-bookworm-dx.yml +++ b/.github/workflows/build-os-bookworm-dx.yml @@ -37,8 +37,6 @@ jobs: - none - segmenter-only base_variant_name: - # We haven't set up networking on bookworm, so the lite image is useless and hard to make - # usable; so we just build the desktop image: - desktop permissions: contents: read diff --git a/.github/workflows/build-os-bookworm.yml b/.github/workflows/build-os-bookworm.yml new file mode 100644 index 000000000..2d3dc0548 --- /dev/null +++ b/.github/workflows/build-os-bookworm.yml @@ -0,0 +1,53 @@ +--- +name: build-os-bookworm +on: + push: + branches: + - 'master' + - 'software/beta' + - 'software/stable' + tags: + - 'software/v*' + paths: + - 'software/**' + - '!software/CHANGELOG.md' + - '.github/workflows/build-os*.yml' + pull_request: + paths: + - 'software/**' + - '.github/workflows/build-os*.yml' + #merge_group: + workflow_dispatch: + inputs: + git-ref: + description: 'Git ref (optional)' + required: false + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: build + strategy: + fail-fast: false + matrix: + variant: + - none + - segmenter-only + base_variant_name: + - lite + permissions: + contents: read + packages: write + id-token: write + uses: ./.github/workflows/build-os.yml + secrets: inherit + with: + name: rpi-forklift-demo + base_release_name: bookworm + base_image_variant: ${{ matrix.base_variant_name }} + base_release_date: 2024-11-19 + arch: arm64 + variant: ${{ matrix.variant }} diff --git a/.github/workflows/build-os.yml b/.github/workflows/build-os.yml index 549e5eaa9..fc29bf60b 100644 --- a/.github/workflows/build-os.yml +++ b/.github/workflows/build-os.yml @@ -252,7 +252,7 @@ jobs: # RUN OS SETUP SCRIPTS - - name: Run OS setup scripts in a booted container + - name: Run OS setup scripts in an unbooted container uses: ethanjli/pinspawn-action@v0.1.4 with: image: ${{ steps.expand-image.outputs.destination }} diff --git a/documentation/docs/operation/index.md b/documentation/docs/operation/index.md index a0d5b7d55..92ed3d837 100644 --- a/documentation/docs/operation/index.md +++ b/documentation/docs/operation/index.md @@ -22,7 +22,7 @@ As you can see, the name of your PlanktoScope's Wi-Fi hotspot will be of the for !!! tip - You should write down the PlanktoScope's machine name in a note (or on your PlanktoScope) for future reference; this will be especially important if you might have multiple PlanktoScopes in the future or if you might need to access the PlanktoScope via an indirect connection (e.g. from a device connected to the same network router as the PlanktoScope). + You may want to write down the PlanktoScope's machine name in a note (or on your PlanktoScope) for future reference; this will be especially important if you might have multiple PlanktoScopes in the future. Unless you have [changed the password](./networking.md#change-the-wi-fi-hotspots-password) of your PlanktoScope's Wi-Fi hotspot, the password should be `copepode`. @@ -30,23 +30,25 @@ Unless you have [changed the password](./networking.md#change-the-wi-fi-hotspots Once you connect your computer (or phone/tablet/etc.) to your PlanktoScope, you will need to access your PlanktoScope's software from a web browser on your computer. -If you know the machine name of your PlanktoScope (which you can determine from the name of your PlanktoScope's Wi-Fi hotspot, as described above), you can try using your web browser to open a URL of format `http://pkscope-{machine-name}.local`, where `{machine-name}` should be substituted with your PlanktoScope's machine name. This URL should work unless you're on a device or web browser without mDNS support; notably, older versions of Android did not support mDNS, and web browsers installed on Linux computers via Flatpak [do not yet support mDNS](https://github.com/flatpak/xdg-desktop-portal/discussions/1365). +If you know the machine name of your PlanktoScope (which you can determine from the name of your PlanktoScope's Wi-Fi hotspot, as described above), you can try using your web browser to open a URL of format `http://pkscope-{machine-name}.local`, where `{machine-name}` should be substituted with your PlanktoScope's machine name. If you don't know your PlanktoScope's machine name, you can try using your web browser to open or . All three of these URLs should work unless you're on a computer or web browser without mDNS support; notably, older versions of Android did not support mDNS, and web browsers installed on Linux computers via Flatpak [do not yet support mDNS](https://github.com/flatpak/xdg-desktop-portal/discussions/1365). +!!! warning + + and may behave in weird ways if your computer is directly connected to multiple PlanktoScopes or if it's connected to a Wi-Fi router or Ethernet router (or to a VPN or other LAN) which has multiple PlanktoScopes connected to it. In such a situation, those URLs may cause you to access the software for a different PlanktoScope connected to your computer than the one you had intended to access. You should instead use the machine-specific URL of format `http://pkscope-{machine-name}.local`. -If you are connecting your computer directly to your PlanktoScope (i.e. via your PlanktoScope's Wi-Fi hotspot, or via an Ethernet cable from your computer to your PlanktoScope's Ethernet port) but you don't know (or you don't want to type out) your PlanktoScope's machine name, then you can try opening the following URLs in your web browser instead (try opening them in the following order, and just use the first one which works): +If those URLs don't work for you and you're connecting your computer directly to your PlanktoScope (i.e. via your PlanktoScope's Wi-Fi hotspot, or via an Ethernet cable from your computer to your PlanktoScope's Ethernet port), then you can try opening the following URLs in your web browser instead (try opening them in the following order, and just use the first one which works): -- [http://planktoscope.local](http://planktoscope.local) (this should work unless you're on a device or web browser without mDNS support) -- [http://pkscope.local](http://pkscope.local) (this should work unless you're on a device or web browser without mDNS support) -- [http://home.pkscope](http://home.pkscope) (this should work unless your web browser is configured to use a Private DNS provider) -- [http://192.168.4.1](http://192.168.4.1) (this should always work on devices connected directly to the PlanktoScope, especially for devices connected directly to the PlanktoScope's Wi-Fi hotspot) -- [http://192.168.5.1](http://192.168.5.1) (this should always work on devices connected directly to the PlanktoScope, especially for devices connected directly to the PlanktoScope's Ethernet port) +- a URL of format `http://{machine-name}.pkscope`, where `{machine-name}` should be substituted with your PlanktoScope's machine name (this should work unless your web browser is configured to use a Private DNS provider) +- (this should work unless your web browser is configured to use a Private DNS provider) +- (this should always work on computers connected to the PlanktoScope's Wi-Fi hotspot, unless your computer is also connected to some other device meant to be accessed with that URL) +- (this should always work on computers connected directly to the PlanktoScope's Ethernet port, unless your computer is also connected to some other device meant to be accessed with that URL) !!! warning - The five URLs listed directly above **will not work** if you're trying to connect to your PlanktoScope indirectly, e.g. via a [Wi-Fi router providing internet access to the PlanktoScope](./networking.md#connect-your-planktoscope-to-an-existing-wi-fi-network). In that case, you will instead need to use a URL with your PlanktoScope's machine name which has format `http://pkscope-{machine-name}.local`. If you might want to do that in the future, then you should write down the machine name of your PlanktoScope as declared on your PlanktoScope's landing page! + The four URLs listed directly above **will not work** if you're trying to connect to your PlanktoScope indirectly, e.g. via a [Wi-Fi router providing internet access to the PlanktoScope](./networking.md#connect-your-planktoscope-to-an-existing-wi-fi-network). In that case, you will instead need to try one of the previously-described URLs ending in `.local`. !!! warning - The five URLs listed directly above may behave in weird ways if your computer is directly connected to multiple PlanktoScopes. In such a situation, those URLs may cause you to access the software for a different PlanktoScope connected to your computer than the one you had intended to access. You should instead use a machine-specific URL, either of format `http://pkscope-{machine-name}.local` (which will only work if your device and web browser support mDNS), or otherwise of format `http://{machine-name}.pkscope` (which will only work if your computer is connected directly to the PlanktoScope). + The four URLs listed directly above may behave in weird ways if your computer is directly connected to multiple PlanktoScopes. In such a situation, those URLs may cause you to access the software for a different PlanktoScope connected to your computer than the one you had intended to access. You should instead use a machine-specific URL, either of format `http://pkscope-{machine-name}.local` (which will only work if your device and web browser support mDNS), or otherwise of format `http://{machine-name}.pkscope` (which will only work if your computer is connected directly to the PlanktoScope). If you still encounter weird behavior, then you will need to disconnect one or more of your other PlanktoScopes from your computer. !!! warning @@ -58,9 +60,9 @@ One of the above URLs should work, and your web browser should show a landing pa !!! tip - If you access the landing page using any URL not of the format `http://pkscope-{machine-name}.local`, then the landing page will suggest its own machine-specific URL for you to try opening; you can see that in the above screenshot, which shows a yellow information box with the URL for the PlanktoScope used to make that screenshot. You should try opening your PlanktoScope's machine-specific URL to see if it works in your web browser, and you may want to bookmark it for future reference for the reasons discussed in the warnings listed earlier in this section of the documentation. + If you access the landing page using any URL not of the format `http://pkscope-{machine-name}.local`, then the landing page will suggest its own machine-specific URL for you to try opening; you can see that in the above screenshot, which shows a yellow information box with the URL for the PlanktoScope used to make that screenshot. You should try opening the corresponding URL on the landing page for your PlanktoScope to see if it works in your web browser. -You should click on the "Node-RED dashboard" link; this will open a new tab with the primary interface for operating your PlanktoScope. Once you have opened the Node-RED dashboard, you should proceed to our [User interface guide](user-interface.md) to understand how to use it. +Next, you should click on the "Node-RED dashboard" link; this will open a new tab with the primary interface for operating your PlanktoScope. Once you have opened the Node-RED dashboard, you should proceed to our [User interface guide](user-interface.md) to understand how to use it. ## Acquire data on plankton samples diff --git a/documentation/docs/operation/networking.md b/documentation/docs/operation/networking.md index c04ad8e36..f2a067dc7 100644 --- a/documentation/docs/operation/networking.md +++ b/documentation/docs/operation/networking.md @@ -2,17 +2,23 @@ By default, your PlanktoScope creates an [isolated Wi-Fi network](./index.md#connect-with-the-planktoscopes-isolated-wi-fi-network) (which we often call "the PlanktoScope's Wi-Fi hotspot network") which devices can connect to in order to access the PlanktoScope's software; and devices can also connect to the PlanktoScope directly [by an Ethernet cable](./index.md#connect-with-an-ethernet-cable). However, you may have reasons to adjust your networking configuration away from this default. This guide provides instructions on how to adjust your networking configuration in various ways. -Currently, all instructions in this guide should be considered as being provided for "advanced users" (and/or for users who are able to ask for help in the PlanktoScope Slack workspace). These instructions will probably change between between successive releases of the PlanktoScope software. All URLs in this guide are written assuming you access your PlanktoScope using [planktoscope.local](http://planktoscope.local) as the domain name; if you need to use a [different domain name](./index.md#access-your-planktoscopes-software) such as [home.pkscope](http://home.pkscope), you should substitute that domain name into the links on this page. +Currently, all instructions in this guide should be considered as being provided for "advanced users" (and/or for users who are able to ask for help in the PlanktoScope Slack workspace). These instructions will probably change between between successive releases of the PlanktoScope software. All URLs in this guide are written assuming you access your PlanktoScope using [planktoscope.local](http://planktoscope.local) as the domain name; if you need to use [some other domain name](./index.md#access-your-planktoscopes-software) such as [home.pkscope](http://home.pkscope), you should replace `planktoscope.local` with that other domain name in all the links on this page. ## Adjust your PlanktoScope's Wi-Fi region settings +By default, the PlanktoScope's Wi-Fi settings are configured in compliance with United States Wi-Fi [regulatory domain](https://en.wikipedia.org/wiki/IEEE_802.11#Regulatory_domains_and_legal_compliance). If you're operating your PlanktoScope in another country, we recommend changing the Wi-Fi settings to match your country, which you can do by editing the file at `/etc/default/crda`. For example, you can do this in the file browser at . You should replace the string `US` with a [ISO/IEC 3166-1 alpha-2 country code](https://en.wikipedia.org/wiki/ISO_3166-1#Codes) corresponding to your country. To apply your changes, restart the PlanktoScope. + +To revert your changes back to the US regulatory domain, we recommend deleting the file at `/var/lib/overlays/overrides/etc/default/crda`. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. + ### Change Wi-Fi hotspot region settings -By default, the PlanktoScope makes its Wi-Fi hotspot network on [WLAN channel](https://en.wikipedia.org/wiki/List_of_WLAN_channels) 8 and in compliance with United States Wi-Fi [regulatory domain](https://en.wikipedia.org/wiki/IEEE_802.11#Regulatory_domains_and_legal_compliance). If your computer/phone/etc.'s connection to the PlanktoScope's Wi-Fi hotspot is unstable and you are operating your PlanktoScope outside the United States, you may need to change your Wi-Fi network's regional settings. For example, if you are operating in France, you should change the Wi-Fi hotspot's [ISO/IEC 3166-1](https://en.wikipedia.org/wiki/ISO_3166-1#Codes)-formatted country code from `US` to `FR`. +By default, the PlanktoScope makes its Wi-Fi hotspot network on [WLAN channel](https://en.wikipedia.org/wiki/List_of_WLAN_channels) 8 in the 2.4 GHz range (802.11b/g), which should generally be compatible with regulations across all countries. However, if your computer/phone/etc.'s connection to the PlanktoScope's Wi-Fi hotspot is unstable and/or you are operating your PlanktoScope outside the United States, you might need to change your Wi-Fi hotspot's regional settings. -To change the regional settings of the PlanktoScope's Wi-Fi hotspot away from the defaults, edit the file at `/etc/hostapd/hostapd.conf.d/50-localization-us.conf`. For example, you can do this by opening the file editor at . To apply your changes, restart the PlanktoScope. +To change the regional settings of the PlanktoScope's Wi-Fi hotspot away from the defaults, edit the file at `/etc/NetworkManager/system-connections.d/wlan0-hotspot/31-wifi-regional.nmconnection`. For example, you can do this by opening the file editor at . To apply your changes, restart the PlanktoScope. -To revert your changes back to the default regional settings, we recommend deleting the file at `/var/lib/overlays/overrides/etc/hostapd/hostapd.conf.d/50-localization-us.conf`. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. +To revert your changes back to the default regional settings, we recommend deleting the file at `/var/lib/overlays/overrides/etc/NetworkManager/system-connections.d/wlan0-hotspot/31-wifi-regional.nmconnection`. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. + +Note that the above instructions are given for the Wi-Fi hotspot generated from the internal Wi-Fi module in your PlanktoScope's Raspberry Pi computer, which is named `wlan0`. If you want to change the regional settings for a Wi-Fi hotspot generated from a USB Wi-Fi dongle plugged into the Raspberry Pi, which will be named `wlan1`, then you should replace `wlan0` with `wlan1` in the above links/filenames. ## Connect your PlanktoScope to the internet @@ -20,55 +26,46 @@ To connect your PlanktoScope to the internet, we recommend one of the following 1. Plug it into an Ethernet port with internet access. 2. Connect an Android phone/tablet by USB to your PlanktoScope and then enabling USB tethering mode so that the Android device shares its internet access (which can be from the Wi-Fi network it's connected to) to the PlanktoScope. Note: we plan to add support for USB tethering from iOS devices, but we haven't figured out how to do that yet. +3. Plug a USB Wi-Fi dongle into your PlanktoScope and then connect to a Wi-Fi network using it, while maintaining a Wi-Fi hotspot from the internal Wi-Fi module in your PlanktoScope's Raspberry Pi computer. +4. Plug a USB Wi-Fi dongle into your PlanktoScope to generate a Wi-Fi hotspot from it, and then connect to a Wi-Fi network using the internal Wi-Fi module in your PlanktoScope's Raspberry Pi computer. -Once your PlanktoScope is connected to the internet, by default it will attempt to share its internet access with any devices connected to the PlanktoScope by Wi-Fi or Ethernet. +Once your PlanktoScope is connected to the internet, by default it will attempt to share its internet access with any devices connected to the PlanktoScope by Wi-Fi hotspot or by Ethernet. If you need to connect to a network which requires you to register your PlanktoScope's MAC address (also known as the "hardware address" of your PlanktoScope's network adapter(s)), please refer to [a subsection below](#check-your-planktoscopes-mac-addresses). -### Connect your PlanktoScope to an existing Wi-Fi network - -It's also possible to connect the PlanktoScope to an existing Wi-Fi network with internet access, but the PlanktoScope will be unable to make its Wi-Fi hotspot network while it is connected to an existing Wi-Fi network. Then, as long as the PlanktoScope is within range of that Wi-Fi network, the PlanktoScope software is only accessible either if 1) the existing Wi-Fi network (including its firewall settings) is configured to allow you to access the PlanktoScope via mDNS and your device and web browser support mDNS URLs (i.e. URLs ending in `.local`, like ) or if 2) you connect your device to the PlanktoScope via an Ethernet cable. +### Connect your PlanktoScope to an existing Wi-Fi network with a second Wi-Fi module -!!! info +First, you should plug in your USB Wi-Fi dongle. It needs to have Linux driver support to work; information about compatible USB Wi-FI dongles which are reasonably compact can be found [here](https://github.com/morrownr/USB-WiFi/blob/main/home/USB_WiFi_Adapters_that_are_supported_with_Linux_in-kernel_drivers.md#single-band-usb-wifi-adapters-that-are-supported-with-linux-in-kernel-drivers); we have had good results with the [Panda PAU03 N150 adapter](https://www.amazon.com/Panda-Ultra-150Mbps-Wireless-Adapter/dp/B00762YNMG). If the Wi-Fi dongle works, then by default your PlanktoScope should create a Wi-Fi hotspot on it; you can check this by running the command `sudo nmcli conn` in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default) and checking whether the `wlan1-hotspot` connection is active (i.e. listed in bright green rather than white). If the Wi-Fi dongle works, then you can create a new Wi-Fi connection for it. - To be precise, the PlanktoScope will ping google.com to check if it can access the internet once it connects to an existing Wi-Fi network; if it still cannot reach google.com after ~10 failed attempts over a period of ~20-30 seconds, then the PlanktoScope will disconnect from the Wi-Fi network and instead make its own Wi-Fi hotspot. This is meant to help prevent PlanktoScopes from getting stuck when connecting to Wi-Fi networks whose firewall settings do not allow the PlanktoScope to access the network, as the PlanktoScopes may be entirely inaccessible on such networks. +To create a new Wi-Fi connection, run the command `sudo nmtui edit` and follow the dialogs to add a new Wi-Fi connection. In the "Edit Connection" window, you should specify either `wlan0` or `wlan1` as the device for that connection, depending on whether you want to connect to that Wi-Fi network with your Raspberry Pi's internal Wi-Fi module (`wlan0`) or with your USB Wi-Fi dongle (`wlan1`); then you should avoid creating any Wi-Fi connections for the other Wi-Fi module, so that it will still make a Wi-Fi hotspot. Then you can activate that new connection by rebooting or by running the command `sudo nmcli conn up "{connection name}"`, where `{connection name}` should be replaced with the connection profile name you had set when you created that connection. For example, if you created a Wi-Fi connection named `My Favorite Connection`, then you would run `sudo nmcli conn up "My Favorite Connection"`. It will take a while to finish, after which either the connection will have activated successfully or the command will have printed an error message. You can run `sudo nmtui conn` again to check what connections are active on your PlanktoScope. -Once you take the PlanktoScope out of range of the existing Wi-Fi network, within approximately two minutes it should automatically revert to making its own Wi-Fi hotspot network. If you then take the PlanktoScope back in range of the existing Wi-Fi network, within approximately two minutes it should automatically attempt to connect to the existing Wi-Fi network again (and, if it succeeds, it will stop the Wi-Fi hotspot network). +Once your PlanktoScope is connected to that external Wi-Fi network, you should try to access it through that network via your PlanktoScope's machine-specific mDNS URL, which has format `http://pkscope-{machine-name}.local`; in order to test whether this works, you will need to disconnect any direct connections your computer might have to your PlanktoScope (e.g. via a Wi-Fi hotspot on the other Wi-Fi module, or via a direct Ethernet connection). Note that your PlanktoScope might not be reachable over the external Wi-Fi network if your Wi-Fi network has restrictive firewall settings, in which case you will only be able to connect to your PlanktoScope directly via Ethernet cable. Note also that this URL only works if your device and web browser both support mDNS. -!!! warning +You may also want to check whether you can access the internet on the PlanktoScope. If the Wi-Fi network has a captive portal (e.g. for device registration or for a terms-of-service agreement), you should disconnect your computer from any Wi-Fi/Ethernet networks with internet access (in order to force it to use the PlanktoScope for internet access) and then try to open a webpage (e.g. ) in your computer's web browser to see if you can access the captive portal. If the Wi-Fi network doesn't have a captive portal, or if you successfully proceed through the captive portal, then you should be able to load web-pages using the internet access shared by the PlanktoScope. - Because you can only undo this configuration change by accessing the PlanktoScope's software (or by running a Linux computer which can open and edit the `/etc/wpa_supplicant/wpa_supplicant.conf` configuration file for Wi-Fi network connections in your PlanktoScope's SD card), we only recommend configuring your PlanktoScope to connect to an existing Wi-Fi network if 1) you also have a way to connect your device to the PlanktoScope via an Ethernet cable or if 2) you are also able to run your PlanktoScope in a location beyond the range of the existing Wi-Fi network (as that the PlanktoScope will revert to making a Wi-Fi hotspot when it cannot detect the existing Wi-Fi network) or if 3) you have a Linux computer which is able to edit files on your PlanktoScope's SD card. +As long as the configuration exists for this Wi-Fi connection, your PlanktoScope will then prefer to use the specified Wi-Fi module to connect to that Wi-Fi network if it's in range during startup; this will remain true even if you reboot your PlanktoScope. If the PlanktoScope loses its connection to the Wi-Fi network (e.g. because it is moved out of the range of that network), then it will automatically revert back to making its Wi-Fi hotspot using that Wi-Fi module, until the next reboot. If you want to revert back to always making a Wi-Fi hotspot from that Wi-Fi module, then you can run the command `sudo nmtui edit` and delete that Wi-Fi connection. -!!! warning +### Connect your PlanktoScope to an existing Wi-Fi network with a single Wi-Fi module - Before you make any changes, you should first write down your PlanktoScope's machine name (which is listed in your PlanktoScope's landing page), and you should also check whether your web browser allows you to use the machine-specific URL of format `http://pkscope-{machine-name}.local` for accessing your PlanktoScope's landing page. This is because other URLs from our [standard software setup guide](../setup/software/standard-install.md#connect-to-the-planktoscope) and our [basic operation guide](./index.md#access-your-planktoscopes-software) which you may be more familiar with, such as or or , will not work for accessing your PlanktoScope through the existing Wi-Fi network! Only a machine-specific URL of format `http://pkscope-{machine-name}.local` may work. +It's also possible to connect a PlanktoScope with a single Wi-Fi module to an existing Wi-Fi network with internet access (using the same instructions as given above, but specifying `wlan0` as the device for your Wi-Fi connection and skipping the steps related to plugging in a USB Wi-Fi dongle), but the PlanktoScope will be unable to make its Wi-Fi hotspot network while it is connected to an existing Wi-Fi network. Then, as long as the PlanktoScope boots up and stays within range of that Wi-Fi network, the PlanktoScope software is only accessible either if 1) the existing Wi-Fi network (including its firewall settings) is configured to allow you to access the PlanktoScope via mDNS and your computer and web browser both support mDNS URLs (i.e. URLs ending in `.local`, like ) or if 2) you connect your device to the PlanktoScope via an Ethernet cable. -For now, the only way we recommend for connecting your PlanktoScope to an existing Wi-Fi network is by editing the `/etc/wpa_supplicant/wpa_supplicant.conf` file; the PlanktoScope's Node-RED dashboard has a "Wifi" page which is partially broken (including writing an error message into the `/etc/wpa_supplicant/wpa_supplicant.conf` file if you attempt to add a Wi-Fi network without a password) and will be removed in a future release of the PlanktoScope's software. For example, you can edit the `wpa_supplicant.conf` file by opening the file editor at . After saving your changes to that file, you can apply them by either 1) using Cockpit's service manager to start the `autohotspot` service (e.g. at ), 2) waiting a few minutes for the `autohotspot` service to automatically run again, or 3) rebooting your PlanktoScope. +Once you take the PlanktoScope out of range of the existing Wi-Fi network, it should automatically revert to making its own Wi-Fi hotspot network. If you then take the PlanktoScope back in range of the existing Wi-Fi network, nothing will change until you either restart or manually switch back to the existing Wi-Fi network. -You can find more information on how to edit the `wpa_supplicant.conf` file in [this guide](https://fleetstack.io/blog/raspberry-pi-etc-wpa-supplicant-wpa-supplicant-conf-file). For quick reference, here are some network configuration snippets from that link: +!!! warning -``` -# For password-protected networks: -network={ - ssid="YourNetworkSSID" -  psk="YourNetworkPassword" -  key_mgmt=WPA-PSK -} + Because you can only undo this configuration change by accessing the PlanktoScope's software (or by running a Linux computer which can open and edit the configuration files in `/var/lib/overlays/overrides/etc/NetworkManager/system-connections/` for Wi-Fi network connections in your PlanktoScope's SD card), we only recommend configuring a PlanktoScope with a single Wi-Fi module to connect to an existing Wi-Fi network if 1) you also have a way to connect your device to the PlanktoScope via an Ethernet cable or if 2) you are also able to run your PlanktoScope in a location beyond the range of the existing Wi-Fi network (as that the PlanktoScope will revert to making a Wi-Fi hotspot when it cannot detect the existing Wi-Fi network) or if 3) you have a Linux computer which is able to edit files on your PlanktoScope's SD card. -# For passwordless networks: -network={ - ssid="YourNetworkSSID" - key_mgmt=NONE -} -``` +!!! warning + Before you make any changes, you should first write down your PlanktoScope's machine name (which is listed in your PlanktoScope's landing page), and you should also check whether your web browser allows you to use the machine-specific URL of format `http://pkscope-{machine-name}.local` for accessing your PlanktoScope's landing page. This is because other URLs from our [standard software setup guide](../setup/software/standard-install.md#connect-to-the-planktoscope) and our [basic operation guide](./index.md#access-your-planktoscopes-software) which you may be more familiar with, such as or , will not work for accessing your PlanktoScope through the existing Wi-Fi network! Only a URL like or or a machine-specific URL of format `http://pkscope-{machine-name}.local` has the possibility of working. +FIXME: update this for NetworkManager with `sudo nmtui`. Does Cockpit make it easy to do this? Once your PlanktoScope is connected to an existing Wi-Fi network, you should try to access it via your PlanktoScope's machine-specific mDNS URL, which has format `http://pkscope-{machine-name}.local`. Note that this may fail if your Wi-Fi network has restrictive firewall settings, in which case you will only be able to connect to your PlanktoScope directly via Ethernet cable. Note also that this URL only works if your device and web browser both support mDNS. ### Check your PlanktoScope's MAC addresses -If you need to check the MAC addresses of your PlanktoScope's network interfaces, you can open an auto-generated report of MAC addresses at `/run/mac-address.yml`, for example by opening the file viewer at . That file viewer is also linked to from your PlanktoScope's landing page, in a link named "MAC address viewer" in the "For advanced users" section of the landing page. +If you need to check the MAC addresses (also sometimes called "hardware addresses") of your PlanktoScope's network interfaces (e.g. in order to register those network interfaces with a network firewall), you can open an auto-generated report of MAC addresses at `/run/mac-address.yml`, for example by opening the file viewer at . That file viewer is also linked to from your PlanktoScope's landing page, in a link named "MAC address viewer" in the "For advanced users" section of the landing page. Note that this is a temporary file which only exists while the PlanktoScope is running - it is not stored on the PlanktoScope's SD card. Usually you will only care about the MAC addresses for `wlan0` or `eth0`, which are the built-in Wi-Fi module and Ethernet port of the PlanktoScope's embedded Raspberry Pi computer, respectively. If you connect additional network adapters (such as a USB Wi-Fi dongle, an Ethernet-to-USB adapter, or a phone in USB tethering mode), then you may also see additional network interfaces named something like `wlan1`, `eth1`, or `usb0` around two minutes later. @@ -83,23 +80,45 @@ If you can connect your PlanktoScope to the internet, then it will behave like a 3. Connect a phone/tablet by USB to your computer and enable your phone/tablet's USB tethering mode to share its internet access to your computer, and then connect your computer to the PlanktoScope's Wi-Fi hotspot or to an Ethernet cable plugged into the PlanktoScope. 4. Add one or two USB Wi-Fi dongles to your computer so that it has a total of two Wi-Fi adapters (one of which may be internal to your computer) and thus can connect to two Wi-Fi networks simultaneously; then configure your computer to connect to the internet by an existing Wi-Fi network using one of the Wi-Fi adapters and to simultaneously connect to the PlanktoScope's Wi-Fi hotspot using the other Wi-Fi adapter. 5. Add one or two USB Ethernet adapters to your computer so that it has a total of two Ethernet adapters (one of which may be internal to your computer) and thus can connect to two Ethernet ports simultaneously; then connect one of your computer's Ethernet adapters to an Ethernet port with internet access, and connect your computer's other Ethernet adapter to the PlanktoScope. -6. Connect both your computer and the PlanktoScope to an [Ethernet switch](https://en.wikipedia.org/wiki/Network_switch), then connect your computer to the internet by an existing Wi-Fi network. Note 1: an Ethernet switch is a different kind of device than an Ethernet router. Note 2: if you then connect your Ethernet switch to an Ethernet or Wi-Fi router, then the the PlanktoScope will only be accessible if the router's firewall allows network connections to the PlanktoScope from other devices on the router's Local Area Network (LAN). +6. Connect both your computer and the PlanktoScope to an [Ethernet switch](https://en.wikipedia.org/wiki/Network_switch), then connect your computer to the internet by an existing Wi-Fi network. Note 1: an Ethernet switch is a different kind of device than an Ethernet router. Note 2: if you then connect your Ethernet switch to an Ethernet or Wi-Fi router, then the PlanktoScope will only be accessible if the router's firewall allows network connections to the PlanktoScope from other devices on the router's Local Area Network (LAN). ### Don't allow the PlanktoScope to be used as a default gateway to the internet Because by default your PlanktoScope is configured to act as a router so that it can share its internet access with all connected devices, it will advertise itself as a default gateway to the internet, so that connected devices will use the PlanktoScope as a network router for internet access. However, some computers (e.g. those running macOS or Windows) may behave as if the PlanktoScope is the *only* router with internet access even if the PlanktoScope actually has no internet access while the computer is also connected to a router for another network which *does* have internet access; then such computers will fail to access the internet while they're connected to the PlanktoScope. -You can work around this unfortunate behavior of your computer's operating system by disabling a setting in the PlanktoScope's networking configuration so that the PlanktoScope no longer advertises itself as a router with internet access; note that doing so will prevent the PlanktoScope from being able to share its own internet access with connected devices as long as this setting is disabled. To disable this setting, run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default) and then restart the PlanktoScope: +You can work around this unfortunate behavior of your computer's operating system by disabling one or two settings in the PlanktoScope's networking configuration so that the PlanktoScope no longer advertises itself as a router with internet access; note that doing so will prevent the PlanktoScope from being able to share its own internet access with connected devices as long as these settings are disabled. + +There are two settings, one controlling the behavior for Ethernet connections to the PlanktoScope and one controlling the behavior for the PlanktoScope's Wi-Fi hotspot. You can change just one setting, or you can change both settings together (in which case you can run both of the `forklift pallet disable-deployment-feature`/`forklift pallet enable-deployment-feature` commands consecutively and then just run the `forklift pallet stage --no-cache-img` command afterwards). + +#### For Ethernet connections + +To disable this setting for Ethernet connections to the PlanktoScope, run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default) and then restart the PlanktoScope: ``` -forklift pallet disable-deployment-feature host/networking/interface-forwarding planktoscope-dhcp-default-route +forklift pallet disable-deployment-feature host/networking/networkmanager-dnsmasq dhcp-default-route forklift pallet stage --no-cache-img ``` To revert this setting back to the default behavior (which is for the PlanktoScope to advertise itself as a router with internet access, so that the PlanktoScope can share its internet access with all connected devices), run the following commands in the Cockpit Terminal and then restart the PlanktoScope: ``` -forklift pallet enable-deployment-feature host/networking/interface-forwarding planktoscope-dhcp-default-route +forklift pallet enable-deployment-feature host/networking/networkmanager-dnsmasq dhcp-default-route +forklift pallet stage --no-cache-img +``` + +#### For the Wi-Fi hotspot + +To disable this setting for the PlanktoScope's Wi-Fi hotspots, run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default) and then restart the PlanktoScope: + +``` +forklift pallet disable-deployment-feature host/networking/networkmanager-hotspot dhcp-default-route +forklift pallet stage --no-cache-img +``` + +To revert this setting back to the default behavior (which is for the PlanktoScope to advertise itself as a router with internet access, so that the PlanktoScope can share its internet access with all connected devices), run the following commands in the Cockpit Terminal and then restart the PlanktoScope: + +``` +forklift pallet enable-deployment-feature host/networking/interface-forwarding dhcp-default-route forklift pallet stage --no-cache-img ``` @@ -107,58 +126,229 @@ forklift pallet stage --no-cache-img This section provides instructions on various things you may want to do to improve the networking-related security of your PlanktoScope. -### Change the Wi-Fi hotspot's password - -To change the password of the PlanktoScope's Wi-Fi hotspot away from the default of `copepode`, edit the file at `/etc/hostapd/hostapd.conf.d/30-auth-30-planktoscope-passphrase.conf`. For example, you can do this by opening the file editor at . To apply your changes, restart the PlanktoScope. +### Change the `pi` user's password -To revert your changes back to the default password, we recommend deleting the file at `/var/lib/overlays/overrides/etc/hostapd/hostapd.conf.d/30-auth-30-planktoscope-passphrase.conf`. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. +You can log in to the Cockpit system administration dashboard as the `pi` user with that user's password (which is `copepode` by default), and you can also log in to the PlanktoScope via SSH as the `pi` user with that same password. You may want to change the `pi` user to have some other password, in order to limit who can log in as the `pi` user. You can do this in the Cockpit account management interface at . ### Disable the Wi-Fi hotspot +Your PlanktoScope can make two separate Wi-Fi hotspots: one from the internal Wi-Fi module of your PlanktoScope's Raspberry Pi computer, and another from a USB Wi-Fi dongle plugged into the Raspberry Pi computer (but only when such a dongle exists). These hotspots can be enabled or disabled separately or together. + +Note that disabling the hotspots does not affect whether the PlanktoScope is able to connect to external Wi-Fi networks: if the PlanktoScope is already configured to connect to such networks, then it will continue attempting to connect to them. + +#### Disable both hotspots together + +The simplest way to disable both hotspots together is to run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default) and then restart the PlanktoScope: + +``` +forklift pallet disable-deployment host/networking/networkmanager-hotspot +forklift pallet stage --no-cache-img +``` + +!!! warning + + Because you can only undo this configuration change by accessing the PlanktoScope's software, we only recommend configuring your PlanktoScope to disable its Wi-Fi hotspots if 1) you also have a way to connect your device to the PlanktoScope via an Ethernet cable or 2) you can connect to the PlanktoScope when it's connected to an existing Wi-Fi network. + +To undo this change (and allow the hotspots to be enabled or disabled separately), run the following commands in the Cockpit Terminal and then restart the PlanktoScope: + +``` +forklift pallet enable-deployment host/networking/networkmanager-hotspot +forklift pallet stage --no-cache-img +``` + +#### Disable the internal Wi-Fi module's hotspot + To disable the PlanktoScope's Wi-Fi hotspot, run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default) and then restart the PlanktoScope: ``` -forklift pallet disable-deployment host/networking/autohotspot +forklift pallet disable-deployment-feature host/networking/networkmanager-hotspot wlan0 forklift pallet stage --no-cache-img ``` -To revert your changes back to the default behavior (which is for the PlanktoScope to make its own Wi-Fi hotspot when it doesn't detect any known existing Wi-Fi networks to connect to), run the following commands in the Cockpit Terminal and then restart the PlanktoScope: +To revert your changes back to the default behavior (which is for the PlanktoScope to make its own Wi-Fi hotspot when it doesn't detect any known existing Wi-Fi networks to connect to with the internal Wi-Fi module), run the following commands in the Cockpit Terminal and then restart the PlanktoScope: ``` -forklift pallet enable-deployment host/networking/autohotspot +forklift pallet enable-deployment-feature host/networking/networkmanager-hotspot wlan0 forklift pallet stage --no-cache-img ``` !!! warning - Because you can only undo this configuration change by accessing the PlanktoScope's software, we only recommend configuring your PlanktoScope to disable its Wi-Fi hotspot if 1) you also have a way to connect your device to the PlanktoScope via an Ethernet cable or 2) you can connect to the PlanktoScope when it's connected to an existing Wi-Fi network. + Because you can only undo this configuration change by accessing the PlanktoScope's software, we only recommend configuring your PlanktoScope to disable its internal Wi-Fi module's hotspot if 1) you also have a way to connect your device to the PlanktoScope via an Ethernet cable or 2) you can connect to the PlanktoScope when it's connected to an existing Wi-Fi network or 3) you also have a USB Wi-Fi dongle which can create a Wi-Fi hotspot. + +#### Disable the USB Wi-Fi dongle's hotspot + +The instructions are the same as for the internal Wi-Fi module's hotspot, except you should replace `wlan0` with `wlan1` in the commands listed above. + +### Change the Wi-Fi hotspot's password + +Your PlanktoScope has two files specifying the passwords of the Wi-Fi hotspot(s) it can generate: one file for the hotspot which is generated by the internal Wi-Fi module of your PlanktoScope's Raspberry Pi computer, and a different file for the hotspot which is generated by a USB Wi-Fi dongle plugged into the Raspberry Pi computer when such a dongle exists. You can change the names of both of these hotspots, or you can change the name of just one of them. If you don't have a USB Wi-Fi dongle, you can safely ignore the instructions for it. + +#### Change the password of the internal Wi-Fi module's hotspot + +To change the password of the PlanktoScope's Wi-Fi hotspot away from the default of `copepode`, edit the file at `/etc/NetworkManager/system-connections.d/wlan0-hotspot/51-wifi-security-password.nmconnection`. For example, you can do this by opening the file editor at . To apply your changes, restart the PlanktoScope. + +To revert your changes back to the default password, we recommend deleting the file at `/var/lib/overlays/overrides/etc/NetworkManager/system-connections.d/wlan0-hotspot/51-wifi-security-password.nmconnection`. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. + +#### Change the password of the USB Wi-Fi dongle's hotspot + +The instructions are the same as for the internal Wi-Fi module's hotspot, except you should replace `wlan0` with `wlan1` in all file paths and URLs mentioned above. + +### Make the firewall more restrictive + +The PlanktoScope has an internal firewall for network connections. By default, the PlanktoScope's firewall is configured to allow access to various specific ports: + +1. for devices connected directly to the PlanktoScope (which are in the `nm-shared` firewall zone) by Ethernet cable or through the PlanktoScope's Wi-Fi hotspot + +2. for devices on the same Local Area Network (LAN) as the PlanktoScope (which are all in the `public` firewall zone), if the PlanktoScope is connected to an Ethernet/Wi-Fi router + +In both of these firewall zones, the firewall blocks access to any ports which are not explicitly allowed by the PlanktoScope's configuration. For usability/recoverability/operability reasons, you will probably want to maintain the default level of access to all of the PlanktoScope's software from directly-connected devices (i.e. in the `nm-shared` zone); but for security reasons, you may want to further restrict access for devices from an untrusted LAN (i.e. in the `public` zone). + +You can view the currently-active firewall rules in Cockpit's firewall configuration panel at ; however, we do not recommend changing rules in that panel if you are able to instead follow the instructions provided below. This is because making changes with Cockpit will override all future changes which would otherwise be made with Forklift (e.g. in future software updates). If you would like to relax firewall restrictions (for example by opening up additional ports), you should instead add drop-in configuration files (as XML fragments) in `/etc/firewalld/zones.d/nm-shared/` and `/etc/firewalld/zones.d/public/` and then reboot to apply your changes; in the future, we create another operation guide with more detailed instructions for how to do this. + +#### Restrict access in the `public` zone + +It is easy to restrict access in the `public` firewall zone for the following ports/protocols: + +- SSH (port 22), whose access is provided by a Forklift package deployment named `host/sshd`: restricting access to this will prevent access to the PlanktoScope over SSH from devices in the `public` zone. +- Direct-access fallback for the Cockpit system administration dashboard (port 9090), whose access is provided by a Forklift package deployment named `host/cockpit`: restricting direct access to this will prevent access to the PlanktoScope's Cockpit dashboard from devices in the `public` zone. Note that by default an HTTP reverse proxy also provides indirect access to Cockpit through port 80. +- MQTT (port 1883), whose access is provided by a Forklift package deployment named `infra/mosquitto`: restricting access to this will prevent access to the PlanktoScope's [MQTT API](../reference/software/interfaces/mqtt.md) from devices in the `public` zone. +- HTTP (port 80), whose access is provided by a Forklift package deployment named `infra/caddy-ingress`: restricting access to this will prevent access to the PlanktoScope's [web browser user interfaces](../reference/software/architecture/os.md#user-interface) and HTTP APIs from devices in the `public` zone. +- mDNS (port 5353), whose access is provided by a Forklift package deployment named `host/networking/avahi-daemon`: restricting access to this will prevent devices in the `public` zone from discovering the PlanktoScope's IP address(es) by querying the PlanktoScope's mDNS hostnames (i.e. hostnames ending in `.local`, such as `planktoscope.local`). +- ICMP, whose access is provided by a Forklift package deployment named `host/networking/networkmanager`: restricting access this will prevent devices in the `public` zone from pinging the PlanktoScope to check whether it exists. + +To restrict access for the `public` zone for one or more of these ports/protocols, run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default), replacing the instances of `{package deployment}` with the names of the various package deployments providing access to ports/protocols which you want to restrict: + +``` +forklift pallet disable-deployment-feature {package deployment} firewall-allow-public +forklift pallet disable-deployment-feature {package deployment} firewall-allow-public +forklift pallet disable-deployment-feature {package deployment} firewall-allow-public +forklift pallet stage --no-cache-img +``` + +For example, to restrict ports 22 and 9090, you would run the following commands: + +``` +forklift pallet disable-deployment-feature host/sshd firewall-allow-public +forklift pallet disable-deployment-feature host/cockpit firewall-allow-public +forklift pallet stage --no-cache-img +``` + +After running these commands, you can apply your changes by rebooting. + +!!! warning + + It is easy to restrict access so much that you will become unable to access the PlanktoScope over its network interfaces through a router connected to your PlanktoScope. You should make sure you have some way to directly connect to the PlanktoScope, e.g. by a direct Ethernet cable connection or with a keyboard and display. + +To undo your changes, run the following commands, replacing the instances of `{package deployment}` with the names of the various package deployments which you want to restore access to: + +``` +forklift pallet enable-deployment-feature {package deployment} firewall-allow-public +forklift pallet enable-deployment-feature {package deployment} firewall-allow-public +forklift pallet enable-deployment-feature {package deployment} firewall-allow-public +forklift pallet stage --no-cache-img +``` + +For example, to restore access to ports 22 and 9090, you would run the following commands: + +``` +forklift pallet enable-deployment-feature host/sshd firewall-allow-public +forklift pallet enable-deployment-feature host/cockpit firewall-allow-public +forklift pallet stage --no-cache-img +``` + +After running these commands, you should apply the resulting changes by rebooting. + +#### Restrict access in the `nm-shared` zone + +If you really want to limit access in the `nm-shared` zone, you can follow the above instructions given for the `public` zone but replace `firewall-allow-public` with `firewall-allow-nm-shared`. + +!!! warning + + It is easy to restrict access so much that you will become unable to access the PlanktoScope over its network interfaces from devices with direct network connections to the PlanktoScope. If that happens, then the only way to access your PlanktoScope (e.g. to restore its previous settings) will be by connecting a display and keyboard to your PlanktoScope. + +### Disable all access to privileged web browser apps + +The following web browser apps provide full administrative (i.e. root or superuser) access to the system on port 80 via the HTTP reverse-proxy server: + +- Cockpit: a system administration dashboard which includes a terminal for running arbitrary commands (including privileged commands with `sudo`); access to this is provided by a feature flag named `frontend` in a Forklift package deployment named `apps/cockpit`. +- System file manager: provides privileged access for viewing and editing arbitrary files across the entire filesystem; access to this is provided by a feature flag named `frontend` in a Forklift package deployment named `apps/filebrowser-root`. +- Node-RED dashboard editor: provides a mechanism for running arbitrary privileged commands; access to this is provided by a feature flag named `editor` in a Forklift package deployment named `apps/ps/node-red-dashboard`. + +To disable access to one or more of these apps, run the following commands in the Cockpit Terminal at (which you should log in to with the username `pi` and the `pi` user's password, which is `copepode` by default), replacing the instances of `{package deployment}` with the names of the various package deployments providing access which you want to restrict, and replacing the instances of `{feature flag}` with the names of the feature flags which specifically provide the access you want to restrict: + +``` +forklift pallet disable-deployment-feature {package deployment} {feature flag} +forklift pallet disable-deployment-feature {package deployment} {feature flag} +forklift pallet disable-deployment-feature {package deployment} {feature flag} +forklift pallet stage --no-cache-img +``` + +For example, to restrict access to the system file manager and the Node-RED dashboard editor, you would run the following commands: + +``` +forklift pallet disable-deployment-feature apps/filebrowser-root frontend +forklift pallet disable-deployment-feature apps/ps/node-red-dashboard editor +forklift pallet stage --no-cache-img +``` + +After running these commands, you can apply your changes by rebooting. Note that the landing page will still link to those apps, but the links won't work anymore. + +To undo your changes, run the following commands, replacing the instances of `{package deployment}` and `{feature flag}` with the names of the various package deployments and their respective feature flags which you want to re-enable to restore access: + +``` +forklift pallet enable-deployment-feature {package deployment} {feature flag} +forklift pallet enable-deployment-feature {package deployment} {feature flag} +forklift pallet enable-deployment-feature {package deployment} {feature flag} +forklift pallet stage --no-cache-img +``` + +For example, to restore access to the system file manager and the Node-RED dashboard editor, you would run the following commands: + +``` +forklift pallet enable-deployment-feature apps/filebrowser-root frontend +forklift pallet enable-deployment-feature apps/ps/node-red-dashboard editor +forklift pallet stage --no-cache-img +``` + +After running these commands, you should apply the resulting changes by rebooting. ## Allow access to Cockpit from additional domain names or IP addresses For security reasons, [Cockpit](http://planktoscope.local/admin/cockpit/) only allows you to access Cockpit from a list of known domain names and ports (e.g. [planktoscope.local](http://planktoscope.local/admin/cockpit/) and [planktoscope.local:9090](http://planktoscope.local:9090/admin/cockpit/)) and known IP addresses and ports (e.g. [192.168.4.1](http://192.168.4.1/admin/cockpit/) and [192.168.4.1:9090](http://192.168.4.1:9090/admin/cockpit/)). If you have connected the PlanktoScope to a network or to the internet so that you can reach the PlanktoScope from some other domain name or IP address, and you also want to access Cockpit from that other domain name or IP address, then you will need to add that domain name or IP address into the list of Cockpit's known domain names and ports. -To add additional known domain names or IP addresses where Cockpit should be accessible, create and edit a file at `/etc/cockpit/origins.d/`, following the instructions/notes at `/etc/cockpit/origins.d/10-README` and referring to `/etc/cockpit/origins.d/20-localhost` as a reference example. For example, you can do this in the file browser at , and you can view the instructions/notes at , and you can view the reference example at . To apply your changes, restart the PlanktoScope. +To add additional known domain names or IP addresses where Cockpit should be accessible, create and edit a file at `/etc/cockpit/origins.d/`, following the instructions/notes at `/etc/cockpit/origins.d/10-README` and referring to `/etc/cockpit/origins.d/20-localhost` as a reference example. For example, you can do this in the file browser at , and you can view the instructions/notes at , and you can view the reference example at . To apply your changes, restart the PlanktoScope. To revert your changes back to the default list of known domain names and IP addresses where Cockpit can be accessed, we recommend deleting the files in `/var/lib/overlays/overrides/etc/cockpit/origins.d/` and then restarting the PlanktoScope. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. ## Change your PlanktoScope's name -Your PlanktoScope has a semi-unique machine name which by default is stably and automatically generated from your Raspberry Pi's semi-unique serial number. This machine name has format `{adjective}-{noun}-{number up to five digits long}`, e.g. `metal-slope-23501` or `safe-minute-6738`. The machine name is used to generate the hostname of the PlanktoScope with the format `pkscope-{machine-name}`; and the PlanktoScope's hostname is also used to generate the name (to be precise, the SSID) of its Wi-Fi hotspot as well as the machine-specific mDNS URLs you can use to access the PlanktoScope (e.g. [metal-slope-23501.local](http://metal-slope-23501.local) or [safe-minute-6738.local](http://safe-minute-6738.local)). +Your PlanktoScope has a semi-unique machine name which by default is stably and automatically generated from your Raspberry Pi's semi-unique serial number. This machine name has format `{adjective}-{noun}-{number up to five digits long}`, e.g. `metal-slope-23501` or `safe-minute-6738`. The machine name is used to generate the hostname of the PlanktoScope with the format `pkscope-{machine-name}`; and the PlanktoScope's hostname is also used to generate the name (to be precise, the SSID) of its Wi-Fi hotspot as well as the machine-specific mDNS URLs you can use to access the PlanktoScope (e.g. [pkscope-metal-slope-23501.local](http://pkscope-metal-slope-23501.local) or [pkscope-safe-minute-6738.local](http://pkscope-safe-minute-6738.local)). + +If you're unhappy with these names, you can change any or all of them. ### Change the machine name -To change your PlanktoScope's machine name, create and edit a file at `/etc/machine-name`. For example, you can do this in the file browser at . To apply your changes, restart the PlanktoScope. +To change your PlanktoScope's machine name, create and edit a file at `/etc/machine-name`; the contents of that file should be the name you want to assign to the PlanktoScope. For example, you can do this in the file browser at . To apply your changes, restart the PlanktoScope. Note: this change will also affect your PlanktoScope's hostname and Wi-Fi hotspot name, unless you've also customized them separately (as described below). To revert your changes back to the default auto-generated machine name based on the Raspberry Pi's serial number, delete the file at `/var/lib/overlays/overrides/etc/machine-name` and then restart the PlanktoScope. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. ### Change the hostname -To change your PlanktoScope's hostname away from the format `{adjective}-{noun}-{number up to five digits long}` without also changing the machine name, edit the file at `/etc/hostname-template`. For example, you can do this by opening the file editor at . To apply your changes, restart the PlanktoScope. +To change your PlanktoScope's hostname away from the format `{adjective}-{noun}-{number up to five digits long}` without also changing the machine name, edit the file at `/etc/hostname-template`. For example, you can do this by opening the file editor at . To apply your changes, restart the PlanktoScope. Note: this change will also affect your PlanktoScope's Wi-Fi hotspot name, unless you've also customized it separately (as described below). To revert your changes back to the default hostname based on the machine name, we recommend deleting the file at `/var/lib/overlays/overrides/etc/hostname-template` and then restarting the PlanktoScope. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. ### Change the Wi-Fi hotspot name -To change your PlanktoScope's Wi-Fi hotspot name away from the format `{the first 32 characters of the hostname}` without also changing the hostname, edit the file at `/etc/hostapd/hostapd.conf-templates.d/20-ssid.conf`, following the instructions/notes at `/etc/hostapd/hostapd.conf-templates.d/10-README.conf`. For example, you can do this by opening the file editor at , and you can view the instructions/notes at . To apply your changes, restart the PlanktoScope. +Your PlanktoScope has two files specifying the names of the Wi-Fi hotspot(s) it can generate: one file for the hotspot which is generated by the internal Wi-Fi module of your PlanktoScope's Raspberry Pi computer, and a different file for the hotspot which is generated by a USB Wi-Fi dongle plugged into the Raspberry Pi computer when such a dongle exists. You can change the names of both of these hotspots, or you can change the name of just one of them. If you don't have a USB Wi-Fi dongle, you can safely ignore the instructions for it. + +#### Change the name of the internal Wi-Fi module's hotspot + +To change your PlanktoScope's Wi-Fi hotspot name away from the format `{the first 32 characters of the hostname}` without also changing the hostname, edit the file at `/etc/NetworkManager/system-connections-templates.d/wlan0-hotspot/20-ssid.nmconnection`, following the instructions/notes at `/etc/NetworkManager/system-connections-templates.d/wlan0-hotspot/10-README.nmconnection`. For example, you can do this by opening the file editor at , and you can view the instructions/notes at . To apply your changes, restart the PlanktoScope. + +To revert your changes back to the default Wi-Fi hotspot name based on the hostname, we recommend deleting the file at `/var/lib/overlays/overrides/etc/NetworkManager/system-connections-templates.d/wlan0-hotspot/20-ssid.nmconnection` and then restarting the PlanktoScope. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. + +#### Change the name of the USB Wi-Fi dongle's hotspot -To revert your changes back to the default Wi-Fi hotspot name based on the hostname, we recommend deleting the file at `/var/lib/overlays/overrides/etc/hostapd/hostapd.conf-templates.d/20-ssid.conf` and then restarting the PlanktoScope. For example, you can do this in the file browser at . Then restart the PlanktoScope immediately. +The instructions are the same as for the internal Wi-Fi module's hotspot, except you should replace `wlan0` with `wlan1` in all file paths and URLs mentioned above. diff --git a/documentation/docs/operation/software-upgrades.md b/documentation/docs/operation/software-upgrades.md index d4d65498c..f2aff7867 100644 --- a/documentation/docs/operation/software-upgrades.md +++ b/documentation/docs/operation/software-upgrades.md @@ -79,7 +79,7 @@ Eventually (i.e. if/when it becomes feasible and safe), we may make it possible If you don't know what `apt` or `apt-get` refer to, then please skip this section and just remember to avoid running `apt` or `apt-get` commands on your PlanktoScope! -Most of the "interesting" software in the PlanktoScope OS (with Cockpit being a notable exception) is not managed using Raspberry Pi OS's APT package-management system, [for various reasons](../reference/software/architecture/os.md#system-upgrades). It's *probably* safe to run APT commands to upgrade most packages installed in the PlanktoScope OS (at least for software which doesn't run during [early boot](../reference/software/architecture/os.md#boot-sequence), because of when the PlanktoScope OS's [filesystem overlay](../reference/software/architecture/os.md#filesystem) for `/usr` is initialized), but we cannot make any guarantees or provide any support if you choose to do that. This is an issue of practicality: APT does not make it easy for us to exactly reproduce the changes to installed versions of packages caused by running `apt`/`apt-get`'s install/upgrade commands, when those commands are run at very different times; so it is not necessarily feasible for us to troubleshoot any resulting problems. If you want to undo the changes caused by running any APT commands, you should try to delete everything in `/var/lib/overlays/overrides/usr` and reboot immediately afterwards. +Most of the "interesting" software in the PlanktoScope OS (with Cockpit being a notable exception) is not managed using Raspberry Pi OS's APT package-management system, [for various reasons](../reference/software/architecture/os.md#system-upgrades). It's safe to run APT commands to install new packages in the PlanktoScope OS, at least for software which doesn't run during [early boot](../reference/software/architecture/os.md#boot-sequence), because of when the PlanktoScope OS's [filesystem overlay](../reference/software/architecture/os.md#filesystem) for `/usr` is initialized). It's *probably* safe to run APT commands to upgrade most packages installed in the PlanktoScope OS (at least for software which doesn't run during early boot), but we cannot make any guarantees or provide any support if you choose to do that. This is an issue of practicality: APT does not make it easy for us to exactly reproduce the changes to installed versions of packages caused by running `apt`/`apt-get`'s install/upgrade commands, when those commands are run at very different times; so it is not necessarily feasible for us to troubleshoot any resulting problems. If you want to undo the changes caused by running any APT commands, you should try to delete everything in `/var/lib/overlays/overrides/usr` (and maybe also `/var/lib/overlays/overrides/etc`) and reboot immediately afterwards. ## Restore your data & settings diff --git a/documentation/docs/reference/software/architecture/os.md b/documentation/docs/reference/software/architecture/os.md index 42e7c5e8c..2e3ad7009 100644 --- a/documentation/docs/reference/software/architecture/os.md +++ b/documentation/docs/reference/software/architecture/os.md @@ -70,7 +70,7 @@ Traditional Linux distros such as the Raspberry Pi OS are designed to run softwa - Making certain customizations to the OS, such as adding additional programs/libraries or modifying system configuration files, increases the risk of *configuration drift* in which the system's actual state increasingly diverges over time from the state expected by the PlanktoScope's software maintainers, and thus becomes harder to understand, troubleshoot, or replace. User customizations to the OS cannot be easily separated from the default configuration of the OS, so it is complicated to copy only those customizations in order to drop them onto a fresh installation of the OS from a newer release - especially if the updated OS includes changes to default configurations which conflict with the user customizations. -- Some Python packages required by PlanktoScope-specific programs (namely the PlanktoScope hardware controller and the PlanktoScope segmenter, which are both described in later sections of this document), such as [picamera2](https://github.com/raspberrypi/picamera2) and [opencv-python-headless](https://github.com/opencv/opencv-python), can only be installed as [pre-built wheels](https://pythonwheels.com/) from [piwheels](https://www.piwheels.org/) (which is used instead of PyPi because the PlanktoScope OS is not yet able to run as a 64-bit operating system) when certain versions of system libraries are installed, or else they must be re-compiled from source (which is prohitively slow on the Raspberry Pi for the affected Python packages). This makes dependencies more complicated to manage in maintenance of the PlanktoScope OS for creating and releasing new SD card images with updated software. The reliance on system libraries also increases the risk that a user-initiated upgrade or removal of some of the system's installed APT packages could cause breakage of some `pip`-managed Python packages which had been installed before the change. +- Some Python packages required by PlanktoScope-specific programs (namely the PlanktoScope hardware controller and the PlanktoScope segmenter, which are both described in later sections of this document), such as [picamera2](https://github.com/raspberrypi/picamera2) and [opencv-python-headless](https://github.com/opencv/opencv-python), can only be installed as [pre-built wheels](https://pythonwheels.com/) from PyPI when certain versions of system libraries are installed, or else they must be re-compiled from source (which is prohitively slow on the Raspberry Pi for the affected Python packages). This makes dependencies more complicated to manage in maintenance of the PlanktoScope OS for creating and releasing new SD card images with updated software. The reliance on system libraries also increases the risk that a user-initiated upgrade or removal of some of the system's installed APT packages could cause breakage of some `pip`-managed Python packages which had been installed before the change. All of the factors listed above increase the perceived risk (and/or the required effort for sufficient mitigation of that risk) of accidentally degrading system integrity by keeping all software on the OS up-to-date, which makes it harder for users to receive bugfixes, security patches, and new features in a timely manner. Indeed, outside of systems like phones and Chromebooks (whose operating systems [automatically update themselves](https://chromium.googlesource.com/aosp/platform/system/update_engine/+/HEAD/README.md)), it is common for users of operating systems to avoid installing security updates or OS upgrades out of a fear of breaking their installed software; this is especially common for users who rely on software to operate other scientific instruments, and for good reasons! But the PlanktoScope project currently does not have enough resources to be able to support users stuck on old versions of the PlanktoScope OS; instead, we want to make it easy and safe for all users to always keep their PlanktoScopes - even with customizations to the OS - automatically updated to the latest version of the PlanktoScope OS. We intend to achieve this by: @@ -86,7 +86,7 @@ We have implemented most of the systems necessary for these goals. Much of the P - PlanktoScope-specific systemd services. - PlanktoScope-specific OS configuration files. -Everything managed by `forklift` is version-controlled in a [Git](https://git-scm.com/) repository with a special file structure (so that the repository is called a *pallet*), enabling easy backup and restoration of `forklift`-managed configurations even if the PlanktoScope's SD card is wiped and re-flashed. Forklift is designed so that a pallet is effectively a version-controlled, configurable, declarative [bill of materials](https://en.wikipedia.org/wiki/Bill_of_materials) for software/configuration modules which are composed together by Forklift into a significant layer of the PlanktoScope OS. Performing an OS upgrade/downgrade with Forklift is just a matter of running a `forklift` command to switch to a different version of a pallet, as described [in our OS upgrade operations guide](../../../operation/software-upgrades.md#perform-an-in-place-upgradedowngrade). +Everything managed by `forklift` is version-controlled in a [Git](https://git-scm.com/) repository with a special file structure (so that the repository is called a *pallet*), enabling easy backup and restoration of `forklift`-managed configurations even if the PlanktoScope's SD card is wiped and re-flashed. Forklift is designed so that a pallet is effectively a version-controlled, configurable, declarative [bill of materials](https://en.wikipedia.org/wiki/Bill_of_materials) for software/configuration modules which are composed together by Forklift into a significant layer of the PlanktoScope OS; many kinds of customizations can be performed through a feature-flagging interface which is uniform regardless of the underlying formats of OS configurations involved in those customizations (e.g. as illustrated in our networking operations guides for [adjusting firewalld zone rules](../../../operation/networking.md#make-the-firewall-more-restrictive) and for [disabling one or both Wi-Fi hotspots](../../../operation/networking.md#disable-the-wi-fi-hotspot) and for [disabling access to one or more browser apps](../../../operation/networking.md#disable-all-access-to-privileged-web-browser-apps)). Performing an OS upgrade/downgrade with Forklift is just a matter of running a `forklift` command to switch to a different version of a pallet and then rebooting, as described [in our OS upgrade operations guide](../../../operation/software-upgrades.md#perform-an-in-place-upgradedowngrade). !!! info @@ -94,11 +94,11 @@ Everything managed by `forklift` is version-controlled in a [Git](https://git-sc In an ideal world, we would not need to use/maintain Forklift in the PlanktoScope OS for achieving the goals which originally motivated the creation of Forklift...or at least Forklift could outsource so much functionality to externally-maintained systems that Forklift could be reduced to a UI wrapper. Or maybe the PlanktoScope OS's goals will later be reduced to the point that Forklift will no longer be very useful for the PlanktoScope OS. -### Package management with `forklift` +### System customization & configuration -When you're just experimenting and you can tolerate the challenges mentioned above, it's fine to customize the PlanktoScope OS by installing software packages using `pip` directly on the OS and/or by making extensive changes to OS configuration files. However, once you actually care about keeping your customizations around - and especially if/when you want to share your customizations with other people - we recommend migrating those customizations into Forklift packages, which are just files and configuration files stored in a specially-structured Git repository which is also published online (e.g. on GitHub, GitLab, Gitea, etc.). `forklift` provides an easy way to [package, publish](https://github.com/PlanktoScope/forklift/blob/main/docs/design.md#app-packaging-and-distribution), [combine, and apply](https://github.com/PlanktoScope/forklift/blob/main/docs/design.md#app-deployment-configuration) customizations via YAML configuration files in Git repositories; this enables easy sharing, configuration, (re-)composition, and downloading of Docker Compose applications, systemd services, and OS configuration files. Configurations of all deployments of Forklift packages on a computer running the PlanktoScope OS are specified and integrated in a single Git repository, a *Forklift pallet*. At any given time, each PlanktoScope has exactly one Forklift pallet applied; switching between Forklift pallets (whether to try out a different set of customizations or to upgrade/downgrade all programs and OS configurations managed by Forklift) is easy and can be done by running just one command (`forklift pallet switch`, described below in the [Applying published customizations](#applying-published-customizations) subsection). +When you're just experimenting and you can tolerate the challenges mentioned above, it's fine to customize the PlanktoScope OS by installing software packages using `pip` directly on the OS and/or by making extensive changes directly to OS configuration files in `/etc`. However, once you actually care about keeping your customizations around - and especially if/when you want to share your customizations with other people - we recommend migrating those customizations into Forklift packages, which are just files and configuration files stored in a specially-structured Git repository which is also published online (e.g. on GitHub, GitLab, Gitea, etc.). `forklift` provides an easy way to [package, publish](https://github.com/PlanktoScope/forklift/blob/main/docs/design.md#app-packaging-and-distribution), [combine, and apply](https://github.com/PlanktoScope/forklift/blob/main/docs/design.md#app-deployment-configuration) customizations via YAML configuration files in Git repositories; this enables easy sharing, configuration, (re-)composition, and downloading of Docker Compose applications, systemd services, and OS configuration files. Configurations of all deployments of Forklift packages on a computer running the PlanktoScope OS are specified and integrated in a single Git repository, a *Forklift pallet*. At any given time, each PlanktoScope has exactly one Forklift pallet applied; switching between Forklift pallets (whether to try out a different set of customizations or to upgrade/downgrade all programs and OS configurations managed by Forklift) is easy and can be done by running just one command (`forklift pallet switch`, described below in the [Applying published customizations](#applying-published-customizations) subsection). -`forklift` is used very differently compared to traditional Linux system package managers like APT, for which you must run step-by-step commands in order to modify the state of your system (e.g. to install some package or install some other package). When using `forklift`, you instead edit configuration files which declare the desired state of your system (or you can instead run some commands provided by `forklift` for common operations, such as in [this example](../../../operation/networking.md#dont-allow-the-planktoscope-to-be-used-as-a-default-gateway-to-the-internet)), and then you ask `forklift` to prepare to make your system match the desired state on its next boot. +`forklift` organizes apps and configurations into modules called *packages*, but `forklift` is used very differently compared to traditional Linux system package managers like APT, for which you must run step-by-step commands in order to modify the state of your system (e.g. to install some package or install some other package). When using `forklift`, you instead edit configuration files which declare the desired state of your system (or you can instead run some commands provided by `forklift` for common operations, such as in [this example](../../../operation/networking.md#dont-allow-the-planktoscope-to-be-used-as-a-default-gateway-to-the-internet)), and then you ask `forklift` to prepare to make your system match the desired state on its next boot. #### (No traditional) dependency management @@ -108,7 +108,7 @@ This design is intended to facilitate replacement of particular programs with mo #### Making & publishing customizations -The workflow with `forklift` for developing/testing OS customizations, such as new package deployments or non-standard configurations of existing package deployments or substitutions of existing package deployments, is as follows: +The workflow with `forklift` for developing/testing new OS customizations, such as new package deployments or non-standard configurations of existing package deployments or substitutions of existing package deployments, is as follows: - Initialize a custom pallet based on (i.e. layered over) an existing pallet, using the `forklift pallet init` command (e.g. `forklift pallet init --from github.com/PlanktoScope/pallet-standard@stable --as github.com/ethanjli/custom-pallet` to make a starter which will be a customization of the latest stable version of the [github.com/PlanktoScope/pallet-standard](https://github.com/PlanktoScope/pallet-standard) pallet, and which can be published to `github.com/ethanjli/custom-pallet`). (Note: the `forklift pallet init` command is not yet implemented, so currently a new pallet can only be created by manually initializing a new Git repository and creating a few YAML files inside it) @@ -116,6 +116,8 @@ The workflow with `forklift` for developing/testing OS customizations, such as n - Optionally, add one or more files which override files from the existing pallet, in order to override the configurations specified by those files. +- Optionally, add one or more new files, in order to add entirely new configurations. + - Stage the pallet to be applied on the next boot of the PlanktoScope OS, with the `forklift pallet stage` command; when Forklift applies a pallet, it makes the PlanktoScope OS match the configuration of Forklift package deployments specified by the pallet. - Use `git` to commit changes and (ideally) push them to GitHub, in order to publish your customizations for other people to try out. @@ -146,7 +148,7 @@ PlanktoScope-specific hardware modules are abstracted by PlanktoScope-specific p Traditional operating systems provide a desktop environment with a graphical user interface for operating the computer. By contrast, the PlanktoScope OS provides a set of web browser-based graphical user interfaces for operating the PlanktoScope. This approach was chosen for the following reasons: -- Most people already have a personal computing device (e.g. a phone or laptop). By relying on the user's personal computing device as the graphical interface for the PlanktoScope's software, the PlanktoScope project can reduce hardware costs by omitting a display from the PlanktoScope hardware. +- Most people already have a personal computing device (e.g. a phone or laptop). By relying on the user's personal computing device as the graphical interface for the PlanktoScope's software, the PlanktoScope project can reduce hardware costs by omitting a full display from the PlanktoScope hardware. - The PlanktoScope's computational resources are limited and may often need to be fully used for [data processing](#data-processing) tasks. By offloading real-time interaction (such as rendering of the graphical display, and handling of mouse and keyboard events) to a separate device, we can ensure a smooth user experience even when the PlanktoScope's Raspberry Pi computer is busy with other work. @@ -158,9 +160,7 @@ The PlanktoScope OS adds the following network services which provide web browse - A datasets [file browser](https://filebrowser.org/) for viewing, managing, uploading, and downloading image dataset files on the PlanktoScope. These files are generated and used by the PlanktoScope hardware controller and the PlanktoScope segmenter. -- [device-portal](https://github.com/PlanktoScope/device-portal): a landing page with links for end-users to quickly access the various web browser-based interfaces mentioned above. - -Note: we will probably simplify things by consolidating some of these components together into the PlanktoScope's Node-RED dashboard. +- [device-portal](https://github.com/PlanktoScope/device-portal): a PlanktoScope-specific landing page with links for end-users to quickly access the various web browser-based interfaces mentioned above. The PlanktoScope OS also provides various tools with web browser-based interfaces to aid with system administration and troubleshooting: @@ -172,9 +172,11 @@ The PlanktoScope OS also provides various tools with web browser-based interface - [Dozzle](https://dozzle.dev/): for viewing and monitoring logs of Docker containers. -- [Grafana](https://grafana.com/): for monitoring and exploring metrics stored in Prometheus. +- [Grafana](https://grafana.com/): for monitoring and exploring metrics stored in Prometheus. Note that this app will no longer be default included on the PlanktoScope OS starting with a future version of the PlanktoScope OS; instead, it will become an optional app which the user would have to manually enable via a Forklift command. -Finally, the PlanktoScope OS adds some command-line tools (beyond what is already provided by the default installation of the Raspberry Pi OS) for administrative tasks which system administrators, software developers, and advanced users may need to use: +Finally, the PlanktoScope OS adds some terminal tools (beyond what is already provided by the default installation of the Raspberry Pi OS) for administrative tasks which system administrators, software developers, and advanced users may need to use: + +- [`forklift`](https://github.com/PlanktoScope/forklift): for configuring, customizing, and updating the PlanktoScope OS. - [`vim`](https://www.vim.org/): for editing text files. @@ -182,35 +184,31 @@ Finally, the PlanktoScope OS adds some command-line tools (beyond what is alread - [`git`](https://git-scm.com/): for interacting with Git repositories. -- [`w3m`](https://tracker.debian.org/pkg/w3m) and [`lynx`](https://lynx.invisible-island.net/): for interacting with web pages (such as Wi-Fi network captive portals) from the PlanktoScope. - -- [`docker`](https://docs.docker.com/reference/cli/docker/): for managing and inspecting Docker containers. +- [`docker`](https://docs.docker.com/reference/cli/docker/) (including [Docker Compose](https://docs.docker.com/compose/)) and [`lazydocker`](https://github.com/jesseduffield/lazydocker): for inspecting Docker containers. -## Networking - -The PlanktoScope is often deployed in settings with limited or unstable internet access, and also in settings with no internet access at all. The PlanktoScope also needs to be deployable in remote settings where the user needs to control the PlanktoScope without being physically present. In both types of situations, the PlanktoScope's web browser-based interfaces need to remain accessible. +- [`w3m`](https://tracker.debian.org/pkg/w3m) and [`lynx`](https://lynx.invisible-island.net/): for interacting with web pages (such as Wi-Fi network captive portals) directly from the PlanktoScope. -We solve this problem by allowing the PlanktoScope to connect to the internet over a known Wi-Fi network, and/or over Ethernet, so that the PlanktoScope's web browser-based interfaces can be accessed over the internet; and by making the PlanktoScope bring up a Wi-Fi hotspot (more formally, a [wireless access point](https://en.wikipedia.org/wiki/Wireless_access_point)) using the Raspberry Pi's integrated Wi-Fi module in the absence of any known Wi-Fi network, so that the web browser-based interfaces can be accessed over the Wi-Fi hotspot. +- Various (relatively) low-level tools provided by [`net-tools`](https://manpages.debian.org/bookworm/net-tools/index.html), [`netcat-openbsd`](https://manpages.debian.org/bookworm/netcat-openbsd/index.html), [`nmap`](https://manpages.debian.org/bookworm/nmap/index.html), [`bind9-dnsutils`](https://manpages.debian.org/bookworm/bind9-dnsutils/index.html), and [`avahi-utils`](https://manpages.debian.org/bookworm/avahi-utils/index.html), for troubleshooting (relatively) low-level networking issues. -When a device connects directly to the PlanktoScope (e.g. via the PlanktoScope's Wi-Fi hotspot, or via an Ethernet cable), the PlanktoScope acts as a [DHCP](https://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol) server to assign itself certain static IP addresses (e.g. 192.168.4.1) and as a DNS server to assign itself certain domain names (e.g. `home.pkscope`), so that user can locate and open the PlanktoScope's web browser-based interfaces via those domain names. The PlanktoScope also announces itself under certain [mDNS](https://en.wikipedia.org/wiki/Multicast_DNS) names (e.g. `planktoscope.local`) which may work on networks where the PlanktoScope does not have a static IP address (e.g. because the PlanktoScope is connected to an existing Wi-Fi network). +## Networking -When the PlanktoScope both has internet access and has devices connected to it (e.g. over a Wi-Fi hotspot or over Ethernet), the PlanktoScope shares its internet access with all connected devices, to enable the user to access web pages even when connected to the PlanktoScope. This is implemented in the PlanktoScope OS with network configurations for the PlanktoScope to act as a network router using [Network Address Translation](https://en.wikipedia.org/wiki/Network_address_translation) when it has internet access. +The PlanktoScope is often operated with limited or sporadic internet access, and also in settings with no internet access at all. The PlanktoScope also needs to be deployable in remote settings where the user needs to control the PlanktoScope without being physically present. In both types of situations, the PlanktoScope's web browser-based interfaces need to remain accessible. -The standard PlanktoScope OS adds the following systemd services (beyond what is already provided by the default installation of the Raspberry Pi OS) for managing the PlanktoScope's network connectivity: +We solve this problem by allowing the PlanktoScope to connect to the internet over a known Wi-Fi network, and/or over Ethernet (or USB tethering to a phone sharing its internet access), so that the PlanktoScope's web browser-based interfaces can be accessed over the internet; and by making the PlanktoScope bring up a Wi-Fi hotspot (more formally, a [wireless access point](https://en.wikipedia.org/wiki/Wireless_access_point)) using the Raspberry Pi's integrated Wi-Fi module (or an optional USB Wi-Fi dongle) in the absence of any known Wi-Fi network, so that the web browser-based interfaces can be accessed over the Wi-Fi hotspot. The PlanktoScope implements these functionalities on the available networking-related hardware by using the sofware systems described in the subsections below. -- `autohotspot` (which in turn launches `hostapd`): a PlanktoScope-specific daemon for automatically checking the presence of known Wi-Fi networks, automatically connecting to any known Wi-Fi networks, and falling back to creating a Wi-Fi hotspot when no known Wi-Fi networks are present. +In addition to systemd services enumerated in the subsections below, the standard PlanktoScope OS also adds the following networking-related systemd services: -- `enable-interface-forwarding-between`: configures the Linux kernel firewall's IP packet filter rules to forward packets between the Raspberry Pi's network interfaces, to allow the Raspberry Pi to act as a network router. +- `assemble-hosts-templated`: generates a temporary hosts drop-in snippet (which is used by a symlink at `/etc/hosts.d/50-generated-templated`) from drop-in hosts snippet templates at `/etc/hosts-templates.d`. -- `enable-interface-forwarding-inbound`: configures the Linux kernel firewall's IP packet filter rules to forward packets targeted at `192.168.4.1`, `192.168.5.1`, etc., all to `localhost`, so that the PlanktoScope can be accessed from a client device's web browser at any such static IP address regardless of which network interface the client device is actually connected to. +- `assemble-hosts` generates a temporary hosts file (which is used by a symlink at `/etc/hosts`) from drop-in snippets at `/etc/hosts-templates.d`. -- `dnsmasq`: for allowing computers connected to the PlanktoScope over a network to access the PlanktoScope using domain names defined on the PlanktoScope. +- `report-mac-addresses`: generates a temporary file at `/run/mac-addresses.yml` which enumerates the system's network interfaces and their respective MAC addresses -- `firewalld`: a network firewall (currently disabled by default). +### Machine naming -- `planktoscope-mdns-alias@pkscope.service` and `planktoscopemdns-alias@planktoscope.service` configure the Avahi daemon (provided by the Raspberry Pi OS) to also resolve mDNS names `pkscope.local` and `planktoscope.local`, respectively, to an IP address (192.168.4.1) which is usable by devices connected to the PlanktoScope by a direct connection between their respective network interfaces. +Because multiple PlanktoScopes may be placed next to each other and each PlanktoScope needs to generate its own Wi-Fi hotspot with its own name, each PlanktoScope has a semi-unique machine name which is also used for the name of its Wi-Fi hotspot. This machine name, which has the format `{word}-{word}-{number up to five digits long}`, is generated by a stable mapping from the serial number of the PlanktoScope's Raspberry Pi. These serial numbers are randomly generated from a pool of approximately 4.2 billion serial numbers - so while it is possible for the Raspberry Pis of two PlanktoScopes to have the same serial numbers (and thus the same machine names), such a situation is extremely unlikely (and we provide a guide to [manually override the auto-generated machine name](../../../operation/networking.md#change-the-machine-name) if you are unlucky enough to encounter this situation, or if you prefer to choose your own name for your PlanktoScope). -The standard PlanktoScope OS also adds the following systemd services for dynamically updating the system's network configuration during boot: +The standard PlanktoScope OS adds the following systemd services (beyond what is already provided by the default installation of Raspberry Pi OS 12) for dynamically generating/updating the hostname at boot: - `generate-machine-name`: generates a human-readable machine name at `/run/machine-name` from the Raspberry Pi's serial number (or, if that's missing, from `/etc/machine-d`). @@ -218,37 +216,105 @@ The standard PlanktoScope OS also adds the following systemd services for dynami - `update-hostname`: updates `systemd-hostnamed` so that the hostname matches what is specified by `/etc/hostname`. -- `assemble-dnsmasq-config-templated`: generates a temporary dnsmasq drop-in config file (which is used by a symlink at `/etc/dnsmasq.d/40-generated-templated-config`) from drop-in config file templates at `/etc/dnsmasq-templates.d`. +### NetworkManager connections -- `assemble-hostapd-config-templated`: generates a temporary hostapd drop-in config file (which is used by a symlink at `/etc/hostapd/hostapd.conf.d/60-generated-templated.conf`) from drop-in config file templates at `/etc/hostapd/hostapd.conf-templates.d`. +The PlanktoScope uses [NetworkManager](https://networkmanager.dev/) to handle the different network connection modes which the PlanktoScope supports on its various network interfaces (internal Ethernet port, internal Wi-Fi module, optional USB Ethernet adapter, optional USB Wi-FI dongle, optional USB connection to a phone in tethering mode). -- `assemble-hostapd-config`: generates a temporary hostapd config file (which is used by a symlink at `/etc/hostapd/hostapd.conf`) from drop-in config files at `/etc/hostapd/hostapd.conf.d`. +When the PlanktoScope both has internet access and has devices connected to it (e.g. over a Wi-Fi hotspot or over Ethernet), the PlanktoScope shares its internet access with all connected devices; this behavior enables the user to access web pages even when connected to the PlanktoScope. This is implemented in the PlanktoScope OS with NetworkManager's "shared" IPv4 connection type for the PlanktoScope to act as a network router using [Network Address Translation](https://en.wikipedia.org/wiki/Network_address_translation) when it has internet access. When the PlanktoScope shares its connection to a router with a captive portal, connected devices opening the captive portal's web page will appear to the router as if the PlanktoScope itself was opening that captive portal's web page. -- `assemble-hosts-templated`: generates a temporary hosts drop-in snippet (which is used by a symlink at `/etc/hosts.d/50-generated-templated`) from drop-in hosts snippet templates at `/etc/hosts-templates.d`. +The standard PlanktoScope OS provides the following NetworkManager connection profiles, each bound to a specific network interface: -- `assemble-hosts` generates a temporary hosts file (which is used by a symlink at `/etc/hosts`) from drop-in snippets at `/etc/hosts-templates.d`. +- For interface `eth0`, the Raspberry Pi's internal Ethernet port: connection profile `eth0-default` sets a network configuration which attempts to connect to an Ethernet router and share its internet connection with all other devices connected to the PlanktoScope; if no such router exists, then profile `eth0-default` becomes inactive, and instead profile `eth0-static` (which configures the PlanktoScope to serve as an Ethernet router) becomes active. -The standard PlanktoScope OS also adds the following common services for integrating network APIs provided by various programs, and to facilitate communication among programs running on the PlanktoScope OS: +- For interface `eth1`, which only appears if a USB-to-Ethernet adapter is plugged in to the Raspberry Pi: connection profile `eth1-default` sets a network configuration which attempts to connect to an Ethernet router and share its internet connection with all other devices connected to the PlanktoScope; if no such router exists, then profile `eth1-default` becomes inactive, and instead profile `eth1-static` (which configures the PlanktoScope to serve as an Ethernet router) becomes active. -- [Mosquito](https://mosquitto.org/): a server which acts as an MQTT broker. This is used by the PlanktoScope hardware controller and segmenter (described below) to receive commands and broadcast notifications. This is also used by the PlanktoScope's Node-RED dashboard (described below) to send commands and receive notifications. +- For interface `usb0`, which only appears if a phone in USB tethering mode is plugged in to the Raspberry Pi, connection profile `usb0-default` sets a network configuration which should connect to the phone and share its internet connection with all other devices connected to the Planktoscope. -- [Caddy](https://caddyserver.com/) with the [caddy-docker-proxy plugin](https://github.com/lucaslorentz/caddy-docker-proxy): an HTTP server which acts as a [reverse proxy](https://en.wikipedia.org/wiki/Reverse_proxy) to route all HTTP requests on port 80 from HTTP clients (e.g. web browsers) to the appropriate HTTP servers (e.g. the Node-RED server, Prometheus, and the PlanktoScope hardware controller's HTTP-MJPEG camera preview stream) running on the PlanktoScope. +- For interface `wlan0`, the Raspberry Pi's internal Wi-Fi module: connection profile `wlan0-hotspot` creates a Wi-Fi hotspot (strictly speaking, a Wi-Fi access point network) from the Wi-Fi module, in effect acting like a Wi-Fi router. -The standard PlanktoScope OS also adds the following systemd services for reporting information about the system for easy access: +- For interface `wlan1`, which only appears if a USB Wi-Fi dongle is plugged in to the Raspberry Pi: connection profile `wlan0-hotspot` creates a Wi-Fi hotspot (strictly speaking, a Wi-Fi access point network) from the USB Wi-Fi dongle, in effect acting like a Wi-Fi router. -- `report-mac-addresses`: generates a temporary file at `/run/mac-addresses.yml` which enumerates the system's network interfaces and their respective MAC addresses +The standard PlanktoScope OS adds the following systemd services (beyond what is already provided by the default installation of Raspberry Pi OS 12) for dynamically updating the NetworkManager connection profiles during boot: + +- `assemble-networkmanager-connection@eth0-default.service`, `assemble-networkmanager-connection@eth1-default.service`, and `assemble-networkmanager-connection@usb0-default.service`: generate NetworkManager connection profiles from drop-in connection profile snippets in `/etc/NetworkManager/system-connections.d/eth0-default`, `/etc/NetworkManager/system-connections.d/eth1-default`, `/etc/NetworkManager/system-connections.d/usb0-default`, respectively. + +- `assemble-networkmanager-connection@eth0-static.service` and `assemble-networkmanager-connection@eth1-static.service`: generate NetworkManager connection profiles from drop-in connection profile snippets in `/etc/NetworkManager/system-connections.d/eth0-static` and `/etc/NetworkManager/system-connections.d/eth1-static`, respectively. + +- `assemble-networkmanager-connection-templated@wlan0-hotspot.service` and `assemble-networkmanager-connection-templated@wlan1-hotspot.service`: generate temporary drop-in NetworkManager connection profile snippets (which are used by symlinks at `/etc/NetworkManager/system-connections.d/wlan0-hotspot/40-generated-templated.nmconnection` and `/etc/NetworkManager/system-connections.d/wlan1-hotspot/40-generated-templated.nmconnection`, respectively) from drop-in connection profile snippet templates in `/etc/NetworkManager/system-connections-templates.d/wlan0-hotspot` and `/etc/NetworkManager/system-connections-templates.d/wlan1-hotspot`, respectively. + +- `assemble-networkmanager-connection@wlan0-hotspot.service` and `assemble-networkmanager-connection@wlan1-hotspot.service`: generate NetworkManager connection profiles from drop-in connection profile snippets in `/etc/NetworkManager/system-connections.d/wlan0-hotspot` and `/etc/NetworkManager/system-connections.d/wlan1-hotspot`, respectively. + +### DHCP & DNS server + +When a device connects directly to the PlanktoScope (e.g. via the PlanktoScope's Wi-Fi hotspot, or via an Ethernet cable), it will rely on the one of the PlanktoScope's NetworkManager `-hotspot` or `-static` connection profiles depending on the network interface being used; these connection profiles are of the `shared` IPv4 connection method so that NetworkManager will: + +- configure NAT for internet connection sharing. +- launch an instance of [dnsmasq](https://thekelleys.org.uk/dnsmasq/doc.html) as a [DHCP](https://en.wikipedia.org/wiki/Dynamic_Host_Configuration_Protocol) server for the PlanktoScope to assign itself certain static IP addresses (e.g. 192.168.4.1) and as a [DNS](https://en.wikipedia.org/wiki/Domain_Name_System) server for the PlanktoScope to assign itself certain domain names (e.g. `home.pkscope`), so that the user can locate and open the PlanktoScope's web browser-based interfaces via those domain names. + +The standard PlanktoScope OS adds the following systemd services (beyond what is already provided by the default installation of Raspberry Pi OS 12) for dynamically updating the NetworkManager's `shared`-network dnsmasq configuration during boot: + +- `assemble-dnsmasq-config-templated`: generates a temporary dnsmasq drop-in config file (which is used by a symlink at `/etc/NetworkManager/dnsmasq-shared.d/40-generated-templated-config.conf`) from drop-in config file templates at `/etc/NetworkManager/dnsmasq-shared-templates.d`. + +### mDNS + +In addition to the `.pkscope` domain names set by dnsmasq as a DNS server (which only work on devices connected directly to the PlanktoScope), the PlanktoScope also uses [Avahi](https://avahi.org/) to announce itself under certain [mDNS](https://en.wikipedia.org/wiki/Multicast_DNS) names (e.g. `planktoscope.local`) for all devices it's connected to; these mDNS names may work on networks where the PlanktoScope does not have a static IP address (e.g. because the PlanktoScope is connected to an existing Wi-Fi network). By default, Avahi is configured so that the PlanktoScope announces itself with the mDNS name `{hostname}.local`. For user convenience (and to provide a possibility of guessing a usable URL for accessing the PlanktoScope in situations where the user doesn't remember or can't type the PlanktoScope's full machine name), the PlanktoScope also announces `planktoscope.local` and `pkscope.local` as CNAME aliases for `{hostname}.local`. + +The standard PlanktoScope OS adds the following systemd services (beyond what is already provided by the default installation of Raspberry Pi OS 12) to provide mDNS aliases: + +- `avahi-publish-cname@pkscope.local` and `avahi-publish-cname@planktoscope.local` configure the Avahi daemon (provided by the Raspberry Pi OS) to also resolve mDNS names `pkscope.local` and `planktoscope.local`, respectively, as aliases for the PlanktoScope's mDNS-based hostname (which has format `{hostname}.local`). + +### Firewall + +For security reasons, the PlanktoScope includes a network firewall ([firewalld](https://firewalld.org/)) which is active by default with a default configuration in which incoming traffic is only allowed on explicitly-allowed ports/protocols (i.e. [default deny + enumerate goodness](https://www.ranum.com/security/computer_security/editorials/dumb/)). These ports/protocols are: + +- ICMP +- DHCP: UDP port 67 +- DHCPv6: UDP port 547 +- DHCPv6 Client: UDP port 546 +- DNS: TCP/UDP port 53 +- mDNS: UDP port 5353 +- SSH: TCP port 22 +- Direct-access fallback for the Cockpit system administration dashboard: TCP port 9090 +- HTTP: TCP port 80 +- HTTPS: TCP port 443 +- MQTT: TCP port 1883 + +By default, these ports/protocols are allowed on two firewalld zones: + +1. `nm-shared`, a firewall zone for devices connecting to the PlanktoScope as a network router (e.g. connected to the PlanktoScope's Wi-Fi hotspot, or connected directly to the PlanktoScope via Ethernet cable) +2. `public`, a firewall zone for devices accessing the PlanktoScope through a network router to which the PlanktoScope is connected. + +For more information about these ports/protocols and zones, and for information about disabling the rules for allowing one or more of the ports/protocols listed above, refer to the networking operations guide for [making the firewall more restrictive](http://localhost:8000/operation/networking/#make-the-firewall-more-restrictive). + +The standard PlanktoScope OS adds the following systemd services (beyond what is already provided by the default installation of Raspberry Pi OS 12) for the firewall: + +- `firewalld`: the firewalld daemon. + +- `assemble-firewalld-zone@nm-shared` and `assemble-firewalld-zone@public`: generate firewalld zone XML files for zones `nm-shared` and `public` from drop-in XML snippets in `/etc/firewalld/zones.d/nm-shared` and `/etc/firewalld/zones.d/public`, respectively. + +## App/API integration + +The standard PlanktoScope OS adds the following common services for integrating network APIs provided by various programs, and to facilitate communication among programs running on the PlanktoScope OS: + +- [Mosquitto](https://mosquitto.org/): a server which acts as an MQTT broker. This is used by the PlanktoScope hardware controller and segmenter (described below) to receive commands and broadcast notifications. This is also used by the PlanktoScope's Node-RED dashboard (described below) to send commands and receive notifications. Apps run in Docker containers can access this server over port 1883 through the `mosquitto` Docker bridge network; apps running outside Docker can access this server from host port 1883. + +- [Caddy](https://caddyserver.com/) with the [caddy-docker-proxy plugin](https://github.com/lucaslorentz/caddy-docker-proxy): an HTTP server which acts as a [reverse proxy](https://en.wikipedia.org/wiki/Reverse_proxy) to route all HTTP requests on port 80 from HTTP clients (e.g. web browsers) to the appropriate HTTP servers (e.g. the Node-RED server, Prometheus, and the PlanktoScope hardware controller's HTTP-MJPEG camera preview stream) running on the PlanktoScope. Apps deployed on the PlanktoScope are integrated with the reverse-proxy server by having the Docker containers for those apps include container labels parsed by the caddy-docker-proxy plugin to configure the reverse-proxy server; for apps not exposed in a port on the host, they can be attached to the `caddy-ingress` Docker bridge network so that the reverse proxy can connectly connect to such apps. Currently, the Caddy [site addresses](https://caddyserver.com/docs/caddyfile/concepts#addresses) for the reverse-proxy are for port 80 as an HTTP catch-all; the equivalent Caddyfile site address would be `:80`. Note that this means that HTTPS is currently not supported by the reverse-proxy server's configurations. ## Filesystem -The PlanktoScope OS's filesystem makes some changes from the default Debian/Raspberry Pi OS filesystem structure so that various sets of files in `/etc` and `/usr` can be atomically upgraded/downgraded/replaced together (using Forklift) while still being directly customizable by the system administrator. Specifically, a number of systemd services in the PlanktoScope OS run during early boot to: +The PlanktoScope OS's filesystem makes some changes from the default Debian/Raspberry Pi OS filesystem structure to make the OS easier to upgrade cleanly with customizations. + +### Overlays + + The biggest change is that the PlanktoScope OS splits the filesystem up into layers combined with the Linux kernel's [overlay-filesystem](https://docs.kernel.org/filesystems/overlayfs.html) functionality, so that various sets of files in `/etc` and `/usr` can be atomically upgraded/downgraded/replaced together (using Forklift) while still being directly customizable by the system administrator. Specifically, a number of systemd services in the PlanktoScope OS run during early boot to: - Make a read-only mount (via the `overlay-sysroot` systemd service) of the initial root filesystem, at `/sysroot` (this layout is loosely inspired by [OSTree's filesystem layout](https://ostreedev.github.io/ostree/adapting-existing/)). - Make a read-only mount of the next Forklift pallet to be applied (via the `bindro-run-forklift-stages-current.service`) from a subdirectory within `/var/lib/forklift/stages` to `/run/forklift/stages/current`. -- Remount `/usr` (via the `overlay-usr` systemd service) as a writable overlay with a Forklift-managed intermediate layer (in a subdirectory within `/var/lib/forklift/stages` which can also be accessed at `/run/forklift/stages/current/exports/overlays/usr`) and `/sysroot/usr` as a base layer; any changes made by the system administrator to files in `/usr` will be transparently stored by the overlay in `/var/lib/overlays/overrides/usr`. This allows Forklift to provide extra files in `/usr` in an atomic way, while overrides made by the system administrator are stored separately. +- Remount `/usr` (via the `overlay-usr` systemd service) as a writable overlay with a Forklift-managed intermediate layer (in a subdirectory within `/var/lib/forklift/stages` which can also be accessed at `/run/forklift/stages/current/exports/overlays/usr`) and `/sysroot/usr` as a base layer; any changes made by the system administrator to files in `/usr` will be transparently stored by the overlay in `/var/lib/overlays/overrides/usr`. This allows Forklift to provide extra files in `/usr` in an atomic way (for atomic upgrades), while overrides made by the system administrator are stored separately. This restructuring of `/usr` also enables factory resetting of the OS even if more packages have been installed using APT, e.g. as discussed in the software reset/upgrade operation guide's [discussion of `apt`/`apt-get`](../../../operation/software-upgrades.md#avoid-touching-aptapt-get). -- Remount `/etc` (via the `overlay-etc` systemd service) as a writable overlay with a Forklift-managed intermediate layer (in a subdirectory within `/var/lib/forklift/stages` which can also be accessed at `/run/forklift/stages/current/exports/overlays/etc`) and `/sysroot/etc` as a base layer; any changes made by the system administrator to files in `/etc` will be transparently stored by the overlay in `/var/lib/overlays/overrides/etc`. This allows Forklift to provide extra files in `/etc` in an atomic way, while overrides made by the system administrator are stored separately. +- Remount `/etc` (via the `overlay-etc` systemd service) as a writable overlay with a Forklift-managed intermediate layer (in a subdirectory within `/var/lib/forklift/stages` which can also be accessed at `/run/forklift/stages/current/exports/overlays/etc`) and `/sysroot/etc` as a base layer; any changes made by the system administrator to files in `/etc` will be transparently stored by the overlay in `/var/lib/overlays/overrides/etc`. This allows Forklift to provide extra files in `/etc` in an atomic way (for atomic upgrades), while overrides made by the system administrator are stored separately. This restructuring of `/etc` also enables factory resetting of the OS even if the user has manually edited any files in `/etc`, e.g. as discussed in the software reset/upgrade operation guide for [resetting the OS](../../../operation/software-upgrades.md#reset-the-planktoscope-os). - Make a writable mount of `/var/lib/forklift/stages` to `/home/pi/.local/share/forklift/stages` (via the `bind-.local-share-forklift-stages@home-pi` systemd service) so that, when the `pi` user runs `forklift` commands like `forklift pallet switch`, those commands will update `/var/lib/forklift/stages` - and without requiring the use of `sudo`. @@ -256,17 +322,53 @@ The PlanktoScope OS's filesystem makes some changes from the default Debian/Rasp Beyond what is required by the Linux [Filesystem Hierarchy Standard](https://refspecs.linuxfoundation.org/FHS_3.0/fhs-3.0.html), the PlanktoScope OS sets the following conventions related to filesystem paths: -- Scripts which are provided by Forklift and only used as part of systemd services should be provided in `/usr/libexec`, Forklift packages should export those scripts to `overlays/usr/libexec` (so, for example, they will be accessible in `/run/forklift/stages/current/exports/overlays/usr/libexec`). +- Scripts which are provided by Forklift and only used as part of systemd services should be provided in `/usr/libexec`, and Forklift packages should export those scripts to `overlays/usr/libexec` (so, for example, they will be accessible in `/run/forklift/stages/current/exports/overlays/usr/libexec`). -- Systemd units provided by Forklift should be provided in `/usr/lib/systemd/system`, and Forklift packages should export those units to `overlays/usr/lib/systemd/system`. Symlinks to enable those units should be provided in `/etc/systemd/system`, and Forklift packages should export those scripts to `overlays/etc/systemd/system`. +- Systemd units provided by Forklift should be provided in `/usr/lib/systemd/system`, and Forklift packages should export those units to `overlays/usr/lib/systemd/system`. Symlinks to enable those units should be provided in `/etc/systemd/system`, and Forklift packages should export those symlinks to directories within `overlays/etc/systemd/system`. - Forklift-provided systemd services which dynamically generate temporary files meant to be used in `/etc` should generate those temporary files at stable paths in `/run/overlays/generated/etc`. Forklift packages which provide such systemd services should also provide relative symlinks into those temporary files in `/run/overlays/generated/etc` to be exported into `overlays/etc` as overlays for the corresponding paths in `/etc`. For example, if a package provides a service to dynamically generate a hosts file meant to be used as `/etc/hosts`, that service should generate the file in `/run/overlays/generated/etc/hosts` and the package should export a symlink at `overlays/etc/hosts` which points to `../../run/overlays/generated/etc/hosts`, so that `/etc/hosts` will be a symlink pointing to `/run/overlays/generated/etc/hosts`. +### Config file assembly from drop-in fragments + +In Debian Linux (and thus also the standard Raspberry Pi OS), various system services are configured using monolithic configuration files (i.e. all settings are stored in a single file) instead of [drop-in configuration directories](https://unix.stackexchange.com/a/589303) (i.e. settings can be split across multiple files within a directory). The PlanktoScope OS generates some of those monolothic configuration files from drop-in configuration directories, in order to make it easy for users to toggle and/or override individual settings (or individual groups of settings) while continuing to follow any changes for other settings made in future software updates delivered via Forklift. PlanktoScope OS introduces drop-in configuration for the following types of configurations: + +- Cockpit: the `/etc/cockpit/cockpit.conf` file is generated from drop-in snippets in `/etc/cockpit/cockpit.conf.d`, and its `WebService` section's `Origins` setting is also generated from drop-in snippets in `/etc/cockpit/origins.d`. + +- firewalld: firewall zones in `/etc/firewalld` can be generated from drop-in snippets; for a zone generated at `/etc/firewalld/zones/{zone-name}.xml`, the drop-in snippets are loaded from the `/etc/firewalld/zones.d/{zone-name}` directory. + +- The `/etc/hosts` file is generated from drop-in snippets in `/etc/hosts.d`. + +- NetworkManager: NetworkManager connection files can be generated from drop-in snippets; for a connection file generated at `/etc/NetworkManager/system-connections/{connection-name}.nmconnection`, the drop-in snippets are loaded from the `/etc/NetworkManager/system-connections.d/{connection-name}` directory. + +PlanktoScope OS's general pattern for introducing drop-in configuration is as follows: + +- The configuration file `/etc/some/path/here` is replaced (via filesystem overlay over `/etc`) with a symlink to `/run/overlays/generated/etc/some/path/here`. This symlink is delivered via Forklift. + +- A systemd service (whose name starts with `assemble-`) runs a script which loads all drop-in files from `/etc/some/path/here.d` in lexicographic order, concatenates them, and writes the output to `/run/overlays/generated/etc/some/path/here`. This systemd service is delivered via Forklift + +- Drop-in files in `/etc/some/path/here.d` are delivered (and thus also updated) via Forklift. The delivery of each drop-in file (or logical groups of drop-in files) can be toggled via Forklift package feature flags. + +- The user can manually add a new drop-in file in `/etc/some/path/here.d`, and it will be preserved even if other drop-in files are changed via Forklift (whether via OS upgrades/downgrades or via feature flag toggles). + +- The user can manually override a Forklift-delivered drop-in file (e.g. by deleting that file or by editing it), and those overrides will continue to override any changes to that file which would otherwise take effect via Forklift (e.g. via OS upgrades/downgrades or via feature flag toggles). The override can be removed by deleting the corresponding file within `/var/lib/overlays/overrides/etc/some/path/here.d`. + +For certain system services, certain variables (e.g. the machine name or the hostname) can also be interpolated (by a corresponding systemd service whose name starts with `assemble-` or `generate-` and ends with `-templated`) into drop-in configuration files using a simple templating system: the string `{machine-name}` will be replaced with the machine's auto-generated machine name as specified in `/run/machine-name`, the string `{hostname}` will be replaced with the machine's machine name as specified in `/etc/hostname`, etc. Templating is provided for the following types of configurations: + +- The `/etc/hostname` file is replaced (by overlay with Forklift) with a symlink pointing to `/run/overlays/generated/etc/hostname`, which is generated by template-interpolating the template loaded from `/etc/hostname-template`. + +- Cockpit: a drop-in snippet at `/etc/cockpit/origins.d/60-generated-templated-names` (which is actually a symlink pointing to a corresponding path within `/run/overlays/generated/etc/...`) is generated by concatenating and template-interpolating the snippets templates loaded from `/etc/cockpit/origins-templates.d/`. + +- `/etc/hosts`: a drop-in snippet at `/etc/hosts.d/50-generated-templated` (which is actually a symlink pointing to a corresponding path within `/run/overlays/generated/etc/...`) is generated by concatenating and template-interpolating the snippets templates loaded from `/etc/hosts-templates.d`. + +- NetworkManager: drop-in snippets at `/etc/NetworkManager/system-connections.d/wlan0-hotspot/40-generated-templated.nmconnection` and `/etc/NetworkManager/system-connections.d/wlan1-hotspot/40-generated-templated.nmconnection` are generated by concatenating and template-interpolating the snippets templates loaded from `/etc/NetworkManager/system-connections-templates.d/wlan0-hotspot` and `/etc/NetworkManager/system-connections-templates.d/wlan1-hotspot`, respectively. Notably, this is used to set the SSID of the Wi-Fi hotspot according to the PlanktoScope's hostname (which by default is the PlanktoScope's semi-unique machine name). + +- dnsmasq (specifically the dnsmasq instance launched by NetworkManager for shared connections): a dnsmasq drop-in configuration file at `/etc/NetworkManager/dnsmasq-shared.d/40-generated-templated-config.conf` (which is actually a symlink pointing to a corresponding path within `/run/overlays/generated/etc/...`) can be generated from templates in `/etc/NetworkManager/dnsmasq-shared-templates.d`. + ## Observability & telemetry Although it is not a high priority yet, we would like to enable operators of large (>10) collections of PlanktoScopes to easily log and monitor the health and utilization of each PlanktoScope and to investigate issues with their PlanktoScopes, regardless of whether each PlanktoScope is deployed locally or remotely. The PlanktoScope OS currently includes the following common services to support system observability and telemetry both for the PlanktoScope OS as a system and for programs running on the PlanktoScope OS: -- [Prometheus](https://prometheus.io/): a server for collecting and storing metrics and for exposing metrics over an HTTP API. +- [Prometheus](https://prometheus.io/): a server for collecting and storing metrics and for exposing metrics over an HTTP API. Apps deployed on the PlanktoScope can be integrated with the Prometheus server by having the Docker containers for those apps include container labels parsed by Prometheus's [Docker container service discovery mechanism](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#docker_sd_config); for apps not exposed in a port (or via the Caddy reverse-proxy server) on the host, they can be attached to the `prometheus-metrics` Docker bridge network so that the Prometheus server can connectly connect to such apps. - [Prometheus node exporter](https://github.com/prometheus/node_exporter): for measuring computer hardware and OS monitoring metrics and exposing them over a Prometheus-compatible HTTP API. @@ -282,7 +384,7 @@ Note: in the future, the PlanktoScope OS will add more on-board services for pro ## Security -Currently, the PlanktoScope OS lacks basic security measures to make it safe for them to be connected to the internet; currently it is the responsibility of system administrators to add extremely basic risk mitigations, for example by: +Currently, the PlanktoScope OS lacks basic security measures to make it safe for them to be connected to the internet; currently it is the responsibility of system administrators to add extremely basic risk mitigations, for example by following the relevant instructions in the [Networking operation guide](../../../operation/networking.md#secure-your-planktoscope): - Changing the password of the `pi` user away from the default of `copepode`. @@ -300,4 +402,4 @@ Other risk mitigations will require deeper changes to the PlanktoScope OS, such - Password-protecting network [APIs](https://en.wikipedia.org/wiki/API). -We would like to start taking even just the very basic steps listed above to improve security, but security is not yet a high-enough priority for us to work on it with the limited resources available to us 🙃 - if you're interested in computer/network security and you'd like to help us as a volunteer on this project, please contact us! +We would like to implement these additional risk mitigations, but security is not yet a high-enough priority for us to work on it with the limited resources available to us 🙃 - if you're interested in computer/network security and you'd like to help us as a volunteer on this project, please contact us! diff --git a/documentation/docs/reference/software/subsystems/installation.md b/documentation/docs/reference/software/subsystems/installation.md index 1a98b4757..7daeff18a 100644 --- a/documentation/docs/reference/software/subsystems/installation.md +++ b/documentation/docs/reference/software/subsystems/installation.md @@ -94,13 +94,13 @@ This phase performs steps which might (in theory) be useful for other projects w - Installation of base tools: Docker, Cockpit, and various command-line tools are installed. -- Installation of `forklift` and a Forklift pallet: a hard-coded version of `forklift` is downloaded to `/usr/bin/forklift` , a hard-coded version of a hard-coded pallet (namely, [github.com/PlanktoScope/pallet-standard](https://github.com/PlanktoScope/pallet-standard)) is downloaded and prepared for deployment, and the `forklift-apply.service` systemd service is created and enabled. (Note: in the future, it will be possible to specify the pallet to be installed as a command-line argument.) +- Installation of `forklift` and a Forklift pallet: a hard-coded version of `forklift` is downloaded to `/usr/bin/forklift` , a hard-coded version of a hard-coded pallet (namely, [github.com/PlanktoScope/pallet-standard](https://github.com/PlanktoScope/pallet-standard)) is downloaded and prepared for deployment, and some Forklift-specific systemd services are created and enabled. (Note: in the future, it will be possible to specify the pallet to be installed as a command-line argument.) - Partial configuration of Raspberry Pi-specific hardware: the SPI and I2C hardware interfaces are enabled, and the serial port and serial port console are enabled (note: the serial port console will be disabled by the PlanktoScope application environment setup phase so that the serial port can be used for the PlanktoScope's GPS receiver instead), and legacy camera support is disabled. - Configuration of the system locale: the system's language is changed to `en_US.UTF-8`, but the time and measurement formats are changed to `en_DK.UTF-8` so that the date format is `yyyy-mm-dd` and units are metric. The system timezone is set to UTC. -- Partial configuration of networking: various system services are installed and configured, namely `dhcpcd`, `dnsmasq`, `hostapd`, and `firewalld`. The `enable-interface-forwarding.service` and `autohotspot.service` systemd services are created and enabled. The Raspberry Pi's Wi-Fi country is set to the US. +- Partial configuration of networking: various system components are installed, namely `network-manager`, `dnsmasq-base`, and `firewalld`. The Raspberry Pi's Wi-Fi country is set to the US. - Cleanup: SSH keys are reset to be regenerated on the next boot, unnecessary APT files are removed, and the OS [machine ID](https://wiki.debian.org/MachineId) is reset to be regenerated on the next boot. @@ -108,11 +108,9 @@ This phase performs steps which might (in theory) be useful for other projects w This phase performs steps specific to the PlanktoScope's hardware: -- Remaining configuration of networking: a hard-coded version of [`machine-name`](https://github.com/PlanktoScope/machine-name) is downloaded to `/usr/bin/machine-name`, `avahi-utils` is installed using APT, and various systemd services are created and enabled to update the PlanktoScope OS's networking configurations based on a machine name which will be determined by `machine-name` from the Raspberry Pi's serial number at every boot. Additional systemd services are created and enabled so that the PlanktoScope will be accessible over some additional mDNS names (namely, `pkscope.local` and `planktoscope.local`). - - Setup of the PlanktoScope hardware controller: various Python tools (`pip`, `venv`, and `poetry`) are installed using APT, a hard-coded version of a hard-coded Git repository (namely [github.com/PlanktoScope/device-backend](https://github.com/PlanktoScope/device-backend)) is cloned, and various dependencies (both system libraries and Python packages) of the hardware controller are installed. The `planktoscope-org.device-backend.controller-adafruithat.service` and `planktoscope-org.device-backend.controller-planktoscopehat.service` systemd services are created, and the appropriate one is enabled depending on which HAT the PlanktoScope OS is being installed for. The appropriate hardware configuration file will also be copied into the location expected by the hardware controller. (Note: once the PlanktoScope hardware controller is containerized and managed in Forklift, this step will be eliminated.) -- Setup of GPIO stepper initialization at boot: a systemd service is created to release the stepper motors at startup. (Note: this service currently doesn't work and will eventually be deleted or replaced.) +- Setup of GPIO stepper initialization at boot: a systemd service is created to release the stepper motors at startup. (Note: this service currently doesn't work and will eventually be fixed, deleted, or replaced.) - Setup of the PlanktoScope Node-RED dashboard: Node-RED is installed, as well as a Python package required by the `adafruithat` version of the PlanktoScope Node-RED dashboard (Note: the dependency on that package will eventually be removed.). The appropriate version of the PlanktoScope Node-RED dashboard and will be copied to the location expected by Node-RED depending on which HAT the PlanktoScope OS is being installed for, along with the appropriate configuration file. Finally, `npm` packages required by the Node-RED dashboard are installed. (Note: once the Node-RED dashboard is containerized and managed in Forklift, this step will be eliminated.) diff --git a/documentation/docs/reference/software/subsystems/startup.md b/documentation/docs/reference/software/subsystems/startup.md index 5484dc3c0..412b7eff2 100644 --- a/documentation/docs/reference/software/subsystems/startup.md +++ b/documentation/docs/reference/software/subsystems/startup.md @@ -34,27 +34,23 @@ The PlanktoScope OS's setup scripts provide some system services which are not m For descriptions of the various targets (e.g. `sysinit.target`, `network-pre.target`) referred to below, see [systemd's bootup process](https://www.freedesktop.org/software/systemd/man/latest/bootup.html) and [systemd's special targets](https://www.freedesktop.org/software/systemd/man/latest/systemd.special.html): -- `generate-machine-name.service` and `generate-hostname-templated.service` runs before `sysinit.target`. +- `generate-machine-name.service` and `generate-hostname-templated.service` runs after `local-fs.target` but before `sysinit.target` and `systemd-hostnamed.service`. -- `update-hostname.service` runs after `generate-hostname-templated.service` and `systemd-hostnamed.service` but before `network-pre.target`. +- `update-hostname.service` runs after `generate-hostname-templated.service` and `systemd-hostnamed.service` but before `network-pre.target` and `avahi-daemon.service`. -- `assemble-dnsmasq-config-templated.service` runs after `generate-machine-name.service` and `generate-hostname-templated.service` but before `dnsmasq.service`. +- `assemble-firewalld-zone@nm-shared.service` and `assemble-firewalld-zone@public.service` run before `firewalld.service` and `NetworkManager.service`. -- `assemble-hosts-templated.service` and `assemble-hosts.service` run after `generate-machine-name.service` and `generate-hostname-templated.service` but before `dnsmasq.service` and `network-pre.target`. +- `assemble-hosts-templated.service` and `assemble-hosts.service` run after `generate-machine-name.service` and `generate-hostname-templated.service` but before `NetworkManager.service` and `network-pre.target`. -- `enable-interface-forwarding-between.service` runs before `network-online.target`. +- `assemble-dnsmasq-config-templated.service` runs after `generate-machine-name.service` and `generate-hostname-templated.service` but before `NetworkManager.service`. -- `enable-interface-forwarding-inbound.service` runs before `network-online.target`. +- `assemble-networkmanager-connection-templated@wlan0-hotspot.service`, `assemble-networkmanager-connection-templated@wlan1-hotspot.service`, `assemble-networkmanager-connection@wlan0-hotspot.service` and `assemble-networkmanager-connection@wlan1-hotspot.service` run after `generate-machine-name.service` and `generate-hostname-templated.service` but before `NetworkManager.service`. -- `report-mac-addresses.service` runs before `network-online.target`. It is re-run every two minutes by `report-mac-addresses.timer`. +- `assemble-networkmanager-connection@eth0-default.service`, `assemble-networkmanager-connection@eth0-static.service`, `assemble-networkmanager-connection@eth1-default.service`, `assemble-networkmanager-connection@eth1-static.service`, and `assemble-networkmanager-connection@usb0-default.service` run before `NetworkManager.service`. -- `assemble-hostapd-config-templated.service` and `assemble-hostapd-config.service` run after `generate-machine-name.service` and `generate-hostname-templated.service` but before `hostapd.service`. +- `avahi-publish-cname@pkscope.local.service` and `avahi-publish-cname@planktoscope.local.service` run after `update-hostname.service` and `avahi-daemon.service`. -- The `hostapd` daemon is manually started and stopped by `autohotspot.service`. - -- `autohotspot.service` runs after `forklift-apply.service` and `enable-interface-forwarding-between.service` and ``enable-interface-forwarding-inbound.service`` have started (so that the PlanktoScope's web browser-based user interfaces are ready for connections before the PlanktoScope's Wi-Fi hotspot is started) and before network connectivity is considered to have been established. It is re-run every two minutes by `autohotspot.timer`. - -- `planktoscope-mdns-alias@pkscope.service` and `planktoscopemdns-alias@planktoscope.service` run after `avahi-daemon.service`. +- `report-mac-addresses.service` runs after `network-online.target`. It is re-run every two minutes by `report-mac-addresses.timer`. ### User interface @@ -62,8 +58,8 @@ For descriptions of the various targets (e.g. `sysinit.target`, `network-pre.tar - `ensure-ssh-host-keys.service` regenerates the SSH server's host keys if the keys are missing, and runs before `ssh.service`. -- The PlanktoScope Node-RED dashboard (managed by `nodered.service`) starts after `planktoscope-org.update-machine-name.service` has started, to ensure that the Node-RED dashboard has the correct machine name. (In the future the PlanktoScope Node-RED dashboard will instead be run as a Docker container and will be managed by `forklift`.) +- The PlanktoScope Node-RED dashboard (managed by `nodered.service`) starts after `generate-machine-name.service` has started, to ensure that the Node-RED dashboard has the correct machine name. (In the future the PlanktoScope Node-RED dashboard will instead be run as a Docker container and will be managed by `forklift`.) ### PlanktoScope-specific hardware abstraction -- The PlanktoScope hardware controller (managed by `planktoscope-org.device-backend.controller-{adafruithat or planktoscopehat}.service`) starts after `forklift-apply.service` (which manages Mosquito) and `nodered.service` have started, to ensure that the PlanktoScope hardware controller broadcasts the detected camera model name only after the PlanktoScope Node-RED dashboard is ready to receive that broadcast. (In the future the PlanktoScope hardware controller will instead be run as a Docker container and will be managed by `forklift`.) +- The PlanktoScope hardware controller (managed by `planktoscope-org.device-backend.controller-{adafruithat or planktoscopehat}.service`) starts after `forklift-apply.service` (which manages Mosquitto) and `nodered.service` have started, to ensure that the PlanktoScope hardware controller broadcasts the detected camera model name only after the PlanktoScope Node-RED dashboard is ready to receive that broadcast. (In the future the PlanktoScope hardware controller will instead be run as a Docker container and will be managed by `forklift`.) diff --git a/documentation/docs/setup/software/nonstandard-install.md b/documentation/docs/setup/software/nonstandard-install.md index ee525f774..9a6dd23e6 100644 --- a/documentation/docs/setup/software/nonstandard-install.md +++ b/documentation/docs/setup/software/nonstandard-install.md @@ -141,7 +141,7 @@ You will only be able to access the PlanktoScope's graphical user interface by p - If you plan to connect another device directly to your PlanktoScope via its Wi-Fi hotspot or via an Ethernet cable, follow the same instructions for connecting to your PlanktoScope as in the [standard installation guide](./standard-install.md#connect-to-the-planktoscope). -- If you had previously configured your PlanktoScope's Raspberry Pi to connect to a Wi-Fi network, it will not make its own Wi-Fi hotspot. On the Wi-Fi network it's connected to, it will only be accessible by its machine-specific mDNS URL, which has the format `http://pkscope-{machine-name}.local`, where `{machine-name}` should be replaced by your PlanktoScope's specific machine name (which you should have recorded in the previous step). +- If you had previously configured your PlanktoScope's Raspberry Pi to connect to a Wi-Fi network, it will not make its own Wi-Fi hotspot. On the Wi-Fi network it's connected to, it will only be accessible by its mDNS URLs (i.e. URLs ending in `.local`), such as the machine-specific URL which has the format `http://pkscope-{machine-name}.local`, where `{machine-name}` should be replaced by your PlanktoScope's specific machine name (which you should have recorded in the previous step). should also work, though if your PlanktoScope is connected to a Wi-Fi network which also has other PlanktoScopes connected, then may show you the landing page for one of those other PlanktoScopes; in that case, you should use your PlanktoScope's machine-specific URL. ## Next steps diff --git a/documentation/docs/setup/software/standard-install.md b/documentation/docs/setup/software/standard-install.md index 2b9627a67..440289ee3 100644 --- a/documentation/docs/setup/software/standard-install.md +++ b/documentation/docs/setup/software/standard-install.md @@ -46,13 +46,13 @@ Power on your PlanktoScope, and wait for it to start up. Note that it may take a You will not be able to access the PlanktoScope's graphical user interface by plugging in a display to the Raspberry Pi. This is because the SD card image we provide does not include a graphical desktop or web browser, in order to keep the SD card image file smaller and to keep the PlanktoScope's Raspberry Pi running more efficiently. -Once your PlanktoScope has created its Wi-Fi hotspot, you can connect another device (e.g. a phone or a computer) directly to the PlanktoScope - either through its Wi-Fi hotspot or through an Ethernet cable directly to the PlanktoScope's Ethernet port. Afterwards, you can open a web browser on the device to access the PlanktoScope's graphical user interface at one of the following URLs (try them in the following order, and just use the first one which works): +Once your PlanktoScope has created its Wi-Fi hotspot, you can connect your computer (or phone/tablet/etc.) directly to the PlanktoScope - either through its Wi-Fi hotspot or through an Ethernet cable directly to the PlanktoScope's Ethernet port. Afterwards, you can open a web browser on your computer to access the PlanktoScope's graphical user interface at one of the following URLs (try them in the following order, and just use the first one which works): -- (this should work unless you're on a device or web browser without mDNS support; notably, older versions of Android did not support mDNS, and web browsers installed on Linux computers via Flatpak [do not yet support mDNS](https://github.com/flatpak/xdg-desktop-portal/discussions/1365)) -- (this should work unless you're on a device or web browser without mDNS support) +- (this should work unless you're on a computer or web browser without mDNS support; notably, older versions of Android did not support mDNS, and web browsers installed on Linux computers via Flatpak [do not yet support mDNS](https://github.com/flatpak/xdg-desktop-portal/discussions/1365)) +- (this should work unless you're on a computer or web browser without mDNS support) - (this should work unless your web browser is configured to use a Private DNS provider) -- (this should always work on devices connected directly to the PlanktoScope, especially for devices connected directly to the PlanktoScope's Wi-Fi hotspot) -- (this should always work on devices connected directly to the PlanktoScope, especially for devices connected directly to the PlanktoScope's Ethernet port) +- (this should always work on computers connected to the PlanktoScope's Wi-Fi hotspot, unless your computer is also connected to some other device meant to be accessed with that URL) +- (this should always work on computers connected directly to the PlanktoScope's Ethernet port, unless your computer is also connected to some other device meant to be accessed with that URL) !!! warning diff --git a/software/CHANGELOG.md b/software/CHANGELOG.md index 1a2a0d844..6a831269a 100644 --- a/software/CHANGELOG.md +++ b/software/CHANGELOG.md @@ -12,18 +12,24 @@ All dates in this file are given in the [UTC time zone](https://en.wikipedia.org ### Added - (Application: GUI) The landing page now has a link to a new page (actually a filebrowser file viewer) which lists the MAC addresses of all network interfaces, to make it easier to figure out MAC addresses for registering the Raspberry Pi on networks which require such registrations as a requirement for internet access. +- (System: networking) If you plug in a supported USB Wi-Fi dongle into the PlanktoScope, now it will by default automatically create a Wi-Fi hotspot network from that Wi-Fi dongle - regardless of whether the PlanktoScope's internal Wi-Fi module is configured to also create the same Wi-Fi hotspot network or to connect to some external Wi-Fi network. This means that the PlanktoScope now supports creating its own Wi-Fi hotspot while simultaneously being connected to the internet via a Wi-Fi network, if you plug in a USB Wi-Fi dongle. +- (System: networking) If the PlanktoScope is connected to a Wi-Fi network with a captive portal, you should be able to access and proceed through the captive portal from a computer/phone connected to the PlanktoScope. +- (System: networking) Firewalld is now enabled, and default firewall policies are provided (via Forklift) for the `public` and `nm-shared` firewall zones. This means that if you want to access any additional ports besides the ports for programs provided with the standard PlanktoScope OS) from other devices, now you must add configurations to open those additional ports, e.g. via drop-in configuration snippets in `/etc/firewalld/zones.d`. ### Changed - (Application: GUI) The Node-RED dashboard now initializes the Sample page's Dilution Factor field to 1.0, instead of leaving it empty. +- (System: networking) Wi-Fi hotspot behavior and network connection management is now based on NetworkManager, in preparation for an upgrade to Raspberry Pi OS 12 (bookworm). Thus, NetworkManager is installed on bullseye-based images, while dhcpcd is now uninstalled on bullseye-based images. As part of this change the previous autohotspot service has been removed, as it's redundant with functionality provided by NetworkManager. ### Removed - (Application: backend) The old raspimjpeg-based imager has now been completely removed, following a deprecation in v2024.0.0-alpha.2. - (Application: GUI) Various elements of the Node-RED dashboard which were deprecated in v2024.0.0 and in v2024.0.0-alpha.2 have now been removed, including the old USB backup functionality. +- (Application: GUI) Portainer (whose default enablement was deprecated in v2024.0.0-alpha.2) is now disabled by default. ### Fixed +- (System: networking) `planktoscope.local` and `pkscope.local` should now work on local area networks (i.e. when the PlanktoScope is connected to a router) and not just on direct connections. - (Application: GUI) The Node-RED dashboard's sample page's "Dilution Factor" input field has been renamed to "Concentration Factor", which is a less misleading name for what that input field actually represents. ## v2024.0.0 - 2024-12-25 diff --git a/software/distro/setup/base-os/cockpit/install.sh b/software/distro/setup/base-os/cockpit/install.sh index 7c08ef503..955f8c6e0 100755 --- a/software/distro/setup/base-os/cockpit/install.sh +++ b/software/distro/setup/base-os/cockpit/install.sh @@ -3,5 +3,4 @@ # Install cockpit sudo -E apt-get install -y --no-install-recommends -o Dpkg::Progress-Fancy=0 \ - cockpit -# TODO: after we switch to NetworkManager, add cockpit-networkmanager + cockpit cockpit-networkmanager cockpit-storaged diff --git a/software/distro/setup/base-os/docker/install.sh b/software/distro/setup/base-os/docker/install.sh index 17923fce7..2368ca9bd 100755 --- a/software/distro/setup/base-os/docker/install.sh +++ b/software/distro/setup/base-os/docker/install.sh @@ -1,19 +1,19 @@ #!/bin/bash -eux # Docker enables deployment of containerized applications. -config_files_root=$(dirname $(realpath $BASH_SOURCE)) +config_files_root=$(dirname "$(realpath "$BASH_SOURCE")") sudo install -m 0755 -d /etc/apt/keyrings if [ ! -f /etc/apt/keyrings/docker.gpg ]; then - curl -fsSL "https://download.docker.com/linux/debian/gpg" | \ + curl -fsSL "https://download.docker.com/linux/debian/gpg" | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg fi sudo chmod a+r /etc/apt/keyrings/docker.gpg DISTRO_VERSION_CODENAME="$(. /etc/os-release && echo "$VERSION_CODENAME")" echo \ - "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \ - "$DISTRO_VERSION_CODENAME" stable" | \ - sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg]" \ + "https://download.docker.com/linux/debian $DISTRO_VERSION_CODENAME stable" | + sudo tee /etc/apt/sources.list.d/docker.list >/dev/null sudo -E apt-get update -y -o Dpkg::Progress-Fancy=0 # get the list of packages from the docker repo # The following command may fail with a post-install error if the system installed kernel updates # via apt upgrade but was not rebooted before installing docker-ce; however, even if this error @@ -27,4 +27,4 @@ sudo -E apt-get remove -y -o Dpkg::Progress-Fancy=0 \ # Allow running Docker commands without sudo. Before the next reboot, subsequent setup scripts will # still need to use sudo to run commands which interact with the Docker daemon # (see https://docs.docker.com/engine/install/linux-postinstall/): -sudo usermod -aG docker $USER +sudo usermod -aG docker "$USER" diff --git a/software/distro/setup/base-os/forklift/ensure-docker.sh b/software/distro/setup/base-os/forklift/ensure-docker.sh index 1338964a4..5621d52d5 100755 --- a/software/distro/setup/base-os/forklift/ensure-docker.sh +++ b/software/distro/setup/base-os/forklift/ensure-docker.sh @@ -8,6 +8,7 @@ if ! [ -S /var/run/docker.sock ] && echo "Warning: couldn't start docker the easy way!" echo "Starting containerd daemon directly..." sudo /usr/bin/containerd & + sleep 5 echo "Starting docker daemon directly..." sudo /usr/bin/dockerd & sleep 5 diff --git a/software/distro/setup/base-os/forklift/forklift-pallet-version b/software/distro/setup/base-os/forklift/forklift-pallet-version index d15f492b7..a30d7df3a 100644 --- a/software/distro/setup/base-os/forklift/forklift-pallet-version +++ b/software/distro/setup/base-os/forklift/forklift-pallet-version @@ -1 +1 @@ -8b21c46 +35b9c55 diff --git a/software/distro/setup/base-os/forklift/forklift-version b/software/distro/setup/base-os/forklift/forklift-version index 90dd9847e..e819a7317 100644 --- a/software/distro/setup/base-os/forklift/forklift-version +++ b/software/distro/setup/base-os/forklift/forklift-version @@ -1 +1 @@ -0.8.0-alpha.5 +0.8.0-alpha.6 diff --git a/software/distro/setup/base-os/forklift/install.sh b/software/distro/setup/base-os/forklift/install.sh index b0cee679c..02b7c61ac 100755 --- a/software/distro/setup/base-os/forklift/install.sh +++ b/software/distro/setup/base-os/forklift/install.sh @@ -14,16 +14,15 @@ config_files_root=$(dirname "$(realpath "$BASH_SOURCE")") sudo cp "$config_files_root"/usr/lib/systemd/system/* /usr/lib/systemd/system/ sudo cp "$config_files_root"/usr/lib/systemd/system-preset/* /usr/lib/systemd/system-preset/ -# Make the stage store at /var/lib/forklift/stages available for non-root access in the current -# (i.e. default) user's default Forklift workspace, both in the current boot and subsequent boots: -local_stage_store="$HOME/.local/share/forklift/stages" -mkdir -p "$local_stage_store" +# Make the stage store at /var/lib/forklift/stages available for easy access in the root user's +# default Forklift workspace, both in the current boot and subsequent boots: +mkdir -p "$HOME/.local/share/forklift/stages" sudo mkdir -p /var/lib/forklift/stages # TODO: maybe we should instead make a new "forklift" group which owns everything in # /var/lib/forklift? sudo chown "$USER" /var/lib/forklift/stages -sudo systemctl enable "bind-.local-share-forklift-stages@-home-$USER.service" -if ! sudo systemctl start "bind-.local-share-forklift-stages@-home-$USER.service" 2>/dev/null; then +sudo systemctl enable "bind-.local-share-forklift-stages@home-$USER.service" +if ! sudo systemctl start "bind-.local-share-forklift-stages@home-$USER.service" 2>/dev/null; then echo "Warning: the system's Forklift stage store is not mounted to $USER's Forklift stage store." echo "As long as you don't touch the Forklift stage store before the next boot, this is fine." fi diff --git a/software/distro/setup/base-os/forklift/load-precached-image.sh b/software/distro/setup/base-os/forklift/load-precached-image.sh index 9aca2de9f..6a9869ba2 100755 --- a/software/distro/setup/base-os/forklift/load-precached-image.sh +++ b/software/distro/setup/base-os/forklift/load-precached-image.sh @@ -6,4 +6,4 @@ precached_image="$cache_path/$image.tar" echo "Loading $image from $precached_image..." sudo docker image load --quiet --input "$precached_image" -rm "$precached_image" +sudo rm "$precached_image" diff --git a/software/distro/setup/base-os/forklift/precache-image.sh b/software/distro/setup/base-os/forklift/precache-image.sh index b660037b8..7cd88e43d 100755 --- a/software/distro/setup/base-os/forklift/precache-image.sh +++ b/software/distro/setup/base-os/forklift/precache-image.sh @@ -1,7 +1,7 @@ #!/bin/bash -eu image="$1" cache_path="$2" # e.g. ~/.cache/containers -platform="$3" # e.g. `linux/arm64` +platform="$3" # e.g. `linux/arm64` precached_image="$cache_path/$image.tar" @@ -10,18 +10,17 @@ if [ -f "$precached_image" ]; then exit 0 fi -crane="crane" -if ! mkdir -p "$(dirname "$precached_image")"; then +crane="$(command -v crane)" +if ! mkdir -p "$(dirname "$precached_image")" &>/dev/null; then sudo mkdir -p "$(dirname "$precached_image")" - crane="$sudo crane" + crane="sudo $crane" fi echo "Downloading $image to $precached_image..." # We pull images as Docker (legacy) tarballs so that they can be inspected with `dive` (which does # not support OCI tarballs) - see https://github.com/google/go-containerregistry/issues/621 for # details: -if ! $crane --platform "$platform" pull --format legacy "$image" "$precached_image" -then +if ! $crane --platform "$platform" pull --format legacy "$image" "$precached_image"; then echo "Encountered error, trying one more time to download $image..." rm -f "$precached_image" || sudo rm -f "$precached_image" $crane --platform "$platform" pull --format legacy "$image" "$precached_image" diff --git a/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bind-.local-share-forklift-stages@.service b/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bind-.local-share-forklift-stages@.service index 23c0e151c..d3407fd95 100644 --- a/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bind-.local-share-forklift-stages@.service +++ b/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bind-.local-share-forklift-stages@.service @@ -1,5 +1,5 @@ [Unit] -Description=Mount the Forklift stage store into the default path for the Forklift workspace at %I +Description=Mount the Forklift stage store into the default path for the Forklift workspace at %f Requires=-.mount After=-.mount Wants=home.mount @@ -19,7 +19,7 @@ ExecStartPre=mkdir -p $TARGET ExecStart=bash -c 'mount -o bind $SOURCE $TARGET' # Note: `umount -l` is not recommended in general (see https://unix.stackexchange.com/a/390057) # because it just removes the mounts from the namespace while writes to open files can continue; -# however, we have a read-only mount so it doesn't matter +# however, this is probably acceptable behavior for our use-case. ExecStopPost=umount -l $TARGET [Install] diff --git a/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bindro-run-forklift-stages-current.service b/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bindro-run-forklift-stages-current.service index 6255113b0..8ec483ce9 100644 --- a/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bindro-run-forklift-stages-current.service +++ b/software/distro/setup/base-os/forklift/usr/lib/systemd/system/bindro-run-forklift-stages-current.service @@ -29,8 +29,7 @@ ExecStart=bash -c -e '\ ' # Note: `umount -l` is not recommended in general (see https://unix.stackexchange.com/a/390057) # because it just removes the mounts from the namespace while writes to open files can continue; -# however, this is probably acceptable behavior for the overlay at shutdown, because any writes -# will be in our upperdir anyways. +# however, we have a read-only mount so it doesn't matter ExecStopPost=umount -l $TARGET diff --git a/software/distro/setup/base-os/forklift/usr/lib/systemd/system/forklift-apply.service b/software/distro/setup/base-os/forklift/usr/lib/systemd/system/forklift-apply.service index dcdac0ef1..507a76d94 100644 --- a/software/distro/setup/base-os/forklift/usr/lib/systemd/system/forklift-apply.service +++ b/software/distro/setup/base-os/forklift/usr/lib/systemd/system/forklift-apply.service @@ -2,8 +2,6 @@ Description=Docker Compose applications specified by the local Forklift pallet Wants=docker.service After=docker.service -# Delay the autohotspot until the Forklift pallet's network services are up: -Before=autohotspot.service [Service] Type=oneshot diff --git a/software/distro/setup/base-os/networking/install.sh b/software/distro/setup/base-os/networking/install.sh index 07e3f4b40..658d00d1b 100755 --- a/software/distro/setup/base-os/networking/install.sh +++ b/software/distro/setup/base-os/networking/install.sh @@ -7,31 +7,20 @@ config_files_root=$(dirname "$(realpath "$BASH_SOURCE")") # Install dependencies sudo -E apt-get install -y -o Dpkg::Progress-Fancy=0 \ - firewalld dnsmasq hostapd avahi-utils + network-manager firewalld dnsmasq-base +sudo systemctl enable NetworkManager.service -# By default hostapd.service is masked and enabled (which causes two symlinks to exist), which -# prevents Forklift from being able to disable hostapd via a filesystem overlay. We override this by -# manually removing those symlinks by default, since our autohotspot relies on hostapd being -# unmaske and disabled. -sudo systemctl unmask hostapd.service -sudo systemctl disable hostapd.service - -# Disable firewalld for now -# FIXME: enable firewalld and set up firewall rules -if sudo systemctl disable firewalld.service --now 2>/dev/null; then - if systemctl list-units --full -all | grep -Fq "docker.service"; then - # Restart docker to integrate with firewalld - sudo systemctl restart docker - fi -else - # We can't stop it because we're not booted, so we don't need to stop it or restart Docker: - sudo systemctl disable firewalld.service +# Uninstall dhcpcd if we're on bullseye +DISTRO_VERSION_ID="$(. /etc/os-release && echo "$VERSION_ID")" +if [ "$DISTRO_VERSION_ID" -le 11 ]; then # Support Raspberry Pi OS 11 (bullseye) + sudo -E apt-get purge -y -o Dpkg::Progress-Fancy=0 \ + dhcpcd5 fi # Set the wifi country # FIXME: instead have the user set the wifi country via a first-launch setup wizard, and do it # without using raspi-config. It should also be updated if the user changes the wifi country. -# This should also update the hostapd config (maybe via a new template variable) +# Or maybe we don't need to set the wifi country? if command -v raspi-config &>/dev/null; then sudo raspi-config nonint do_wifi_country US else diff --git a/software/distro/setup/base-os/tools/install.sh b/software/distro/setup/base-os/tools/install.sh index f046bc5f8..7987e878e 100755 --- a/software/distro/setup/base-os/tools/install.sh +++ b/software/distro/setup/base-os/tools/install.sh @@ -12,3 +12,7 @@ sudo -E apt-get install -y -o DPkg::Lock::Timeout=60 -o Dpkg::Progress-Fancy=0 \ # Install some tools for dealing with captive portals sudo -E apt-get install -y -o Dpkg::Progress-Fancy=0 \ w3m lynx + +# Install some tools for troubleshooting networking stuff +sudo -E apt-get install -y -o Dpkg::Progress-Fancy=0 \ + net-tools bind9-dnsutils netcat-openbsd nmap avahi-utils