From b7056cedef4a1c63775f00993bc9e77e78d460f8 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Wed, 20 Apr 2022 22:43:47 -0400 Subject: [PATCH 01/16] Improve Windows compatibility Add command to Dockerfile to automatically remove any CRLF line termination from the entry point script. They are sometimes added by git when it runs on Windows and they prevent the script from running when the file is copied back on a Linux image. Add code in cmqttd.py that is triggered when the Python code is run directly on Windows. It deals with an issue related to windows ports' behavior within the event loop that results in an error. The issue is documented here: Configuration of windows event loop for libraries python/cpython#81554 It is useful when porting the python code to a Windows system. --- Dockerfile | 1 + cbus/daemon/cmqttd.py | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/Dockerfile b/Dockerfile index 093ac44..3251b2d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ RUN pip3 install 'parameterized' && \ # cmqttd runner image FROM base as cmqttd COPY COPYING COPYING.LESSER Dockerfile README.md entrypoint-cmqttd.sh / +RUN sed -i 's/\r$//' entrypoint-cmqttd.sh COPY --from=builder /cbus/dist/cbus-0.2.generic.tar.gz / RUN tar zxf /cbus-0.2.generic.tar.gz && rm /cbus-0.2.generic.tar.gz diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index c647207..3932706 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -22,6 +22,11 @@ from typing import Any, BinaryIO, Dict, Optional, Text, TextIO import paho.mqtt.client as mqtt +import sys + +if sys.platform == 'win32': + from asyncio import set_event_loop_policy, WindowsSelectorEventLoopPolicy + set_event_loop_policy(WindowsSelectorEventLoopPolicy()) try: from serial_asyncio import create_serial_connection From a1a7e8edfaf7e66486adc9d761a8f9e53406a8f2 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Thu, 21 Apr 2022 11:18:27 -0400 Subject: [PATCH 02/16] Provide way to bundle option files with the docker image Files in directory cmqttd__config are copied to image director etc/cmqttd. --- .gitignore | 5 +++++ Dockerfile | 1 + cmqttd_config/README.md | 16 ++++++++++++++++ 3 files changed, 22 insertions(+) create mode 100644 cmqttd_config/README.md diff --git a/.gitignore b/.gitignore index bae68ee..42ca6d7 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,8 @@ example.* home.* img .coverage +*.cbz +*.pem +*.key +auth +certificates diff --git a/Dockerfile b/Dockerfile index 3251b2d..4d5ae71 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ COPY COPYING COPYING.LESSER Dockerfile README.md entrypoint-cmqttd.sh / RUN sed -i 's/\r$//' entrypoint-cmqttd.sh COPY --from=builder /cbus/dist/cbus-0.2.generic.tar.gz / RUN tar zxf /cbus-0.2.generic.tar.gz && rm /cbus-0.2.generic.tar.gz +COPY cmqttd_config/ /etc/cmqttd/ # Runs cmqttd itself CMD /entrypoint-cmqttd.sh diff --git a/cmqttd_config/README.md b/cmqttd_config/README.md new file mode 100644 index 0000000..2acfdf4 --- /dev/null +++ b/cmqttd_config/README.md @@ -0,0 +1,16 @@ +# This folder should contain optional cmqttd configuration files. + +## project.cbz (file) +This file provides cbus group labels. Create a backup of the cbus project with the cbus setup tool, copy the back up file in this folder and rename it: project.cbz + +## auth (file) +Username and password to use to connect to an MQTT broker, separated by a newline character. +If this file is not present, then cmqttd will try to use the MQTT broker without authentication. + +## certificates (directory) +A directory of CA certificates to trust when connecting with TLS. +If this directory is not present, the default (Python) CA store will be used instead. + +## client.pem client.key (files) +Client certificate (pem) and private key (key) to use to connect to the MQTT broker. + From ef30f308ea722e0935d5edd10a25766e77e9090c Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Thu, 21 Apr 2022 17:21:24 -0400 Subject: [PATCH 03/16] Correct bug preventing parsing of XML project file ElementTree.parse(...) does not accept a bufferReader as argument but accepts a file path --- .gitignore | 2 ++ cbus/toolkit/cbz.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 42ca6d7..7cc6ad2 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,5 @@ img *.key auth certificates +.vscode +project.xml diff --git a/cbus/toolkit/cbz.py b/cbus/toolkit/cbz.py index 187e883..376e7d6 100644 --- a/cbus/toolkit/cbz.py +++ b/cbus/toolkit/cbz.py @@ -219,7 +219,7 @@ def __init__(self, fh: BinaryIO): zip_h = ZipFile(fh, 'r') except BadZipFile: # Try to load as XML instead. - xml_fh = fh + xml_fh = fh.name if zip_h: files = zip_h.namelist() From 155d2543c6e8dc721c300d61fa510da60fccb4a8 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Fri, 22 Apr 2022 20:22:03 -0400 Subject: [PATCH 04/16] Add option to select a bus network Add option to select a specific cbus network if the project file contains multiple networks. Backward compatibility is retained The argument --bus-network add to the main cmqttd call and the argument CMQTTD_CBUS_NETWORK is added to the entrypoint-cmqttd.sh --- cbus/daemon/cmqttd.py | 17 +++++++++++++++-- entrypoint-cmqttd.sh | 7 +++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index 3932706..96a7931 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -276,7 +276,7 @@ def read_auth(client: mqtt.Client, auth_file: TextIO): client.username_pw_set(username, password) -def read_cbz_labels(cbz_file: BinaryIO) -> Dict[int, Text]: +def read_cbz_labels(cbz_file: BinaryIO, network_name = None) -> Dict[int, Text]: """Reads group address names from a given Toolkit CBZ file.""" labels = {} # type: Dict[int, Text] cbz = CBZ(cbz_file) @@ -285,6 +285,10 @@ def read_cbz_labels(cbz_file: BinaryIO) -> Dict[int, Text]: # Look for 1 direct network networks = [n for n in cbz.installation.project.network if n.interface.interface_type != 'bridge'] + + if network_name: + networks = [n for n in networks if n.tag_name == network_name] + if len(networks) != 1: logger.warning('Expected exactly 1 non-bridge network in project file, ' 'got %d instead! Labels will be unavailable.', @@ -421,8 +425,17 @@ async def _main(): 'generated names like "C-Bus Light 001" will be used instead.' ) + group.add_argument( + '-N', '--cbus-network', + nargs='*', + help='Name of the C-Bus network to be used in case a project file is provided' + 'and the project contains multiple networks.' + ) + option = parser.parse_args() + option.cbus_network = " ".join(option.cbus_network) + if bool(option.broker_client_cert) != bool(option.broker_client_key): return parser.error( 'To use client certificates, both -k and -K must be specified.') @@ -433,7 +446,7 @@ async def _main(): loop = get_event_loop() connection_lost_future = loop.create_future() - labels = (read_cbz_labels(option.project_file) + labels = (read_cbz_labels(option.project_file,option.cbus_network) if option.project_file else None) def factory(): diff --git a/entrypoint-cmqttd.sh b/entrypoint-cmqttd.sh index e1d719b..33ecff9 100755 --- a/entrypoint-cmqttd.sh +++ b/entrypoint-cmqttd.sh @@ -82,6 +82,13 @@ else echo "${CMQTTD_PROJECT_FILE} not found; using generated labels." fi +echo ">${CMQTTD_CBUS_NETWORK}<" + +if [ -n "${CMQTTD_CBUS_NETWORK}" ]; then + echo "Loading C-Bus network ${CMQTTD_CBUS_NETWORK}" + CMQTTD_ARGS="${CMQTTD_ARGS} --cbus-network ${CMQTTD_CBUS_NETWORK}" +fi + echo "" # Announce what we think local time is on start-up. This will be sent to the C-Bus network. From 01a7ddfb23145496aa0c1e9c940dfeafa86a053b Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 20:37:52 -0400 Subject: [PATCH 05/16] Preparation for multi application --- .gitignore | 1 + cbus/common.py | 24 ++++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/.gitignore b/.gitignore index 7cc6ad2..6e9e1f3 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,4 @@ auth certificates .vscode project.xml +venv diff --git a/cbus/common.py b/cbus/common.py index ca56a3b..cc800c1 100644 --- a/cbus/common.py +++ b/cbus/common.py @@ -116,10 +116,28 @@ class Application(IntEnum): LIGHTING_5d = 0x5d LIGHTING_5e = 0x5e LIGHTING_LAST = LIGHTING_5f = 0x5f + HVACACTUATOR_73 = 0x73 + HVACACTUATOR_74 = 0x74 + HVAC = 0xCA CLOCK = 0xDF + TRIGGER = 0xCA ENABLE = 0xCB MASTER_APPLICATION = STATUS_REQUEST = 0xff + def isLighting(val): + if isinstance(val,Application): + val = int(val) + if isinstance(val,int): + return int(Application.LIGHTING_FIRST)<=val<=int(Application.LIGHTING_LAST) + return False + + def isHvacActuator(val): + if isinstance(val,Application): + val = int(val) + if isinstance(val,int): + return int(Application.HVACACTUATOR_73)==val or int(Application.HVACACTUATOR_74)==val + return False + class CAL(IntEnum): RESET = 0x08 @@ -429,3 +447,9 @@ def check_ga(group_addr: int) -> None: raise ValueError( 'Group Address out of range ({}..{}), got {}'.format( MIN_GROUP_ADDR, MAX_GROUP_ADDR, group_addr)) + + +def check_aa_lighting(app_addr: int | Application) -> None: + if not Application.isLighting(app_addr): + raise ValueError( + 'Expected lighting application address, got {}'.format(app_addr)) \ No newline at end of file From 218c09c3ee2d50286100118c999454e948d6135e Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 20:45:24 -0400 Subject: [PATCH 06/16] Add "application" argument to SAL class initialization This necessary to enable to the lightingSAL class to work with multilple lighting applications --- cbus/protocol/application/clock.py | 4 +-- cbus/protocol/application/enable.py | 4 +-- cbus/protocol/application/lighting.py | 37 +++++++++++---------- cbus/protocol/application/sal.py | 2 +- cbus/protocol/application/status_request.py | 4 +-- cbus/protocol/application/temperature.py | 4 +-- cbus/protocol/pciprotocol.py | 24 ++++++------- cbus/protocol/pm_packet.py | 2 +- 8 files changed, 42 insertions(+), 39 deletions(-) diff --git a/cbus/protocol/application/clock.py b/cbus/protocol/application/clock.py index 70809b1..b0f207b 100644 --- a/cbus/protocol/application/clock.py +++ b/cbus/protocol/application/clock.py @@ -50,7 +50,7 @@ def application(self) -> Application: return Application.CLOCK @staticmethod - def decode_sals(data: bytes) -> Sequence[ClockSAL]: + def decode_sals(data: bytes,_=None) -> Sequence[ClockSAL]: """ Decodes a clock broadcast application packet and returns it's SAL(s). @@ -251,7 +251,7 @@ def supported_applications() -> Set[Application]: return {Application.CLOCK} @staticmethod - def decode_sals(data: bytes) -> Sequence[SAL]: + def decode_sals(data: bytes,_=None) -> Sequence[SAL]: """ Decodes a clock and timekeeping application packet and returns its SAL(s). diff --git a/cbus/protocol/application/enable.py b/cbus/protocol/application/enable.py index cdafffb..9d6f545 100644 --- a/cbus/protocol/application/enable.py +++ b/cbus/protocol/application/enable.py @@ -43,7 +43,7 @@ def application(self) -> Application: return Application.ENABLE @staticmethod - def decode_sals(data: bytes) -> List[EnableSAL]: + def decode_sals(data: bytes,_=None) -> List[EnableSAL]: """ Decodes a enable control application packet and returns it's SAL(s). @@ -150,7 +150,7 @@ def supported_applications() -> Set[Application]: return {Application.ENABLE} @classmethod - def decode_sals(cls, data: bytes) -> List[EnableSAL]: + def decode_sals(cls, data: bytes, _ = None) -> List[EnableSAL]: """ Decodes a enable broadcast application packet and returns its SAL(s). """ diff --git a/cbus/protocol/application/lighting.py b/cbus/protocol/application/lighting.py index 2dcdfb6..bb67afc 100644 --- a/cbus/protocol/application/lighting.py +++ b/cbus/protocol/application/lighting.py @@ -49,23 +49,26 @@ class LightingSAL(SAL, abc.ABC): Base type for lighting application SALs. """ - def __init__(self, group_address: int): + def __init__(self, group_address: int, application_address: Union[Application,int]): """ This should not be called directly by your code! Use one of the subclasses of cbus.protocol.lighting.LightingSAL instead. """ + #TODO: modify to avoid redundancy + if not Application.isLighting(application_address): + raise ValueError('Expected light Application address, got {}'.format(application_address)) check_ga(group_address) + self.application_address = application_address self.group_address = group_address @property def application(self) -> Union[int, Application]: - # TODO: Support other application IDs - return Application.LIGHTING + return self.application_address @staticmethod - def decode_sals(data: bytes) -> List[LightingSAL]: + def decode_sals(data: bytes, application: int | Application ) -> List[LightingSAL]: """ Decodes a lighting application packet and returns it's SAL(s). @@ -98,7 +101,7 @@ def decode_sals(data: bytes) -> List[LightingSAL]: break sal, data = _SAL_HANDLERS[command_code].decode( - data, command_code, group_address) + data, command_code, group_address,application) if sal: output.append(sal) @@ -128,7 +131,7 @@ class LightingRampSAL(LightingSAL): """ - def __init__(self, group_address: int, duration: int, level: int): + def __init__(self, group_address: int,application_address: Union[int,Application] , duration: int, level: int ): """ Creates a new SAL Lighting Ramp message. @@ -142,14 +145,14 @@ def __init__(self, group_address: int, duration: int, level: int): indicating full brightness. :type level: int """ - super().__init__(group_address=group_address) + super().__init__(group_address,application_address ) self.duration = duration self.level = level @staticmethod def decode(data: bytes, command_code: int, - group_address: int) -> Tuple[Optional[LightingSAL], bytes]: + group_address: int, application_address: int | Application) -> Tuple[Optional[LightingSAL], bytes]: """ Do not call this method directly -- use LightingSAL.decode """ @@ -165,7 +168,7 @@ def decode(data: bytes, command_code: int, data = data[1:] return LightingRampSAL( - group_address=group_address, duration=duration, level=level), data + group_address=group_address, application_address=application_address,duration=duration, level=level), data def encode(self) -> bytes: if self.level < 0 or self.level > 255: @@ -192,11 +195,11 @@ class LightingOnSAL(LightingSAL): @staticmethod def decode(data: bytes, command_code: int, - group_address: int) -> Tuple[LightingOnSAL, bytes]: + group_address: int, application_address: int | Application) -> Tuple[LightingOnSAL, bytes]: """ Do not call this method directly -- use LightingSAL.decode """ - return LightingOnSAL(group_address), data + return LightingOnSAL(group_address,application_address), data def encode(self): return super().encode() + bytes([LightCommand.ON, self.group_address]) @@ -214,11 +217,11 @@ class LightingOffSAL(LightingSAL): @staticmethod def decode(data: bytes, command_code: int, - group_address: int) -> Tuple[LightingOffSAL, bytes]: + group_address: int, application_address: int | Application) -> Tuple[LightingOffSAL, bytes]: """ Do not call this method directly -- use LightingSAL.decode """ - return LightingOffSAL(group_address), data + return LightingOffSAL(group_address,application_address), data def encode(self): return super().encode() + bytes([LightCommand.OFF, self.group_address]) @@ -237,11 +240,11 @@ class LightingTerminateRampSAL(LightingSAL): @staticmethod def decode(data: bytes, command_code: int, - group_address: int) -> Tuple[LightingTerminateRampSAL, bytes]: + group_address: int, application_address: int | Application) -> Tuple[LightingTerminateRampSAL, bytes]: """ Do not call this method directly -- use LightingSAL.decode """ - return LightingTerminateRampSAL(group_address), data + return LightingTerminateRampSAL(group_address,application_address), data def encode(self): return super().encode() + bytes([ @@ -276,8 +279,8 @@ class LightingApplication(BaseApplication): """ @staticmethod - def decode_sals(data: bytes) -> List[SAL]: - return LightingSAL.decode_sals(data) + def decode_sals(data: bytes, application: int | Application ) -> List[SAL]: + return LightingSAL.decode_sals(data,application) @staticmethod def supported_applications() -> FrozenSet[int]: diff --git a/cbus/protocol/application/sal.py b/cbus/protocol/application/sal.py index 152e28d..719a54d 100644 --- a/cbus/protocol/application/sal.py +++ b/cbus/protocol/application/sal.py @@ -57,7 +57,7 @@ def supported_applications() -> FrozenSet[Union[int, Application]]: @staticmethod @abc.abstractmethod - def decode_sals(data: bytes) -> Sequence[SAL]: + def decode_sals(data: bytes,application: int| Application=None) -> Sequence[SAL]: """ Decodes a SAL message diff --git a/cbus/protocol/application/status_request.py b/cbus/protocol/application/status_request.py index 153a356..80c1f51 100644 --- a/cbus/protocol/application/status_request.py +++ b/cbus/protocol/application/status_request.py @@ -36,7 +36,7 @@ def supported_applications() -> FrozenSet[int]: return _SUPPORTED_APPLICATIONS @staticmethod - def decode_sals(data: bytes) -> Sequence[SAL]: + def decode_sals(data: bytes,_=None) -> Sequence[SAL]: return StatusRequestSAL.decode_sals(data) @@ -47,7 +47,7 @@ class StatusRequestSAL(SAL): child_application: int @classmethod - def decode_sals(cls, data: bytes) -> Sequence[StatusRequestSAL]: + def decode_sals(cls, data: bytes,_=None) -> Sequence[StatusRequestSAL]: output = [] while data: diff --git a/cbus/protocol/application/temperature.py b/cbus/protocol/application/temperature.py index 176b1fe..753835a 100644 --- a/cbus/protocol/application/temperature.py +++ b/cbus/protocol/application/temperature.py @@ -53,7 +53,7 @@ def application(self) -> Union[int, Application]: return Application.TEMPERATURE @staticmethod - def decode_sals(data: bytes) -> Sequence[TemperatureSAL]: + def decode_sals(data: bytes, _ = None) -> Sequence[TemperatureSAL]: """ Decodes a temperature broadcast application packet and returns its SAL(s). @@ -169,7 +169,7 @@ def supported_applications() -> Set[Application]: return {Application.TEMPERATURE} @staticmethod - def decode_sals(data: bytes) -> Sequence[TemperatureSAL]: + def decode_sals(data: bytes, _ = None) -> Sequence[TemperatureSAL]: """ Decodes a temperature broadcast application packet and returns it's SAL(s). diff --git a/cbus/protocol/pciprotocol.py b/cbus/protocol/pciprotocol.py index 08df01d..bf30285 100644 --- a/cbus/protocol/pciprotocol.py +++ b/cbus/protocol/pciprotocol.py @@ -114,17 +114,17 @@ def handle_cbus_packet(self, p: BasePacket) -> None: # lighting application if isinstance(s, LightingRampSAL): self.on_lighting_group_ramp(p.source_address, - s.group_address, + s.group_address,s.application_address, s.duration, s.level) elif isinstance(s, LightingOnSAL): self.on_lighting_group_on(p.source_address, - s.group_address) + s.group_address,s.application_address) elif isinstance(s, LightingOffSAL): self.on_lighting_group_off(p.source_address, - s.group_address) + s.group_address,s.application_address) elif isinstance(s, LightingTerminateRampSAL): self.on_lighting_group_terminate_ramp( - p.source_address, s.group_address) + p.source_address, s.group_address,s.application_address) else: logger.debug(f'hcp: unhandled lighting SAL type: {s!r}') elif isinstance(s, ClockSAL): @@ -430,7 +430,7 @@ def identify(self, unit_address, attribute): unit_address=unit_address, cals=[IdentifyCAL(attribute)]) return self._send(p) - def lighting_group_on(self, group_addr: Union[int, Iterable[int]]): + def lighting_group_on(self, group_addr: Union[int, Iterable[int]],application_addr: Union[int,Application] ): """ Turns on the lights for the given group_id. @@ -453,10 +453,10 @@ def lighting_group_on(self, group_addr: Union[int, Iterable[int]]): f'group_addr iterable length is > 9 ({group_addr_count})') p = PointToMultipointPacket( - sals=[LightingOnSAL(ga) for ga in group_addr]) + sals=[LightingOnSAL(ga,application_addr) for ga in group_addr]) return self._send(p) - def lighting_group_off(self, group_addr: Union[int, Iterable[int]]): + def lighting_group_off(self, group_addr: Union[int, Iterable[int]],application_addr: Union[int,Application] ): """ Turns off the lights for the given group_id. @@ -480,11 +480,11 @@ def lighting_group_off(self, group_addr: Union[int, Iterable[int]]): f'group_addr iterable length is > 9 ({group_addr_count})') p = PointToMultipointPacket( - sals=[LightingOffSAL(ga) for ga in group_addr]) + sals=[LightingOffSAL(ga,application_addr) for ga in group_addr]) return self._send(p) def lighting_group_ramp( - self, group_addr: int, duration: int, level: int = 255): + self, group_addr: int, application_addr: Union[int,Application], duration: int, level: int = 255 ): """ Ramps (fades) a group address to a specified lighting level. @@ -506,11 +506,11 @@ def lighting_group_ramp( """ p = PointToMultipointPacket( - sals=LightingRampSAL(group_addr, duration, level)) + sals=LightingRampSAL(group_addr, application_addr, duration, level)) return self._send(p) def lighting_group_terminate_ramp( - self, group_addr: Union[int, Iterable[int]]): + self, group_addr: Union[int, Iterable[int]], application_addr: Union[int,Application]): """ Stops ramping a group address at the current point. @@ -533,7 +533,7 @@ def lighting_group_terminate_ramp( f'group_addr iterable length is > 9 ({group_addr_count})') p = PointToMultipointPacket( - sals=[LightingTerminateRampSAL(ga) for ga in group_addr]) + sals=[LightingTerminateRampSAL(ga,application_addr) for ga in group_addr]) return self._send(p) def clock_datetime(self, when: Optional[datetime] = None): diff --git a/cbus/protocol/pm_packet.py b/cbus/protocol/pm_packet.py index 8aafd6a..747f9fa 100644 --- a/cbus/protocol/pm_packet.py +++ b/cbus/protocol/pm_packet.py @@ -107,7 +107,7 @@ def decode_packet( # find an application handler handler = get_application(application) data = data[2:] - sals = handler.decode_sals(data) + sals = handler.decode_sals(data,application) return cls( checksum=checksum, priority_class=priority_class, From 4056315ec466be038bec4075c23b48b47a142ab9 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 20:47:52 -0400 Subject: [PATCH 07/16] CNI set up for multi applications Change CNI parameters to enable relaying of all applications packets. --- cbus/protocol/pciprotocol.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/cbus/protocol/pciprotocol.py b/cbus/protocol/pciprotocol.py index bf30285..5ca5516 100644 --- a/cbus/protocol/pciprotocol.py +++ b/cbus/protocol/pciprotocol.py @@ -384,10 +384,17 @@ def pci_reset(self): self._send(ResetPacket()) # serial user interface guide sect 10.2 - # Set application address 1 to 38 (lighting) - # self._send('A3210038', encode=False, checksum=False) + # Set application address 1 to ALL applications + # self._send('A32100FF', encode=False, checksum=False) self._send(DeviceManagementPacket( - checksum=False, parameter=0x21, value=0x38), + checksum=False, parameter=0x21, value=0xFF), + basic_mode=True) + + # serial user interface guide sect 10.2 + # Set application address 2 to USED applications + # self._send('A32200FF', encode=False, checksum=False) + self._send(DeviceManagementPacket( + checksum=False, parameter=0x22, value=0xFF), basic_mode=True) # Interface options #3 From 06287742cb2015705c9b0785e8dd90fc40faa425 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 20:53:51 -0400 Subject: [PATCH 08/16] Expand daemon to handle all Lighting application Application argument added to mttq client and Cbus handler --- cbus/daemon/cmqttd.py | 199 +++++++++++++++++++++++++----------------- 1 file changed, 120 insertions(+), 79 deletions(-) diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index 96a7931..aaae6db 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -24,6 +24,8 @@ import paho.mqtt.client as mqtt import sys +from cbus.protocol import application + if sys.platform == 'win32': from asyncio import set_event_loop_policy, WindowsSelectorEventLoopPolicy set_event_loop_policy(WindowsSelectorEventLoopPolicy()) @@ -34,7 +36,7 @@ async def create_serial_connection(*_, **__): raise ImportError('Serial device support requires pyserial-asyncio') -from cbus.common import MIN_GROUP_ADDR, MAX_GROUP_ADDR, check_ga, Application +from cbus.common import MIN_GROUP_ADDR, MAX_GROUP_ADDR, check_aa_lighting, check_ga, Application from cbus.paho_asyncio import AsyncioHelper from cbus.protocol.pciprotocol import PCIProtocol from cbus.toolkit.cbz import CBZ @@ -48,45 +50,57 @@ async def create_serial_connection(*_, **__): _TOPIC_CONF_SUFFIX = '/config' _TOPIC_STATE_SUFFIX = '/state' _META_TOPIC = 'homeassistant/binary_sensor/cbus_cmqttd' - +_APPLICATION_GROUP_SEPARATOR = "_" def ga_range(): return range(MIN_GROUP_ADDR, MAX_GROUP_ADDR + 1) +def ga_string(group_addr: int, app_addr: int | Application, zeros=True) -> Text: + #note: the logic is necessary to ensure backward compatibility with previous version + if(app_addr==Application.LIGHTING): + return f'{group_addr:03d}' if zeros else f'{group_addr}' + else: + return f'{app_addr:03d}{_APPLICATION_GROUP_SEPARATOR}{group_addr:03d}' + +def default_light_name(group_addr,app_addr): + return f'C-Bus Light {ga_string(group_addr,app_addr,True)}' -def get_topic_group_address(topic: Text) -> int: +def get_topic_group_address(topic: Text) -> tuple[int,int | Application]: """Gets the group address for the given topic.""" if not topic.startswith(_LIGHT_TOPIC_PREFIX): raise ValueError( f'Invalid topic {topic}, must start with {_LIGHT_TOPIC_PREFIX}') - ga = int(topic[len(_LIGHT_TOPIC_PREFIX):].split('/', maxsplit=1)[0]) + a1,*a2 = topic[len(_LIGHT_TOPIC_PREFIX):].split('/', maxsplit=1)[0].split(_APPLICATION_GROUP_SEPARATOR) + aa,ga = (a1,a2[0]) if a2 else (Application.LIGHTING,a1) + aa,ga = (int(aa),int(ga)) check_ga(ga) - return ga - + check_aa_lighting(aa) + return ga,aa -def set_topic(group_addr: int) -> Text: +def set_topic(group_addr: int, app_addr: int | Application) -> Text: """Gets the Set topic for a group address.""" - return _LIGHT_TOPIC_PREFIX + str(group_addr) + _TOPIC_SET_SUFFIX + return (_LIGHT_TOPIC_PREFIX + ga_string(group_addr,app_addr,False) + _TOPIC_SET_SUFFIX ) -def state_topic(group_addr: int) -> Text: +def state_topic(group_addr: int, app_addr: int | Application) -> Text: """Gets the State topic for a group address.""" - return _LIGHT_TOPIC_PREFIX + str(group_addr) + _TOPIC_STATE_SUFFIX + return _LIGHT_TOPIC_PREFIX + ga_string(group_addr,app_addr,False) + _TOPIC_STATE_SUFFIX -def conf_topic(group_addr: int) -> Text: +def conf_topic(group_addr: int, app_addr: int | Application) -> Text: """Gets the Config topic for a group address.""" - return _LIGHT_TOPIC_PREFIX + str(group_addr) + _TOPIC_CONF_SUFFIX + return _LIGHT_TOPIC_PREFIX + ga_string(group_addr,app_addr,False) + _TOPIC_CONF_SUFFIX -def bin_sensor_state_topic(group_addr: int) -> Text: +def bin_sensor_state_topic(group_addr: int, app_addr: int | Application) -> Text: """Gets the Binary Sensor State topic for a group address.""" - return _BINSENSOR_TOPIC_PREFIX + str(group_addr) + _TOPIC_STATE_SUFFIX + return _BINSENSOR_TOPIC_PREFIX + ga_string(group_addr,app_addr,False) + _TOPIC_STATE_SUFFIX -def bin_sensor_conf_topic(group_addr: int) -> Text: +def bin_sensor_conf_topic(group_addr: int, app_addr: int | Application) -> Text: """Gets the Binary Sensor Config topic for a group address.""" - return _BINSENSOR_TOPIC_PREFIX + str(group_addr) + _TOPIC_CONF_SUFFIX + return _BINSENSOR_TOPIC_PREFIX + ga_string(group_addr,app_addr,False) + _TOPIC_CONF_SUFFIX + class CBusHandler(PCIProtocol): @@ -95,27 +109,30 @@ class CBusHandler(PCIProtocol): """ mqtt_api = None - def __init__(self, labels: Optional[Dict[int, Text]], *args, **kwargs): + def __init__(self, labels: Optional[Dict[int, Dict]], *args, **kwargs): super().__init__(*args, **kwargs) self.labels = ( - labels if labels is not None else {}) # type: Dict[int, Text] + labels if labels is not None else {56:{}}) # type: Dict[int, Text] - def on_lighting_group_ramp(self, source_addr, group_addr, duration, level): + def on_lighting_group_ramp(self, source_addr, group_addr, app_addr,duration, level): if not self.mqtt_api: return self.mqtt_api.lighting_group_ramp( - source_addr, group_addr, duration, level) + source_addr, group_addr, app_addr,duration, level) + #need to address application - def on_lighting_group_on(self, source_addr, group_addr): + def on_lighting_group_on(self, source_addr, group_addr,app_addr): + #need to address application if not self.mqtt_api: return - self.mqtt_api.lighting_group_on(source_addr, group_addr) + self.mqtt_api.lighting_group_on(source_addr, group_addr,app_addr) - def on_lighting_group_off(self, source_addr, group_addr): + def on_lighting_group_off(self, source_addr, group_addr,app_addr): + #need to address application if not self.mqtt_api: return - self.mqtt_api.lighting_group_off(source_addr, group_addr) + self.mqtt_api.lighting_group_off(source_addr, group_addr,app_addr) # TODO: on_lighting_group_terminate_ramp @@ -128,9 +145,10 @@ class MqttClient(mqtt.Client): def on_connect(self, client, userdata: CBusHandler, flags, rc): logger.info('Connected to MQTT broker') userdata.mqtt_api = self - self.subscribe([(set_topic(ga), 2) for ga in ga_range()]) + self.groupDB = {} self.publish_all_lights(userdata.labels) + def on_message(self, client, userdata: CBusHandler, msg: mqtt.MQTTMessage): """Handle a message from an MQTT subscription.""" if not (msg.topic.startswith(_LIGHT_TOPIC_PREFIX) and @@ -138,7 +156,7 @@ def on_message(self, client, userdata: CBusHandler, msg: mqtt.MQTTMessage): return try: - ga = get_topic_group_address(msg.topic) + ga, aa = get_topic_group_address(msg.topic) except ValueError: # Invalid group address logging.error(f'Invalid group address in topic {msg.topic}') @@ -164,23 +182,69 @@ def on_message(self, client, userdata: CBusHandler, msg: mqtt.MQTTMessage): if light_on: if brightness == 255 and transition_time == 0: # lighting on - userdata.lighting_group_on(ga) - self.lighting_group_on(None, ga) + userdata.lighting_group_on(ga,aa) + self.lighting_group_on(None, ga,aa) else: # ramp - userdata.lighting_group_ramp(ga, transition_time, brightness) - self.lighting_group_ramp(None, ga, transition_time, brightness) + userdata.lighting_group_ramp(ga, aa, transition_time, brightness) + self.lighting_group_ramp(None, ga, aa, transition_time, brightness) else: # lighting off - userdata.lighting_group_off(ga) - self.lighting_group_off(None, ga) + userdata.lighting_group_off(ga,aa) + self.lighting_group_off(None, ga,aa) def publish(self, topic: Text, payload: Dict[Text, Any]): """Publishes a payload as JSON.""" payload = json.dumps(payload) return super().publish(topic, payload, 1, True) - def publish_all_lights(self, labels: Dict[int, Text]): + def publish_light(self, group_addr : int, app_addr :int | Application, app_labels:dict = None): + default_name = default_light_name(group_addr,app_addr) + UID = f'cbus_light_{ga_string(group_addr,app_addr,False)}' + name = default_name + if app_labels: + _,labels = app_labels.get(app_addr,(None,{})) + if labels: + name = labels.get(group_addr,name) + UID = f'cbus_light_{ga_string(group_addr,app_addr,False)}' + self.subscribe(set_topic(group_addr,app_addr), 2 ) + self.publish(conf_topic(group_addr,app_addr), { + 'name': name, + 'unique_id': UID, + 'cmd_t': set_topic(group_addr,app_addr), + 'stat_t': state_topic(group_addr,app_addr), + 'schema': 'json', + 'brightness': True, + 'device': { + 'identifiers': [UID], + 'connections': [['cbus_group_address', str(group_addr)],['cbus_application_address', str(app_addr)]], + 'sw_version': 'cmqttd https://github.com/micolous/cbus', + 'name': default_name, + 'manufacturer': 'Clipsal', + 'model': 'C-Bus Lighting Application', + 'via_device': 'cmqttd', + }, + }) + Sensor_UID = f'cbus_bin_sensor_{ga_string(group_addr,app_addr,False)}' + self.publish(bin_sensor_conf_topic(group_addr,app_addr), { + 'name': f'{name} (as binary sensor)', + 'unique_id': Sensor_UID, + 'stat_t': bin_sensor_state_topic(group_addr,app_addr), + 'device': { + 'identifiers': [Sensor_UID], + 'connections': [['cbus_group_address', str(group_addr)],['cbus_application_address', str(app_addr)]], + 'sw_version': 'cmqttd https://github.com/micolous/cbus', + 'name': default_name, + 'manufacturer': 'Clipsal', + 'model': 'C-Bus Lighting Application', + 'via_device': 'cmqttd', + }, + }) + self.groupDB.setdefault(app_addr,{})[group_addr] = True + + + + def publish_all_lights(self, app_labels): """Publishes a configuration topic for all lights.""" # Meta-device which holds all the C-Bus group addresses self.publish(_META_TOPIC + _TOPIC_CONF_SUFFIX, { @@ -197,76 +261,53 @@ def publish_all_lights(self, labels: Dict[int, Text]): }, }) - for ga in ga_range(): - name = labels.get(ga, f'C-Bus Light {ga:03d}') - self.publish(conf_topic(ga), { - 'name': name, - 'unique_id': f'cbus_light_{ga}', - 'cmd_t': set_topic(ga), - 'stat_t': state_topic(ga), - 'schema': 'json', - 'brightness': True, - 'device': { - 'identifiers': [f'cbus_light_{ga}'], - 'connections': [['cbus_group_address', str(ga)]], - 'sw_version': 'cmqttd https://github.com/micolous/cbus', - 'name': f'C-Bus Light {ga:03d}', - 'manufacturer': 'Clipsal', - 'model': 'C-Bus Lighting Application', - 'via_device': 'cmqttd', - }, - }) - - self.publish(bin_sensor_conf_topic(ga), { - 'name': f'{name} (as binary sensor)', - 'unique_id': f'cbus_bin_sensor_{ga}', - 'stat_t': bin_sensor_state_topic(ga), - 'device': { - 'identifiers': [f'cbus_bin_sensor_{ga}'], - 'connections': [['cbus_group_address', str(ga)]], - 'sw_version': 'cmqttd https://github.com/micolous/cbus', - 'name': f'C-Bus Light {ga:03d}', - 'manufacturer': 'Clipsal', - 'model': 'C-Bus Lighting Application', - 'via_device': 'cmqttd', - }, - }) - - def publish_binary_sensor(self, group_addr: int, state: bool): + for aa,(_,labels) in app_labels.items(): + for ga in labels.keys(): + self.publish_light(ga,aa,app_labels) + + def check_published(self,group_addr: int, app_addr: int | Application): + if not self.groupDB.setdefault(app_addr,{}).get(group_addr,False): + self.publish_light(group_addr,app_addr) + + + def publish_binary_sensor(self, group_addr: int, app_addr: int | Application, state: bool): payload = 'ON' if state else 'OFF' return super().publish( - bin_sensor_state_topic(group_addr), payload, 1, True) + bin_sensor_state_topic(group_addr,app_addr), payload, 1, True) - def lighting_group_on(self, source_addr: Optional[int], group_addr: int): + def lighting_group_on(self, source_addr: Optional[int], group_addr: int, app_addr: int | Application): """Relays a lighting-on event from CBus to MQTT.""" - self.publish(state_topic(group_addr), { + self.check_published(group_addr,app_addr) + self.publish(state_topic(group_addr,app_addr), { 'state': 'ON', 'brightness': 255, 'transition': 0, 'cbus_source_addr': source_addr, }) - self.publish_binary_sensor(group_addr, True) + self.publish_binary_sensor(group_addr,app_addr, True) - def lighting_group_off(self, source_addr: Optional[int], group_addr: int): + def lighting_group_off(self, source_addr: Optional[int], group_addr: int, app_addr: int | Application): """Relays a lighting-off event from CBus to MQTT.""" - self.publish(state_topic(group_addr), { + self.check_published(group_addr,app_addr) + self.publish(state_topic(group_addr,app_addr), { 'state': 'OFF', 'brightness': 0, 'transition': 0, 'cbus_source_addr': source_addr, }) - self.publish_binary_sensor(group_addr, False) + self.publish_binary_sensor(group_addr,app_addr, False) - def lighting_group_ramp(self, source_addr: Optional[int], group_addr: int, + def lighting_group_ramp(self, source_addr: Optional[int], group_addr: int, app_addr:int|Application, duration: int, level: int): """Relays a lighting-ramp event from CBus to MQTT.""" - self.publish(state_topic(group_addr), { + self.check_published(group_addr,app_addr) + self.publish(state_topic(group_addr,app_addr), { 'state': 'ON', 'brightness': level, 'transition': duration, 'cbus_source_addr': source_addr, }) - self.publish_binary_sensor(group_addr, level > 0) + self.publish_binary_sensor(group_addr, app_addr, level > 0) def read_auth(client: mqtt.Client, auth_file: TextIO): From c639d2b668305727976fa91158cd7784e9c93f77 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 21:28:41 -0400 Subject: [PATCH 09/16] Update label extraction Extract label from all lighting applications. Default label generation is moved here (from MQTT onconnect event. Default labels only generated if no label availabl for application --- cbus/daemon/cmqttd.py | 64 +++++++++++++++++++++++++++---------------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index aaae6db..ce91957 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -319,11 +319,13 @@ def read_auth(client: mqtt.Client, auth_file: TextIO): def read_cbz_labels(cbz_file: BinaryIO, network_name = None) -> Dict[int, Text]: """Reads group address names from a given Toolkit CBZ file.""" - labels = {} # type: Dict[int, Text] + class obj(): + pass + + labels = {56:{}} # type: Dict[int,Dict[int, Text]] + cbz = CBZ(cbz_file) - # TODO: support multiple networks/applications - # Look for 1 direct network networks = [n for n in cbz.installation.project.network if n.interface.interface_type != 'bridge'] @@ -331,29 +333,43 @@ def read_cbz_labels(cbz_file: BinaryIO, network_name = None) -> Dict[int, Text]: networks = [n for n in networks if n.tag_name == network_name] if len(networks) != 1: - logger.warning('Expected exactly 1 non-bridge network in project file, ' - 'got %d instead! Labels will be unavailable.', - len(networks)) - return labels - - # Look for - applications = [a for a in networks[0].applications - if a.address == Application.LIGHTING] - if len(applications) != 1: - logger.warning('Could not find lighting application %x in project ' - 'file. Labels will be unavailable.', - Application.LIGHTING) - return labels - - for group in applications[0].groups: - name = group.tag_name.strip() - - # Ignore default names - if not name or name in ('', f'Group {group.address}'): - continue + if network_name: + logger.warning('Could not find a non-bridge network with name "%s" in project file.',network_name) + else: + logger.warning('Expected one non-bridge network in project file, found %d instead',len(networks)) + app = obj() + app.address = 56 + app.tag_name = "Lighting" + app.groups = [] + net = obj() + net.applications = [app] + networks = [net] - labels[group.address] = name + applications = [a for a in networks[0].applications if Application.isLighting(a.address)] + + if len(applications) == 0: + logger.warning('Could not find any lighting application in project file.') + app = obj() + app.address = 56 + app.tag_name = "Lighting" + app.groups = [] + applications = [app] + + for a in applications: + l = {} + if a.groups: + for group in a.groups: + name = group.tag_name.strip() + if not name or name in ('', f'Group {group.address}'): + name = default_light_name(group.address,a.address) + l[group.address] = name + else: + logger.warning('No label available in project file for application %d.',a.address) + logger.warning('Will use default labels.') + for ga in ga_range(): + l[ga] = default_light_name(ga,a.address) + labels[a.address]=(a.tag_name,l) return labels From 6a52e1f6102ef4831508bf047b4e617ed6bebe0f Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 21:30:59 -0400 Subject: [PATCH 10/16] Update Python and packages Python 3.10 required. At this time only alpine edge includes the release. Docker pull will need to be modified after next regular release --- Dockerfile | 7 ++++--- requirements-cmqttd.txt | 4 ++-- requirements.txt | 6 +++--- setup.cfg | 2 +- setup.py | 6 +++--- 5 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4d5ae71..86022f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,12 +8,13 @@ # $ docker build -t cmqttd . # $ docker run --device /dev/ttyUSB0 -e "SERIAL_PORT=/dev/ttyUSB0" \ # -e "MQTT_SERVER=192.2.0.1" -e "TZ=Australia/Adelaide" -it cmqttd -FROM alpine:3.11 as base +FROM alpine:edge as base +# python 3.10 required, at date this file is created only available in alpine:edge # Install most Python deps here, because that way we don't need to include build tools in the # final image. -RUN apk add --no-cache python3 py3-cffi py3-paho-mqtt py3-six tzdata && \ - pip3 install 'pyserial==3.4' 'pyserial_asyncio==0.4' +RUN apk add --no-cache python3 py-pip py3-cffi py3-paho-mqtt py3-six tzdata && \ + pip3 install 'pyserial==3.5' 'pyserial_asyncio==0.6' # Runs tests and builds a distribution tarball FROM base as builder diff --git a/requirements-cmqttd.txt b/requirements-cmqttd.txt index a8f9910..e67a24a 100644 --- a/requirements-cmqttd.txt +++ b/requirements-cmqttd.txt @@ -1,4 +1,4 @@ pyserial==3.4 -pyserial_asyncio (==0.4) +pyserial_asyncio (==0.6) six -paho-mqtt==1.5.0 +paho-mqtt==1.6.1 diff --git a/requirements.txt b/requirements.txt index 2394302..9beddbb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -pyserial==3.4 -pyserial_asyncio==0.4 +pyserial==3.5 +pyserial_asyncio==0.6 six # official protocol documentation downloading @@ -9,7 +9,7 @@ requests pydot # required for cmqttd -paho-mqtt==1.5.0 +paho-mqtt==1.6.1 # required for cbz lxml diff --git a/setup.cfg b/setup.cfg index fcb3456..d468769 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,5 +3,5 @@ based_on_style = google [pytype] inputs = cbus -version = 3.7 +version = 3.10 exclude = cbus/toolkit/cbz.py diff --git a/setup.py b/setup.py index bb6a456..ba69795 100755 --- a/setup.py +++ b/setup.py @@ -3,12 +3,12 @@ from setuptools import setup, find_packages deps = [ - 'pyserial (==3.4)', - 'pyserial_asyncio (==0.4)', + 'pyserial (==3.5)', + 'pyserial_asyncio (==0.6)', 'lxml (>=2.3.2)', 'six', 'pydot', - 'paho_mqtt (==1.5.0)' + 'paho_mqtt (==1.6.1)' ] tests_require = [ From 41e74ccda3733da6e0c5ab7e84e003bee4a83a5f Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Mon, 25 Apr 2022 21:33:12 -0400 Subject: [PATCH 11/16] Updated test routines Updated test code with application argument where necessary. No change/addition to the test case. --- tests/test_cmqttd.py | 20 ++++++++++---------- tests/test_lighting.py | 18 ++++++++++-------- tests/test_pm_packet.py | 6 +++--- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/tests/test_cmqttd.py b/tests/test_cmqttd.py index d24fe7e..3e295e0 100644 --- a/tests/test_cmqttd.py +++ b/tests/test_cmqttd.py @@ -23,7 +23,7 @@ from typing import Optional, Text, cast import unittest -from cbus.common import check_ga +from cbus.common import Application, check_ga from cbus.daemon import cmqttd @@ -59,22 +59,22 @@ def test_valid_topic_group_address(self, ga): bin_topic_len = len(bin_topic) # base topic path -> ga - self.assertEqual(ga, cmqttd.get_topic_group_address(light_topic)) + self.assertEqual((ga,Application.LIGHTING), cmqttd.get_topic_group_address(light_topic)) # Generating a set topic - set_topic = cmqttd.set_topic(ga) + set_topic = cmqttd.set_topic(ga,Application.LIGHTING) self.assertEqual(light_topic, set_topic[:light_topic_len]) - self.assertEqual(ga, cmqttd.get_topic_group_address(set_topic)) + self.assertEqual((ga,Application.LIGHTING), cmqttd.get_topic_group_address(set_topic)) # Generating a state topic - state_topic = cmqttd.state_topic(ga) + state_topic = cmqttd.state_topic(ga,Application.LIGHTING) self.assertEqual(light_topic, state_topic[:light_topic_len]) - self.assertEqual(ga, cmqttd.get_topic_group_address(state_topic)) + self.assertEqual((ga,Application.LIGHTING), cmqttd.get_topic_group_address(state_topic)) # Generating a conf topic - conf_topic = cmqttd.conf_topic(ga) + conf_topic = cmqttd.conf_topic(ga,Application.LIGHTING) self.assertEqual(light_topic, conf_topic[:light_topic_len]) - self.assertEqual(ga, cmqttd.get_topic_group_address(conf_topic)) + self.assertEqual((ga,Application.LIGHTING), cmqttd.get_topic_group_address(conf_topic)) # Ensure all the topics are unique self.assertNotEqual(set_topic, state_topic) @@ -83,10 +83,10 @@ def test_valid_topic_group_address(self, ga): # Binary sensors are read only, so get_topic_group_address doesn't # support them. - bin_state_topic = cmqttd.bin_sensor_state_topic(ga) + bin_state_topic = cmqttd.bin_sensor_state_topic(ga,Application.LIGHTING) self.assertTrue(bin_topic, bin_state_topic[:bin_topic_len]) - bin_conf_topic = cmqttd.bin_sensor_conf_topic(ga) + bin_conf_topic = cmqttd.bin_sensor_conf_topic(ga,Application.LIGHTING) self.assertTrue(bin_topic, bin_conf_topic[:bin_topic_len]) # Uniqueness check diff --git a/tests/test_lighting.py b/tests/test_lighting.py index f4804e8..e3f62ec 100644 --- a/tests/test_lighting.py +++ b/tests/test_lighting.py @@ -16,8 +16,10 @@ # along with this library. If not, see . from __future__ import absolute_import +from email.mime import application import unittest +from cbus.common import Application from cbus.protocol.pm_packet import PointToMultipointPacket from cbus.protocol.application.lighting import ( @@ -127,7 +129,7 @@ class InternalLightingTest(CBusTestCase): def test_lighting_encode_decode(self): """test of encode then decode""" - orig = PointToMultipointPacket(sals=LightingOnSAL(27)) + orig = PointToMultipointPacket(sals=LightingOnSAL(27,Application.LIGHTING)) orig.source_address = 5 data = orig.encode_packet() + b'\r\n' @@ -142,7 +144,7 @@ def test_lighting_encode_decode(self): def test_lighting_encode_decode_client(self): """test of encode then decode, with packets from a client""" - orig = PointToMultipointPacket(sals=LightingOnSAL(27)) + orig = PointToMultipointPacket(sals=LightingOnSAL(27,Application.LIGHTING)) data = b'\\' + orig.encode_packet() + b'\r' @@ -155,23 +157,23 @@ def test_lighting_encode_decode_client(self): def test_invalid_ga(self): """test argument validation""" with self.assertRaises(ValueError): - PointToMultipointPacket(sals=LightingOnSAL(999)) + PointToMultipointPacket(sals=LightingOnSAL(999,Application.LIGHTING)) with self.assertRaises(ValueError): - PointToMultipointPacket(sals=LightingOffSAL(-1)) + PointToMultipointPacket(sals=LightingOffSAL(-1,Application.LIGHTING)) def test_slow_ramp(self): """test very slow ramps""" p1 = PointToMultipointPacket( - sals=LightingRampSAL(1, 18*60, 255)).encode_packet() + sals=LightingRampSAL(1,Application.LIGHTING, 18*60, 255)).encode_packet() p2 = PointToMultipointPacket( - sals=LightingRampSAL(1, 17*60, 255)).encode_packet() + sals=LightingRampSAL(1,Application.LIGHTING, 17*60, 255)).encode_packet() self.assertEqual(p1, p2) def test_brightness_bounds(self): with self.assertRaisesRegex(ValueError, r'Ramp level .+ bounds'): - LightingRampSAL(1, 10, -1).encode() + LightingRampSAL(1,Application.LIGHTING, 10, -1).encode() with self.assertRaisesRegex(ValueError, r'Ramp level .+ bounds'): - LightingRampSAL(1, 10, 256).encode() + LightingRampSAL(1,Application.LIGHTING, 10, 256).encode() class LightingRegressionTest(CBusTestCase): diff --git a/tests/test_pm_packet.py b/tests/test_pm_packet.py index 1c7b5a0..4372776 100644 --- a/tests/test_pm_packet.py +++ b/tests/test_pm_packet.py @@ -64,14 +64,14 @@ def test_invalid_multiple_application_sal(self): with self.assertRaisesRegex( ValueError, r'SAL .+ of application ff, .+ has application 38'): PointToMultipointPacket(sals=[ - LightingOffSAL(1), + LightingOffSAL(1,Application.LIGHTING), StatusRequestSAL(level_request=True, group_address=1, child_application=Application.LIGHTING), ]) def test_remove_sals(self): # create a packet - p = PointToMultipointPacket(sals=LightingOffSAL(1)) + p = PointToMultipointPacket(sals=LightingOffSAL(1,Application.LIGHTING)) self.assertEqual(1, len(p)) p.clear_sal() @@ -84,7 +84,7 @@ def test_remove_sals(self): # Adding another lighting SAL should fail with self.assertRaisesRegex(ValueError, r'has application ff$'): - p.append_sal(LightingOffSAL(1)) + p.append_sal(LightingOffSAL(1,Application.LIGHTING)) self.assertEqual(1, len(p)) def test_invalid_sal(self): From 1a8946e78b49175b82ce69a16a5d1712a2547a67 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Wed, 27 Apr 2022 13:44:22 -0400 Subject: [PATCH 12/16] Clean obsolete comments --- cbus/daemon/cmqttd.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index ce91957..3f113f0 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -120,16 +120,13 @@ def on_lighting_group_ramp(self, source_addr, group_addr, app_addr,duration, lev return self.mqtt_api.lighting_group_ramp( source_addr, group_addr, app_addr,duration, level) - #need to address application def on_lighting_group_on(self, source_addr, group_addr,app_addr): - #need to address application if not self.mqtt_api: return self.mqtt_api.lighting_group_on(source_addr, group_addr,app_addr) def on_lighting_group_off(self, source_addr, group_addr,app_addr): - #need to address application if not self.mqtt_api: return self.mqtt_api.lighting_group_off(source_addr, group_addr,app_addr) From 7fa0ffe002767f2b8865b1905df31c7340fddb62 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Wed, 27 Apr 2022 13:51:13 -0400 Subject: [PATCH 13/16] Small update gitignore --- .gitignore | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 6e9e1f3..d580c5e 100644 --- a/.gitignore +++ b/.gitignore @@ -16,10 +16,11 @@ example.* home.* img .coverage -*.cbz -*.pem -*.key -auth +cmqttd_config/*.cbz +cmqttd_config/*.pem +cmqttd_config/*.key¸ +cmqttd_config/*.xml +cmqttd_config/auth certificates .vscode project.xml From 393036092b0697ff3f8b1b7cd6ba5aceb5a5273d Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Wed, 27 Apr 2022 21:20:17 -0400 Subject: [PATCH 14/16] Removing redundant function isLighting(val) not needed. --- cbus/common.py | 18 ------------------ cbus/daemon/cmqttd.py | 14 ++++++++++---- cbus/protocol/application/lighting.py | 2 +- cbus/protocol/pciprotocol.py | 2 +- 4 files changed, 12 insertions(+), 24 deletions(-) diff --git a/cbus/common.py b/cbus/common.py index cc800c1..93b89e9 100644 --- a/cbus/common.py +++ b/cbus/common.py @@ -124,19 +124,6 @@ class Application(IntEnum): ENABLE = 0xCB MASTER_APPLICATION = STATUS_REQUEST = 0xff - def isLighting(val): - if isinstance(val,Application): - val = int(val) - if isinstance(val,int): - return int(Application.LIGHTING_FIRST)<=val<=int(Application.LIGHTING_LAST) - return False - - def isHvacActuator(val): - if isinstance(val,Application): - val = int(val) - if isinstance(val,int): - return int(Application.HVACACTUATOR_73)==val or int(Application.HVACACTUATOR_74)==val - return False class CAL(IntEnum): @@ -448,8 +435,3 @@ def check_ga(group_addr: int) -> None: 'Group Address out of range ({}..{}), got {}'.format( MIN_GROUP_ADDR, MAX_GROUP_ADDR, group_addr)) - -def check_aa_lighting(app_addr: int | Application) -> None: - if not Application.isLighting(app_addr): - raise ValueError( - 'Expected lighting application address, got {}'.format(app_addr)) \ No newline at end of file diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index 3f113f0..37ae290 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -36,10 +36,11 @@ async def create_serial_connection(*_, **__): raise ImportError('Serial device support requires pyserial-asyncio') -from cbus.common import MIN_GROUP_ADDR, MAX_GROUP_ADDR, check_aa_lighting, check_ga, Application +from cbus.common import MIN_GROUP_ADDR, MAX_GROUP_ADDR, check_ga, Application from cbus.paho_asyncio import AsyncioHelper from cbus.protocol.pciprotocol import PCIProtocol from cbus.toolkit.cbz import CBZ +from cbus.protocol.application import LightingApplication logger = logging.getLogger(__name__) @@ -52,6 +53,12 @@ async def create_serial_connection(*_, **__): _META_TOPIC = 'homeassistant/binary_sensor/cbus_cmqttd' _APPLICATION_GROUP_SEPARATOR = "_" +def check_aa_lighting(aa): + if not aa in LightingApplication.supported_applications(): + raise ValueError( + 'Application ${aa} is not a valid lighting application'.format( + aa)) + def ga_range(): return range(MIN_GROUP_ADDR, MAX_GROUP_ADDR + 1) @@ -74,7 +81,7 @@ def get_topic_group_address(topic: Text) -> tuple[int,int | Application]: aa,ga = (a1,a2[0]) if a2 else (Application.LIGHTING,a1) aa,ga = (int(aa),int(ga)) check_ga(ga) - check_aa_lighting(aa) + return ga,aa def set_topic(group_addr: int, app_addr: int | Application) -> Text: @@ -342,8 +349,7 @@ class obj(): net.applications = [app] networks = [net] - - applications = [a for a in networks[0].applications if Application.isLighting(a.address)] + applications = [a for a in networks[0].applications if a.address in LightingApplication.supported_applications()] if len(applications) == 0: logger.warning('Could not find any lighting application in project file.') diff --git a/cbus/protocol/application/lighting.py b/cbus/protocol/application/lighting.py index bb67afc..80fbd46 100644 --- a/cbus/protocol/application/lighting.py +++ b/cbus/protocol/application/lighting.py @@ -57,7 +57,7 @@ def __init__(self, group_address: int, application_address: Union[Application,in instead. """ #TODO: modify to avoid redundancy - if not Application.isLighting(application_address): + if not application_address in _SUPPORTED_APPLICATIONS: raise ValueError('Expected light Application address, got {}'.format(application_address)) check_ga(group_address) self.application_address = application_address diff --git a/cbus/protocol/pciprotocol.py b/cbus/protocol/pciprotocol.py index 5ca5516..2319e2e 100644 --- a/cbus/protocol/pciprotocol.py +++ b/cbus/protocol/pciprotocol.py @@ -326,7 +326,7 @@ def _get_confirmation_code(self): return int2byte(o) def _send(self, - cmd: Union[BasePacket], + cmd: BasePacket, confirmation: bool = True, basic_mode: bool = False): """ From c48dfbc52f350bc2881df11a6b888eb309333229 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Thu, 28 Apr 2022 23:02:38 -0400 Subject: [PATCH 15/16] Utility function --- cbus/daemon/cmqttd.py | 62 +++++++++++++++++++++--------- cbus/protocol/buffered_protocol.py | 30 ++++++++------- cbus/protocol/pciprotocol.py | 24 +++++++++++- cbus/toolkit/periodic.py | 29 ++++++++++++++ 4 files changed, 113 insertions(+), 32 deletions(-) create mode 100644 cbus/toolkit/periodic.py diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index 37ae290..ebfb7bd 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -15,10 +15,11 @@ # You should have received a copy of the GNU Lesser General Public License # along with this library. If not, see . -from asyncio import get_event_loop, run +from asyncio import get_event_loop, run, sleep from argparse import ArgumentParser, FileType import json import logging +from marshal import load from typing import Any, BinaryIO, Dict, Optional, Text, TextIO import paho.mqtt.client as mqtt @@ -40,7 +41,9 @@ async def create_serial_connection(*_, **__): from cbus.paho_asyncio import AsyncioHelper from cbus.protocol.pciprotocol import PCIProtocol from cbus.toolkit.cbz import CBZ +from cbus.toolkit.periodic import Periodic from cbus.protocol.application import LightingApplication +from cbus.protocol.cal.report import LevelStatusReport,BinaryStatusReport logger = logging.getLogger(__name__) @@ -53,11 +56,11 @@ async def create_serial_connection(*_, **__): _META_TOPIC = 'homeassistant/binary_sensor/cbus_cmqttd' _APPLICATION_GROUP_SEPARATOR = "_" -def check_aa_lighting(aa): - if not aa in LightingApplication.supported_applications(): +def check_aa_lighting(app): + if not app in LightingApplication.supported_applications(): raise ValueError( 'Application ${aa} is not a valid lighting application'.format( - aa)) + app)) def ga_range(): return range(MIN_GROUP_ADDR, MAX_GROUP_ADDR + 1) @@ -137,6 +140,20 @@ def on_lighting_group_off(self, source_addr, group_addr,app_addr): if not self.mqtt_api: return self.mqtt_api.lighting_group_off(source_addr, group_addr,app_addr) + + def on_level_report(self, app_addr, start, report: LevelStatusReport): + groups = self.mqtt_api.groupDB.setdefault(app_addr,{}) + for val in report: + if groups[start]: + if val==None: + pass + elif val==0: + self.on_lighting_group_off(0,start,app_addr) + elif val == 255: + self.on_lighting_group_on(0,start,app_addr) + else: + self.on_lighting_group_ramp(0,start,app_addr,0,val) + start+=1 # TODO: on_lighting_group_terminate_ramp @@ -151,7 +168,10 @@ def on_connect(self, client, userdata: CBusHandler, flags, rc): userdata.mqtt_api = self self.groupDB = {} self.publish_all_lights(userdata.labels) - + for app_addr in self.groupDB.keys(): + for block in range(0,256,32): + pass + Periodic.throttler.enqueue(lambda b= block,a= app_addr:userdata.request_status(b,a)) def on_message(self, client, userdata: CBusHandler, msg: mqtt.MQTTMessage): """Handle a message from an MQTT subscription.""" @@ -160,7 +180,7 @@ def on_message(self, client, userdata: CBusHandler, msg: mqtt.MQTTMessage): return try: - ga, aa = get_topic_group_address(msg.topic) + group_addr, app_addr = get_topic_group_address(msg.topic) except ValueError: # Invalid group address logging.error(f'Invalid group address in topic {msg.topic}') @@ -186,16 +206,16 @@ def on_message(self, client, userdata: CBusHandler, msg: mqtt.MQTTMessage): if light_on: if brightness == 255 and transition_time == 0: # lighting on - userdata.lighting_group_on(ga,aa) - self.lighting_group_on(None, ga,aa) + userdata.lighting_group_on(group_addr,app_addr) + self.lighting_group_on(None, group_addr,app_addr) else: # ramp - userdata.lighting_group_ramp(ga, aa, transition_time, brightness) - self.lighting_group_ramp(None, ga, aa, transition_time, brightness) + userdata.lighting_group_ramp(group_addr, app_addr, transition_time, brightness) + self.lighting_group_ramp(None, group_addr, app_addr, transition_time, brightness) else: # lighting off - userdata.lighting_group_off(ga,aa) - self.lighting_group_off(None, ga,aa) + userdata.lighting_group_off(group_addr,app_addr) + self.lighting_group_off(None, group_addr,app_addr) def publish(self, topic: Text, payload: Dict[Text, Any]): """Publishes a payload as JSON.""" @@ -265,17 +285,17 @@ def publish_all_lights(self, app_labels): }, }) - for aa,(_,labels) in app_labels.items(): - for ga in labels.keys(): - self.publish_light(ga,aa,app_labels) + for app_addr,(_,labels) in app_labels.items(): + for group_addr in labels.keys(): + self.publish_light(group_addr,app_addr,app_labels) def check_published(self,group_addr: int, app_addr: int | Application): if not self.groupDB.setdefault(app_addr,{}).get(group_addr,False): self.publish_light(group_addr,app_addr) - def publish_binary_sensor(self, group_addr: int, app_addr: int | Application, state: bool): - payload = 'ON' if state else 'OFF' + def publish_binary_sensor(self, group_addr: int, app_addr: int | Application, state: Optional[bool]): + payload = "UNK" if state == None else ('ON' if state else 'OFF') return super().publish( bin_sensor_state_topic(group_addr,app_addr), payload, 1, True) @@ -311,7 +331,7 @@ def lighting_group_ramp(self, source_addr: Optional[int], group_addr: int, app_a 'transition': duration, 'cbus_source_addr': source_addr, }) - self.publish_binary_sensor(group_addr, app_addr, level > 0) + self.publish_binary_sensor(group_addr, app_addr, level > 0 if isinstance(level,int) else None) def read_auth(client: mqtt.Client, auth_file: TextIO): @@ -375,8 +395,14 @@ class obj(): labels[a.address]=(a.tag_name,l) return labels +# Wait time between commands emitted by throtller +_PERIOD = 0.97 + async def _main(): + + #throttler is queue used used to stagger commmands + Periodic.throttler = Periodic(_PERIOD) parser = ArgumentParser() group = parser.add_argument_group('Logging options') diff --git a/cbus/protocol/buffered_protocol.py b/cbus/protocol/buffered_protocol.py index e8f03df..b651159 100644 --- a/cbus/protocol/buffered_protocol.py +++ b/cbus/protocol/buffered_protocol.py @@ -67,23 +67,27 @@ def data_received(self, data: bytes) -> None: :param data: new data to add to the buffer :return: None """ - data_size = len(data) - if data_size > self._size_limit: - raise ValueError('Received data exceeds size limit ' + try: + data_size = len(data) + if data_size > self._size_limit: + raise ValueError('Received data exceeds size limit ' '({} bytes)'.format(self._size_limit)) - # Add the data to the buffer - with self._buf_lock: - if len(self._buf) + data_size > self._size_limit: - self._buf = bytearray() - raise ValueError( - 'Received data would make the buffer exceed the maximum ' - 'limit, buffer dropped!') + # Add the data to the buffer + with self._buf_lock: + if len(self._buf) + data_size > self._size_limit: + self._buf = bytearray() + raise ValueError( + 'Received data would make the buffer exceed the maximum ' + 'limit, buffer dropped!') - self._buf.extend(data) + self._buf.extend(data) - # Handle any messages that may be in the buffer. - self._process_buffer() + # Handle any messages that may be in the buffer. + self._process_buffer() + except: + # Clear buffer. + self._buf = bytearray() def _process_buffer(self): with self._buf_lock: diff --git a/cbus/protocol/pciprotocol.py b/cbus/protocol/pciprotocol.py index 2319e2e..1d6ce22 100644 --- a/cbus/protocol/pciprotocol.py +++ b/cbus/protocol/pciprotocol.py @@ -27,6 +27,8 @@ from six import int2byte +from cbus.protocol.cal.report import BinaryStatusReport, LevelStatusReport + try: from serial_asyncio import create_serial_connection except ImportError: @@ -52,6 +54,7 @@ async def create_serial_connection(*_, **__): from cbus.protocol.pm_packet import PointToMultipointPacket from cbus.protocol.pp_packet import PointToPointPacket from cbus.protocol.reset_packet import ResetPacket +from cbus.protocol.cal.extended import ExtendedCAL logger = logging.getLogger(__name__) @@ -134,6 +137,18 @@ def handle_cbus_packet(self, p: BasePacket) -> None: self.on_clock_update(p.source_address, s.val) else: logger.debug(f'hcp: unhandled SAL type: {s!r}') + elif isinstance(p,PointToPointPacket): + for s in p: + if isinstance(s,ExtendedCAL): + if isinstance(s.report,BinaryStatusReport): + pass + elif isinstance(s.report,LevelStatusReport): + self.on_level_report(s.child_application, s.block_start, s.report) + else: + pass + else: + pass + else: logger.debug(f'hcp: unhandled other packet: {p!r}') @@ -416,7 +431,7 @@ def pci_reset(self): # 6: IDMON # self._send('A3300059', encode=False, checksum=False) self._send(DeviceManagementPacket( - checksum=False, parameter=0x30, value=0x59), + checksum=False, parameter=0x30, value=0x79), basic_mode=True) def identify(self, unit_address, attribute): @@ -463,6 +478,13 @@ def lighting_group_on(self, group_addr: Union[int, Iterable[int]],application_ad sals=[LightingOnSAL(ga,application_addr) for ga in group_addr]) return self._send(p) + def request_status(self,group_addr: Union[int, Iterable[int]],application_addr: Union[int,Application] ): + p = PointToMultipointPacket(sals=[ + StatusRequestSAL(level_request=True, group_address=group_addr,child_application=application_addr) + ]) + return self._send(p) + + def lighting_group_off(self, group_addr: Union[int, Iterable[int]],application_addr: Union[int,Application] ): """ Turns off the lights for the given group_id. diff --git a/cbus/toolkit/periodic.py b/cbus/toolkit/periodic.py new file mode 100644 index 0000000..fe620d5 --- /dev/null +++ b/cbus/toolkit/periodic.py @@ -0,0 +1,29 @@ +import asyncio +import queue + +class Periodic: + """ + class that manages a queue of functions to be called while + leaving a interval between two successsive calls + """ + queue = [] + + def __init__(self,period = 1): + self.queue = queue.Queue() + self.period = period + loop = asyncio.get_event_loop() + self.task = loop.create_task(self._work()) + + async def _work(self): + while True: + try: + action = self.queue.get(block=False) + action() + except: + pass + finally: + await asyncio.sleep(self.period) + + def enqueue(self,task): + #talks is a lambda or the name of a function with no argument + self.queue.put(task) \ No newline at end of file From c8c4000b81cfa1258c466d0c45b313f8ff6759d1 Mon Sep 17 00:00:00 2001 From: CABrouwers Date: Thu, 28 Apr 2022 23:11:14 -0400 Subject: [PATCH 16/16] Update cmqttd.py --- cbus/daemon/cmqttd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cbus/daemon/cmqttd.py b/cbus/daemon/cmqttd.py index ebfb7bd..215a1ba 100644 --- a/cbus/daemon/cmqttd.py +++ b/cbus/daemon/cmqttd.py @@ -295,7 +295,7 @@ def check_published(self,group_addr: int, app_addr: int | Application): def publish_binary_sensor(self, group_addr: int, app_addr: int | Application, state: Optional[bool]): - payload = "UNK" if state == None else ('ON' if state else 'OFF') + payload = 'ON' if state else 'OFF' return super().publish( bin_sensor_state_topic(group_addr,app_addr), payload, 1, True)