diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 1e63bf2..a2ed4f8 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -1,24 +1,28 @@ -name: Run tests +name: Run Python unit tests on: push: - branches: [ master ] + branches: [ master, release ] pull_request: - branches: [ master ] - workflow_dispatch: jobs: - test: - runs-on: ubuntu-latest + test-netflow: + runs-on: ubuntu-20.04 + strategy: + matrix: + python: + - "3.5.3" # Debian Stretch + - "3.7.3" # Debian Buster + - "3.9.2" # Debian Bullseye + - "3.11" # Debian Bookworm uses 3.11.1, but it's in a newer pyenv release steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - - name: Set up Python 3.5.3 - uses: gabrielfalcao/pyenv-action@v7 + - uses: actions/checkout@v3 + + - name: Set up Python with pyenv + uses: gabrielfalcao/pyenv-action@v11 with: - default: '3.5.3' # Debian Buster (stable) - + default: "${{ matrix.python }}" + - name: Run Python unittests run: python3 -m unittest diff --git a/README.md b/README.md index 5135098..3593193 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # Python NetFlow/IPFIX library -This package contains libraries and tools for **NetFlow versions 1, 5 and 9, and IPFIX**. +This package contains libraries and tools for **NetFlow versions 1, 5 and 9, and IPFIX**. It is available [on PyPI as "netflow"](https://pypi.org/project/netflow/). Version 9 is the first NetFlow version using templates. Templates make dynamically sized and configured NetFlow data flowsets possible, which makes the collector's job harder. The library provides the `netflow.parse_packet()` function as the main API point (see below). By importing `netflow.v1`, `netflow.v5` or `netflow.v9` you have direct access to the respective parsing objects, but at the beginning you probably will have more success by running the reference collector (example below) and look into its code. IPFIX (IP Flow Information Export) is based on NetFlow v9 and standardized by the IETF. All related classes are contained in `netflow.ipfix`. -Copyright 2016-2020 Dominik Pataky +![Data flow diagram](nf-workflow.png) + +Copyright 2016-2023 Dominik Pataky Licensed under MIT License. See LICENSE. @@ -34,7 +36,7 @@ assert p.header.version == 5 # NetFlow v5 packet assert p.flows[0].PROTO == 1 # ICMP flow ``` -In NetFlow v9 and IPFIX, templates are used instead of a fixed set of fields (like `PROTO`). See `collector.py` on how to handle these. +In NetFlow v9 and IPFIX, templates are used instead of a fixed set of fields (like `PROTO`). See `collector.py` on how to handle these. You **must** store received templates in between exports and pass them to the parser when new packets arrive. Not storing the templates will always result in parsing failures. ## Using the collector and analyzer Since v0.9.0 the `netflow` library also includes reference implementations of a collector and an analyzer as CLI tools. @@ -67,20 +69,20 @@ The test files contain tests for all use cases in the library, based on real sof 1. Run tcpdump/Wireshark on your public-facing interface (with tcpdump, save the pcap to disk). 2. Produce some sample flows, e.g. surf the web and refresh your mail client. With Wireshark, save the captured packets to disk. - 4. Run tcpdump/Wireshark again on a local interface. + 3. Run tcpdump/Wireshark again on a local interface. 4. Run `softflowd` with the `-r ` flag. softflowd reads the captured traffic, produces the flows and exports them. Use the interface you are capturing packets on to send the exports to. E.g. capture on the localhost interface (with `-i lo` or on loopback) and then let softflowd export to `127.0.0.1:1337`. 5. Examine the captured traffic. Use Wireshark and set the `CFLOW` "decode as" dissector on the export packets (e.g. based on the port). The `data` fields should then be shown correctly as Netflow payload. 6. Extract this payload as hex stream. Anonymize the IP addresses with a hex editor if necessary. A recommended hex editor is [bless](https://github.com/afrantzis/bless). Second, a Docker way: - 2. Run a softflowd daemon in the background inside a Docker container, listening on `eth0` and exporting to e.g. `172.17.0.1:1337`. - 3. On your host start Wireshark to listen on the Docker bridge. - 4. Create some traffic from inside the container. - 5. Check the softflow daemon with `softflowctl dump-flows`. - 6. If you have some flows shown to you, export them with `softflowctl expire-all`. - 7. Your Wireshark should have picked up the epxort packets (it does not matter if there's a port unreachable error). - 8. Set the decoder for the packets to `CFLOW` and copy the hex value from the NetFlow packet. + 1. Run a softflowd daemon in the background inside a Docker container, listening on `eth0` and exporting to e.g. `172.17.0.1:1337`. + 2. On your host start Wireshark to listen on the Docker bridge. + 3. Create some traffic from inside the container. + 4. Check the softflow daemon with `softflowctl dump-flows`. + 5. If you have some flows shown to you, export them with `softflowctl expire-all`. + 6. Your Wireshark should have picked up the export packets (it does not matter if there's a port unreachable error). + 7. Set the decoder for the packets to `CFLOW` and copy the hex value from the NetFlow packet. Your exported hex string should begin with `0001`, `0005`, `0009` or `000a`, depending on the version. diff --git a/netflow/analyzer.py b/netflow/analyzer.py index 71338b7..7448b84 100644 --- a/netflow/analyzer.py +++ b/netflow/analyzer.py @@ -204,7 +204,7 @@ def total_packets(self): if args.match_host: try: match_host = ipaddress.ip_address(args.match_host) - except ValueError as ex: + except ValueError: exit("IP address '{}' is neither IPv4 nor IPv6".format(args.match_host)) # Using a file and using stdin differ in their further usage for gzip.open diff --git a/netflow/collector.py b/netflow/collector.py index 0cc9538..81c27fc 100644 --- a/netflow/collector.py +++ b/netflow/collector.py @@ -19,9 +19,9 @@ import time from collections import namedtuple -from .ipfix import IPFIXTemplateNotRecognized -from .utils import UnknownExportVersion, parse_packet -from .v9 import V9TemplateNotRecognized +from netflow.ipfix import IPFIXTemplateNotRecognized +from netflow.utils import UnknownExportVersion, parse_packet +from netflow.v9 import V9TemplateNotRecognized RawPacket = namedtuple('RawPacket', ['ts', 'client', 'data']) ParsedPacket = namedtuple('ParsedPacket', ['ts', 'client', 'export']) diff --git a/netflow/ipfix.py b/netflow/ipfix.py index c800f94..88c0334 100644 --- a/netflow/ipfix.py +++ b/netflow/ipfix.py @@ -514,9 +514,9 @@ def get_type_unpack(cls, key: Union[int, str]) -> Optional[DataType]: :return: """ item = None - if type(key) == int: + if type(key) is int: item = cls.by_id(key) - elif type(key) == str: + elif type(key) is str: item = cls.by_name(key) if not item: return None @@ -575,7 +575,7 @@ def is_signed(cls, dt: Union[DataType, str]) -> bool: :return: """ fields = ["signed8", "signed16", "signed32", "signed64"] - if type(dt) == DataType: + if type(dt) is DataType: return dt.type in fields return dt in fields @@ -587,7 +587,7 @@ def is_float(cls, dt: Union[DataType, str]) -> bool: :return: """ fields = ["float32", "float64"] - if type(dt) == DataType: + if type(dt) is DataType: return dt.type in fields return dt in fields @@ -601,7 +601,7 @@ def is_bytes(cls, dt: Union[DataType, str]) -> bool: fields = ["octetArray", "string", "macAddress", "ipv4Address", "ipv6Address", "dateTimeMicroseconds", "dateTimeNanoseconds"] - if type(dt) == DataType: + if type(dt) is DataType: return dt.type in fields return dt in fields @@ -635,6 +635,10 @@ class IPFIXTemplateNotRecognized(KeyError): pass +class PaddingCalculationError(Exception): + pass + + class IPFIXHeader: """The header of the IPFIX export packet """ @@ -663,9 +667,6 @@ def __init__(self, data): offset += offset_add if len(self.fields) != self.field_count: raise IPFIXMalformedRecord - - # TODO: if padding is needed, implement here - self._length = offset def get_length(self): @@ -697,8 +698,6 @@ def __init__(self, data): raise IPFIXMalformedRecord offset += offset_add - # TODO: if padding is needed, implement here - self._length = offset def get_length(self): @@ -739,7 +738,7 @@ def __init__(self, data, template: List[Union[TemplateField, TemplateFieldEnterp raise NotImplementedError("Field type with ID {} is not implemented".format(field_type_id)) datatype = field_type.type # type: str - discovered_fields.append((field_type.name, field_type_id)) + discovered_fields.append((datatype, field_type_id)) # Catch fields which are meant to be raw bytes and skip the rest if IPFIXDataTypes.is_bytes(datatype): @@ -766,15 +765,18 @@ def __init__(self, data, template: List[Union[TemplateField, TemplateFieldEnterp pack = struct.unpack(unpacker, data[0:offset]) # Iterate through template again, but taking the unpacked values this time - for index, ((field_type_name, field_type_id), value) in enumerate(zip(discovered_fields, pack)): + for index, ((field_datatype, field_type_id), value) in enumerate(zip(discovered_fields, pack)): if type(value) is bytes: # Check if value is raw bytes, so no conversion happened in struct.unpack - if field_type_name in ["string"]: - value = str(value) + if field_datatype in ["string"]: + try: + value = value.decode() + except UnicodeDecodeError: + value = str(value) # TODO: handle octetArray (= does not have to be unicode encoded) - elif field_type_name in ["boolean"]: + elif field_datatype in ["boolean"]: value = True if value == 1 else False # 2 = false per RFC - elif field_type_name in ["dateTimeMicroseconds", "dateTimeNanoseconds"]: + elif field_datatype in ["dateTimeMicroseconds", "dateTimeNanoseconds"]: seconds = value[:4] fraction = value[4:] value = (int.from_bytes(seconds, "big"), int.from_bytes(fraction, "big")) @@ -809,17 +811,29 @@ def __init__(self, data: bytes, templates): self.records = [] self._templates = {} - offset = IPFIXSetHeader.size + offset = IPFIXSetHeader.size # fixed size + if self.header.set_id == 2: # template set while offset < self.header.length: # length of whole set template_record = IPFIXTemplateRecord(data[offset:]) self.records.append(template_record) if template_record.field_count == 0: + # Should not happen, since RFC says "one or more" self._templates[template_record.template_id] = None else: self._templates[template_record.template_id] = template_record.fields offset += template_record.get_length() + # If the rest of the data is deemed to be too small for another + # template record, check existence of padding + if ( + offset != self.header.length + and self.header.length - offset <= 16 # 16 is chosen as a guess + and rest_is_padding_zeroes(data[:self.header.length], offset) + ): + # Rest should be padding zeroes + break + elif self.header.set_id == 3: # options template while offset < self.header.length: optionstemplate_record = IPFIXOptionsTemplateRecord(data[offset:]) @@ -831,16 +845,47 @@ def __init__(self, data: bytes, templates): optionstemplate_record.scope_fields + optionstemplate_record.fields offset += optionstemplate_record.get_length() + # If the rest of the data is deemed to be too small for another + # options template record, check existence of padding + if ( + offset != self.header.length + and self.header.length - offset <= 16 # 16 is chosen as a guess + and rest_is_padding_zeroes(data[:self.header.length], offset) + ): + # Rest should be padding zeroes + break + elif self.header.set_id >= 256: # data set, set_id is template id - while offset < self.header.length: - template = templates.get( - self.header.set_id) # type: List[Union[TemplateField, TemplateFieldEnterprise]] - if not template: - raise IPFIXTemplateNotRecognized - data_record = IPFIXDataRecord(data[offset:], template) + # First, get the template behind the ID. Returns a list of fields or raises an exception + template_fields = templates.get( + self.header.set_id) # type: List[Union[TemplateField, TemplateFieldEnterprise]] + if not template_fields: + raise IPFIXTemplateNotRecognized + + # All template fields have a known length. Add them all together to get the length of the data set. + dataset_length = functools.reduce(lambda a, x: a + x.length, template_fields, 0) + + # This is the last possible offset value possible if there's no padding. + # If there is padding, this value marks the beginning of the padding. + # Two cases possible: + # 1. No padding: then (4 + x * dataset_length) == self.header.length + # 2. Padding: then (4 + x * dataset_length + p) == self.header.length, + # where p is the remaining length of padding zeroes. The modulo calculates p + no_padding_last_offset = self.header.length - ((self.header.length - IPFIXSetHeader.size) % dataset_length) + + while offset < no_padding_last_offset: + data_record = IPFIXDataRecord(data[offset:], template_fields) self.records.append(data_record) offset += data_record.get_length() - self._length = offset + + # Safety check + if ( + offset != self.header.length + and not rest_is_padding_zeroes(data[:self.header.length], offset) + ): + raise PaddingCalculationError + + self._length = self.header.length def get_length(self): return self._length @@ -922,7 +967,7 @@ def __init__(self, data: bytes, templates: Dict[int, list]): raise IPFIXMalformedPacket @property - def contains_new_templates(self): + def contains_new_templates(self) -> bool: return self._contains_new_templates @property @@ -950,11 +995,11 @@ def parse_fields(data: bytes, count: int) -> (list, int): offset = 0 fields = [] # type: List[Union[TemplateField, TemplateFieldEnterprise]] for ctr in range(count): - if data[offset] & 1 << 7 != 0: # enterprise flag set + if (data[offset] & (1 << 7)) != 0: # enterprise flag set. Bitwise AND checks bit only in the first byte/octet pack = struct.unpack("!HHI", data[offset:offset + 8]) fields.append( TemplateFieldEnterprise( - id=pack[0] & ~(1 << 7), # ID, clear enterprise flag bit + id=(pack[0] & ~(1 << 15)), # clear enterprise flag bit. Bitwise AND and INVERT work on two bytes length=pack[1], # field length enterprise_number=pack[2] # enterprise number ) @@ -963,7 +1008,21 @@ def parse_fields(data: bytes, count: int) -> (list, int): else: pack = struct.unpack("!HH", data[offset:offset + 4]) fields.append( - TemplateField(id=pack[0], length=pack[1]) + TemplateField( + id=pack[0], + length=pack[1] + ) ) offset += 4 return fields, offset + + +def rest_is_padding_zeroes(data: bytes, offset: int) -> bool: + if offset <= len(data): + # padding zeros, so rest of bytes must be summed to 0 + if sum(data[offset:]) != 0: + return False + return True + + # If offset > len(data) there is an error + raise ValueError("netflow.ipfix.rest_is_padding_zeroes received a greater offset value than there is data") diff --git a/netflow/utils.py b/netflow/utils.py index 395eb21..c4603d9 100644 --- a/netflow/utils.py +++ b/netflow/utils.py @@ -31,8 +31,8 @@ def get_export_version(data): return struct.unpack('!H', data[:2])[0] -def parse_packet(data: Union[str, bytes], templates: Dict = None) -> Union[V1ExportPacket, V5ExportPacket, - V9ExportPacket, IPFIXExportPacket]: +def parse_packet(data: Union[str, bytes], templates: Dict = None) \ + -> Union[V1ExportPacket, V5ExportPacket, V9ExportPacket, IPFIXExportPacket]: """ Parse an exported packet, either from string (hex) or from bytes. @@ -66,10 +66,10 @@ def parse_packet(data: Union[str, bytes], templates: Dict = None) -> Union[V1Exp :param templates: The templates dictionary with keys 'netflow' and 'ipfix' (created if not existing). :return: The parsed packet, or an exception. """ - if type(data) == str: + if type(data) is str: # hex dump as string data = bytes.fromhex(data) - elif type(data) == bytes: + elif type(data) is bytes: # check representation based on utf-8 decoding result try: # hex dump as bytes, but not hex @@ -83,8 +83,8 @@ def parse_packet(data: Union[str, bytes], templates: Dict = None) -> Union[V1Exp if version in [9, 10] and templates is None: raise ValueError("{} packet detected, but no templates dict was passed! For correct parsing of packets with " - "templates, create a 'templates' dict and pass it into the 'parse_packet' function.".format( - "NetFlow v9" if version == 9 else "IPFIX")) + "templates, create a 'templates' dict and pass it into the 'parse_packet' function." + .format("NetFlow v9" if version == 9 else "IPFIX")) if version == 1: return V1ExportPacket(data) diff --git a/netflow/v9.py b/netflow/v9.py index 348deb8..aee63db 100644 --- a/netflow/v9.py +++ b/netflow/v9.py @@ -14,6 +14,7 @@ import ipaddress import struct +import sys from .ipfix import IPFIXFieldTypes, IPFIXDataTypes @@ -21,6 +22,8 @@ "V9TemplateFlowSet", "V9TemplateNotRecognized", "V9TemplateRecord", "V9OptionsTemplateFlowSet", "V9OptionsTemplateRecord", "V9OptionsDataRecord"] +V9_FIELD_TYPES_CONTAINING_IP = [8, 12, 15, 18, 27, 28, 62, 63] + V9_FIELD_TYPES = { 0: 'UNKNOWN_FIELD_TYPE', # fallback for unknown field types @@ -204,31 +207,54 @@ def __init__(self, data, template): # As the field lengths are variable V9 has padding to next 32 Bit padding_size = 4 - (self.length % 4) # 4 Byte + # For performance reasons, we use struct.unpack to get individual values. Here + # we prepare the format string for parsing it. The format string is based on the template fields and their + # lengths. The string can then be re-used for every data record in the data stream + struct_format = '!' + struct_len = 0 + for field in template.fields: + # The length of the value byte slice is defined in the template + flen = field.field_length + if flen == 4: + struct_format += 'L' + elif flen == 2: + struct_format += 'H' + elif flen == 1: + struct_format += 'B' + else: + struct_format += '%ds' % flen + struct_len += flen + while offset <= (self.length - padding_size): - new_record = V9DataRecord() + # Here we actually unpack the values, the struct format string is used in every data record + # iteration, until the final offset reaches the end of the whole data stream + unpacked_values = struct.unpack(struct_format, data[offset:offset + struct_len]) - for field in template.fields: + new_record = V9DataRecord() + for field, value in zip(template.fields, unpacked_values): flen = field.field_length fkey = V9_FIELD_TYPES[field.field_type] - # The length of the value byte slice is defined in the template - dataslice = data[offset:offset + flen] - - # Better solution than struct.unpack with variable field length - fdata = 0 - for idx, byte in enumerate(reversed(bytearray(dataslice))): - fdata += byte << (idx * 8) - # Special handling of IP addresses to convert integers to strings to not lose precision in dump # TODO: might only be needed for IPv6 - if fkey in ["IPV4_SRC_ADDR", "IPV4_DST_ADDR", "IPV6_SRC_ADDR", "IPV6_DST_ADDR"]: + if field.field_type in V9_FIELD_TYPES_CONTAINING_IP: try: - ip = ipaddress.ip_address(fdata) + ip = ipaddress.ip_address(value) except ValueError: - print("IP address could not be parsed: {}".format(fdata)) + print("IP address could not be parsed: {}".format(repr(value))) continue new_record.data[fkey] = ip.compressed + elif flen in (1, 2, 4): + # These values are already converted to numbers by struct.unpack: + new_record.data[fkey] = value else: + # Caveat: this code assumes little-endian system (like x86) + if sys.byteorder != "little": + print("v9.py uses bit shifting for little endianness. Your processor is not little endian") + + fdata = 0 + for idx, byte in enumerate(reversed(bytearray(value))): + fdata += byte << (idx * 8) new_record.data[fkey] = fdata offset += flen @@ -373,7 +399,7 @@ def __init__(self, data: bytes, template: V9OptionsTemplateRecord): for scope_type, length in template.scope_fields.items(): type_name = V9_SCOPE_TYPES.get(scope_type, scope_type) # Either name, or unknown int - value = int.from_bytes(data[offset:offset+length], 'big') # TODO: is this always integer? + value = int.from_bytes(data[offset:offset + length], 'big') # TODO: is this always integer? new_options_record.scopes[type_name] = value offset += length @@ -392,9 +418,9 @@ def __init__(self, data: bytes, template: V9OptionsTemplateRecord): value = None if is_bytes: - value = data[offset:offset+length] + value = data[offset:offset + length] else: - value = int.from_bytes(data[offset:offset+length], 'big') + value = int.from_bytes(data[offset:offset + length], 'big') new_options_record.data[type_name] = value @@ -499,6 +525,8 @@ def __init__(self, data: bytes, templates: dict): if id_ not in self._templates: self._new_templates = True self._templates[id_] = template + if tfs.length == 0: + break offset += tfs.length continue @@ -510,6 +538,8 @@ def __init__(self, data: bytes, templates: dict): self._new_templates = True self._templates[id_] = template offset += otfs.flowset_length + if otfs.flowset_length == 0: + break continue # Data / option flowsets @@ -518,6 +548,8 @@ def __init__(self, data: bytes, templates: dict): # Could not be parsed, continue to check for templates skipped_flowsets_offsets.append(offset) offset += flowset_length + if flowset_length == 0: + break continue matched_template = self._templates[flowset_id] @@ -525,11 +557,15 @@ def __init__(self, data: bytes, templates: dict): if isinstance(matched_template, V9TemplateRecord): dfs = V9DataFlowSet(data[offset:], matched_template) self._flows += dfs.flows + if dfs.length == 0: + break offset += dfs.length elif isinstance(matched_template, V9OptionsTemplateRecord): odfs = V9OptionsDataFlowset(data[offset:], matched_template) self._options += odfs.option_data_records + if odfs.length == 0: + break offset += odfs.length else: diff --git a/nf-workflow.png b/nf-workflow.png new file mode 100644 index 0000000..67cdd50 Binary files /dev/null and b/nf-workflow.png differ diff --git a/setup.py b/setup.py index fb2438f..1490218 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ setup( name='netflow', - version='0.10.5', + version='0.12.2', description='NetFlow v1, v5, v9 and IPFIX tool suite implemented in Python 3', long_description=long_description, long_description_content_type='text/markdown', diff --git a/tests/lib.py b/tests/lib.py index 5a06bda..7867132 100644 --- a/tests/lib.py +++ b/tests/lib.py @@ -185,9 +185,37 @@ def single_packet(pkts): "0000bfee0000002f00000000000000000050ae56061b04007f0000017f000001fb3ddbb3fb3c1a18" "00000982000000270000000000000000ae560050061b04007f0000017f000001fb3ddbb3fb3c1a18" "0000130e0000001200000000000000000050e820061b04007f0000017f000001fb3ddbb3fb3c1a18" - "0000059c000000140000000000000000e8200050061b0400" + "0000059c000000140000000000000000e8200050061b0400", ] +PACKET_V9_WITH_ZEROS = ( + "000900057b72e830620b717d78cf34e30102000001040048000000000000006e0000000101000000" + "000a20076a06065c0800000d6b15c80000000b7b72e4487b72e448080000000000000438bf6401c7" + "65ad1e0d6b15c8000000000001040048000000000000006700000001110000c951ac180b0306065c" + "080035010000010000000b7b72e4487b72e448000000000000000443177b01c765ada501000001c3" + "9c00350001040048000000000000004a000000010100000000ac19bc3206065c080000287048cd00" + "00000b7b72e8307b72e83008000000000000048f071a01c765ae42287048cd000000000001040048" + "000000000000004600000001060002cbef0a30681f06065c0801bb142a49180000000b7b72e8307b" + "72e8300000000000000004801c7801c765ae1d142a49185a2b01bb00010400480000000000000046" + "00000001060002fe800a2f601206065c0801bb142a49180000000b7b72e8307b72e8300000000000" + "0000040806b001c765ae28142a4918d4b501bb000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" + "00000000000000000000000000000000000000000000000000000000000000000000000000000000" +) + # Example export for IPFIX (v10) with 4 templates, 1 option template and 8 data flow sets PACKET_IPFIX_TEMPLATE = "000a05202d45a4700000001300000000000200400400000e00080004000c00040016000400150004" \ "0001000400020004000a0004000e000400070002000b00020004000100060001003c000100050001" \ @@ -282,3 +310,16 @@ def single_packet(pkts): "000000400000000100000000000000008800060000000000123456affefeaffeaffeaffe08000054" \ "fde66f14e0f196090000affeaffeaffe2a044e42020000000000000000000223e58bc8ede58be4d8" \ "00000140000000040000000000000000e54c01bb0602060000000000affeaffeaffe123456affefe" + +PACKET_IPFIX_PADDING = "000a01c064e0b1900000000200000000000200480400001000080004000c00040016000400150004" \ + "0001000400020004000a0004000e0004003d00010088000100070002000b00020004000100060001" \ + "003c000100050001000200400401000e00080004000c000400160004001500040001000400020004" \ + "000a0004000e0004003d0001008800010020000200040001003c0001000500010002004808000010" \ + "001b0010001c001000160004001500040001000400020004000a0004000e0004003d000100880001" \ + "00070002000b00020004000100060001003c000100050001000200400801000e001b0010001c0010" \ + "00160004001500040001000400020004000a0004000e0004003d000100880001008b000200040001" \ + "003c00010005000100030022010000060001008f000400a000080131000401320004013000020052" \ + "0010040100547f0000017f000001ffff07d0ffff0ff7000000fc0000000300000000000000000001" \ + "08000104007f0000017f000001ffff07d0ffff0ff7000000fc000000030000000000000000000100" \ + "0001040000000100002a0000b2da0000018a0db59d2e000000010000000000017465737463617074" \ + "7572655f73696e67" diff --git a/tests/test_ipfix.py b/tests/test_ipfix.py index b2bed1b..feac3b6 100644 --- a/tests/test_ipfix.py +++ b/tests/test_ipfix.py @@ -13,7 +13,7 @@ import unittest from tests.lib import send_recv_packets, PACKET_IPFIX_TEMPLATE, PACKET_IPFIX, PACKET_IPFIX_ETHER, \ - PACKET_IPFIX_TEMPLATE_ETHER + PACKET_IPFIX_TEMPLATE_ETHER, PACKET_IPFIX_PADDING class TestFlowExportIPFIX(unittest.TestCase): @@ -98,3 +98,19 @@ def test_ipfix_contents_ether(self): self.assertTrue(hasattr(flow, "postDestinationMacAddress")) self.assertEqual(flow.sourceMacAddress, 0x123456affefe) self.assertEqual(flow.postDestinationMacAddress, 0xaffeaffeaffe) + + def test_ipfix_padding(self): + """ + Checks successful parsing of export packets that contain padding zeroes in an IPFIX set. + The padding in the example data is in between the last two data sets, so the successful parsing of the last + data set indicates correct handling of padding zero bytes. + """ + pkts, _, _ = send_recv_packets([PACKET_IPFIX_PADDING]) + self.assertEqual(len(pkts), 1) + p = pkts[0] + + # Check for length of whole export + self.assertEqual(p.export.header.length, 448) + + # Check a specific value of the last flow in the export. Success means correct handling of padding in the set + self.assertEqual(p.export.flows[-1].meteringProcessId, 45786) diff --git a/tests/test_netflow.py b/tests/test_netflow.py index c145bef..3949687 100755 --- a/tests/test_netflow.py +++ b/tests/test_netflow.py @@ -13,7 +13,7 @@ import unittest from tests.lib import send_recv_packets, NUM_PACKETS, \ - PACKET_INVALID, PACKET_V1, PACKET_V5, \ + PACKET_INVALID, PACKET_V1, PACKET_V5, PACKET_V9_WITH_ZEROS, \ PACKET_V9_TEMPLATE, PACKET_V9_TEMPLATE_MIXED, PACKETS_V9 @@ -119,6 +119,10 @@ def test_recv_v9_packet(self): pkts, _, _ = send_recv_packets([PACKETS_V9[0]]) self.assertEqual(len(pkts), 0) # no export is parsed due to missing template + # send an invalid packet with zero bytes, must fail to parse + pkts, _, _ = send_recv_packets([PACKET_V9_WITH_ZEROS]) + self.assertEqual(len(pkts), 0) # no export is parsed due to missing template + # send packet with two templates and eight flows, should parse correctly since the templates are known pkts, _, _ = send_recv_packets([PACKET_V9_TEMPLATE]) self.assertEqual(len(pkts), 1)