From 4cd7d7a034896b9d7ed5251e08c3066f967956c0 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 3 Jan 2016 14:56:58 -0500 Subject: [PATCH 1/6] Add EXTENDED message ID and extended handshake ID --- BitTornado/Client/Connecter.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/BitTornado/Client/Connecter.py b/BitTornado/Client/Connecter.py index eed57bf..6fdd1ba 100644 --- a/BitTornado/Client/Connecter.py +++ b/BitTornado/Client/Connecter.py @@ -5,20 +5,20 @@ DEBUG2 = False -CHOKE = b'\x00' -UNCHOKE = b'\x01' -INTERESTED = b'\x02' -NOT_INTERESTED = b'\x03' -# index -HAVE = b'\x04' -# index, bitfield -BITFIELD = b'\x05' -# index, begin, length -REQUEST = b'\x06' -# index, begin, piece -PIECE = b'\x07' -# index, begin, piece -CANCEL = b'\x08' +# Message IDs +CHOKE = b'\x00' # no payload +UNCHOKE = b'\x01' # no payload +INTERESTED = b'\x02' # no payload +NOT_INTERESTED = b'\x03' # no payload +HAVE = b'\x04' # index +BITFIELD = b'\x05' # index, bitfield +REQUEST = b'\x06' # index, begin, length +PIECE = b'\x07' # index, begin, piece +CANCEL = b'\x08' # index, begin, piece +EXTENDED = b'\x14' # msg_id, payload + +# Extended Message IDs +EXT_HANDSHAKE = b'\x00' class Connection(object): From adbcf175707dcac873c920cfbb81038f28512eb9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 18 Jan 2016 21:31:31 -0500 Subject: [PATCH 2/6] ENH: Add bitwise operations to Bitfield --- BitTornado/Types/bitfield.py | 87 +++++++++++++++++++++++-- BitTornado/Types/tests/test_bitfield.py | 72 ++++++++++++++++++++ 2 files changed, 154 insertions(+), 5 deletions(-) diff --git a/BitTornado/Types/bitfield.py b/BitTornado/Types/bitfield.py index 6bba976..5f124f9 100644 --- a/BitTornado/Types/bitfield.py +++ b/BitTornado/Types/bitfield.py @@ -1,5 +1,7 @@ """Manipulable boolean list structure and related functions""" +import math +import operator as ops CHARBITMAP = [tuple(bool((integer << nbits) & 0x80) for nbits in range(8)) for integer in range(256)] @@ -8,11 +10,29 @@ class Bitfield(list): - """Allow a sequence of booleans to be used as an indexable bitfield""" + """Indexable sequence of mutable booleans + + Supports bitwise logical operations (&, ^, |, ~). + + Binary operations require the second operand to be interpretable as a + Bitfield of the same size. + + - A Bitfield must be the same size as the first operand (arg1) + - A bytes object must have the same length as bytes(arg1) + - An int must be encodable as bytestring of length len(bytes(arg1)) + + The first (most-significant) bit of a Bitfield is the first bit of its + bytes representation. A Bitfield whose length is not divisible by 8 + will be zero-padded on the right. Binary operations with bytes or int + may not behave intuitively in those cases. + """ def __init__(self, length=None, bitstring=None, copyfrom=None, val=False): if copyfrom is not None: super(Bitfield, self).__init__(copyfrom) - self.numfalse = copyfrom.numfalse + if isinstance(copyfrom, Bitfield): + self.numfalse = copyfrom.numfalse + else: + self.numfalse = len(self) - sum(self) return if length is None: raise ValueError('length must be provided unless copying from ' @@ -20,14 +40,15 @@ def __init__(self, length=None, bitstring=None, copyfrom=None, val=False): if bitstring is not None: extra = len(bitstring) * 8 - length if not 0 <= extra < 8: - raise ValueError + raise ValueError("Bitstring must be `ceiling(length // 8)` " + "bytes long") if isinstance(bitstring, str): bitstring = map(ord, bitstring) bits = [bit for byte in bitstring for bit in CHARBITMAP[byte]] if extra > 0: if bits[-extra:] != [False] * extra: - raise ValueError + raise ValueError("Excess (low order) bits must be zero") del bits[-extra:] self.numfalse = len(bits) - sum(bits) else: @@ -45,11 +66,67 @@ def __repr__(self): return "".format(','.join(str(int(i)) for i in self)) def __bytes__(self): - """Produce a bytestring corresponding to the current bitfield""" + """Produce a bytestring corresponding to the current bitfield + + NB: Zero-pads on the right. The first bit of the bytestring is the + first bit of the bitfield. + """ bits = self + [False] * (-len(self) % 8) return bytes(BITCHARMAP[tuple(bits[x:x + 8])] for x in range(0, len(bits), 8)) + def __invert__(self): + """Flip all bits (~)""" + return Bitfield(length=len(self), val=True) ^ Bitfield(copyfrom=self) + + def __and__(self, val): + """Bitwise AND (&) + + Right-hand-side must be Bitfield, bytes, or int. (See object docs.)""" + return self._bitwise_op(val, ops.and_) + + def __or__(self, val): + """Bitwise OR (|) + + Right-hand-side must be Bitfield, bytes, or int. (See object docs.)""" + return self._bitwise_op(val, ops.or_) + + def __xor__(self, val): + """Bitwise XOR (^) + + Right-hand-side must be Bitfield, bytes, or int. (See object docs.)""" + return self._bitwise_op(val, ops.xor) + + def _bitwise_op(self, val, op): + """Perform bitwise operation""" + nbits = len(self) + nbytes = math.ceil(nbits / 8) + + if isinstance(val, Bitfield): + if len(val) != nbits: + raise ValueError("Cannot perform bitwise operation on " + "differently sized Bitfields") + val = bytes(val) + if isinstance(val, bytes): + if len(val) != nbytes: + raise ValueError("Cannot perform bitwise operation on " + "Bitfield and bytes object of unmatching " + "length") + val = int.from_bytes(val, 'big') + if not isinstance(val, int): + raise ValueError("Can only perform bitwise operations between " + "Bitfield object and Bitfield, bytes, or int") + try: + val.to_bytes(nbytes, 'big') + except OverflowError: + raise ValueError("Integer ({:d}) cannot be represented in {:d} " + "bits".format(val, nbits)) + if (val << nbits) & (2 ** nbits - 1): + raise ValueError("Trailing bits must all be zero.") + + this = int.from_bytes(bytes(self), 'big') + return Bitfield(nbits, op(this, val).to_bytes(nbytes, 'big')) + @property def complete(self): """True if all booleans are True""" diff --git a/BitTornado/Types/tests/test_bitfield.py b/BitTornado/Types/tests/test_bitfield.py index 93c73e9..8b61d7f 100644 --- a/BitTornado/Types/tests/test_bitfield.py +++ b/BitTornado/Types/tests/test_bitfield.py @@ -1,4 +1,6 @@ import unittest +import operator as ops +import random from .. import Bitfield @@ -45,5 +47,75 @@ def test_bitfield(self): self.assertEqual(testx.numfalse, 5) self.assertEqual(bytes(testx), b'\xc4') + def test_bitwise_ops(self): + zeros = Bitfield(8, b'\x00') # 0 0 0 0 0 0 0 0 + ones = Bitfield(8, b'\xff') # 1 1 1 1 1 1 1 1 + aa = Bitfield(8, b'\xaa') # 1 0 1 0 1 0 1 0 + fives = Bitfield(8, b'\x55') # 0 1 0 1 0 1 0 1 + + rands = tuple(Bitfield(8, random.randrange(256).to_bytes(1, 'big')) + for _ in range(10)) + + bitfields = (zeros, ones, aa, fives) + rands + inverses = ((ones, zeros), (aa, fives)) + + # Invert + for bf_a, bf_b in inverses: + self.assertEqual(~bf_a, bf_b) + self.assertEqual(bf_a, ~bf_b) + + # Tautologies + for bitfield in bitfields: + # Double inversion + self.assertEqual(~~bitfield, bitfield) + + # Bitwise AND + # Identity + self.assertEqual(bitfield & ones, bitfield) + self.assertEqual(bitfield & b'\xff', bitfield) + self.assertEqual(bitfield & 0xff, bitfield) + # Zero + self.assertEqual(bitfield & zeros, zeros) + self.assertEqual(bitfield & b'\x00', zeros) + self.assertEqual(bitfield & 0x00, zeros) + # Inverses + self.assertEqual(bitfield & ~bitfield, zeros) + + # Bitwise OR + # Identity + self.assertEqual(bitfield | zeros, bitfield) + self.assertEqual(bitfield | b'\x00', bitfield) + self.assertEqual(bitfield | 0x00, bitfield) + # Zero + self.assertEqual(bitfield | ones, ones) + self.assertEqual(bitfield | b'\xff', ones) + self.assertEqual(bitfield | 0xff, ones) + # Inverses + self.assertEqual(bitfield | ~bitfield, ones) + + # Bitwise XOR + # Identity + self.assertEqual(bitfield ^ zeros, bitfield) + self.assertEqual(bitfield ^ b'\x00', bitfield) + self.assertEqual(bitfield ^ 0x00, bitfield) + # Inversion + self.assertEqual(bitfield ^ ones, ~bitfield) + self.assertEqual(bitfield ^ b'\xff', ~bitfield) + self.assertEqual(bitfield ^ 0xff, ~bitfield) + self.assertEqual(bitfield ^ ~bitfield, ones) + + # Commutativity + for bf_a in bitfields: + for bf_b in bitfields: + self.assertEqual(bf_a & bf_b, bf_b & bf_a) + self.assertEqual(bf_a | bf_b, bf_b | bf_a) + self.assertEqual(bf_a ^ bf_b, bf_b ^ bf_a) + + # Breakage + for op in (ops.and_, ops.or_, ops.xor): + self.assertRaises(ValueError, op, ones, Bitfield(9)) + self.assertRaises(ValueError, op, ones, b'\xff\xff') + self.assertRaises(ValueError, op, ones, 258) + if __name__ == '__main__': unittest.main() From 2db7e2d2e56db205df7b3e3b78ed36a60c03e375 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 18 Jan 2016 22:06:27 -0500 Subject: [PATCH 3/6] RF: Specify protocol options in Network.Protocol --- BitTornado/Network/Encrypter.py | 10 ++++------ BitTornado/Network/NatCheck.py | 4 ++-- BitTornado/Network/Protocol.py | 21 +++++++++++++++++++++ BitTornado/Network/ServerPortHandler.py | 5 ++--- 4 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 BitTornado/Network/Protocol.py diff --git a/BitTornado/Network/Encrypter.py b/BitTornado/Network/Encrypter.py index 44e156d..4556f79 100644 --- a/BitTornado/Network/Encrypter.py +++ b/BitTornado/Network/Encrypter.py @@ -1,14 +1,12 @@ import urllib from binascii import hexlify from .BTcrypto import Crypto, padding +from .Protocol import protocol_name, reserved DEBUG = False MAX_INCOMPLETE = 8 -protocol_name = b'\x13BitTorrent protocol' -option_pattern = bytes(8) - def make_readable(s): if not s: @@ -63,7 +61,7 @@ def __init__(self, Encoder, connection, peerid, self.write(self.encrypter.padded_pubkey()) else: self.encrypted = False - self.write(protocol_name + option_pattern + + self.write(protocol_name + bytes(reserved) + self.Encoder.download_id) self.next_len = len(protocol_name) self.next_func = self.read_header @@ -285,7 +283,7 @@ def read_crypto_block4done(self): if not self.buffer: # oops; check for exceptions to this return None self._end_crypto() - self.write(protocol_name + option_pattern + self.Encoder.download_id) + self.write(protocol_name + bytes(reserved) + self.Encoder.download_id) return len(protocol_name), self.read_encrypted_header ### START PROTOCOL OVER ENCRYPTED CONNECTION ### @@ -306,7 +304,7 @@ def read_download_id(self, s): if not self.locally_initiated: if not self.encrypted: self.Encoder.connecter.external_connection_made += 1 - self.write(protocol_name + option_pattern + + self.write(protocol_name + bytes(reserved) + self.Encoder.download_id + self.Encoder.my_id) return 20, self.read_peer_id diff --git a/BitTornado/Network/NatCheck.py b/BitTornado/Network/NatCheck.py index 22034fc..b0e5b11 100644 --- a/BitTornado/Network/NatCheck.py +++ b/BitTornado/Network/NatCheck.py @@ -1,5 +1,5 @@ from .BTcrypto import Crypto, CRYPTO_OK, padding -from .Encrypter import protocol_name, option_pattern +from .Protocol import protocol_name, reserved CHECK_PEER_ID_ENCRYPTED = True @@ -126,7 +126,7 @@ def read_crypto_block4done(self): if not self.buffer: # oops; check for exceptions to this return None self._end_crypto() - self.write(protocol_name + option_pattern + self.Encoder.download_id) + self.write(protocol_name + bytes(reserved) + self.Encoder.download_id) return len(protocol_name), self.read_encrypted_header ### START PROTOCOL OVER ENCRYPTED CONNECTION ### diff --git a/BitTornado/Network/Protocol.py b/BitTornado/Network/Protocol.py new file mode 100644 index 0000000..a0b38b3 --- /dev/null +++ b/BitTornado/Network/Protocol.py @@ -0,0 +1,21 @@ +from ..Types import Bitfield + +protocol_name = b'\x13BitTorrent protocol' +reserved = Bitfield(64) + +# Flags (See: http://www.bittorrent.org/beps/bep_0004.html) +# Byte Index: 0 1 2 3 4 5 6 7 +DIST_HASH = 0x0000000000000001 # BEP 5: Distributed Hash Table +XBT_PEERX = 0x0000000000000002 # XBT Peer Exchange +FAST_EXTN = 0x0000000000000004 # BEP 6: Fast Extension +NAT_PUNCH = 0x0000000000000008 # NAT Traversal +# Byte Index: 0 1 2 3 4 5 6 7 +EXT_NEG_1 = 0x0000000000010000 # Extension Negotiation Protocol (Deprecated) +EXT_NEG_2 = 0x0000000000020000 # Extension Negotiation Protocol (Deprecated) +EXTENSION = 0x0000000000100000 # BEP 10: Libtorrent Extension Protocol +# Byte Index: 0 1 2 3 4 5 6 7 +LOC_AWARE = 0x0008000000000000 # BitTorrent Location-aware Protocol +COMET_BYT = 0x00ff000000000000 # BitComet Extension Protocol +# Byte Index: 0 1 2 3 4 5 6 7 +AZURE_MSG = 0x8000000000000000 # Azureus Messaging Protocol +COMET_BYT = 0xff00000000000000 # BitComet Extension Protocol diff --git a/BitTornado/Network/ServerPortHandler.py b/BitTornado/Network/ServerPortHandler.py index fcebd0d..6eb632e 100644 --- a/BitTornado/Network/ServerPortHandler.py +++ b/BitTornado/Network/ServerPortHandler.py @@ -1,6 +1,5 @@ from .BTcrypto import Crypto - -from .Encrypter import protocol_name +from .Protocol import protocol_name class SingleRawServer(object): @@ -75,7 +74,7 @@ def __init__(self, multihandler, connection): self.complete = False self.read = self._read self.write = connection.write - self.next_len = 1 + len(protocol_name) + self.next_len = len(protocol_name) self.next_func = self.read_header self.multihandler.rawserver.add_task(self._auto_close, 30) From 43a673e816a2b43b644d5dadee601a45276e0be6 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 31 May 2016 21:29:17 -0400 Subject: [PATCH 4/6] Add extension plumbing --- BitTornado/Client/Connecter.py | 14 ++++ BitTornado/Client/Extension.py | 126 +++++++++++++++++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 BitTornado/Client/Extension.py diff --git a/BitTornado/Client/Connecter.py b/BitTornado/Client/Connecter.py index 6fdd1ba..fbb81fd 100644 --- a/BitTornado/Client/Connecter.py +++ b/BitTornado/Client/Connecter.py @@ -1,3 +1,5 @@ +from .Extension import PeerExtensions +from ..Meta.bencode import bdecode from ..Types import Bitfield from BitTornado.clock import clock @@ -15,6 +17,12 @@ REQUEST = b'\x06' # index, begin, length PIECE = b'\x07' # index, begin, piece CANCEL = b'\x08' # index, begin, piece +DHT_PORT = b'\x09' # port (2 bytes) +FAST_SUGGEST = b'\x0d' # index +FAST_HAVE_ALL = b'\x0e' # no payload +FAST_HAVE_NONE = b'\x0f' # no payload +FAST_REJECT = b'\x10' # index, begin, length +ALLOW_FAST = b'\x11' # index EXTENDED = b'\x14' # msg_id, payload # Extended Message IDs @@ -34,6 +42,7 @@ def __init__(self, connection, connecter, ccount): self.upload = None # Uploader.Upload self.send_choke_queued = False # Bool (togglable) self.just_unchoked = None # None -> 0 <-> clock() + self.supported_exts = {} # Pass-through functions self.get_id = connection.get_id @@ -97,6 +106,9 @@ def send_bitfield(self, bitfield): def send_have(self, index): self._send_message(HAVE + index.to_bytes(4, 'big')) + def send_extended(self, ext_id, payload): + self._send_message(EXTENDED + ext_id.to_bytes(1, 'big') + payload) + def send_keepalive(self): self._send_message(b'') @@ -311,5 +323,7 @@ def got_message(self, connection, message): if c.download.got_piece(i, int.from_bytes(message[5:9], 'big'), message[9:]): self.got_piece(i) + elif t == EXTENDED: + c.got_extended(message) else: connection.close() diff --git a/BitTornado/Client/Extension.py b/BitTornado/Client/Extension.py new file mode 100644 index 0000000..2580c93 --- /dev/null +++ b/BitTornado/Client/Extension.py @@ -0,0 +1,126 @@ +from BitTornado.Meta.bencode import bencode, bdecode +# Design +# +# Fundamental: +# extension_name, msg_id, handler +# +# ext_table[msg_id] = (extension_name, handler) +# extensions[extension_name] = msg_id +# +# Ours: +# msg_id = index +# ext_table = [(extension_name, handler)] +# +# ext_lookup = {extension_name: msg_id} +# +# Theirs: +# msg_id = key +# handlers = {msg_id: (extension_name, handler)} +# extensions = {extension_name: msg_id} +# +# Default: +# handlers = {0: (handshake, handshake_handler)} +# extensions = {handshake: 0} + + +class _SupportedExtensions(object): + def __init__(self): + self.table = [Handshake(None)] + self.lookup = {} + + def register(self, handler): + """Register a handler (may be used as decorator)""" + assert isinstance(handler.name, (bytes, str)) + if handler.name in self.lookup: + # Previously registered, do nothing + if self.table[self.lookup[handler.name]] == handler: + return + + # Previously unregistered, restore + try: + msg_id = self.table.index(handler, 1) + self.lookup[handler.name] = msg_id + except ValueError: + pass + + # New handler (possibly replacing existing handler) + self.lookup[handler.name] = len(self.table) + self.table.append(handler) + return handler + + def unregister(self, item): + if isinstance(item, int): + msg_id = item + handler = self.table[item] + elif isinstance(item, ExtensionHandler): + handler = item + try: + msg_id = self.table.index(item, 1) + except ValueError: + raise ValueError("Handler not registered!") + +EXTENSIONS = _SupportedExtensions() + +class ExtensionHandler(object): + name = None + + def prepare(self): + raise NotImplementedError + + def receive(self, payload): + raise NotImplementedError + + +class Handshake(ExtensionHandler): + """Handle the BEP 10 handshake, registering mutually supported handshakes + for a peer. + """ + def __init__(self, extensions=None): + self.exts = extensions + self.payload = {'m': ext_lookup} + + def prepare(self): + return bencode(self.payload) + + def receive(self, payload): + if self.exts is None: + raise ValueError('This Handshake is not associated with a client.') + message = bdecode(payload) + to_add = [] + to_remove = [] + + for ext_name, msg_id in message['m']: + if msg_id == 0: + to_remove.append(ext_name) + elif ext_name not in ext_table: + continue + elif ext_name not in self.exts.extensions: + to_add.append((ext_name, msg_id)) + elif self.exts.extensions[ext_name] != msg_id: + to_remove.append(ext_name) + to_add.append((ext_name, msg_id)) + + if len(to_add) + len(to_remove) == 0: + return + + handlers = self.exts.handlers.copy() + extensions = self.exts.extensions.copy() + + for ext_name in to_remove: + old_id = extensions[ext_name] + assert handlers[old_id].name == ext_name + del handlers[old_id] + del extensions[ext_name] + + for ext_name, msg_id in to_add: + extensions[ext_name] = msg_id + handlers[msg_id] = ext_table[ext_lookup[ext_name]] + + +class PeerExtensions(object): + def __init__(self): + self.handlers = {0: Handshake(self)} + self.extensions = {} + + def __getitem__(self, key): + return self.handlers[key] From 533d234588d29503a4987fe7844e9772d734675a Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 31 May 2016 21:29:46 -0400 Subject: [PATCH 5/6] WIP: Handle extended message --- BitTornado/Client/Connecter.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/BitTornado/Client/Connecter.py b/BitTornado/Client/Connecter.py index fbb81fd..c67ba42 100644 --- a/BitTornado/Client/Connecter.py +++ b/BitTornado/Client/Connecter.py @@ -179,6 +179,18 @@ def got_request(self, piece_num, pos, length): self.connecter.ratelimiter.ping(clock() - self.just_unchoked) self.just_unchoked = 0 + def got_extended(self, message): + message_id = message[:1] + if message_id == EXT_HANDSHAKE: + payload = bdecode(message[1:]) + supported_exts = payload['m'] + self.supported_exts.update((k, v) + for k, v in supported_exts.items() + if k in SUPPORTED_EXTS) + else: + + + class Connecter(object): def __init__(self, make_upload, downloader, choker, numpieces, totalup, From d156558df0b05e686895004bfad46bfc09f30737 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Sun, 18 Sep 2016 17:43:50 -0400 Subject: [PATCH 6/6] STY: PEP8 --- BitTornado/Client/Connecter.py | 3 +-- BitTornado/Client/Extension.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/BitTornado/Client/Connecter.py b/BitTornado/Client/Connecter.py index c67ba42..f5c4ff9 100644 --- a/BitTornado/Client/Connecter.py +++ b/BitTornado/Client/Connecter.py @@ -188,8 +188,7 @@ def got_extended(self, message): for k, v in supported_exts.items() if k in SUPPORTED_EXTS) else: - - + pass class Connecter(object): diff --git a/BitTornado/Client/Extension.py b/BitTornado/Client/Extension.py index 2580c93..169533d 100644 --- a/BitTornado/Client/Extension.py +++ b/BitTornado/Client/Extension.py @@ -61,6 +61,7 @@ def unregister(self, item): EXTENSIONS = _SupportedExtensions() + class ExtensionHandler(object): name = None