From c7fabc071e897ee82f5c463b927f7f5c4e695096 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 9 Apr 2026 14:01:45 -0600 Subject: [PATCH 1/7] started refactor to use CESR codes for memograms --- src/hio/core/memo/__init__.py | 2 +- src/hio/core/memo/memoing.py | 117 +++++++++++++++++++++------- tests/core/memo/test_memoing.py | 47 +++++------ tests/core/udp/test_peer_memoing.py | 8 +- tests/core/uxd/test_peer_memoing.py | 6 +- 5 files changed, 123 insertions(+), 57 deletions(-) diff --git a/src/hio/core/memo/__init__.py b/src/hio/core/memo/__init__.py index 809660f5..f5bef6fb 100644 --- a/src/hio/core/memo/__init__.py +++ b/src/hio/core/memo/__init__.py @@ -4,7 +4,7 @@ """ -from .memoing import (Versionage, Sizage, GramDex, AuthDex, Keyage, +from .memoing import (Versionage, Sizage, MemoDex, GramDex, AuthDex, Keyage, openMemoer, Memoer, MemoerDoer, openSM, SureMemoer, SureMemoerDoer) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index b61c0cae..282702b2 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -255,9 +255,10 @@ """ Sizage = namedtuple("Sizage", "cz mz oz nz sz hz") + @dataclass(frozen=True) -class GramCodex: - """GramCodex is codex of all Gram Codes. +class MemoGramCodex: + """MemoGramCodex is codex of all varieties of MemoGram Hard Codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ @@ -267,21 +268,36 @@ class GramCodex: def __iter__(self): return iter(astuple(self)) -GramDex = GramCodex() # Make instance +MemoDex = MemoGramCodex() # Make instance @dataclass(frozen=True) -class MemoGramCodex: - """MemoGramCodex is codex of all MemoGram Codes. +class GramCodex: + """GramCodex is codex of all Gram Hard Codes. Only provide defined codes. Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ MemoGram: str = '__' # memogram code - AuthMemoGram: str = '_-' # authenticated memogram code (signed) def __iter__(self): return iter(astuple(self)) -MemoDex = MemoGramCodex() # Make instance +GramDex = GramCodex() # Make instance + + +@dataclass(frozen=True) +class AuthGramCodex: + """AuthGramCodex is codex of all AuthGram (authenticated signed) Hard Codes. + Only provide defined codes. + Undefined are left out so that inclusion(exclusion) via 'in' operator works. + """ + AuthMemoGram: str = '_-' # authenticated memogram code (signed) + + def __iter__(self): + return iter(astuple(self)) + +AuthDex = AuthGramCodex() # Make instance + + #@dataclass(frozen=True) #class AckGramCodex: @@ -297,18 +313,7 @@ def __iter__(self): #AckDex = AckGramCodex() # Make instance -@dataclass(frozen=True) -class AuthGramCodex: - """AuthGramCodex is codex of all AuthGram (authenticated signed) Codes. - Only provide defined codes. - Undefined are left out so that inclusion(exclusion) via 'in' operator works. - """ - AuthMemoGram: str = '_-' # authenticated memogram code (signed) - - def __iter__(self): - return iter(astuple(self)) -AuthDex = AuthGramCodex() # Make instance class Memoer(hioing.Mixin): @@ -495,7 +500,7 @@ class Memoer(hioing.Mixin): _oid (str or None): see oid property """ Version = Versionage(major=0, minor=0) # default version - Codex = GramDex + Codex = MemoDex Codes = asdict(Codex) # map code name to code Names = {val : key for key, val in Codes.items()} # invert map code to code name Sodex = AuthDex # signed gram codex @@ -545,7 +550,7 @@ def _encodeOID(cls, raw, code='B'): qb64 = code + b64.decode() # fully qualified base64 oid with prefix code - _, _, oz, _, _, _ = cls.Sizes[AuthDex.AuthMemoGram] # cz mz oz nz sz hz + _, _, oz, _, _, _ = cls.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz if len(qb64) != oz: hioing.MemoerError(f"Invalid oid qb64 size={len(qb64) != {oz}}") @@ -795,7 +800,7 @@ def _encodeSGN(cls, raw, code='0B'): qb64 = code + b64.decode() # fully qualified base64 with prefixqb64de - _, _, _, _, sz, _ = cls.Sizes[AuthDex.AuthMemoGram] # cz mz oz nz sz hz + _, _, _, _, sz, _ = cls.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz if len(qb64) != sz: hioing.MemoerError(f"Invalid sig qb64 size={len(qb64) != {sz}}") @@ -861,7 +866,7 @@ def __init__(self, *, txms=None, txgs=None, txbs=None, - code=GramDex.MemoGram, + code=MemoDex.MemoGram, curt=False, size=None, verific=False, @@ -1150,7 +1155,7 @@ def wiff(self, gram): """Determines encoding of gram bytes header when parsing grams. The encoding maybe either base2 or base64. - Returns: + Returns:: curt (bool): True means base2 encoding False means base64 encoding Otherwise raises hioing.MemoerError @@ -1180,8 +1185,8 @@ def wiff(self, gram): All Base2 codes for Base64 chars are unique since the B2 codes are just the count from 0 to 63. There is one collision, however, between Base2 and Base 64 for invalid gram headers. This is because 0o27 is the - base2 code for ascii 'V'. This means that Base2 encodings - that begin with 0o27 wich is the Base2 encoding of the ascii 'V, would be + base2 code for ascii 'X'. This means that Base2 encodings + that begin with 0o27 which is the Base2 encoding of the ascii 'X', would be incorrectly detected as text not binary. """ @@ -1193,6 +1198,66 @@ def wiff(self, gram): raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") + def wiffNew(self, gram): + """Determines encoding of gram bytes header when parsing grams. + The encoding maybe either base2 or base64. + + Returns:: + curt (bool): True means base2 encoding + False means base64 encoding + Otherwise raises hioing.MemoerError + + All gram head codes start with '1' in base64 text or in base2 binary. + Given only allowed chars are from the set of base64 then can determine + if header is in base64 or base2. + + 1AAQ, 1AAR, 1AAS, 1AAT, 1AAU, 1AAV, 1AAW, 1AAX, 1AAY, 1AAZ + + First sextet: 0b is binary, 0o is octal 2 octal digits or 6 bits + 0x is hexadecimal 1 sextet is 1.5 hex digits or octets + + 0o14 = 0b001100 means first sextet of '1' in base64 (0b00110001) + 0o65 = 0b110101 means first sextet of '1' in base2 (0b110101) + + Assuming that only '1' is allowed as the first char of a valid gram head, + in either Base64 or Base2 encoding, a parser can unambiguously determine + if the gram header encoding is binary Base2 or text Base64 because + 0o14 != 0o65. + + The sextet 0o14 is not a unique first sextet among Base64 ascii chars. + It is shared with the ascii chars '0', '1', '2', '3'. So an invalidly + encoded gram header when in Base64 encoding is not fully detectable + unless the full first octet (8bits) is examined. But an invalid gram + header could have any of the following sextets also encoded invalidly. + Thise means that in general some other mechanism is required to detect + and invalidly encoded gram header. Therefore some additional tamper + evident mechanism such as a digest or signature is required to detect + and invalidly encoded gram header. But all we really need wiff to do is + to detect if a validly encoded gram header is encoded in Base64 or Base2 + and merely examining the first sextet is enough to unambiguously make that + determination. + + In addition there is one collision (see below) with invalid gram headers + in Base2 encoding. So an erroneously constructed gram might + not be detected merely by looking at the first sextet. + + All Base2 codes for Base64 chars are unique since the B2 codes are just + the count from 0 to 63. There is one collision, however, between Base2 and + Base 64 for the first sextet of an invalid gram header. + This is because 0o14 is the base2 code for ascii 'M'. This means that + invalid Base2 encoded gram headers that begin with 0o14 (the Base2 + encoding of the ascii 'M') would be incorrectly detected by wiff as text + not binary. + """ + + sextet = gram[0] >> 2 + if sextet == 0o14: + return False # base64 text encoding + if sextet == 0o65: + return True # base2 binary encoding + + raise hioing.MemoerError(f"Unexpected {sextet=} at gram head start.") + def verify(self, oid, sig, ser): """Verify signature sig on signed part of gram, ser, using current verkey @@ -2189,7 +2254,7 @@ class SureMemoer(Tymee, Memoer): """ Tymeout = 0.0 # tymeout in seconds, tymeout of 0.0 means ignore tymeout - def __init__(self, *, tymeout=None, code=GramDex.AuthMemoGram, verific=True, **kwa): + def __init__(self, *, tymeout=None, code=MemoDex.AuthMemoGram, verific=True, **kwa): """ Initialization method for instance. Inherited Parameters: diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 3dbeef8f..ca5205ba 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -16,7 +16,8 @@ from hio.help import helping from hio.base import doing, tyming from hio.core.memo import memoing -from hio.core.memo import Versionage, Sizage, GramDex, AuthDex, Memoer, Keyage +from hio.core.memo import (Versionage, Sizage, Keyage, MemoDex, GramDex, AuthDex, + Memoer, ) def _setupKeep(salt=None): @@ -96,7 +97,7 @@ def test_memoer_class(): """Test class attributes of Memoer class""" assert Memoer.Version == Versionage(major=0, minor=0) - assert Memoer.Codex == memoing.GramDex + assert Memoer.Codex == memoing.MemoDex assert Memoer.Codes == \ { @@ -151,7 +152,7 @@ def test_memoer_class(): raw, code = Memoer._decodeOID(oid) assert raw == verkey assert code == 'B' # nontrans AID - _, _, oz, _, _, _ = Memoer.Sizes[AuthDex.AuthMemoGram] # cz mz oz nz sz hz + _, _, oz, _, _, _ = Memoer.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz assert len(oid) == 44 == oz qvk = Memoer._encodeQVK(raw=verkey) @@ -176,7 +177,7 @@ def test_memoer_class(): raw, code = Memoer._decodeSGN(sgntr) assert raw == signature assert code == '0B' - _, _, _, _, sz, _ = Memoer.Sizes[AuthDex.AuthMemoGram] # cz mz oz nz sz hz + _, _, _, _, sz, _ = Memoer.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz assert len(sgntr) == 88 == sz """Done Test""" @@ -231,12 +232,12 @@ def test_memoer_sign_verify(): oid = list(keep.keys())[0] assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' - peer = memoing.Memoer(code=GramDex.AuthMemoGram, keep=keep, oid=oid) + peer = memoing.Memoer(code=MemoDex.AuthMemoGram, keep=keep, oid=oid) assert peer.name == "main" assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.AuthMemoGram == '_-' + assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize @@ -292,7 +293,7 @@ def test_memoer_basic(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.MemoGram == '__' + assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize @@ -442,7 +443,7 @@ def test_memoer_small_gram_size(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.MemoGram == '__' + assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == 33 # can't be smaller than head + neck + 1 @@ -619,7 +620,7 @@ def test_memoer_multiple(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.MemoGram == '__' + assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt assert not peer.verific assert not peer.echoic @@ -698,7 +699,7 @@ def test_memoer_multiple_echoic_service_tx_rx(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.MemoGram == '__' + assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt assert not peer.verific assert peer.echoic @@ -752,7 +753,7 @@ def test_memoer_multiple_echoic_service_all(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.MemoGram == '__' + assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt assert not peer.verific assert peer.echoic @@ -803,12 +804,12 @@ def test_memoer_basic_signed(): oid = list(keep.keys())[0] assert oid == 'BJZTHNWXscuT-SPokPzSeBkShpHj6g8bQrP0Rh7IJNUp' - peer = memoing.Memoer(code=GramDex.AuthMemoGram, keep=keep, oid=oid) + peer = memoing.Memoer(code=MemoDex.AuthMemoGram, keep=keep, oid=oid) assert peer.name == "main" assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.AuthMemoGram == '_-' + assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize @@ -838,7 +839,7 @@ def test_memoer_basic_signed(): assert not peer.wiff(g) # base64 assert g.find(memo.encode()) != -1 assert len(g) == 160 + 4 + len(memo) - assert g[:2].decode() == GramDex.AuthMemoGram + assert g[:2].decode() == MemoDex.AuthMemoGram assert d == dst == 'beta' peer.serviceTxGrams() assert not peer.txgs @@ -893,7 +894,7 @@ def test_memoer_basic_signed(): assert not peer.wiff(g) # base64 assert g.find(memo.encode()) != -1 assert len(g) == 160 + 4 + len(memo) - assert g[:2].decode() == GramDex.AuthMemoGram + assert g[:2].decode() == MemoDex.AuthMemoGram assert d == dst == 'beta' peer.serviceTxGrams(echoic=True) assert not peer.txgs @@ -932,7 +933,7 @@ def test_memoer_basic_signed(): assert peer.wiff(g) # base64 assert g.find(memo.encode()) != -1 assert len(g) == 3 * (160 + 4) // 4 + len(memo) - assert helping.codeB2ToB64(g, 2) == GramDex.AuthMemoGram + assert helping.codeB2ToB64(g, 2) == MemoDex.AuthMemoGram assert d == dst == 'beta' peer.serviceTxGrams() assert not peer.txgs @@ -1001,13 +1002,13 @@ def test_memoer_multiple_signed(): oidBeta = list(keep.keys())[1] assert oidBeta == 'DGORBFFJe5Zj4T1FQHpRFSe41hQuq8HULAMWyc9C07ni' - peer = memoing.Memoer(code=GramDex.AuthMemoGram, size=170, keep=keep, oid=oid) + peer = memoing.Memoer(code=MemoDex.AuthMemoGram, size=170, keep=keep, oid=oid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.AuthMemoGram == '_-' + assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt assert not peer.verific # force rx must be signed assert not peer.echoic @@ -1152,7 +1153,7 @@ def test_memoer_verific(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.MemoGram == '__' + assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert peer.size == peer.MaxGramSize @@ -1231,14 +1232,14 @@ def test_memoer_multiple_signed_verific_echoic_service_all(): # verific forces rx memos to be signed or dropped # to force signed tx then use Signed code - peer = memoing.Memoer(code=GramDex.AuthMemoGram, size=170, verific=True, + peer = memoing.Memoer(code=MemoDex.AuthMemoGram, size=170, verific=True, echoic=True, keep=keep, oid=oid) assert peer.size == 170 assert peer.name == "main" assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.AuthMemoGram == '_-' + assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt assert peer.verific assert peer.echoic @@ -1384,7 +1385,7 @@ def test_sure_memoer_basic(): assert peer.opened == False assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.AuthMemoGram == '_-' + assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt assert peer.verific assert peer.echoic @@ -1557,7 +1558,7 @@ def test_sure_memoer_multiple_echoic_service_all(): assert peer.opened == True assert peer.bc is None assert peer.bs == memoing.Memoer.BufSize == 65535 - assert peer.code == memoing.GramDex.AuthMemoGram == '_-' + assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt assert peer.verific assert peer.echoic diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index 65915d0a..a72cc19e 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -11,7 +11,7 @@ from hio.help import helping from hio.base import doing, tyming -from hio.core.memo import GramDex +from hio.core.memo import MemoDex from hio.core.udp import udping, peermemoing @@ -23,7 +23,7 @@ def test_memoer_peer_basic(): alpha = peermemoing.PeerMemoer(name="alpha", temp=True) assert alpha.name == "alpha" - assert alpha.code == GramDex.MemoGram + assert alpha.code == MemoDex.MemoGram assert not alpha.curt assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 1240 # default MaxGramSize for udp @@ -36,7 +36,7 @@ def test_memoer_peer_basic(): size = 38 # force gram size to be smaller than default so forces segmentation alpha = peermemoing.PeerMemoer(name="alpha", temp=True, size=size, port=alphaPort) assert alpha.name == "alpha" - assert alpha.code == GramDex.MemoGram + assert alpha.code == MemoDex.MemoGram assert not alpha.curt assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == size @@ -173,7 +173,7 @@ def test_memoer_peer_open(): assert alpha.name == "alpha" - assert alpha.code == GramDex.MemoGram + assert alpha.code == MemoDex.MemoGram assert not alpha.curt assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == size diff --git a/tests/core/uxd/test_peer_memoing.py b/tests/core/uxd/test_peer_memoing.py index 9aa8bf19..6b1bd2df 100644 --- a/tests/core/uxd/test_peer_memoing.py +++ b/tests/core/uxd/test_peer_memoing.py @@ -10,7 +10,7 @@ from hio.help import helping from hio.base import doing, tyming -from hio.core.memo import GramDex +from hio.core.memo import MemoDex from hio.core.uxd import uxding, peermemoing @@ -20,7 +20,7 @@ def test_memoer_peer_basic(): return alpha = peermemoing.PeerMemoer(name="alpha", temp=True, size=38) assert alpha.name == "alpha" - assert alpha.code == GramDex.MemoGram + assert alpha.code == MemoDex.MemoGram assert not alpha.curt assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 38 @@ -149,7 +149,7 @@ def test_memoer_peer_open(): with (peermemoing.openPM(name='alpha', size=38) as alpha, peermemoing.openPM(name='beta', size=38) as beta): assert alpha.name == "alpha" - assert alpha.code == GramDex.MemoGram + assert alpha.code == MemoDex.MemoGram assert not alpha.curt assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz assert alpha.size == 38 From 242a985c895c1ac2cd6b89fdbcb4c683f40d77db Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 9 Apr 2026 15:03:00 -0600 Subject: [PATCH 2/7] more setup to refactor to CESR support --- src/hio/core/memo/memoing.py | 6 ++++++ tests/core/memo/test_memoing.py | 14 +++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 282702b2..446be794 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -264,6 +264,10 @@ class MemoGramCodex: """ MemoGram: str = '__' # memogram code AuthMemoGram: str = '_-' # authenticated memogram code (signed) + GramZero: str = '1AAQ' # zeroth gram code + Gram: str = '1AAR' # non-zeroth gram code + GramAuthZero: str = '1AAS' # zeroth authenticated gram code (signed) + GramAuth: str = '1AAT' # non-zeroth authenticated gram code (signed) def __iter__(self): return iter(astuple(self)) @@ -291,6 +295,8 @@ class AuthGramCodex: Undefined are left out so that inclusion(exclusion) via 'in' operator works. """ AuthMemoGram: str = '_-' # authenticated memogram code (signed) + GramAuthZero: str = '1AAS' # zeroth authenticated gram code (signed) + GramAuth: str = '1AAT' # non-zeroth authenticated gram code (signed) def __iter__(self): return iter(astuple(self)) diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index ca5205ba..bc0bca55 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -103,6 +103,10 @@ def test_memoer_class(): { 'MemoGram': '__', 'AuthMemoGram': '_-', + 'GramZero': '1AAQ', + 'Gram': '1AAR', + 'GramAuthZero': '1AAS', + 'GramAuth': '1AAT' , } # Codes table with sizes of code (hard) and full primitive material @@ -135,7 +139,15 @@ def test_memoer_class(): if oz: assert sz # sz must not be empty if oz not empty - assert Memoer.Names == {'__': 'MemoGram', '_-': 'AuthMemoGram'} + assert Memoer.Names == \ + { + '__': 'MemoGram', + '_-': 'AuthMemoGram', + '1AAQ': 'GramZero', + '1AAR': 'Gram', + '1AAS': 'GramAuthZero', + '1AAT': 'GramAuth', + } assert Memoer.Sodex == AuthDex # Base2 Binary index representation of Text Base64 Char Codes From d5fa35c539d53638962a565acd842f3dc23f32cd Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 9 Apr 2026 16:57:48 -0600 Subject: [PATCH 3/7] added zz field to Sizes refactored field names --- src/hio/core/memo/memoing.py | 226 ++++++++++++++-------------- tests/core/memo/test_memoing.py | 62 ++++---- tests/core/udp/test_peer_memoing.py | 6 +- tests/core/uxd/test_peer_memoing.py | 4 +- 4 files changed, 155 insertions(+), 143 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 446be794..569b933d 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -94,12 +94,12 @@ HeadCode hc = b'__' HeadCodeSize hcs = 2 MemoIDSize mis = 22 - Gram OID oz =0 + Gram OID xz =0 GramNumSize gns = 4 GramCntSize gcs = 4 GramHeadSize ghs = 28 short GramHeadSize ghs = 32 long - Gram Sig size sz = 0 + Gram Sig size az = 0 GramHeadNeckLongSize ghns = 28 + 4 = 32 Gram Header Fields signed:: @@ -107,21 +107,21 @@ HeadCode hc = b'_-' HeadCodeSize hcs = 2 MemoIDSize mis = 22 - Gram OID oz = 44 + Gram OID xz = 44 GramNumSize gns = 4 GramCntSize gcs = 4 GramHeadSize ghs = 28 + 44 = 72 short GramHeadSize ghs = 32+ 44 = 76 long GramHeadNeckSize ghns = 28 + 4 = 32 - GramSigSize sz = 88 + GramSigSize az = 88 GramHeadNeckSigSize ghnss = 72 + 88 = 160 short GramHeadNeckSigSize ghnss = 76 + 88 = 164 long Sizes:: { - '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), - '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), + '__': Sizage(hz=2, mz=22, xz=0, nz=4, zz=0, az=0, oz=28), + '_-': Sizage(hz=2, mz=22, xz=44, nz=4, zz=0, az=88, oz=160), } Sizing: @@ -236,24 +236,29 @@ """ """Sizage: namedtuple for gram header part size entries in Memoer code tables -cz is the serialization code part size int number of chars in code part +hz is the serialization hard code part size int number of chars in hard code part mz is the mid (memo ID) part size int number of chars in the mid part -oz is the oid (origin ID) part size int number of chars in the oid part -nz is the neck part size int number of chars for the gram number in all grams - it is also the size of the gram count that only appears in the zeroth gram - The zeroth gram has a long neck consiting of two nz sized fields - All other grams have ashort neck consisting of 1 nz sized field -sz is the signature part size int number of chars in the signature part - the signature part is discontinuously attached after the body part but +xz is the oid (origin ID) part size int number of chars in the oid part +nz is the gram num (neck) part size int number of chars for the gram number +zz is the gram count part size int count of grams in memo. It only appears in the + zeroth gram in a memo. +az is the authenticator (signature) part size int number of chars in the auth part + the authenticator part is discontinuously attached after the body part but its size is included in the overhead size computation for the body size -hz is the head (header) size int number of chars:: +oz is the overhead size int number of chars:: - head with short neck - hz = cz + mz + oz + nz + sz - zeroth head with long neck is calculated from short neck head - zhz = hz + nz + overhead is header with short neck and authenticator + oz = hz + mz + xz + nz + zz+ az + + zeroth overhead is header with long neck relative to short neck + zoz = oz (short) + zz (note zz in zeroth gram = nz otherwise 0) + +The gram num and gram count both have the same size i.e. the neck size. +The gram count only appears in the zeroth gram +The zeroth gram has a long neck consiting of two nz sized fields +All other grams have a short neck consisting of 1 nz sized field """ -Sizage = namedtuple("Sizage", "cz mz oz nz sz hz") +Sizage = namedtuple("Sizage", "hz mz xz nz zz az oz") @dataclass(frozen=True) @@ -510,10 +515,11 @@ class Memoer(hioing.Mixin): Codes = asdict(Codex) # map code name to code Names = {val : key for key, val in Codes.items()} # invert map code to code name Sodex = AuthDex # signed gram codex - # dict of gram header part sizes keyed by gram codes: cz mz oz nz sz hz + # dict of gram header part sizes keyed by gram codes: hz mz xz nz zz az oz Sizes = { - '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), - '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), + '__': Sizage(hz=2, mz=22, xz=0, nz=4, zz=0, az=0, oz=28), + '_-': Sizage(hz=2, mz=22, xz=44, nz=4, zz=0, az=88, oz=160), + '1AAQ': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), } # Base2 Binary index representation of Text Base64 Char Codes @@ -556,9 +562,9 @@ def _encodeOID(cls, raw, code='B'): qb64 = code + b64.decode() # fully qualified base64 oid with prefix code - _, _, oz, _, _, _ = cls.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz - if len(qb64) != oz: - hioing.MemoerError(f"Invalid oid qb64 size={len(qb64) != {oz}}") + _, _, xz, _, _, _, _ = cls.Sizes[MemoDex.AuthMemoGram] # hz mz xz nz zz az oz + if len(qb64) != xz: + hioing.MemoerError(f"Invalid oid qb64 size={len(qb64) != {xz}}") return qb64 # fully qualified base64 oid with prefix code @@ -588,8 +594,8 @@ def _decodeOID(cls, qb64): if hasattr(qb64, "decode"): # bytes qb64 = qb64.decode() # convert to str - cz = 1 # only support qb64 length 44 - code = qb64[:cz] + hz = 1 # only support qb64 length 44 + code = qb64[:hz] if code not in ('B', 'D', 'E'): raise hioing.MemoerError(f"Invalid oid {code=}") @@ -597,12 +603,12 @@ def _decodeOID(cls, qb64): if qz != 44: raise hioing.MemoerError(f"Invalid oid text size {qz=} not 44") - cz = len(code) - pz = cz % 4 # net pad size given cz - if cz != pz != 1: # special case here for now we only accept cz=1 - raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + hz = len(code) + pz = hz % 4 # net pad size given hz + if hz != pz != 1: # special case here for now we only accept hz=1 + raise hioing.MemoerError(f"Invalid {hz=} not equal {pz=} not equal 1") - base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + base = pz * b'A' + qb64[hz:].encode() # strip code from b64 and prepad pz 'A's paw = decodeB64(base) # now should have pz leading sextexts of zeros raw = paw[pz:] # remove prepad midpad bytes to invert back to raw # ensure midpad bytes are zero @@ -610,7 +616,7 @@ def _decodeOID(cls, qb64): if pi != 0: raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") - if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + if len(raw) != ((qz - hz) * 3 // 4): # exact lengths raise hioing.MemoerError(f"Improperly qualified material = {oid}") return raw, code @@ -668,8 +674,8 @@ def _decodeQVK(cls, qb64): if hasattr(qb64, "decode"): # bytes qb64 = qb64.decode() # convert to str - cz = 1 # only support qb64 length 44 - code = qb64[:cz] + hz = 1 # only support qb64 length 44 + code = qb64[:hz] if code not in ('B'): raise hioing.MemoerError(f"Invalid qvk {code=}") @@ -677,12 +683,12 @@ def _decodeQVK(cls, qb64): if qz != 44: raise hioing.MemoerError(f"Invalid qvk text size {qz=} not 44") - cz = len(code) - pz = cz % 4 # net pad size given cz - if cz != pz != 1: # special case here for now we only accept cz=1 - raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + hz = len(code) + pz = hz % 4 # net pad size given hz + if hz != pz != 1: # special case here for now we only accept hz=1 + raise hioing.MemoerError(f"Invalid {hz=} not equal {pz=} not equal 1") - base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + base = pz * b'A' + qb64[hz:].encode() # strip code from b64 and prepad pz 'A's paw = decodeB64(base) # now should have pz leading sextexts of zeros raw = paw[pz:] # remove prepad midpad bytes to invert back to raw # ensure midpad bytes are zero @@ -690,7 +696,7 @@ def _decodeQVK(cls, qb64): if pi != 0: raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") - if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + if len(raw) != ((qz - hz) * 3 // 4): # exact lengths raise hioing.MemoerError(f"Improperly qualified material = {qss}") return raw, code @@ -748,8 +754,8 @@ def _decodeQSS(cls, qb64): if hasattr(qb64, "decode"): # bytes qb64 = qb64.decode() # convert to str - cz = 1 # only support qb64 length 44 - code = qb64[:cz] + hz = 1 # only support qb64 length 44 + code = qb64[:hz] if code not in ('A'): raise hioing.MemoerError(f"Invalid qss {code=}") @@ -757,12 +763,12 @@ def _decodeQSS(cls, qb64): if qz != 44: raise hioing.MemoerError(f"Invalid qss text size {qz=} not 44") - cz = len(code) - pz = cz % 4 # net pad size given cz - if cz != pz != 1: # special case here for now we only accept cz=1 - raise hioing.MemoerError(f"Invalid {cz=} not equal {pz=} not equal 1") + hz = len(code) + pz = hz % 4 # net pad size given hz + if hz != pz != 1: # special case here for now we only accept hz=1 + raise hioing.MemoerError(f"Invalid {hz=} not equal {pz=} not equal 1") - base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + base = pz * b'A' + qb64[hz:].encode() # strip code from b64 and prepad pz 'A's paw = decodeB64(base) # now should have pz leading sextexts of zeros raw = paw[pz:] # remove prepad midpad bytes to invert back to raw # ensure midpad bytes are zero @@ -770,7 +776,7 @@ def _decodeQSS(cls, qb64): if pi != 0: raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") - if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + if len(raw) != ((qz - hz) * 3 // 4): # exact lengths raise hioing.MemoerError(f"Improperly qualified material = {qss}") return raw, code @@ -806,9 +812,9 @@ def _encodeSGN(cls, raw, code='0B'): qb64 = code + b64.decode() # fully qualified base64 with prefixqb64de - _, _, _, _, sz, _ = cls.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz - if len(qb64) != sz: - hioing.MemoerError(f"Invalid sig qb64 size={len(qb64) != {sz}}") + _, _, _, _, _, az, _ = cls.Sizes[MemoDex.AuthMemoGram] # hz mz xz nz zz az oz + if len(qb64) != az: + hioing.MemoerError(f"Invalid sig qb64 size={len(qb64) != {az}}") return qb64 # fully qualified base64 oid with prefix code @@ -832,8 +838,8 @@ def _decodeSGN(cls, qb64): if hasattr(qb64, "decode"): # bytes qb64 = qb64.decode() # convert to str - cz = 2 # only support qb64 length 88 - code = qb64[:cz] + hz = 2 # only support qb64 length 88 + code = qb64[:hz] if code not in ('0B'): raise hioing.MemoerError(f"Invalid signature {code=}") @@ -841,11 +847,11 @@ def _decodeSGN(cls, qb64): if qz != 88: raise hioing.MemoerError(f"Invalid sig text size {qz=} not 88") - pz = cz % 4 # net pad size given cz - if pz != cz: - raise hioing.MemoerError(f"Invalid {pz=} not equal {cz=}") + pz = hz % 4 # net pad size given hz + if pz != hz: + raise hioing.MemoerError(f"Invalid {pz=} not equal {hz=}") - base = pz * b'A' + qb64[cz:].encode() # strip code from b64 and prepad pz 'A's + base = pz * b'A' + qb64[hz:].encode() # strip code from b64 and prepad pz 'A's paw = decodeB64(base) # now should have pz leading sextexts of zeros raw = paw[pz:] # remove prepad midpad bytes to invert back to raw # ensure midpad bytes are zero @@ -853,7 +859,7 @@ def _decodeSGN(cls, qb64): if pi != 0: raise hioing.MemoerError(f"Nonzero midpad bytes=0x{pi:0{(pz)*2}x}.") - if len(raw) != ((qz - cz) * 3 // 4): # exact lengths + if len(raw) != ((qz - hz) * 3 // 4): # exact lengths raise hioing.MemoerError(f"Improperly qualified material = {sig}") return raw, code # qualified base64 sigseed @@ -1070,14 +1076,14 @@ def size(self, size): size (int or None): gram size for rending memo """ - _, _, _, nz, _, hz = self.Sizes[self.code] # cz mz oz nz sz hz + _, _, _, nz, _, _, oz = self.Sizes[self.code] # hz mz xz nz zz az oz size = size if size is not None else self.MaxGramSize if self.curt: # minimum header smaller when in base2 curt - hz = 3 * hz // 4 + oz = 3 * oz // 4 nz = 3 * nz // 4 # mininum size must be big enough for first gram header and 1 body byte - self._size = max(min(size, self.MaxGramSize), hz + nz + 1) + self._size = max(min(size, self.MaxGramSize), oz + nz + 1) @property @@ -1356,42 +1362,43 @@ def pick(self, gram): raise hioing.MemoerError(f"Unsigned gram {code =} when signed " f"required.") - cz, mz, oz, nz, sz, hz = self.Sizes[code] # cz mz oz nz sz hz + hz, mz, xz, nz, zz, az, oz = self.Sizes[code] # hz mz xz nz zz az oz pz = (3 - ((mz) % 3)) % 3 # net pad size for mid - cms = 3 * (cz + mz) // 4 # cz + mz are aligned on 24 bit boundary - hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 - nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + cms = 3 * (hz + mz) // 4 # hz + mz are aligned on 24 bit boundary oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 - sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + zz = 3 * zz // 4 # encoding b2 means head part sizes smaller by 3/4 + xz = 3 * xz // 4 # encoding b2 means head part sizes smaller by 3/4 + az = 3 * az // 4 # encoding b2 means head part sizes smaller by 3/4 - if len(gram) < (hz + 1): # not big enough for non-first gram + if len(gram) < (oz + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b2 gram" - f" < {hz + 1}.") + f" < {oz + 1}.") mid = encodeB64(gram[:cms]) # convert to b64b with prefix code - oid = encodeB64(gram[cms:cms+oz]) # convert to b64b - neck = gram[cms+oz:cms+oz+nz] # bytearray to bytes short part of neck + oid = encodeB64(gram[cms:cms+xz]) # convert to b64b + neck = gram[cms+xz:cms+xz+nz] # bytearray to bytes short part of neck gn = int.from_bytes(neck) # gram number convert to int #neck = encodeB64(neck) # convert to b64b #head = mid + oid + neck # bytes if gn == 0: # first (zeroth) gram so long neck - if len(gram) < hz + nz + 1: + if len(gram) < oz + nz + 1: raise hioing.MemoerError(f"Not enough rx bytes for b2 " - f"gram < {hz + nz + 1}.") - lneck = bytes(gram[cms+oz+nz:cms+oz+2*nz]) # bytearray to bytes + f"gram < {oz + nz + 1}.") + lneck = bytes(gram[cms+xz+nz:cms+xz+2*nz]) # bytearray to bytes gc = int.from_bytes(lneck) # gram count convert to int lneck = encodeB64(lneck) # convert to b64b - sig = encodeB64(gram[-sz if sz else len(gram):]) # last ss bytes are signature - del gram[-sz if sz else len(gram):] # strip sig if any + sig = encodeB64(gram[-az if az else len(gram):]) # last ss bytes are signature + del gram[-az if az else len(gram):] # strip sig if any sgram = bytes(gram[:]) # signed part make bytes copy to sign - del gram[:hz-sz+nz] # strip off fore head leaving body in gram + del gram[:oz-az+nz] # strip off fore head leaving body in gram #head += lneck else: # non-zeroth gram so short neck gc = None - sig = encodeB64(gram[-sz if sz else len(gram):]) - del gram[-sz if sz else len(gram):] # strip sig if any + sig = encodeB64(gram[-az if az else len(gram):]) + del gram[-az if az else len(gram):] # strip sig if any sgram = bytes(gram[:]) # signed part make bytes copy to sign - del gram[:hz-sz] # strip of fore head leaving body in gram + del gram[:oz-az] # strip of fore head leaving body in gram else: # base64 text encoding if len(gram) < 2: # assumes len(code) must be 2 @@ -1401,34 +1408,34 @@ def pick(self, gram): if self.verific and code not in self.Sodex: # must be signed raise hioing.MemoerError(f"Unsigned gram {code =} when signed " f"required.") - cz, mz, oz, nz, sz, hz = self.Sizes[code] # cz mz oz nz sz hz + hz, mz, xz, nz, zz, az, oz = self.Sizes[code] # hz mz xz nz zz az oz - if len(gram) < (hz + 1): # not big enough for non-first gram + if len(gram) < (oz + 1): # not big enough for non-first gram raise hioing.MemoerError(f"Not enough rx bytes for b64 gram" - f" < {hz + 1}.") + f" < {oz + 1}.") - mid = bytes(gram[:cz+mz]) # bytearray to bytes qb64b with prefix - oid = bytes(gram[cz+mz:cz+mz+oz]) #bytearry to bytes qb64b convert to qb64 - neck = bytes(gram[cz+mz+oz:cz+mz+oz+nz]) # qb64b short part of neck + mid = bytes(gram[:hz+mz]) # bytearray to bytes qb64b with prefix + oid = bytes(gram[hz+mz:hz+mz+xz]) #bytearry to bytes qb64b convert to qb64 + neck = bytes(gram[hz+mz+xz:hz+mz+xz+nz]) # qb64b short part of neck gn = helping.b64ToInt(neck) #head = mid + oid + neck # bytes if gn == 0: # first (zeroth) gram so long neck - if len(gram) < hz + nz + 1: + if len(gram) < oz + nz + 1: raise hioing.MemoerError(f"Not enough rx bytes for b64 " - f"gram < {hz + nz + 1}.") - lneck = bytes(gram[cz+mz+oz+nz:cz+mz+oz+2*nz]) # bytearray to bytes copy + f"gram < {oz + nz + 1}.") + lneck = bytes(gram[hz+mz+xz+nz:hz+mz+xz+2*nz]) # bytearray to bytes copy gc = helping.b64ToInt(lneck) # convert to int - sig = bytes(gram[-sz if sz else len(gram):]) # last sz bytes signature - del gram[-sz if sz else len(gram):] # strip sig if any + sig = bytes(gram[-az if az else len(gram):]) # last az bytes signature + del gram[-az if az else len(gram):] # strip sig if any sgram = bytes(gram[:]) # signed part make bytes copy to sign - del gram[:hz-sz+nz] # strip of fore head leaving body in gram + del gram[:oz-az+nz] # strip of fore head leaving body in gram #head += lneck # bytes else: # non-zeroth gram short neck gc = None - sig = bytes(gram[-sz if sz else len(gram):]) - del gram[-sz if sz else len(gram):] # strip sig if any + sig = bytes(gram[-az if az else len(gram):]) + del gram[-az if az else len(gram):] # strip sig if any sgram = bytes(gram[:]) # signed part make bytes copy to sign - del gram[:hz-sz] # strip of fore head leaving body in gram + del gram[:oz-az] # strip of fore head leaving body in gram if sig: # signature not empty #sgram = head + bytes(gram) # bytearray to bytes @@ -1750,38 +1757,39 @@ def rend(self, memo, oid=None): lookup sigkey to sign. None means not signable - Note zeroth gram has head + neck overhead, zhz = hz + nz + Note zeroth gram has head + neck overhead, zhz = oz + nz so bz that fits is smaller by nz relative to non-zeroth - non-zeroth grams just head overhead hz + non-zeroth grams just head overhead oz so bz that fits is bigger by nz relative to zeroth """ grams = [] memo = bytearray(memo.encode()) # convert and copy to bytearray # self.size is max gram size - cz, mz, oz, nz, sz, hz = self.Sizes[self.code] # cz mz oz nz sz hz + hz, mz, xz, nz, zz, az, oz = self.Sizes[self.code] # hz mz xz n zz az oz oid = oid if oid is not None else self.oid - if oz and (not oid or len(oid) != oz): - raise hioing.MemoerError(f"Missing or invalid {oid=} for {oz=}") + if xz and (not oid or len(oid) != xz): + raise hioing.MemoerError(f"Missing or invalid {oid=} for {xz=}") pz = (3 - ((mz) % 3)) % 3 # net pad size for mid # memo ID is 16 byte random UUID converted to 22 char Base64 right aligned mid = encodeB64(bytes([0] * pz) + uuid.uuid1().bytes)[pz:] #pzrepad, convert, and prestrip - if cz != pz or cz != len(self.code): - raise hioing.MemoerError(f"Invalid code size {cz=} for {pz=} or " + if hz != pz or hz != len(self.code): + raise hioing.MemoerError(f"Invalid code size {hz=} for {pz=} or " f"code={self.code}") mid = self.code.encode() + mid # fully qualified mid with prefix code ml = len(memo) if self.curt: # rend header parts in base2 instead of base64 - hz = 3 * hz // 4 # encoding b2 means head part sizes smaller by 3/4 - nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 oz = 3 * oz // 4 # encoding b2 means head part sizes smaller by 3/4 - sz = 3 * sz // 4 # encoding b2 means head part sizes smaller by 3/4 + nz = 3 * nz // 4 # encoding b2 means head part sizes smaller by 3/4 + zz = 3 * zz // 4 # encoding b2 means head part sizes smaller by 3/4 + xz = 3 * xz // 4 # encoding b2 means head part sizes smaller by 3/4 + az = 3 * az // 4 # encoding b2 means head part sizes smaller by 3/4 mid = decodeB64(mid) - bz = (self.size - hz) # max standard gram body size without neck + bz = (self.size - oz) # max standard gram body size without neck # compute gram count based on overhead note added neck overhead in first gram # first gram is special its header is longer by ns than the other grams # which means its payload body is shorter by ns than the other gram bodies @@ -1808,7 +1816,7 @@ def rend(self, memo, oid=None): else: num = helping.intToB64b(gn, l=nz) # num size must always be neck size - if oz: # need oid part, but can't mod here may need below to sign + if xz: # need oid part, but can't mod here may need below to sign if self.curt: oidp = decodeB64(oid.encode()) # oid part b2 else: @@ -1824,7 +1832,7 @@ def rend(self, memo, oid=None): gram = head + memo[:bz] # copy slice past end just copies to end del memo[:bz] # del slice past end just deletes to end - if sz: # signed gram, .sign returns proper sig format when .curt + if az: # signed gram, .sign returns proper sig format when .curt sig = self.sign(oid, gram) # raises MemoerError if invalid gram = gram + sig diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index bc0bca55..8ec3d639 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -112,32 +112,36 @@ def test_memoer_class(): # Codes table with sizes of code (hard) and full primitive material assert Memoer.Sizes == \ { - '__': Sizage(cz=2, mz=22, oz=0, nz=4, sz=0, hz=28), - '_-': Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160), + '__': Sizage(hz=2, mz=22, xz=0, nz=4, zz=0, az=0, oz=28), + '_-': Sizage(hz=2, mz=22, xz=44, nz=4, zz=0, az=88, oz=160), + '1AAQ': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), } # verify Sizes and Codes for code, val in Memoer.Sizes.items(): - cz = val.cz + hz = val.hz mz = val.mz - oz = val.oz + xz = val.xz nz = val.nz - sz = val.sz - hz = val.hz + zz = val.zz + az = val.az + oz = val.oz - assert len(code) == cz == 2 - assert code[0] == '_' - code[1] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_' - assert hz > 0 - assert hz == cz + mz + oz + sz + nz + assert len(code) == hz + assert hz == 2 or hz == 4 + assert code[0] == '_' or code[0] == '1' + assert code[1] in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_' + assert oz > 0 + assert oz == hz + mz + xz + zz + az + nz assert mz # ms must not be empty pz = (3 - ((mz) % 3)) % 3 # net pad size for mid - assert pz == (cz % 4) # combined code + mid size must lie on 24 bit boundary - assert not oz % 4 # oid size must be on 24 bit boundary - assert not sz % 4 # sig size must be on 24 bit boundary - assert nz and not nz % 4 # neck (num or cnt) size must be on 24 bit boundary - assert hz and not hz % 4 # head size must be on 24 bit boundary - if oz: - assert sz # sz must not be empty if oz not empty + assert pz == (hz % 4) # combined code + mid size must lie on 24 bit boundary + assert not xz % 4 # oid size must be on 24 bit boundary + assert not az % 4 # sig size must be on 24 bit boundary + assert nz and not nz % 4 # neck (num) size must be on 24 bit boundary + assert not zz % 4 # when zeroth cnt size must be on 24 bit boundary + assert oz and not oz % 4 # head size must be on 24 bit boundary + if xz: + assert az # az must not be empty if xz not empty assert Memoer.Names == \ { @@ -164,8 +168,8 @@ def test_memoer_class(): raw, code = Memoer._decodeOID(oid) assert raw == verkey assert code == 'B' # nontrans AID - _, _, oz, _, _, _ = Memoer.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz - assert len(oid) == 44 == oz + _, _, xz, _, _, _, _ = Memoer.Sizes[MemoDex.AuthMemoGram] # hz mz xz nz zz az oz + assert len(oid) == 44 == xz qvk = Memoer._encodeQVK(raw=verkey) assert qvk == 'BG-R9L4kTXULe33Tqidn0c-W-x6xU4lIXCdhZQYrsih2' @@ -189,8 +193,8 @@ def test_memoer_class(): raw, code = Memoer._decodeSGN(sgntr) assert raw == signature assert code == '0B' - _, _, _, _, sz, _ = Memoer.Sizes[MemoDex.AuthMemoGram] # cz mz oz nz sz hz - assert len(sgntr) == 88 == sz + _, _, _, _, _, az, _ = Memoer.Sizes[MemoDex.AuthMemoGram] # hz mz xz nz zz az oz + assert len(sgntr) == 88 == az """Done Test""" @@ -251,7 +255,7 @@ def test_memoer_sign_verify(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt - assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 0, 88, 160) # hz mz xz nz zz az oz assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic @@ -307,7 +311,7 @@ def test_memoer_basic(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt - assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic @@ -457,7 +461,7 @@ def test_memoer_small_gram_size(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt - assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert peer.size == 33 # can't be smaller than head + neck + 1 assert not peer.verific assert not peer.echoic @@ -823,7 +827,7 @@ def test_memoer_basic_signed(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.MemoDex.AuthMemoGram == '_-' assert not peer.curt - assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 0, 88, 160) # hz mz xz nz zz az oz assert peer.size == peer.MaxGramSize assert not peer.verific assert not peer.echoic @@ -1167,7 +1171,7 @@ def test_memoer_verific(): assert peer.bs == memoing.Memoer.BufSize == 65535 assert peer.code == memoing.MemoDex.MemoGram == '__' assert not peer.curt - assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert peer.Sizes[peer.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert peer.size == peer.MaxGramSize assert peer.verific assert not peer.echoic @@ -1404,8 +1408,8 @@ def test_sure_memoer_basic(): assert peer.keep == keep assert peer.oid == oid - assert peer.Sizes[peer.code] == Sizage(cz=2, mz=22, oz=44, nz=4, sz=88, hz=160) - assert peer.Sizes[peer.code] == (2, 22, 44, 4, 88, 160) # cz mz oz nz sz hz + assert peer.Sizes[peer.code] == Sizage(hz=2, mz=22, xz=44, nz=4, zz=0, az=88, oz=160) + assert peer.Sizes[peer.code] == (2, 22, 44, 4, 0, 88, 160) # hz mz xz nz zz az oz assert peer.size == peer.MaxGramSize assert peer.tymeout == 0.0 assert peer.tymers == {} diff --git a/tests/core/udp/test_peer_memoing.py b/tests/core/udp/test_peer_memoing.py index a72cc19e..85150436 100644 --- a/tests/core/udp/test_peer_memoing.py +++ b/tests/core/udp/test_peer_memoing.py @@ -25,7 +25,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == MemoDex.MemoGram assert not alpha.curt - assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert alpha.size == 1240 # default MaxGramSize for udp assert alpha.bc == 1024 assert not alpha.opened @@ -38,7 +38,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == MemoDex.MemoGram assert not alpha.curt - assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert alpha.size == size assert alpha.bc == 1024 assert not alpha.opened @@ -175,7 +175,7 @@ def test_memoer_peer_open(): assert alpha.name == "alpha" assert alpha.code == MemoDex.MemoGram assert not alpha.curt - assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert alpha.size == size assert alpha.bc == 1024 diff --git a/tests/core/uxd/test_peer_memoing.py b/tests/core/uxd/test_peer_memoing.py index 6b1bd2df..d715e223 100644 --- a/tests/core/uxd/test_peer_memoing.py +++ b/tests/core/uxd/test_peer_memoing.py @@ -22,7 +22,7 @@ def test_memoer_peer_basic(): assert alpha.name == "alpha" assert alpha.code == MemoDex.MemoGram assert not alpha.curt - assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert alpha.size == 38 assert alpha.bc == 4 @@ -151,7 +151,7 @@ def test_memoer_peer_open(): assert alpha.name == "alpha" assert alpha.code == MemoDex.MemoGram assert not alpha.curt - assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 28) # cz mz oz nz sz hz + assert alpha.Sizes[alpha.code] == (2, 22, 0, 4, 0, 0, 28) # hz mz xz nz zz az oz assert alpha.size == 38 assert alpha.bc == 4 From 095309c13bda1ee8c316b70ef8b240b5079951c4 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 9 Apr 2026 17:06:25 -0600 Subject: [PATCH 4/7] added correct Sizes for new CESR codes --- src/hio/core/memo/memoing.py | 5 ++++- tests/core/memo/test_memoing.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 569b933d..93e14373 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -519,7 +519,10 @@ class Memoer(hioing.Mixin): Sizes = { '__': Sizage(hz=2, mz=22, xz=0, nz=4, zz=0, az=0, oz=28), '_-': Sizage(hz=2, mz=22, xz=44, nz=4, zz=0, az=88, oz=160), - '1AAQ': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), + '1AAQ': Sizage(hz=4, mz=24, xz=0, nz=4, zz=4, az=0, oz=36), + '1AAR': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), + '1AAS': Sizage(hz=4, mz=24, xz=44, nz=4, zz=4, az=88, oz=168), + '1AAT': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), } # Base2 Binary index representation of Text Base64 Char Codes diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 8ec3d639..5f6fcd37 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -114,7 +114,10 @@ def test_memoer_class(): { '__': Sizage(hz=2, mz=22, xz=0, nz=4, zz=0, az=0, oz=28), '_-': Sizage(hz=2, mz=22, xz=44, nz=4, zz=0, az=88, oz=160), - '1AAQ': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), + '1AAQ': Sizage(hz=4, mz=24, xz=0, nz=4, zz=4, az=0, oz=36), + '1AAR': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), + '1AAS': Sizage(hz=4, mz=24, xz=44, nz=4, zz=4, az=88, oz=168), + '1AAT': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), } # verify Sizes and Codes for code, val in Memoer.Sizes.items(): From 1779cc4e4db712e2ef6845132b20833c76fedc09 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Thu, 9 Apr 2026 17:08:48 -0600 Subject: [PATCH 5/7] added another test check --- tests/core/memo/test_memoing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 5f6fcd37..485cccb5 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -142,6 +142,7 @@ def test_memoer_class(): assert not az % 4 # sig size must be on 24 bit boundary assert nz and not nz % 4 # neck (num) size must be on 24 bit boundary assert not zz % 4 # when zeroth cnt size must be on 24 bit boundary + assert zz == (nz if zz else 0) assert oz and not oz % 4 # head size must be on 24 bit boundary if xz: assert az # az must not be empty if xz not empty From 7bd1dd4e89f0c915e7d94d1bb3eb36f750ac34a7 Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Fri, 10 Apr 2026 10:17:30 -0600 Subject: [PATCH 6/7] added rest of the codes --- src/hio/core/memo/memoing.py | 22 ++++++++++++++++++++++ tests/core/memo/test_memoing.py | 27 ++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index 93e14373..f37160bf 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -273,6 +273,12 @@ class MemoGramCodex: Gram: str = '1AAR' # non-zeroth gram code GramAuthZero: str = '1AAS' # zeroth authenticated gram code (signed) GramAuth: str = '1AAT' # non-zeroth authenticated gram code (signed) + GramSureZero: str = '1AAU' # zeroth reliable gram code (acked) + GramSure: str = '1AAV' # non-zeroth reliable gram code (acked) + GramSureAuthZero: str = '1AAW' # zeroth reliable authenticated gram code (acked & signed) + GramSureAuth: str = '1AAX' # non-zeroth reliable authenticated gram code (acked & signed) + Ack: str = '1AAY' # ack code to enable reliable grams + AckAuth: str = '1AAZ' # authenticated ack code to enable reliable grams (signed) def __iter__(self): return iter(astuple(self)) @@ -302,6 +308,9 @@ class AuthGramCodex: AuthMemoGram: str = '_-' # authenticated memogram code (signed) GramAuthZero: str = '1AAS' # zeroth authenticated gram code (signed) GramAuth: str = '1AAT' # non-zeroth authenticated gram code (signed) + GramSureAuthZero: str = '1AAW' # zeroth reliable authenticated gram code (acked & signed) + GramSureAuth: str = '1AAX' # non-zeroth reliable authenticated gram code (acked & signed) + AckAuth: str = '1AAZ' # authenticated ack code to enable reliable grams (signed) def __iter__(self): return iter(astuple(self)) @@ -523,8 +532,21 @@ class Memoer(hioing.Mixin): '1AAR': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), '1AAS': Sizage(hz=4, mz=24, xz=44, nz=4, zz=4, az=88, oz=168), '1AAT': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), + '1AAU': Sizage(hz=4, mz=24, xz=0, nz=4, zz=4, az=0, oz=36), + '1AAV': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), + '1AAW': Sizage(hz=4, mz=24, xz=44, nz=4, zz=4, az=88, oz=168), + '1AAX': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), + '1AAY': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), + '1AAZ': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), } + Pairs = dict() # pair the zeroth code with the non-zeroth code of same type + Pairs[MemoDex.GramZero] = MemoDex.Gram + Pairs[MemoDex.GramAuthZero] = MemoDex.GramAuth + Pairs[MemoDex.GramSureZero] = MemoDex.GramSure + Pairs[MemoDex.GramSureAuthZero] = MemoDex.GramSureAuth + + # Base2 Binary index representation of Text Base64 Char Codes #Bodes = ({helping.codeB64ToB2(c): c for n, c in Codes.items()}) # big enough to hold a CESR big frame with code diff --git a/tests/core/memo/test_memoing.py b/tests/core/memo/test_memoing.py index 485cccb5..3a0a40a5 100644 --- a/tests/core/memo/test_memoing.py +++ b/tests/core/memo/test_memoing.py @@ -107,6 +107,12 @@ def test_memoer_class(): 'Gram': '1AAR', 'GramAuthZero': '1AAS', 'GramAuth': '1AAT' , + 'GramSureZero': '1AAU', + 'GramSure': '1AAV', + 'GramSureAuthZero': '1AAW', + 'GramSureAuth': '1AAX', + 'Ack': '1AAY', + 'AckAuth': '1AAZ', } # Codes table with sizes of code (hard) and full primitive material @@ -118,6 +124,12 @@ def test_memoer_class(): '1AAR': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), '1AAS': Sizage(hz=4, mz=24, xz=44, nz=4, zz=4, az=88, oz=168), '1AAT': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), + '1AAU': Sizage(hz=4, mz=24, xz=0, nz=4, zz=4, az=0, oz=36), + '1AAV': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), + '1AAW': Sizage(hz=4, mz=24, xz=44, nz=4, zz=4, az=88, oz=168), + '1AAX': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), + '1AAY': Sizage(hz=4, mz=24, xz=0, nz=4, zz=0, az=0, oz=32), + '1AAZ': Sizage(hz=4, mz=24, xz=44, nz=4, zz=0, az=88, oz=164), } # verify Sizes and Codes for code, val in Memoer.Sizes.items(): @@ -143,7 +155,7 @@ def test_memoer_class(): assert nz and not nz % 4 # neck (num) size must be on 24 bit boundary assert not zz % 4 # when zeroth cnt size must be on 24 bit boundary assert zz == (nz if zz else 0) - assert oz and not oz % 4 # head size must be on 24 bit boundary + assert oz and not oz % 4 # overhead size must be on 24 bit boundary if xz: assert az # az must not be empty if xz not empty @@ -155,9 +167,22 @@ def test_memoer_class(): '1AAR': 'Gram', '1AAS': 'GramAuthZero', '1AAT': 'GramAuth', + '1AAU': 'GramSureZero', + '1AAV': 'GramSure', + '1AAW': 'GramSureAuthZero', + '1AAX': 'GramSureAuth', + '1AAY': 'Ack', + '1AAZ': 'AckAuth', } + + assert Memoer.Sodex == AuthDex + assert Memoer.Pairs[MemoDex.GramZero] == MemoDex.Gram + assert Memoer.Pairs[MemoDex.GramAuthZero] == MemoDex.GramAuth + assert Memoer.Pairs[MemoDex.GramSureZero] == MemoDex.GramSure + assert Memoer.Pairs[MemoDex.GramSureAuthZero] == MemoDex.GramSureAuth + # Base2 Binary index representation of Text Base64 Char Codes #assert Memoer.Bodes == {b'\xff\xf0': '__', b'\xff\xe0': '_-'} From f8fb2cadc05a943ca8195d95a595daa3547d729b Mon Sep 17 00:00:00 2001 From: Samuel M Smith Date: Sat, 11 Apr 2026 10:51:51 -0600 Subject: [PATCH 7/7] update doc string --- src/hio/core/memo/memoing.py | 52 +++++++++++++++++------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/hio/core/memo/memoing.py b/src/hio/core/memo/memoing.py index f37160bf..db9ab550 100644 --- a/src/hio/core/memo/memoing.py +++ b/src/hio/core/memo/memoing.py @@ -358,33 +358,6 @@ class Memoer(hioing.Mixin): in order to create a new subclass that adds memogram support to the datagram transport class. For example MemoerUdp or MemoerUxd - Each direction of dataflow uses a tiered set of buffers that respect - the constraints of non-blocking asynchronous IO with datagram transports. - - On the transmit side memos are placed in a memo deque (double ended queue). - Each memo is then segmented into grams (memograms) that respect the size constraints - of the underlying datagram transport. These grams are placed in the outgoing - gram deque. Each entry in this deque is a duple of form: - (gram: bytes, dst: str). Each duple is pulled off the self._serviceOneRxMemo()deque and its - gram is put in bytearray for transport. - - Flow:: - - memo -> .txms deque -> rend -> grams -> .txgs deque -> send -> .txbs - - On the receive side each complete memogram (gram) is put in a gram receive - deque as a memogram (datagram sized) segment of a memo. - These deques are indexed by the sender's source addr. - The grams in the gram receive deque are then desegmented into a memo - and placed in the memo deque for consumption by the application or some other - higher level protocol. - - Receive flow:: - - receive -> (gram, src) -> grams parsed to .rxgs .counts .oids .sources -> - signing key pair sigkey and verifier key -> - fuse -> memo .rxms deque - When using non-blocking IO, asynchronous datagram transport protocols may have hidden buffering constraints that result in fragmentation of the sent datagram which means the whole datagram is not sent at once via @@ -406,6 +379,31 @@ class Memoer(hioing.Mixin): The encapsulated unpartitioned memogram with its signature it what is meant for archival not the partitioned gram and per gram signature. + Each direction of dataflow uses a tiered set of buffers that respect + the constraints of non-blocking asynchronous IO with datagram transports. + + Transmit Flow:: + memo to .txms deque -> .rend -> grams to .txgs deque -> .send to .txbs + + On the transmit side memos are placed in a memo deque (double ended queue). + Each memo is then segmented into grams (memograms) that respect the size + constraints of the underlying datagram transport. These grams are placed + in the outgoing gram deque. Each entry in this deque is a duple of form: + (gram: bytes, dst: str). Each duple is pulled off the self._serviceOneRxMemo()deque and its + gram is put in bytearray for transport. + + Receive Flow:: + .receive -> (gram, src) -> grams parsed to .rxgs .counts .oids .sources -> + signing key pair sigkey and verifier key -> + .fuse -> memo .rxms deque + + On the receive side each complete memogram (gram) is put in a gram receive + deque as a memogram (datagram sized) segment of a memo. + These deques are indexed by the sender's source addr. + The grams in the gram receive deque are then desegmented into a memo + and placed in the memo deque for consumption by the application or some other + higher level protocol. + Inherited Class Attributes:: MaxGramSize (int): absolute max gram size on tx with overhead, override