Skip to content

Parsing - Optimize token consumption #164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
igorkramaric merged 1 commit into from
Feb 26, 2026
Merged

Parsing - Optimize token consumption #164

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions daffodil/parser.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import string
from collections import deque
from datetime import datetime, timezone, timedelta
from .exceptions import ParseError
from .predicate cimport DictionaryPredicateDelegate
Expand Down Expand Up @@ -472,13 +473,13 @@ cdef class DaffodilParser:

cdef object _read_val(tokens):
cdef _ArrayToken array_token
cdef Token token = tokens.pop(0)
cdef Token token = tokens.popleft()

if isinstance(token, ArrayStart):
array_token = _ArrayToken([])
while True:
if isinstance(tokens[0], ArrayEnd):
tokens.pop(0)
tokens.popleft()
array_token.raw_content = array_token.content
array_token.content = [
token.content
Expand All @@ -494,6 +495,8 @@ cdef object _read_val(tokens):

cdef class Daffodil:
def __init__(self, source, BaseDaffodilDelegate delegate=DictionaryPredicateDelegate()):
cdef object tokens

if isinstance(source, DaffodilParser):
self.parse_result = source
else:
Expand All @@ -503,7 +506,8 @@ cdef class Daffodil:

self.keys = set()
self.delegate = delegate
self.predicate = self.make_predicate(self.parse_result.tokens)
tokens = deque(self.parse_result.tokens)
self.predicate = self.make_predicate(tokens)

def clean_input_source(self, source):
return ''.join(
Expand All @@ -523,20 +527,20 @@ cdef class Daffodil:
cdef Token key_token, test_token

if parent is None:
parent = tokens[0]
return self.make_predicate(tokens[1:], parent)
parent = tokens.popleft()
return self.make_predicate(tokens, parent)

children = []
while tokens:
token = tokens.pop(0)
token = tokens.popleft()
if parent.is_end(token):
return self._handle_group(parent, children)
elif isinstance(token, Key):
self.keys.add(token.content)
children.append(
self.delegate.mk_cmp(
token,
tokens.pop(0),
tokens.popleft(),
_read_val(tokens),
)
)
Expand Down