tali-python/lex.py at master · tali-software-foundation/tali-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
'''
Lexical analysis transforms a stream of characters into a stream
of tokens.
'''

class Token():
    def name(self):
        return None

    def value(self):
        return None

    def __repr__(self):
        return f'({self.name()}, "{self.value()}")'


class LPAREN(Token):
    def name(self):
        return "LPAREN"

    def value(self):
        return "("


class RPAREN(Token):
    def name(self):
        return "RPAREN"

    def value(self):
        return ")"


class LSQUARE(Token):
    def name(self):
        return "LSQUARE"

    def value(self):
        return "["


class RSQUARE(Token):
    def name(self):
        return "RSQUARE"

    def value(self):
        return "]"


class COLON(Token):
    def name(self):
        return "COLON"

    def value(self):
        return ":"


class ATOM(Token):
    def __init__(self, val):
        self.val = val

    def name(self):
        return "ATOM"

    def value(self):
        return self.val


def tokenize(cs):
    '''
    Given a character stream (file), yield a stream of tokens.
    '''
    reserved = ['(', ')', '[', ']', ':']

    c = cs.read(1)
    while c != '':
        if c.isspace():
            c = cs.read(1)

        elif c == '(':
            c = cs.read(1)
            yield LPAREN()

        elif c == ')':
            c = cs.read(1)
            yield RPAREN()

        elif c == '[':
            c = cs.read(1)
            yield LSQUARE()

        elif c == ']':
            c = cs.read(1)
            yield RSQUARE()

        elif c == ':':
            c = cs.read(1)
            yield COLON()

        else:
            n = cs.read(1)
            while n not in reserved and not n.isspace():
                c = c + n
                n = cs.read(1)
            yield ATOM(c)
            c = n
    return