LaTeXParser/Tokenizer.cpp at master · mxalimetov/LaTeXParser · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#include "Tokenizer.h"

namespace LaTeX
{
namespace TKNZR
{

Content::Content() {}
Content::Content(Type type, std::string content): type(type), content(content) {}
void Content::operator=(const Content& temp) {
	type = temp.type;
	content = temp.content;
}
std::string Content::toString(int i) const {
	std::string result;//(i, ' ');
	if(type==Type::Text or type==Type::Token or type==Type::Symbol) {
		result+= content;
	} else if(type==Type::Command) {
		result+='\\'+ content;
	}
	return result;
}

Tokenizer::Tokenizer(Stream& stream): stream(stream) {}
bool Tokenizer::operator>>(Content& rightObject) {
	produce = true;
	while (produce) {
		if(!(stream >> current)) {
			if(checkIfBlockEnds()) {
				blockEnds(rightObject);
				return true;
			} else {
				return false;
			}
		}
		//std::cout << current;

		if(status==Status::READING_COMMENT) {
			if(current=='\n') status=Status::READY_TO_READ;
		} else if(status==Status::READY_TO_COMMAND and (
				current=='\\' or
				current==' ' or current=='\t' or current=='\n' or
				current=='%' or
				checkIfToken()
			)) {
			recordSymbol(rightObject);
		} else if(current=='\\') {
			if(checkIfBlockEnds()) {
				stream << current;
				blockEnds(rightObject);
			} else {
				status = Status::READY_TO_COMMAND;
			}
		} else if(current==' ' or current=='\t' or current=='\n') {
			if(checkIfBlockEnds()) {
				stream << current;
				blockEnds(rightObject);
			}
		} else if(checkIfToken()) {
			if(checkIfBlockEnds()) {
				stream << current;
				blockEnds(rightObject);
			} else {
				recordToken(rightObject);
			}
		} else if(current=='%') {
			if(checkIfBlockEnds()) {
				stream << current;
				blockEnds(rightObject);
			} else {
				status = Status::READING_COMMENT;
			}
		} else {
			if(status==Status::READY_TO_READ) status = Status::READING_TEXT;
			else if(status==Status::READY_TO_COMMAND) status = Status::READING_COMMAND;
			//std::cout << current;
			buffer += current;
		}
	}
	return true;
}
bool Tokenizer::checkIfToken() {
	if( current=='[' or current==']' or current=='{' or current=='}' or
		current==',' or current=='.' or current=='$' or current==':' or
		current=='(' or current==')' or current=='^' or current=='=' or
		current==';' or current=='-' or current=='*' or current=='/' or
		current=='<' or current=='>' or current=='&' or current=='+' or
		current=='_' or current=='"' or current=='\'' ) return true;
		return false;
}
bool Tokenizer::checkIfBlockEnds() {
	return status==Status::READING_TEXT or status==Status::READING_COMMAND;
}
void Tokenizer::blockEnds(Content& rightObject) {
	produce = false;
	if(status==Status::READING_TEXT) {  // move
		rightObject = Content(Type::Text, buffer);
	} else if(status==Status::READING_COMMAND) {
		rightObject = Content(Type::Command, buffer);
	}
	buffer.clear();
	status=Status::READY_TO_READ;
}
void Tokenizer::recordToken(Content& rightObject) {
	produce = false;
	rightObject = Content(Type::Token, std::string(1, current));
	status=Status::READY_TO_READ;
}
void Tokenizer::recordSymbol(Content& rightObject) {
	produce = false;
	rightObject = Content(Type::Symbol, std::string(1, current));
	status=Status::READY_TO_READ;
}

}
}