ASR/NGrams.cpp at master · asrAachen/ASR · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
#include <stdio.h>
#include "Ngrams.h"
#include <stdexcept>
#include <iterator>
#include <iostream>


using namespace std;


NGramReader::NGramReader(const std::string& corpusFileName,
                         const std::string& vocabFileName)
{
    frequencies = new NGramsMap();

    if (!vocabFileName.empty()) {
        processVocab(vocabFileName);
        ofs.open("corpus_vocab");
    }

    fs.open(corpusFileName);
    if(fs.fail()) {
        throw "Error reading the corpus";
    }

    cSize = 0;
    sentence = "";
    while (!fs.eof()) {
        getline(fs,sentence);
        std::stringstream  stream(sentence);
        while (!stream.eof()) {
            string word;
            stream >> word;
            try {
                if (!vocabFileName.empty()) {
                    int val = vocabFrequencies->at(word);
                }
                words.push_back(word);
            }
            catch (const std::out_of_range& oor) {
                words.push_back("<unk>");
            }
        }
        if (!vocabFileName.empty()) {
            writeNewCorpus(words);
        }
        processCorpus(words);

        sentence = "";
        words.clear();
    }
    ofs.close();
}

const NGramsMap* NGramReader::frequencyMap()
{
    return frequencies;
}

const NGramsMap* NGramReader::vocab()
{
    return vocabFrequencies;
}

size_t NGramReader::corpusSize()
{
    return cSize;
}


void NGramReader::processCorpus(vector<std::string>& words)
{
    std::vector<string>::iterator it;
    it = words.begin();
    words.insert(it, "<s>");
    words.push_back("</s>");

    std::string builder;
    for (size_t i = 0; i < words.size(); ++i) {
        cSize++;
        builder.clear();
        for (size_t j = i; j < min((i + 3), words.size()); ++j) {
            if (builder.empty()) {
                builder = words[j];
            } else {
                builder+= " "+ words[j];
            }
            (*frequencies)[builder] += 1;
        }
    }
}

void NGramReader::processVocab( const std::string& vocabFile)
{
    fs.open(vocabFile);
    if(fs.fail()) {
        throw "Error reading the vocab file";
    }

    vocabFrequencies = new NGramsMap();

    while (!fs.eof()) {
        getline(fs,sentence);
        (*vocabFrequencies)[sentence] = 0;
    }
    fs.close();

}

void NGramReader::writeNewCorpus(const std::vector<std::string>& words)
{
    const char* const delim = " ";

    std::ostringstream imploded;
    std::copy(words.begin(), words.end(),
              std::ostream_iterator<std::string>(imploded, delim));
    ofs <<imploded.str();
    ofs <<"\n";
}


NGramReader::~NGramReader()
{
    if (vocabFrequencies) {
        vocabFrequencies->clear();
    }
    frequencies->clear();
    fs.close();
}