linker/Tokenizer.cpp at master · begumcig/linker · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
#include <iostream>
#include "Tokenizer.h"

Tokenizer::Tokenizer(char *fileName)
{
    input.open(fileName);
    lineCount = 0;
    tokenOffset = 1;
}

int Tokenizer::getLineCount()
{
    return lineCount;
}

int Tokenizer::getTokenOffset()
{
    return tokenOffset;
}

string Tokenizer::getToken()
{
    if (input.is_open() && peekToken())
    {

        tokenOffset += currentToken.length();
        iss >> currentToken;
        //cout << "....Currently at word offset: " << tokenOffset << " ....." << endl;
        //cout << ".... The token is: " << currentToken << " ....." << endl;
        return currentToken;
    }
    else
    {
        cout << "There is an error with the input file." << endl;
        return TOKEN_ERR;
    }
}

void Tokenizer::close()
{
    input.close();
}

bool Tokenizer::peekToken()
{
    if (tokenOffset + currentToken.length() > currentLine.length())
    {
        if (lineCount != 0){
            //cout << ".....Line ending offset: " << tokenOffset + currentToken.length() << " ....." << endl;
            tokenOffset += currentToken.length();
        }

        if (getline(input, currentLine))
        {
            lineCount++;
            tokenOffset = 1;
            currentToken = "";
            //cout << ".........Currently processing line: " << lineCount << " ........." << endl;
            iss.clear();
            iss.str(currentLine);
        }
        else
        {
            return false;
        }
    }

    //---Handling white spaces at the beginning of a line.--//
    while (iss.peek() == 32 || iss.peek() == 9)
    {
        iss.get();
        tokenOffset++;
        //--The case that a line is completely white space or ends with white space.--//
        if (tokenOffset > currentLine.length())
        {
            tokenOffset -= currentToken.length();
            return peekToken();
        }
    }
    return true;
}