Frequency--DFA/Stokenizer.h at master · MarisabelC/Frequency--DFA · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
#ifndef STOKENIZER
#define STOKENIZER
#include "Token_function.h"
#include <iostream>
using namespace std;

class STokenizer{

public:

    //Constructor
    STokenizer();
    STokenizer(string s);

    bool more();
    //postcondition: return true if exist more token in the string

    //Mutator
    void operator=(string _string);
    //postcondition:Overloaded function. Set the private member equal to
    //string. Position is reset to zero.

    friend Token & operator>>(STokenizer& s,Token &t);
    //postcondition:Return the next token

    Token next(int start=*START);
    //postcondition: return a token

private:

    string token;
    int _pos;
    Table _table;
    STable success;
};

#endif // TOKENIZER

STokenizer::STokenizer():token(string()),_pos(0){

}

STokenizer::STokenizer(string s):_pos(0){
    token=s;
}

void STokenizer::operator=(string _string){

    token=_string;
    _pos=0;
}

bool STokenizer::more(){

    if(token[_pos]!='\0')
        return true;
    return false;
}

Token STokenizer::next(int start){
    Token t;
    switch (start){

    case 0://Number

        t=Process(_table,token,_pos,START[0],success,TYPE[0]);
        if (!t.empty())
            return t;

    case 1: //AlphaNumeric

        t=Process(_table,token,_pos,START[1],success,TYPE[1]);
        if (!t.empty())
            return t;

    case 2://Space

        t=Process(_table,token,_pos,START[2],success,TYPE[2]);
        if  (!t.empty())
            break;

    case 3: //OPERATOR
        t=Process(_table,token,_pos,START[3],success,TYPE[3]);
        if (!t.empty())
            return t;

    case 4: //PUNCTUATION

        t=Process(_table,token,_pos,START[4],success,TYPE[4]);
        if (!t.empty())
            return t;


    case 5: //UNKNOWN

        t=Process(_table,token,_pos,START[5],success,TYPE[5]);
        if (!t.empty())
            return t;

    default:
        _pos++;
        break;
    }

}

Token & operator>>(STokenizer& s,Token &t){

    t=s.next();

    return t;
}