Frequency--DFA/stokenize.h at master · MarisabelC/Frequency--DFA · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#ifndef STOKENIZE
#define STOKENIZE
#include "token.h"

class Stokenize{

public:
    Stokenize();
    Stokenize(string s);
    Token GetNextToken();
    bool More();

private:

    char* str;
    int _pos;

};

Stokenize::Stokenize():str(NULL),_pos(0){}

Stokenize::Stokenize(string s):_pos(0){
    str=String2Cstring(s);
}


Token Stokenize::GetNextToken(){

    char* input=str+_pos,*walker=str+_pos;
    char* token=new char[20];
    *token='\0';
    if (S5(walker,token)){
        Token t(token,SPACE);
        _pos++;
        return t;
    }
    while(!S5(walker,token)){
       _pos++;
       walker++;
    }

    if(S1(input,token)){

        Token t(token,INTEGER);
        return t;
    }

    else if(S2(input,token)){

        Token t(token,ALPHA);
        return t;
    }

    else if(S6(input,token)){

        Token t(token,FLOAT);
        return t;
    }


}

bool Stokenize::More(){

    if(*str+_pos!=END_OF_LINE)
        return true;
    return false;
}


#endif // STOKENIZE