lisp-interpreter-project/tokenize.cpp at master · Halfwake/lisp-interpreter-project · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#include <list>
#include <string>
#include <ostream>
#include <cctype>
#include <cstdio>

#include "tokenize.hpp"

namespace token {
  std::list<Token> tokenize(std::istream & stream) {
    std::list<Token> tokens;
    std::list<char> word;
    size_t lineNumber = 1;
    while (!stream.eof()) {
      switch (stream.peek()) {
      case '(':
	if (!word.empty()) {
	  std::string text(word.begin(), word.end());
	  tokens.push_back(Token(ATOM, text, lineNumber));
	  word.clear();
	}
	tokens.push_back(Token(OPEN_PAREN, "(", lineNumber));
	stream.get();
	break;
      case ')':
	if (!word.empty()) {
	  std::string text(word.begin(), word.end());
	  tokens.push_back(Token(ATOM, text, lineNumber));
	  word.clear();
	}
	tokens.push_back(Token(CLOSE_PAREN, ")", lineNumber));
	stream.get();
	break;
      case ';':
	if (!word.empty()) {
	  std::string text(word.begin(), word.end());
	  tokens.push_back(Token(ATOM, text, lineNumber));
	  word.clear();
	}
	while((stream.get() != '\n') && !stream.eof());
	lineNumber++;
	break;
      case ' ':
      case '\t':
      case '\r':
      case '\n':
	if (!word.empty()) {
	  std::string text(word.begin(), word.end());
	  tokens.push_back(Token(ATOM, text, lineNumber));
	  word.clear();
	}
	while (isspace(stream.peek())) {
	  if(stream.get() == '\n') {
	    lineNumber++;
	  }
	}
	break;
      case EOF:
	if (!word.empty()) {
	  std::string text(word.begin(), word.end());
	  tokens.push_back(Token(ATOM, text, lineNumber));
	  word.clear();
	}
	break;
      default:
	word.push_back(stream.get());
      }
    }
    return tokens;
  }

  Token::Token(Type type, std::string text, size_t lineNumber) {
    this->type = type;
    this->text = text;
    this->lineNumber = lineNumber;
  }

  Token::Token(const Token & other) {
    this->type = other.getType();
    this->text = other.getText();
    this->lineNumber = other.getLineNumber();
  }

  bool Token::operator==(const Token & other) const {
    return (type == other.type) && (text == other.text) && (lineNumber == other.lineNumber);
  }

  bool Token::operator!=(const Token & other) const {
    return !((*this) == other);
  }

  Type Token::getType() const {
    return this->type;
  }

  std::string Token::getText() const {
    return this->text;
  }

  size_t Token::getLineNumber() const {
    return lineNumber;
  }

  std::ostream & operator << (std::ostream & stream, const token::Token & token) {
    stream << "(" << token.getType()
	   << "|" << token.getText()
	   << "|" << token.getLineNumber()
	   << ")";
    return stream;
  }
}