lexer.cpp

#include "lexer.h"
#include <cctype>

namespace AR437 {

Lexer::Lexer(const std::string& source) : source(source), current(0), line(1), column(1) {}

std::vector<Token> Lexer::tokenize() {
    std::vector<Token> tokens;
    
    while (!isAtEnd()) {
        Token token = nextToken();
        if (token.type != TokenType::INVALID) {
            tokens.push_back(token);
        }
    }
    
    tokens.push_back(Token(TokenType::EOF_TOK, "", line, column));
    return tokens;
}

Token Lexer::nextToken() {
    skipWhitespace();
    
    if (isAtEnd()) {
        return Token(TokenType::EOF_TOK, "", line, column);
    }
    
    char c = peek();
    int startLine = line;
    int startColumn = column;
    
    // Handle newlines
    if (c == '\n') {
        advance();
        return Token(TokenType::NEWLINE, "\\n", startLine, startColumn);
    }
    
    // Handle comments
    if (c == '#') {
        skipComment();
        return nextToken(); // Skip the comment and get the next token
    }
    
    if (c == '/' && peek(1) == '/') {
        skipComment();
        return nextToken(); // Skip the comment and get the next token
    }
    
    // Handle strings
    if (c == '"') {
        return makeString();
    }
    
    // Handle numbers
    if (isDigit(c)) {
        return makeNumber();
    }
    
    // Handle identifiers and keywords
    if (isAlpha(c) || c == '_') {
        return makeIdentifier();
    }
    
    // Handle operators
    if (TokenUtil::isOperator(c)) {
        return makeOperator();
    }
    
    // Handle delimiters
    if (TokenUtil::isDelimiter(c)) {
        char delimiter = advance();
        std::string value(1, delimiter);
        
        switch (delimiter) {
            case ';': return Token(TokenType::SEMICOLON, value, startLine, startColumn);
            case ',': return Token(TokenType::COMMA, value, startLine, startColumn);
            case '(': return Token(TokenType::LPAREN, value, startLine, startColumn);
            case ')': return Token(TokenType::RPAREN, value, startLine, startColumn);
            case '{': return Token(TokenType::LBRACE, value, startLine, startColumn);
            case '}': return Token(TokenType::RBRACE, value, startLine, startColumn);
            case '[': return Token(TokenType::LBRACKET, value, startLine, startColumn);
            case ']': return Token(TokenType::RBRACKET, value, startLine, startColumn);
        }
    }
    
    // Invalid character
    advance();
    return Token(TokenType::INVALID, std::string(1, c), startLine, startColumn);
}

char Lexer::peek(int offset) const {
    size_t pos = current + offset;
    if (pos >= source.length()) return '\0';
    return source[pos];
}

char Lexer::advance() {
    if (isAtEnd()) return '\0';
    
    char c = source[current++];
    if (c == '\n') {
        line++;
        column = 1;
    } else {
        column++;
    }
    return c;
}

void Lexer::skipWhitespace() {
    while (!isAtEnd()) {
        char c = peek();
        if (c == ' ' || c == '\t' || c == '\r') {
            advance();
        } else {
            break;
        }
    }
}

void Lexer::skipComment() {
    char c = peek();
    
    if (c == '#') {
        // Skip single-line comment starting with #
        while (!isAtEnd() && peek() != '\n') {
            advance();
        }
    } else if (c == '/' && peek(1) == '/') {
        // Skip '//'
        advance();
        advance();
        
        // Skip until end of line
        while (!isAtEnd() && peek() != '\n') {
            advance();
        }
    }
}

Token Lexer::makeNumber() {
    int startLine = line;
    int startColumn = column;
    std::string value;
    
    while (!isAtEnd() && (isDigit(peek()) || peek() == '.')) {
        value += advance();
    }
    
    return Token(TokenType::NUMBER, value, startLine, startColumn);
}

Token Lexer::makeString() {
    int startLine = line;
    int startColumn = column;
    std::string value;
    
    // Skip opening quote
    advance();
    
    while (!isAtEnd() && peek() != '"') {
        if (peek() == '\\') {
            advance(); // Skip escape character
            if (!isAtEnd()) {
                char escaped = advance();
                switch (escaped) {
                    case 'n': value += '\n'; break;
                    case 't': value += '\t'; break;
                    case 'r': value += '\r'; break;
                    case '\\': value += '\\'; break;
                    case '"': value += '"'; break;
                    default: value += escaped; break;
                }
            }
        } else {
            value += advance();
        }
    }
    
    if (!isAtEnd()) {
        advance(); // Skip closing quote
    }
    
    return Token(TokenType::STRING, value, startLine, startColumn);
}

Token Lexer::makeIdentifier() {
    int startLine = line;
    int startColumn = column;
    std::string value;
    
    while (!isAtEnd() && (isAlphaNumeric(peek()) || peek() == '_')) {
        value += advance();
    }
    
    TokenType type = TokenUtil::getKeywordType(value);
    return Token(type, value, startLine, startColumn);
}

Token Lexer::makeOperator() {
    int startLine = line;
    int startColumn = column;
    char first = advance();
    
    // Check for two-character operators
    if (!isAtEnd()) {
        char second = peek();
        std::string twoChar = std::string(1, first) + second;
        
        if (twoChar == "==") {
            advance();
            return Token(TokenType::EQUAL, twoChar, startLine, startColumn);
        } else if (twoChar == "!=") {
            advance();
            return Token(TokenType::NOT_EQUAL, twoChar, startLine, startColumn);
        } else if (twoChar == "<=") {
            advance();
            return Token(TokenType::LESS_EQUAL, twoChar, startLine, startColumn);
        } else if (twoChar == ">=") {
            advance();
            return Token(TokenType::GREATER_EQUAL, twoChar, startLine, startColumn);
        } else if (twoChar == "&&") {
            advance();
            return Token(TokenType::AND, twoChar, startLine, startColumn);
        } else if (twoChar == "||") {
            advance();
            return Token(TokenType::OR, twoChar, startLine, startColumn);
        }
    }
    
    // Single-character operators
    std::string value(1, first);
    switch (first) {
        case '+': return Token(TokenType::PLUS, value, startLine, startColumn);
        case '-': return Token(TokenType::MINUS, value, startLine, startColumn);
        case '*': return Token(TokenType::MULTIPLY, value, startLine, startColumn);
        case '/': return Token(TokenType::DIVIDE, value, startLine, startColumn);
        case '%': return Token(TokenType::MODULO, value, startLine, startColumn);
        case '=': return Token(TokenType::ASSIGN, value, startLine, startColumn);
        case '!': return Token(TokenType::NOT, value, startLine, startColumn);
        case '<': return Token(TokenType::LESS, value, startLine, startColumn);
        case '>': return Token(TokenType::GREATER, value, startLine, startColumn);
        default: return Token(TokenType::INVALID, value, startLine, startColumn);
    }
}

bool Lexer::isAtEnd() const {
    return current >= source.length();
}

bool Lexer::isAlpha(char c) const {
    return std::isalpha(c) || c == '_';
}

bool Lexer::isDigit(char c) const {
    return std::isdigit(c);
}

bool Lexer::isAlphaNumeric(char c) const {
    return isAlpha(c) || isDigit(c);
}

} // namespace AR437