Skip to content

FEAT: implement lexer #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Nykenik24 merged 1 commit into from
Mar 16, 2026
Merged

FEAT: implement lexer #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
458 changes: 0 additions & 458 deletions docs/.oldspec.md
View file
Open in desktop

This file was deleted.

26 changes: 0 additions & 26 deletions examples/test.rvn
View file
Open in desktop

This file was deleted.

127 changes: 127 additions & 0 deletions include/csquare/lexer/lexer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#ifndef _LEXER_H
#define _LEXER_H

#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#define T(NAME, STR) NAME,
#define TOKEN_TYPES \
T(T_EOF, "EOF") \
T(T_ERROR, "ERROR") \
T(T_IDENTIFIER, "IDENTIFIER") \
T(T_DECIMAL, "DECIMAL") \
T(T_UNSIGNED, "UNSIGNED DECIMAL") \
T(T_DOUBLE, "DOUBLE") \
T(T_FLOAT, "FLOAT") \
T(T_QUAD, "QUAD") \
T(T_DECIMAL_EXPO, "DECIMAL EXPONENT") \
T(T_UNSIGNED_EXPO, "UNSIGNED DECIMAL EXPONENT") \
T(T_DOUBLE_EXPO, "DOUBLE EXPONENT") \
T(T_FLOAT_EXPO, "FLOAT EXPONENT") \
T(T_QUAD_EXPO, "QUAD EXPONENT") \
T(T_STRING, "STRING") \
\
T(T_EQ, "EQUALS") \
T(T_NEQ, "NOT EQUALS") \
T(T_ASSIGN, "ASSIGN") \
T(T_GREATER, "GREATER") \
T(T_LESS, "LESS") \
T(T_GREATER_EQUALS, "GREATER OR EQUALS") \
T(T_LESS_EQUALS, "LESS OR EQUALS") \
T(T_ADD, "ADD") \
T(T_SUB, "SUBTRACT") \
T(T_DIV, "DIVIDE") \
T(T_MUL, "MULTIPLY") \
T(T_ADD_ASSIGN, "ADD AND ASSIGN") \
T(T_SUB_ASSIGN, "SUBTRACT AND ASSIGN") \
T(T_DIV_ASSIGN, "DIVIDE AND ASSIGN") \
T(T_MUL_ASSIGN, "MULTIPLY AND ASSIGN") \
T(T_OPEN_PAREN, "OPEN PARENTHESES") \
T(T_CLOSE_PAREN, "CLOSE PARENTHESES") \
T(T_OPEN_BRACE, "OPEN BRACE") \
T(T_CLOSE_BRACE, "CLOSE BRACE") \
T(T_OPEN_BRACKET, "OPEN BRACKET") \
T(T_CLOSE_BRACKET, "CLOSE BRACKET") \
T(T_PERIOD, "PERIOD") \
T(T_COMMA, "COMMA") \
T(T_COLON, "COLON") \
T(T_SEMICOLON, "SEMICOLON") \
T(T_AND, "AND") \
T(T_OR, "OR") \
\
T(T_KW_DO, "DO") \
T(T_KW_IF, "IF") \
T(T_KW_FOR, "FOR") \
T(T_KW_INT, "INT") \
T(T_KW_CHAR, "CHAR") \
T(T_KW_VOID, "VOID") \
T(T_KW_ELSE, "ELSE") \
T(T_KW_ENUM, "ENUM") \
T(T_KW_LONG, "LONG") \
T(T_KW_QUAD, "QUAD") \
T(T_KW_BOOL, "BOOL") \
T(T_KW_CASE, "CASE") \
T(T_KW_CONST, "CONST") \
T(T_KW_TYPE, "TYPE") \
T(T_KW_FLOAT, "FLOAT") \
T(T_KW_GOTO, "GOTO") \
T(T_KW_INFER, "INFER") \
T(T_KW_SHORT, "SHORT") \
T(T_KW_UCHAR, "UCHAR") \
T(T_KW_UINT, "UINT") \
T(T_KW_ULONG, "ULONG") \
T(T_KW_ERROR, "ERROR") \
T(T_KW_RETURN, "RETURN") \
T(T_KW_STRUCT, "STRUCT") \
T(T_KW_DOUBLE, "DOUBLE") \
T(T_KW_STATIC, "STATIC") \
T(T_KW_WHILE, "WHILE") \
T(T_KW_DEFAULT, "DEFAULT") \
T(T_KW_SWITCH, "SWITCH") \
T(T_KW_USHORT, "USHORT") \
T(T_KW_CONTINUE, "CONTINUE")

typedef enum { TOKEN_TYPES T__COUNT } token_type;

#undef T

extern const char *token_type_str[T__COUNT];

struct token {
const char *start;
int length;
token_type type;
};

typedef struct token token;

token *new_token(const char *start, int length, token_type type);
void free_token(token *tk);
token *error_token(const char *msg);

typedef struct {
token **tokens;
size_t count;
size_t capacity;
} token_list;

void init_token_list(token_list *list);
void free_token_list(token_list *list);
void add_token(token_list *list, token *tk);

#define isws(c) (c == ' ' || c == '\t' || c == '\n' || c == '\r')
#define isdigit(c) (c >= '0' && c <= '9')
#define isalpha(c) (c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z')

#define LEX_FUNC_ARGS const char *p, int *len
token *lex_symbol(LEX_FUNC_ARGS);
token *lex_digit(LEX_FUNC_ARGS);
token *lex_string(LEX_FUNC_ARGS);
token *lex_ident(LEX_FUNC_ARGS);
#undef LEX_FUNC_ARGS
token_list *lex(const char *src);

void print_token(token *tk);

#endif
36 changes: 18 additions & 18 deletions src/csquare/opt-common.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,35 +4,35 @@
#include <stddef.h>
static void handle_info(csq_options *opts, const char *val);

static const opt_map_t opts[] = {
{"--info", "-i", OPT_KIND_FUNC, offsetof(csq_options, show_info), handle_info}};

static void handle_info(csq_options *opts, const char *val) {
(void)opts;
printf("Csquared - %s (%s)\n", CSQ_VERSION, __DATE__);
printf("Authors: %s\n", CSQ_AUTHORS);
#ifdef CSQ_DEBUG
printf("Build: Debug\n");
#else
printf("Build: Release\n");
#endif
opts->show_info = true;
static const opt_map_t opts[] = {{"--info", "-i", OPT_KIND_FUNC,
offsetof(csq_options, show_info),
handle_info}};

static void handle_info(csq_options *opts, const char *_) {
(void)opts;
printf("Csquared - %s (%s)\n", CSQ_VERSION, __DATE__);
printf("Authors: %s\n", CSQ_AUTHORS);
#ifdef CSQ_DEBUG
printf("Build: Debug\n");
#else
printf("Build: Release\n");
#endif
opts->show_info = true;
}
csq_options *options_parse(int argc, char *argv[]) {
csq_options *opt = calloc(1, sizeof(csq_options));
if (!opt)
return NULL;
for (int i = 1; i < argc; ++i) {
const char *arg = argv[i];
bool opt_found = false;
// bool opt_found = false;

for (int j = 0; opts[j].long_name != NULL; ++j) {
const opt_map_t *m = &opts[j];
if (STRCMP(arg, m->long_name) == 0 ||
(m->short_name && STRCMP(arg, m->short_name) == 0)) {
opt_found = true;
const char* val = NULL;
// opt_found = true;
const char *val = NULL;
if (m->kind == OPT_KIND_FLAG) {
*(bool *)((char *)opt + m->offset) = true;
} else if (m->kind == OPT_KIND_VAL) {
Expand All @@ -42,8 +42,8 @@ csq_options *options_parse(int argc, char *argv[]) {
fprintf(stderr, "Error: %s requires an argument\n", arg);
goto error;
}
} else if(m->kind == OPT_KIND_FUNC && m->func) {
m->func(opt, val);
} else if (m->kind == OPT_KIND_FUNC && m->func) {
m->func(opt, val);
}
break;
}
Expand Down
46 changes: 46 additions & 0 deletions src/lexer/lex_digit.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#include "csquare/lexer/lexer.h"
#include <stdbool.h>

token *lex_digit(const char *p, int *len) {
const char *start = p;
token_type type = T_DECIMAL;
bool has_dot = false;
bool has_exp = false;

while (isdigit(*p))
p++;

if (*p == '.') {
has_dot = true;
p++;
while (isdigit(*p))
p++;
}

if (*p == 'e' || *p == 'E') {
has_exp = true;
p++;
if (*p == '+' || *p == '-')
p++;
while (isdigit(*p))
p++;
}

if (*p == 'u') {
type = has_exp ? T_UNSIGNED_EXPO : T_UNSIGNED;
p++;
} else if (*p == 'f') {
type = has_exp ? T_FLOAT_EXPO : T_FLOAT;
p++;
} else if (*p == 'q') {
type = has_exp ? T_QUAD_EXPO : T_QUAD;
p++;
} else if (has_dot) {
type = has_exp ? T_DOUBLE_EXPO : T_DOUBLE;
} else if (!has_dot) {
type = has_exp ? T_DECIMAL_EXPO : T_DECIMAL;
}

*len = (int)(p - start);
return new_token(start, *len, type);
}
80 changes: 80 additions & 0 deletions src/lexer/lex_ident.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,80 @@
#include "csquare/lexer/lexer.h"

const struct {
const char *kw;
token_type type;
int len;
} keyword_table[] = {{"do", T_KW_DO, 2},
{"if", T_KW_IF, 2},
{"for", T_KW_FOR, 3},
{"int", T_KW_INT, 3},
{"char", T_KW_CHAR, 4},
{"void", T_KW_VOID, 4},
{"else", T_KW_ELSE, 4},
{"enum", T_KW_ENUM, 4},
{"long", T_KW_LONG, 4},
{"quad", T_KW_QUAD, 4},
{"bool", T_KW_BOOL, 4},
{"case", T_KW_CASE, 4},
{"type", T_KW_TYPE, 4},
{"goto", T_KW_GOTO, 4},
{"uint", T_KW_UINT, 4},
{"const", T_KW_CONST, 5},
{"float", T_KW_FLOAT, 5},
{"infer", T_KW_INFER, 5},
{"short", T_KW_SHORT, 5},
{"uchar", T_KW_UCHAR, 5},
{"ulong", T_KW_ULONG, 5},
{"error", T_KW_ERROR, 5},
{"while", T_KW_WHILE, 5},
{"return", T_KW_RETURN, 6},
{"struct", T_KW_STRUCT, 6},
{"double", T_KW_DOUBLE, 6},
{"static", T_KW_STATIC, 6},
{"switch", T_KW_SWITCH, 6},
{"ushort", T_KW_USHORT, 6},
{"default", T_KW_DEFAULT, 7},
{"continue", T_KW_CONTINUE, 8}};

int keyword_count = sizeof(keyword_table) / sizeof(keyword_table[0]);

token *lex_ident(const char *p, int *len) {
char buf[64];
int bufi = 0;
token_type type = T_IDENTIFIER;

if (isalpha(*p) || *p == '_') {
buf[bufi] = *p;
bufi++;
p++;
}

while ((isalpha(*p) || isdigit(*p) || *p == '_' || *p == '?') &&
(size_t)bufi < sizeof(buf) - 1) {
buf[bufi] = *p;
bufi++;
p++;
}

buf[bufi] = '\0';
int skip_kw = (buf[bufi - 1] == '?');

if (!skip_kw) {
for (size_t i = 0; i < (size_t)keyword_count; i++) {
if (bufi != keyword_table[i].len)
continue;

if (buf[0] != keyword_table[i].kw[0])
continue;

if (memcmp(buf, keyword_table[i].kw, bufi) == 0) {
type = keyword_table[i].type;
break;
}
}
}

const char *start = p - bufi;
*len = bufi;
return new_token(start, bufi, type);
}
28 changes: 28 additions & 0 deletions src/lexer/lex_string.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
#include "csquare/lexer/lexer.h"

token *lex_string(const char *p, int *len) {
const char *start = p;
char delim = *p;
p++;

while (*p != delim && *p != '\0') {
if (*p == '\\' && *(p + 1) != '\0')
p++;
p++;
}

if (*p != delim) {
*len = (int)(p - start);

const char *prefix = "Unterminated string: ";
char *msg = malloc(strlen(prefix) + *len + 1);

sprintf(msg, "Unterminated string: %.*s", *len, start);
return error_token(msg);
}

p++;

*len = (int)(p - start);
return new_token(start, *len, T_STRING);
}
Loading
Loading