36 lines
609 B
C
36 lines
609 B
C
|
|
#include <wchar.h>
|
|
|
|
struct token;
|
|
|
|
struct tokenizer
|
|
{
|
|
struct wcistream* stream;
|
|
|
|
struct position* position;
|
|
|
|
struct token* token;
|
|
|
|
struct token* put_back;
|
|
|
|
struct {
|
|
wchar_t data[1024];
|
|
size_t i, n;
|
|
} buffer;
|
|
|
|
struct {
|
|
wchar_t* data;
|
|
size_t n, cap;
|
|
} rawtoken;
|
|
};
|
|
|
|
// keep track of line and column position
|
|
|
|
// all tokens want a line number and start and end column position
|
|
// and file name
|
|
|
|
// I think we can make it work if the tokenizer does NOT keep references to \
|
|
tokens, only expressions and statements do.
|
|
|
|
|
|
|