#include <wchar.h>

struct token;

struct tokenizer
{
    struct wcistream* stream;
    
    struct position* position;
    
    struct token* token;
    
    struct token* put_back;
    
    struct {
        wchar_t data[1024];
        size_t i, n;
    } buffer;
    
    struct {
        wchar_t* data;
        size_t n, cap;
    } rawtoken;
};

// keep track of line and column position

// all tokens want a line number and start and end column position
    // and file name

// I think we can make it work if the tokenizer does NOT keep references to \
   tokens, only expressions and statements do.