#include <debug.h>

#include <parse/wcistream/inc.h>

#include <parse/position/inc.h>

#include <memory/smalloc.h>

#include "struct.h"
#include "new.h"

struct tokenizer* new_tokenizer(
    struct wcistream* istream,
    struct position* position)
{
    ENTER;
    
    struct tokenizer* this = smalloc(sizeof(*this));
    
    this->stream = inc_wcistream(istream);
    
    this->position = inc_position(position);
    
    this->token = NULL;
    
    this->put_back = NULL;
    
    this->rawtoken.data = NULL;
    this->rawtoken.n = 0;
    this->rawtoken.cap = 0;
    
    this->buffer.i = 0;
    this->buffer.n = 0;
    
    EXIT;
    return this;
}