53 lines
667 B
C
53 lines
667 B
C
|
|
#include <debug.h>
|
|
|
|
#include <wcistream/inc.h>
|
|
|
|
#include <parse/position/inc.h>
|
|
|
|
#include <memory/smalloc.h>
|
|
|
|
#include "struct.h"
|
|
#include "new.h"
|
|
|
|
struct tokenizer* new_tokenizer(
|
|
struct wcistream* istream,
|
|
struct position* position)
|
|
{
|
|
ENTER;
|
|
|
|
struct tokenizer* this = smalloc(sizeof(*this));
|
|
|
|
this->stream = inc_wcistream(istream);
|
|
|
|
this->position = inc_position(position);
|
|
|
|
this->token = NULL;
|
|
|
|
this->put_back = NULL;
|
|
|
|
this->rawtoken.data = NULL;
|
|
this->rawtoken.n = 0;
|
|
this->rawtoken.cap = 0;
|
|
|
|
this->buffer.i = 0;
|
|
this->buffer.n = 0;
|
|
|
|
EXIT;
|
|
return this;
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|