21 lines
281 B
C
21 lines
281 B
C
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include "token.h"
|
|
|
|
struct tokenizer
|
|
{
|
|
struct istream* stream; // needs to free after.
|
|
|
|
enum token token;
|
|
|
|
struct {
|
|
uint8_t* data;
|
|
|
|
size_t n, cap;
|
|
} rawtoken;
|
|
|
|
unsigned refcount;
|
|
};
|
|
|