#include <stdlib.h>

#include <debug.h>

#include "../wcistream/free.h"

#include "../position/free.h"

#include "../token/free.h"

#include "struct.h"
#include "free.h"

void free_tokenizer(
    struct tokenizer* this)
{
    ENTER;
    
    if (this)
    {
        free_wcistream(this->stream);
        
        free_position(this->position);
        
        free_token(this->token);
        
        free_token(this->put_back);
        
        free(this->rawtoken.data);
        
        free(this);
    }
    
    EXIT;
}