token.cpp File Reference

#include "token.hpp"
#include <algorithm>
#include <cctype>
#include <cstddef>
#include <functional>
#include <initializer_list>
#include <iterator>
#include <llvm/ADT/StringExtras.h>
#include <llvm/Support/Format.h>
#include <llvm/Support/raw_ostream.h>
#include <sstream>
#include <stdexcept>
#include <tuple>
#include <type_traits>
#include <utility>
#include <vector>

Include dependency graph for token.cpp:

Go to the source code of this file.

Classes
struct	yume::TokenState
class	yume::Tokenizer
	Contains the state while the tokenizer is running, such as the position within the file currently being read. More...

Namespaces
namespace	yume

Concepts
concept	yume::char_fn

Typedefs
using	yume::char_raw_fn = bool(char)

Functions
auto	yume::tokenize_preserve_skipped (std::istream &in, const string &source_file) -> vector< Token >
	Consume the contents of the input stream and create corresponding tokens, preserving every token, including whitespace. This is usually undesired. More...
auto	yume::tokenize (std::istream &in, const string &source_file) -> vector< Token >
	Consume the contents of the input stream and create corresponding tokens, ignoring insignificant whitespace. More...
auto	yume::operator<< (llvm::raw_ostream &os, const Token &token) -> llvm::raw_ostream &