Boost C++ Libraries of the most highly regarded and expertly designed C++ library projects in the world. Herb Sutter and Andrei Alexandrescu, C++ Coding Standards


//  Copyright (c) 2001-2009 Hartmut Kaiser
//  Distributed under the Boost Software License, Version 1.0. (See accompanying 
//  file LICENSE_1_0.txt or copy at


//  Token definition: We keep the base class for the token definition as a 
//                    template parameter to allow this class to be used for
//                    both: the code generation and the lexical analysis
enum tokenids 
    IDANY = boost::spirit::lex::min_token_id + 1,

// This token definition class can be used without any change for all three
// possible use cases: a dynamic lexical analyzer, a code generator, and a
// static lexical analyzer.
template <typename BaseLexer>
struct word_count_tokens : boost::spirit::lex::lexer<BaseLexer> 
      : word_count_tokens::base_type(
        // define tokens and associate them with the lexer
        word = "[^ \t\n]+";
        this->self = word | '\n' | boost::spirit::lex::token_def<>(".", IDANY);

    boost::spirit::lex::token_def<std::string> word;