Boost C++ Libraries

...one of the most highly regarded and expertly designed C++ library projects in the world. — Herb Sutter and Andrei Alexandrescu, C++ Coding Standards

This is the documentation for an old version of Boost. Click here to view this page for the latest version.

boost/test/utils/iterator/token_iterator.hpp

//  (C) Copyright Gennadiy Rozental 2004-2008.
//  Distributed under the Boost Software License, Version 1.0.
//  (See accompanying file LICENSE_1_0.txt or copy at 
//  http://www.boost.org/LICENSE_1_0.txt)

//  See http://www.boost.org/libs/test for the library home page.
//
//  File        : $RCSfile$
//
//  Version     : $Revision: 54633 $
//
//  Description : token iterator for string and range tokenization
// ***************************************************************************

#ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER
#define BOOST_TOKEN_ITERATOR_HPP_071894GER

// Boost
#include <boost/config.hpp>
#include <boost/detail/workaround.hpp>

#include <boost/iterator/iterator_categories.hpp>
#include <boost/iterator/iterator_traits.hpp>

#include <boost/test/utils/iterator/input_iterator_facade.hpp>
#include <boost/test/utils/basic_cstring/basic_cstring.hpp>
#include <boost/test/utils/named_params.hpp>
#include <boost/test/utils/foreach.hpp>

// STL
#include <iosfwd>
#include <cctype>

#include <boost/test/detail/suppress_warnings.hpp>

//____________________________________________________________________________//

#ifdef BOOST_NO_STDC_NAMESPACE
namespace std{ using ::ispunct; using ::isspace; }
#endif

namespace boost {

namespace unit_test {

// ************************************************************************** //
// **************               ti_delimeter_type              ************** //
// ************************************************************************** //

enum ti_delimeter_type { 
    dt_char,        // character is delimeter if it among explicit list of some characters
    dt_ispunct,     // character is delimeter if it satisfies ispunct functor
    dt_isspace,     // character is delimeter if it satisfies isspace functor
    dt_none         // no character is delimeter
};

namespace ut_detail {

// ************************************************************************** //
// **************             default_char_compare             ************** //
// ************************************************************************** //

template<typename CharT>
class default_char_compare {
public:
    bool operator()( CharT c1, CharT c2 )
    {
#ifdef BOOST_CLASSIC_IOSTREAMS
        return std::string_char_traits<CharT>::eq( c1, c2 );
#else
        return std::char_traits<CharT>::eq( c1, c2 );
#endif
    }
};

// ************************************************************************** //
// **************                 delim_policy                 ************** //
// ************************************************************************** //

template<typename CharT,typename CharCompare>
class delim_policy {
    typedef basic_cstring<CharT const> cstring;
public:
    // Constructor
    explicit    delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() )
    : m_type( t )
    {
        set_delimeters( d );
    }

    void        set_delimeters( ti_delimeter_type t ) { m_type = t; }
    template<typename Src>
    void        set_delimeters( Src d )
    {
        nfp::optionally_assign( m_delimeters, d );
        
        if( !m_delimeters.is_empty() )
            m_type = dt_char;
    }

    bool        operator()( CharT c )
    {
        switch( m_type ) {
        case dt_char: {
            BOOST_TEST_FOREACH( CharT, delim, m_delimeters )
                if( CharCompare()( delim, c ) )
                    return true;

            return false;
        }
        case dt_ispunct:
            return (std::ispunct)( c ) != 0;
        case dt_isspace:
            return (std::isspace)( c ) != 0;
        case dt_none:
            return false;
        }

        return false;
    }

private:
    // Data members
    cstring             m_delimeters;
    ti_delimeter_type   m_type;
};

// ************************************************************************** //
// **************                 token_assigner               ************** //
// ************************************************************************** //

template<typename TraversalTag>
struct token_assigner {
#if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 )
    template<typename Iterator, typename C, typename T>
    static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t )
    { for( ; b != e; ++b ) t += *b; }
    
    template<typename Iterator, typename C>
    static void assign( Iterator b, Iterator e, basic_cstring<C>& t )  { t.assign( b, e ); }
#else
    template<typename Iterator, typename Token>
    static void assign( Iterator b, Iterator e, Token& t )  { t.assign( b, e ); }
#endif
    template<typename Iterator, typename Token>
    static void append_move( Iterator& b, Token& )          { ++b; }
};

//____________________________________________________________________________//

template<>
struct token_assigner<single_pass_traversal_tag> {
    template<typename Iterator, typename Token>
    static void assign( Iterator b, Iterator e, Token& t )  {}

    template<typename Iterator, typename Token>
    static void append_move( Iterator& b, Token& t )        { t += *b; ++b; }
};

} // namespace ut_detail

// ************************************************************************** //
// **************                  modifiers                   ************** //
// ************************************************************************** //

namespace {
nfp::keyword<struct dropped_delimeters_t >           dropped_delimeters;
nfp::keyword<struct kept_delimeters_t >              kept_delimeters;
nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens;
nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens;
}

// ************************************************************************** //
// **************             token_iterator_base              ************** //
// ************************************************************************** //

template<typename Derived,
         typename CharT,
         typename CharCompare   = ut_detail::default_char_compare<CharT>,
         typename ValueType     = basic_cstring<CharT const>,
         typename Reference     = basic_cstring<CharT const>,
         typename Traversal     = forward_traversal_tag>
class token_iterator_base
: public input_iterator_facade<Derived,ValueType,Reference,Traversal> {
    typedef basic_cstring<CharT const>                                      cstring;
    typedef ut_detail::delim_policy<CharT,CharCompare>                      delim_policy;
    typedef input_iterator_facade<Derived,ValueType,Reference,Traversal>    base;

protected:
    // Constructor
    explicit    token_iterator_base()
    : m_is_dropped( dt_isspace )
    , m_is_kept( dt_ispunct )
    , m_keep_empty_tokens( false )
    , m_tokens_left( static_cast<std::size_t>(-1) )
    , m_token_produced( false )
    {
    }

    template<typename Modifier>
    void
    apply_modifier( Modifier const& m )
    {
        if( m.has( dropped_delimeters ) )
            m_is_dropped.set_delimeters( m[dropped_delimeters] );

        if( m.has( kept_delimeters ) )
            m_is_kept.set_delimeters( m[kept_delimeters] );

        if( m.has( keep_empty_tokens ) )
            m_keep_empty_tokens = true;

        nfp::optionally_assign( m_tokens_left, m, max_tokens );
    }

    template<typename Iter> 
    bool                    get( Iter& begin, Iter end )
    {
        typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner;
        Iter check_point;

        this->m_value.clear();

        if( !m_keep_empty_tokens ) {
            while( begin != end && m_is_dropped( *begin ) )
                ++begin;

            if( begin == end )
                return false;

            check_point = begin;

            if( m_tokens_left == 1 )
                while( begin != end )
                    Assigner::append_move( begin, this->m_value );
            else if( m_is_kept( *begin ) )
                Assigner::append_move( begin, this->m_value );
            else
                while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
                    Assigner::append_move( begin, this->m_value );

            --m_tokens_left;
        } 
        else { // m_keep_empty_tokens is true
            check_point = begin;

            if( begin == end ) {
                if( m_token_produced ) 
                    return false;

                m_token_produced = true;
            }
            if( m_is_kept( *begin ) ) {
                if( m_token_produced ) 
                    Assigner::append_move( begin, this->m_value );

                m_token_produced = !m_token_produced;
            } 
            else if( !m_token_produced && m_is_dropped( *begin ) )
                m_token_produced = true;
            else {
                if( m_is_dropped( *begin ) )
                    check_point = ++begin;

                while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
                    Assigner::append_move( begin, this->m_value );

                m_token_produced = true;
            }
        }

        Assigner::assign( check_point, begin, this->m_value );

        return true;
    }

private:
    // Data members
    delim_policy            m_is_dropped;
    delim_policy            m_is_kept;
    bool                    m_keep_empty_tokens;
    std::size_t             m_tokens_left;
    bool                    m_token_produced;
};

// ************************************************************************** //
// **************          basic_string_token_iterator         ************** //
// ************************************************************************** //

template<typename CharT,
         typename CharCompare = ut_detail::default_char_compare<CharT> >
class basic_string_token_iterator
: public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> {
    typedef basic_cstring<CharT const> cstring;
    typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base;
public:
    explicit    basic_string_token_iterator() {}
    explicit    basic_string_token_iterator( cstring src )
    : m_src( src )
    {
        this->init();
    }

    template<typename Src, typename Modifier>
    basic_string_token_iterator( Src src, Modifier const& m )
    : m_src( src )
    {
        this->apply_modifier( m );

        this->init();
    }

private:
    friend class input_iterator_core_access;

    // input iterator implementation
    bool        get()
    {
        typename cstring::iterator begin = m_src.begin();
        bool res = base::get( begin, m_src.end() );

        m_src.assign( begin, m_src.end() );

        return res;
    }

    // Data members
    cstring     m_src;
};

typedef basic_string_token_iterator<char>       string_token_iterator;
typedef basic_string_token_iterator<wchar_t>    wstring_token_iterator;

// ************************************************************************** //
// **************              range_token_iterator            ************** //
// ************************************************************************** //

template<typename Iter,
         typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
         typename ValueType   = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
         typename Reference   = ValueType const&>
class range_token_iterator
: public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
                             typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> {
    typedef basic_cstring<typename ValueType::value_type> cstring;
    typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
                                typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base;
public:
    explicit    range_token_iterator() {}
    explicit    range_token_iterator( Iter begin, Iter end = Iter() )
    : m_begin( begin ), m_end( end )
    {
        this->init();
    }
    range_token_iterator( range_token_iterator const& rhs )
    : base( rhs )
    {
        if( this->m_valid ) {
            m_begin = rhs.m_begin;
            m_end   = rhs.m_end;
        }
    }

    template<typename Modifier>
    range_token_iterator( Iter begin, Iter end, Modifier const& m )
    : m_begin( begin ), m_end( end )
    {
        this->apply_modifier( m );

        this->init();
    }

private:
    friend class input_iterator_core_access;

    // input iterator implementation
    bool        get()
    {
        return base::get( m_begin, m_end );
    }

    // Data members
    Iter m_begin;
    Iter m_end;
};

// ************************************************************************** //
// **************            make_range_token_iterator         ************** //
// ************************************************************************** //

template<typename Iter>
inline range_token_iterator<Iter>
make_range_token_iterator( Iter begin, Iter end = Iter() )
{
    return range_token_iterator<Iter>( begin, end );
}

//____________________________________________________________________________//

template<typename Iter,typename Modifier>
inline range_token_iterator<Iter>
make_range_token_iterator( Iter begin, Iter end, Modifier const& m )
{
    return range_token_iterator<Iter>( begin, end, m );
}

//____________________________________________________________________________//

} // namespace unit_test

} // namespace boost

//____________________________________________________________________________//

#include <boost/test/detail/enable_warnings.hpp>

#endif // BOOST_TOKEN_ITERATOR_HPP_071894GER