The Token Identifiers

Table of literal token identifiers
Table of white token identifiers
Table of keyword token identifiers
Table of operator token identifiers
Table of preprocessor token identifiers

The following tables contain all those tokens, which should be recognized by a C++ lexer, which is to be used in conjunction with the Wave preprocessing engine. The tokens are grouped into categories to simplify some internal logic and eventually the driver programs. The token identifiers are defined as an enum, and it is recommended to reuse this definition for your own lexer implementations.

Note though, that the preprocessor engine does some token transformation, so that not all of the listet here token identifiers may occur inside tokens, returned from the preprocessing iterator.

List of literal token id's

Token identifier

Token category

Token value

T_CHARLIT 

CharacterLiteralTokenType

'a', '\t', '\u0057'

T_FLOATLIT

FloatingLiteralTokenType

-1.23456e78

T_IDENTIFIER

IdentifierTokenType

C++ identifier

T_OCTALINT

IntegerLiteralTokenType

007

T_DECIMALINT

IntegerLiteralTokenType

10

T_HEXAINT

IntegerLiteralTokenType

0x1234

T_INTLIT

IntegerLiteralTokenType

T_OCTALINT, T_DECIMALINT or T_HEXAINT

T_LONGINTLIT

IntegerLiteralTokenType

0x1234LL

T_STRINGLIT

StringLiteralTokenType

"abc"

T_FALSE

BoolLiteralTokenType

false

T_TRUE

BoolLiteralTokenType

true

List of whitespace token id's

Token identifier

Token category

Token value

T_ANY

UnknownTokenType

any value not matched otherwise

T_CCOMMENT

WhiteSpaceTokenType

/* ... */

T_CPPCOMMENT

WhiteSpaceTokenType

// ... \n

T_SPACE

WhiteSpaceTokenType

blank or '\t'

T_SPACE2

WhiteSpaceTokenType

'\v' or '\f'

T_EOF

EOFTokenType

end of input

T_CONTLINE

EOLTokenType

'\\' followed by '\n'

T_NEWLINE

EOLTokenType

'\n'


List of keyword token id's

Token identifier

Token category

Token value

T_ASM

KeywordTokenType

asm

T_AUTO

KeywordTokenType

auto

T_BOOL

KeywordTokenType

bool

T_BREAK

KeywordTokenType

break

T_CASE

KeywordTokenType

case

T_CATCH

KeywordTokenType

catch

T_CHAR

KeywordTokenType

char

T_CLASS

KeywordTokenType

class

T_CONST

KeywordTokenType

const

T_CONSTCAST

KeywordTokenType

const_cast

T_CONTINUE

KeywordTokenType

continue

T_DEFAULT

KeywordTokenType

default

T_DEFINED

KeywordTokenType

defined

T_DELETE

KeywordTokenType

delete

T_DO

KeywordTokenType

do

T_DOUBLE

KeywordTokenType

double

T_DYNAMICCAST

KeywordTokenType

dynamic_cast

T_ELSE

KeywordTokenType

else

T_ENUM

KeywordTokenType

enum

T_EXPLICIT

KeywordTokenType

explicit

T_EXPORT

KeywordTokenType

export

T_EXTERN

KeywordTokenType

extern

T_FLOAT

KeywordTokenType

float

T_FOR

KeywordTokenType

for

T_FRIEND

KeywordTokenType

friend

T_GOTO

KeywordTokenType

goto

T_IF

KeywordTokenType

if

T_INLINE

KeywordTokenType

inline

T_INT

KeywordTokenType

int

T_LONG

KeywordTokenType

long

T_MUTABLE

KeywordTokenType

mutable

T_NAMESPACE

KeywordTokenType

namespace

T_NEW

KeywordTokenType

new

T_OPERATOR

KeywordTokenType

operator

T_PRIVATE

KeywordTokenType

private

T_PROTECTED

KeywordTokenType

protected

T_PUBLIC

KeywordTokenType

public

T_REGISTER

KeywordTokenType

register

T_REINTERPRETCAST

KeywordTokenType

reinterpret_cast

T_RETURN

KeywordTokenType

return

T_SHORT

KeywordTokenType

short

T_SIGNED

KeywordTokenType

signed

T_SIZEOF

KeywordTokenType

sizeof

T_STATIC

KeywordTokenType

static

T_STATICCAST

KeywordTokenType

static_cast

T_STRUCT

KeywordTokenType

struct

T_SWITCH

KeywordTokenType

switch

T_TEMPLATE

KeywordTokenType

template

T_THIS

KeywordTokenType

this

T_THROW

KeywordTokenType

throw

T_TRY

KeywordTokenType

try

T_TYPEDEF

KeywordTokenType

typedef

T_TYPEID

KeywordTokenType

typeid

T_TYPENAME

KeywordTokenType

typename

T_UNION

KeywordTokenType

union

T_UNSIGNED

KeywordTokenType

unsigned

T_USING

KeywordTokenType

using

T_VIRTUAL

KeywordTokenType

virtual

T_VOID

KeywordTokenType

void

T_VOLATILE

KeywordTokenType

volatile

T_WCHART

KeywordTokenType

wchar_t

T_WHILE

KeywordTokenType

while

List of operator token id's

Token identifier

Token category

Token value

T_AND

OperatorTokenType

&

T_ANDAND

OperatorTokenType

&&

T_ASSIGN

OperatorTokenType

=

T_ANDASSIGN

OperatorTokenType

&=

T_OR

OperatorTokenType

|

T_ORASSIGN

OperatorTokenType

|=

T_XOR

OperatorTokenType

^

T_XORASSIGN

OperatorTokenType

^=

T_COMMA

OperatorTokenType

,

T_COLON

OperatorTokenType

:

T_DIVIDE

OperatorTokenType

/

T_DIVIDEASSIGN

OperatorTokenType

/=

T_DOT

OperatorTokenType

.

T_DOTSTAR

OperatorTokenType

.*

T_ELLIPSIS

OperatorTokenType

...

T_EQUAL

OperatorTokenType

==

T_GREATER

OperatorTokenType

>

T_GREATEREQUAL

OperatorTokenType

>=

T_LEFTBRACE

OperatorTokenType

{

T_LESS

OperatorTokenType

<

T_LESSEQUAL

OperatorTokenType

<=

T_LEFTPAREN

OperatorTokenType

(

T_LEFTBRACKET

OperatorTokenType

[

T_MINUS

OperatorTokenType

-

T_MINUSASSIGN

OperatorTokenType

-=

T_MINUSMINUS

OperatorTokenType

--

T_PERCENT

OperatorTokenType

%

T_PERCENTASSIGN

OperatorTokenType

%=

T_NOT

OperatorTokenType

!

T_NOTEQUAL

OperatorTokenType

!=

T_OROR

OperatorTokenType

||

T_PLUS

OperatorTokenType

+

T_PLUSASSIGN

OperatorTokenType

+=

T_PLUSPLUS

OperatorTokenType

++

T_ARROW

OperatorTokenType

->

T_ARROWSTAR

OperatorTokenType

->*

T_QUESTION_MARK

OperatorTokenType

?

T_RIGHTBRACE

OperatorTokenType

}

T_RIGHTPAREN

OperatorTokenType

)

T_RIGHTBRACKET

OperatorTokenType

]

T_COLON_COLON

OperatorTokenType

::

T_SEMICOLON

OperatorTokenType

;

T_SHIFTLEFT

OperatorTokenType

<<

T_SHIFTLEFTASSIGN

OperatorTokenType

<<=

T_SHIFTRIGHT

OperatorTokenType

>>

T_SHIFTRIGHTASSIGN

OperatorTokenType

>>=

T_STAR

OperatorTokenType

*

T_COMPL

OperatorTokenType

~

T_STARASSIGN

OperatorTokenType

*=

T_POUND_POUND

OperatorTokenType

##

T_POUND

OperatorTokenType

#

T_AND_ALT

OperatorTokenType | AltTokenType

bitand

T_ANDASSIGN_ALT

OperatorTokenType | AltTokenType

and_eq

T_OR_ALT

OperatorTokenType | AltTokenType

or

T_ORASSIGN_ALT

OperatorTokenType | AltTokenType

or_eq

T_XOR_ALT

OperatorTokenType | AltTokenType

xor

T_XORASSIGN_ALT

OperatorTokenType | AltTokenType

xor_eq

T_LEFTBRACE_ALT

OperatorTokenType | AltTokenType

<%

T_LEFTBRACKET_ALT

OperatorTokenType | AltTokenType

<:

T_NOT_ALT

OperatorTokenType | AltTokenType

not

T_NOTEQUAL_ALT

OperatorTokenType | AltTokenType

not_eq

T_RIGHTBRACE_ALT

OperatorTokenType | AltTokenType

%>

T_RIGHTBRACKET_ALT

OperatorTokenType | AltTokenType

:>

T_COMPL_ALT

OperatorTokenType | AltTokenType

compl

T_POUND_POUND_ALT

OperatorTokenType | AltTokenType

%:%:

T_POUND_ALT

OperatorTokenType | AltTokenType

%:

T_OR_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??!

T_XOR_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??'

T_LEFTBRACE_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??<

T_LEFTBRACKET_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??(

T_RIGHTBRACE_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??>

T_RIGHTBRACKET_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??)

T_COMPL_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??-

T_POUND_POUND_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??=??=

T_POUND_TRIGRAPH

OperatorTokenType | TriGraphTokenType

??=


The preprocessor tokens listet in the following table are returned by the C++ lexer only. The preprocessor naturally acts on these tokens and they are not returned by dereferencing the preprocessing iterators.

List of preprocessor token id's

Token identifier

Token category

Token value

T_PP_DEFINE

PPTokenType

#define

T_PP_IF

PPTokenType

#if

T_PP_IFDEF

PPTokenType

#ifdef

T_PP_IFNDEF

PPTokenType

#ifndef

T_PP_ELSE

PPTokenType

#else

T_PP_ELIF

PPTokenType

#elif

T_PP_ENDIF

PPTokenType

#endif

T_PP_ERROR

PPTokenType

#error

T_PP_LINE

PPTokenType

#line

T_PP_PRAGMA

PPTokenType

#pragma

T_PP_UNDEF

PPTokenType

#undef

T_PP_WARNING

PPTokenType

#warning

T_PP_INCLUDE

PPTokenType

#include "..."

T_PP_QHEADER

PPTokenType

#include <...>

T_PP_HHEADER

PPTokenType

#include ...