Module dparse.lexer

Functions

Name	Description
byToken(range)	Creates a token range from the given source code. Creates a default lexer configuration and a GC-managed string cache.
byToken(range, cache)	Creates a token range from the given source code. Uses the given string cache.
byToken(range, config, cache)	Creates a token range from the given source code. Uses the provided lexer configuration and string cache.
getTokensForParser(sourceCode, config, cache)
isBasicType(type)
isIntegerLiteral(type)
isKeyword(type)
isLiteral(type)
isNumberLiteral(type)
isOperator(type)
isProtection(type)
isSpecialToken(type)
isStringLiteral(type)
optimalBucketCount(size)	Helper function used to avoid too much allocations while lexing.
rangeMatch(bytes)
skip(bytes, pindex, pcolumn)	Skips between 0 and 16 bytes that match (or do not match) one of the given chars.

Structs

Name	Description
DLexer	The D lexer struct.
LexerConfig	Lexer configuration struct
StringBehavior	Configure string lexing behavior
StringCache	The string cache is used for string interning.

Enums

Name	Description
CommentBehavior
WhitespaceBehavior	Configure whitespace handling

Templates

Name	Description
TokenTriviaFields

Manifest constants

Name	Type	Description
ByteCombine

Aliases

Name	Type	Description
BasicTypes	AliasSeq!(tok!"int",tok!"bool",tok!"byte",tok!"cdouble",tok!"cent",tok!"cfloat",tok!"char",tok!"creal",tok!"dchar",tok!"double",tok!"float",tok!"idouble",tok!"ifloat",tok!"ireal",tok!"long",tok!"real",tok!"short",tok!"ubyte",tok!"ucent",tok!"uint",tok!"ulong",tok!"ushort",tok!"void",tok!"wchar")	Basic type token types.
IdType	ubyte	Token ID type for the D lexer.
IntegerLiterals	AliasSeq!(tok!"intLiteral",tok!"longLiteral",tok!"uintLiteral",tok!"ulongLiteral")	Number literal token types.
Keywords	AliasSeq!(tok!"abstract",tok!"alias",tok!"align",tok!"asm",tok!"assert",tok!"auto",tok!"break",tok!"case",tok!"cast",tok!"catch",tok!"class",tok!"const",tok!"continue",tok!"debug",tok!"default",tok!"delegate",tok!"delete",tok!"deprecated",tok!"do",tok!"else",tok!"enum",tok!"export",tok!"extern",tok!"false",tok!"final",tok!"finally",tok!"for",tok!"foreach",tok!"foreach_reverse",tok!"function",tok!"goto",tok!"if",tok!"immutable",tok!"import",tok!"in",tok!"inout",tok!"interface",tok!"invariant",tok!"is",tok!"lazy",tok!"macro",tok!"mixin",tok!"module",tok!"new",tok!"nothrow",tok!"null",tok!"out",tok!"override",tok!"package",tok!"pragma",tok!"private",tok!"protected",tok!"public",tok!"pure",tok!"ref",tok!"return",tok!"scope",tok!"shared",tok!"static",tok!"struct",tok!"super",tok!"switch",tok!"synchronized",tok!"template",tok!"this",tok!"throw",tok!"true",tok!"try",tok!"typedef",tok!"typeid",tok!"typeof",tok!"union",tok!"unittest",tok!"version",tok!"while",tok!"with",tok!"__DATE__",tok!"__EOF__",tok!"__FILE__",tok!"__FILE_FULL_PATH__",tok!"__FUNCTION__",tok!"__gshared",tok!"__LINE__",tok!"__MODULE__",tok!"__parameters",tok!"__PRETTY_FUNCTION__",tok!"__TIME__",tok!"__TIMESTAMP__",tok!"__traits",tok!"__vector",tok!"__VENDOR__",tok!"__VERSION__")	Keyword token types.
Literals	AliasSeq!(StringLiterals,NumberLiterals,tok!"characterLiteral",SpecialTokens,tok!"true",tok!"false",tok!"null",tok!"$")
NumberLiterals	AliasSeq!(tok!"doubleLiteral",tok!"floatLiteral",tok!"idoubleLiteral",tok!"ifloatLiteral",tok!"intLiteral",tok!"longLiteral",tok!"realLiteral",tok!"irealLiteral",tok!"uintLiteral",tok!"ulongLiteral")	Number literal token types.
Operators	AliasSeq!(tok!",",tok!".",tok!"..",tok!"...",tok!"/",tok!"/=",tok!"!",tok!"!<",tok!"!<=",tok!"!<>",tok!"!<>=",tok!"!=",tok!"!>",tok!"!>=",tok!"$",tok!"%",tok!"%=",tok!"&",tok!"&&",tok!"&=",tok!"(",tok!")",tok!"*",tok!"*=",tok!"+",tok!"++",tok!"+=",tok!"-",tok!"--",tok!"-=",tok!":",tok!";",tok!"<",tok!"<<",tok!"<<=",tok!"<=",tok!"<>",tok!"<>=",tok!"=",tok!"==",tok!"=>",tok!">",tok!">=",tok!">>",tok!">>=",tok!">>>",tok!">>>=",tok!"?",tok!"@",tok!"[",tok!"]",tok!"^",tok!"^=",tok!"^^",tok!"^^=",tok!"{",tok!"|",tok!"|=",tok!"||",tok!"}",tok!"~",tok!"~=")	Operator token types.
Protections	AliasSeq!(tok!"export",tok!"package",tok!"private",tok!"public",tok!"protected")	Protection token types.
SpecialTokens	AliasSeq!(tok!"__DATE__",tok!"__TIME__",tok!"__TIMESTAMP__",tok!"__VENDOR__",tok!"__VERSION__",tok!"__FILE__",tok!"__FILE_FULL_PATH__",tok!"__LINE__",tok!"__MODULE__",tok!"__FUNCTION__",tok!"__PRETTY_FUNCTION__")
str	tokenStringRepresentation!(IdType,operators,dynamicTokens,keywords)	Function used for converting an IdType to a string.
StringLiterals	AliasSeq!(tok!"dstringLiteral",tok!"stringLiteral",tok!"wstringLiteral")	String literal token types
tok	TokenId!(IdType,operators,dynamicTokens,keywords,token)	Template used to refer to D token types.
Token	TokenStructure!(ubyte,"import dparse.lexer:TokenTriviaFields,TriviaToken; mixin TokenTriviaFields;")	The token type in the D lexer
TriviaToken	TokenStructure!(ubyte,"\x0a import dparse.lexer : Token;\x0a\x0a this(Token token) pure nothrow @safe @nogc {\x0a this(token.type, token.text, token.line, token.column, token.index);\x0a }\x0a\x0a int opCmp(size_t i) const pure nothrow @safe @nogc {\x0a if (index < i) return -1;\x0a if (index > i) return 1;\x0a return 0;\x0a }\x0a\x0a int opCmp(ref const typeof(this) other) const pure nothrow @safe @nogc {\x0a return opCmp(other.index);\x0a }\x0a\x0a string toString() const @safe pure\x0a {\x0a import std.array : appender;\x0a\x0a auto sink = appender!string;\x0a toString(sink);\x0a return sink.data;\x0a }\x0a\x0a void toString(R)(auto ref R sink) const\x0a {\x0a import std.conv : to;\x0a import dparse.lexer : str;\x0a\x0a sink.put(`trivia!"`)	Same as Token, but doesn't contain child TriviaTokens