Wiki

CobraTokenizer.cobra

Revision 2600, 31.2 KB (checked in by Charles.Esterbrook, 9 months ago)
Expand support for escape codes in strings to support \a \b \f \v \' \" and \?.
Property svn:eol-style set to `native`

Line
1	use System.Text.RegularExpressions
2
3
4	class CobraTokenizer inherits Tokenizer
5
6	test
7	# There are plenty of external Cobra source code tests that will exercise the lexer.
8	# But here are a few basic tests to make sure the tokenizer has some viability.
9	t = CobraTokenizer()
10	# TODO:
11	t.startSource('foo bar')
12	tokens = t.allTokens
13	Tokenizer.checkTokens(tokens, 'ID ID EOL')
14
15	t.restart
16	t.startSource('class Foo\n\tdef foo()\n\t\treturn 1')
17	# t.startSource(
18	#'''class Foo
19	# def foo()
20	# return 1
21	#''')
22	tokens = t.allTokens
23	Tokenizer.checkTokens(tokens, 'CLASS ID EOL INDENT DEF OPEN_CALL RPAREN EOL INDENT RETURN INTEGER_LIT EOL DEDENT DEDENT')
24
25	t.restart
26	t.startSource('class Foo\n\tpass\n\nclass Bar\n\tpass')
27	# t.startSource(
28	# '''class Foo
29	# pass
30	#
31	# class Bar
32	# pass
33	# ''')
34	tokens = t.allTokens
35	Tokenizer.checkTokens(tokens, 'CLASS ID EOL INDENT PASS EOL EOL DEDENT CLASS ID EOL INDENT PASS EOL DEDENT')
36
37	t.restart
38	t.startSource('class Foo\n\tdef foo()\n\t\treturn 1\n\nclass Bar\n\tpass\n')
39	# t.startSource(
40	#'''class Foo
41	# def foo()
42	# return 1
43	#
44	#class Bar
45	# pass
46	#''')
47	tokens = t.allTokens
48	Tokenizer.checkTokens(tokens, 'CLASS ID EOL INDENT DEF OPEN_CALL RPAREN EOL INDENT RETURN INTEGER_LIT EOL EOL DEDENT DEDENT CLASS ID EOL INDENT PASS EOL DEDENT')
49
50	var _indentCount as int
51	var _substLBracketCount as int
52	var _inSubstStringSingle = false
53	var _inSubstStringDouble = false
54	var _inDocString = false
55	var _inCommentBlock = 0
56
57	cue init
58	base.init
59
60	cue init(verbosity as int)
61	base.init
62	_verbosity = verbosity
63
64	def addInfo(sb as StringBuilder) is override
65	base.addInfo(sb)
66	sb.append('_indentCount=[_indentCount], ')
67	sb.append('_substLBracketCount=[_substLBracketCount], ')
68	sb.append('_inSubstStringSingle=[_inSubstStringSingle], ')
69	sb.append('_inSubstStringDouble=[_inSubstStringDouble], ')
70	sb.append('_inDocString=[_inDocString]')
71	sb.append('_inCommentBlock=[_inCommentBlock]')
72
73	pro willReturnComments from var as bool
74
75	pro willReturnDirectives from var as bool
76
77	# Note: The Tokenizer class handles it's input one line at a time,
78	# and retains the \n at the end of the line. This affects
79	# the regex design for the tokens below.
80
81	get orderedTokenSpecs as List<of String> is override
82	return [
83	# whitespace
84	r'WHITESPACE_LINE ^[\t ]+$',
85	r'WHITESPACE_COMMENT_1 ^[\t]+[ ]\#.$',
86	r'WHITESPACE_COMMENT_2 ^[ ]+[\t]\#.$',
87	r'COMMENT_BLOCK_START ^[ \t]\/\#.$',
88	r'INDENT_MIXED_TSNS ^[\t]+[ ]+(?=[^\t ])',
89	r'INDENT_MIXED_TS ^[\t]+[ ]+',
90	r'INDENT_MIXED_ST ^[ ]+[\t]+',
91	r'INDENT_ALL_TABS ^[\t]+',
92	r'INDENT_ALL_SPACES ^[ ]+',
93	r'NO_INDENT ^(?=[^\t\n#\/])',
94	r'EOL \n',
95	r'INLINE_COMMENT \/\#.*\#/',
96	r'SINGLE_LINE_COMMENT \#.*',
97	r'AMBIGUOUS_COMMENT \/\#.*',
98	r'SPACE [ \t]+',
99
100	r'AT_ID @[A-Za-z_][A-Za-z0-9_]*',
101
102	r'OPEN_GENERIC [A-Za-z_][A-Za-z0-9_]*<of([ \n\r\t]\|(?=[,>]))',
103	r'OPEN_DO do\(',
104	r'OPEN_IF if\(',
105	r'OPEN_CALL [A-Za-z_][A-Za-z0-9_]*\(',
106
107	r'HEX_LIT_UNSIGN 0x[\dA-Fa-f][\dA-Fa-f]*(_?u)(8\|16\|32\|64)?',
108	r'HEX_LIT_EXPLICIT 0x[\dA-Fa-f][\dA-Fa-f]*_(8\|16\|32\|64)?',
109	r'HEX_LIT 0x[\dA-Fa-f][\dA-Fa-f]*',
110
111	r'FLOAT_LIT_1 \d[\d_]*\.\d+_?f(32\|64)?',
112	r'FLOAT_LIT_2 \d[\d_]*(_?f)(32\|64)?',
113	r'DECIMAL_LIT \d[\d_]*(\.\d+)?(_?d)',
114	r'NUMBER_LIT \d[\d_]*(\.\d+)?(_?n)',
115	r'FRACTIONAL_LIT \d[\d_]*\.\d+',
116	r'INTEGER_LIT_EXPLICIT \d[\d_]*_?[iu](8\|16\|32\|64)?',
117	r'INTEGER_LIT \d[\d_]*',
118
119	r'INT_SIZE int[0-9]+(?=[^A-Za-z0-9_])',
120	r'UINT_SIZE uint[0-9]+(?=[^A-Za-z0-9_])',
121	r'FLOAT_SIZE float[0-9]+(?=[^A-Za-z0-9_])',
122
123	r"CHAR_LIT_SINGLE c'(?:\\'\|\\?[^'])'",
124	r'CHAR_LIT_DOUBLE c"(?:\\"\|\\?[^"])"',
125
126	# doc strings
127	r'DOC_STRING_START """[ \t]*\n',
128	r'DOC_STRING_LINE """."""[ \t]\n',
129
130	# sharp strings
131	r"SHARP_SINGLE sharp'(?:\\.?\|[^'\n])*'",
132	r'SHARP_DOUBLE sharp"(?:\\.?\|[^"\n])*"',
133
134	# raw strings
135	r"STRING_RAW_SINGLE r'(?:\\.?\|[^'\n])*'",
136	r'STRING_RAW_DOUBLE r"(?:\\.?\|[^"\n])*"',
137
138	# substituted strings
139	r'RBRACKET_SPECIAL ]',
140	r"STRING_START_SINGLE '", # see "def makeSTRING_FOO_BAR"
141	r"STRING_PART_SINGLE '",
142	r"STRING_STOP_SINGLE '",
143
144	r'STRING_START_DOUBLE "',
145	r'STRING_PART_DOUBLE "',
146	r'STRING_STOP_DOUBLE "',
147
148	r'STRING_PART_FORMAT :[^X"\n\[]*(?=])'.replace('X', "'"),
149
150	# plain strings
151	r"STRING_NOSUB_SINGLE ns'(?:\\.?\|[^'\n])*'",
152	r'STRING_NOSUB_DOUBLE ns"(?:\\.?\|[^"\n])*"',
153
154	r"STRING_SINGLE '(?:\\.?\|[^'\n])*'",
155	r'STRING_DOUBLE "(?:\\.?\|[^"\n])*"',
156
157	r'TOQ to\?',
158	r'ID [A-Za-z_][A-Za-z0-9_]*',
159	]
160
161	get unorderedTokenSpecs as List<of String> is override
162	return [
163	r'SHARP_OPEN \$sharp\(', # deprecated. $ is reserved for future language level regex support
164	r"SINGLE_QUOTE '",
165	r'DOUBLE_QUOTE "',
166	r'DOT \.',
167	r'DOTDOT \.\.',
168	r'COLON :',
169	r'PLUS \+',
170	r'PLUSPLUS \+\+',
171	r'MINUSMINUS \-\-',
172	r'MINUS -',
173	r'STARSTAR \\',
174	r'STAR \*',
175	r'SLASHSLASH //',
176	r'SLASH /',
177	r'PERCENTPERCENT %%',
178	r'PERCENT %',
179	r'ASSIGN =',
180	r'LPAREN \(',
181	r'RPAREN \)',
182	r'LBRACKET \[',
183	r'RBRACKET \]',
184	r'LCURLY \{',
185	r'RCURLY \}',
186	r'SEMI ;',
187	r'COMMA ,',
188	r'DOUBLE_LT <<',
189	r'DOUBLE_GT >>',
190	r'DICT_OPEN {',
191	r'DICT_CLOSE }',
192	r'QUESTION \?',
193	r'BANG \!',
194	r'ARRAY_OPEN \@\[',
195	r'AMPERSAND \&',
196	r'VERTICAL_BAR \\|',
197	r'CARET \^',
198	r'TILDE \~',
199
200	r'EQ ==',
201	r'NE <>',
202	r'LT <',
203	r'GT >',
204	r'LE <=',
205	r'GE >=',
206
207	r'PLUS_EQUALS \+=',
208	r'MINUS_EQUALS \-=',
209	r'STAR_EQUALS \*=',
210	r'SLASH_EQUALS \/=',
211	r'PERCENT_EQUALS %=',
212
213	r'STARSTAR_EQUALS \\=',
214
215	r'AMPERSAND_EQUALS \&=',
216	r'VERTICAL_BAR_EQUALS \\|=',
217	r'CARET_EQUALS \^=',
218	r'DOUBLE_LT_EQUALS <<=',
219	r'DOUBLE_GT_EQUALS >>=',
220
221	r'QUESTION_EQUALS \?=',
222	r'BANG_EQUALS \!=',
223	]
224
225	get keywords as IList<of String> is override
226	return KeywordSpecs.keywords
227
228	pro typeProvider from var as ITypeProvider?
229
230	def _reset is override
231	base._reset
232	_indentCount = 0
233	_substLBracketCount = 0
234
235	def makeSPACE(definition as String) as TokenDef?
236	return SpaceTokenDef('SPACE', definition)
237
238	# TODO
239	# def makeOPEN_GENERIC(definition as String) as TokenDef?
240	# return OpenGenericTokenDef('OPEN_GENERIC', definition)
241
242	def makeSTRING_START_SINGLE(definition as String) as TokenDef?
243	return StringStartTokenDef('STRING_START_SINGLE', definition[0])
244
245	def makeSTRING_START_DOUBLE(definition as String) as TokenDef?
246	return StringStartTokenDef('STRING_START_DOUBLE', definition[0])
247
248	def makeSTRING_PART_SINGLE(definition as String) as TokenDef?
249	return StringPartTokenDef('STRING_PART_SINGLE', definition[0])
250
251	def makeSTRING_PART_DOUBLE(definition as String) as TokenDef?
252	return StringPartTokenDef('STRING_PART_DOUBLE', definition[0])
253
254	def makeSTRING_STOP_SINGLE(definition as String) as TokenDef?
255	return StringStopTokenDef('STRING_STOP_SINGLE', definition[0])
256
257	def makeSTRING_STOP_DOUBLE(definition as String) as TokenDef?
258	return StringStopTokenDef('STRING_STOP_DOUBLE', definition[0])
259
260	def afterStart is override
261	base.afterStart
262	# CC:
263	# _tokenDefsByWhich['STRING_PART_SINGLE'].isActiveCall = def(tokenizer)=tokenizer.inSubstStringSingle
264	# _tokenDefsByWhich['STRING_STOP_SINGLE'].isActiveCall = def(tokenizer)=tokenizer.inSubstStringSingle
265
266	# recover from multiline comments
267	while _tokenDefsByWhich.containsKey('COMMENT_BLOCK_LINE')
268	.popTokenDefs
269	_inCommentBlock = 0
270
271	inactivate = [
272	'RBRACKET_SPECIAL' ,
273	'STRING_PART_SINGLE', 'STRING_STOP_SINGLE',
274	'STRING_PART_DOUBLE', 'STRING_STOP_DOUBLE',
275	'STRING_PART_FORMAT',
276	]
277	for which in inactivate
278	_tokenDefsByWhich[which].isActive = false
279
280	def isActiveCall(tok as TokenDef) as bool is override
281	if tok.which=='STRING_PART_SINGLE' or tok.which=='STRING_STOP_SINGLE'
282	return _inSubstStringSingle
283	return true
284
285	get _nextToken as IToken? is override
286	# overridden to deliver the final DEDENTS to close out indentation
287	tok = base._nextToken
288	if tok is nil
289	colNum = 0
290	while _indentCount > 0
291	if colNum == 0, colNum = .lastToken.colNum + 1
292	t = Token(.lastToken.fileName, .lastToken.lineNum, colNum, .lastToken.charNum, 'DEDENT', '', '')
293	_tokenQueue.enqueue(t)
294	_indentCount -= 1
295	colNum += 1
296	if _tokenQueue.count
297	return _nextToken
298	else
299	return nil
300	else
301	return tok
302
303	def onWHITESPACE_LINE(tok as IToken) as IToken?
304	# Eat these.
305	# Don't muck with perceived indentation level as
306	# these kinds of lines are irrelevant.
307	#print '<> onWHITESPACE_LINE'
308	return nil
309
310	def onWHITESPACE_COMMENT_1(tok as IToken) as IToken?
311	#print '<> onWHITESPACE_COMMENT_1'
312	if .checkForCommentDirective(tok)
313	return .directiveToken(tok)
314	else
315	return .commentToken(tok)
316
317	def onWHITESPACE_COMMENT_2(tok as IToken) as IToken?
318	#print '<> onWHITESPACE_COMMENT_2'
319	if .checkForCommentDirective(tok)
320	return .directiveToken(tok)
321	else
322	return .commentToken(tok)
323	return nil
324
325	##
326	## Comment out block
327	##
328
329	var _commentBlockDefs as List<of TokenDef>?
330
331	def onCOMMENT_BLOCK_START(tok as IToken) as IToken?
332	#print '<> onCOMMENT_BLOCK_START', tok
333	assert _inCommentBlock >= 0, tok
334	# narrow the tokenizer's token defs to a new shorter set
335	if _inCommentBlock == 0
336	if _commentBlockDefs is nil
337	# CC: _commentBlockDefs = List<of TokenDef>[_commentBlockStop, _commentBlockLine] # instead of the next 5 lines
338	defs = List<of TokenDef>()
339	defs.add(.tokenDefsByWhich['COMMENT_BLOCK_START'])
340	defs.add(TokenRegexDef('COMMENT_BLOCK_STOP', r'[^#]\#\/.$'))
341	defs.add(TokenRegexDef('COMMENT_BLOCK_LINE', '.*\n'))
342	_commentBlockDefs = defs
343	.pushTokenDefs(_commentBlockDefs to !)
344	_inCommentBlock += 1
345	return .commentToken(tok)
346
347	def onCOMMENT_BLOCK_LINE(tok as IToken) as IToken?
348	#print '<> onCOMMENT_BLOCK_LINE', tok.lineNum
349	assert _inCommentBlock > 0, tok
350	return .commentToken(tok)
351
352	def onCOMMENT_BLOCK_STOP(tok as IToken) as IToken?
353	#print '<> onCOMMENT_BLOCK_STOP', tok.lineNum
354	assert _inCommentBlock > 0, tok
355	_inCommentBlock -= 1
356	if _inCommentBlock == 0
357	.popTokenDefs
358	return .commentToken(tok)
359
360	def onINDENT_MIXED_TSNS(tok as IToken) as IToken?
361	# expecting tabs, spaces, non-whitespace
362	assert tok.text.startsWith('\t')
363	assert tok.text.endsWith(' ')
364	# this is okay on continued lines
365	if .justDidLineContinuation
366	indentLevel = tok.text.count(c'\t') + tok.text.count(c' ') // 4
367	return _processNumIndentLevels(indentLevel) # will check continuation indentation rules
368	else
369	return .onINDENT_MIXED_TS(tok)
370
371	def onINDENT_MIXED_TS(tok as IToken) as IToken?
372	sb = StringBuilder()
373	for c in tok.text
374	branch c
375	on c' ', sb.append(r'[SPACE]')
376	on c'\t', sb.append(r'[TAB]')
377	else, sb.append(c)
378	# to-do: for people using tabs a stray space is most common error here,
379	# such as ...[TAB][SPACE][TAB]...
380	# could probably detect this and invoke .recordError so compilation can continue.
381	.throwError('Cannot mix tabs and spaces in indentation. [sb]...')
382	return nil # make compiler happy.
383
384	def onINDENT_MIXED_ST(tok as IToken) as IToken?
385	return .onINDENT_MIXED_TS(tok)
386
387	def onINDENT_ALL_TABS(tok as IToken) as IToken?
388	numTabs = tok.text.count(c'\t')
389	return _processNumIndentLevels(numTabs)
390
391	def onINDENT_ALL_SPACES(tok as IToken) as IToken?
392	numSpaces = tok.text.count(c' ')
393	if numSpaces % 4 and not .justDidLineContinuation # yes, 4. hard coded, intentionally.
394	# TODO: should really just record an error and take (numSpaces/4).round as the indent
395	.throwError('Space-based indentation must be a multiple of 4. This line has a remainder of [numSpaces%4].')
396	return _processNumIndentLevels(numSpaces // 4)
397
398	def onNO_INDENT(tok as IToken) as IToken?
399	require tok.text==''
400	_curTokenDef.ignoreCount = 1
401	t = _processNumIndentLevels(0)
402	return t
403
404	def _processNumIndentLevels(numTabs as int) as IToken?
405	if .justDidLineContinuation
406	if numTabs < _indentCount
407	.recordError('Must indent same amount or more on a continued line.')
408	return nil
409	firstTok as IToken?
410	lastTok as IToken?
411	while numTabs > _indentCount
412	_indentCount += 1
413	newTok = Token(_fileName, _lineNum, 1, _charNum, 'INDENT', '', '')
414	if lastTok
415	lastTok.nextToken = newTok
416	lastTok = newTok
417	else
418	firstTok = lastTok = newTok
419	if firstTok
420	return firstTok
421	while numTabs < _indentCount
422	_indentCount -= 1
423	newTok = Token(_fileName, _lineNum, 1, _charNum, 'DEDENT', '', '')
424	if lastTok
425	lastTok.nextToken = newTok
426	lastTok = newTok
427	else
428	firstTok = lastTok = newTok
429	return firstTok
430
431	var _didLineContinuation as bool # only meaningful after an EOL
432
433	get justDidLineContinuation as bool
434	return .lastToken and .lastToken.which == 'EOL' and _didLineContinuation
435
436	def onEOL(tok as IToken) as IToken?
437	_didLineContinuation = .lastToken and .lastToken.text == '_' and .lastToken.which == 'ID'
438	return tok
439
440	def onSINGLE_LINE_COMMENT(tok as IToken) as IToken?
441	if .checkForCommentDirective(tok)
442	return .directiveToken(tok)
443	else
444	return .commentToken(tok)
445
446	def onINLINE_COMMENT(tok as IToken) as IToken?
447	return .commentToken(tok)
448
449	def onAMBIGUOUS_COMMENT(tok as IToken) as IToken?
450	.throwError('Ambiguous comment at /#. For an end-of-line comment, put a space between / and #. For an inline comment, end it with #/. For a block comment, put /# at the beginning of a line.')
451	return .commentToken(tok)
452
453	def onSPACE(tok as IToken) as IToken?
454	# eat these
455	return nil
456
457	def onAT_ID(tok as IToken) as IToken?
458	tok.value = tok.text[1:]
459	return tok
460
461	def onOPEN_CALL(tok as IToken) as IToken?
462	tok.value = tok.text[:-1]
463	return tok
464
465	def onOPEN_GENERIC(tok as IToken) as IToken?
466	require tok.text.trim.endsWith('<of')
467	s = tok.text.trim
468	tok.value = s[:-3]
469	return tok
470
471	def onID(tok as IToken) as IToken?
472	tok = .keywordOrWhich(tok, 'ID')
473	if tok.which<>'ID'
474	tok.isKeyword = true
475	return tok
476
477	def onFLOAT_LIT_1(tok as IToken) as IToken?
478	ensure
479	result.which == 'FLOAT_LIT'
480	(result.info to int) in [32, 64] # CC: axe cast
481	body
482	s = tok.text.replace('_', '')
483	if s.endsWith('f')
484	size = 64
485	s = s[:-1]
486	else if s.endsWith('f32')
487	size = 32
488	s = s[:-3]
489	else if s.endsWith('f64')
490	size = 64
491	s = s[:-3]
492	else
493	# cannot have other size specs given regex
494	size = 64
495	try
496	tok.value = float.parse(s, Utils.cultureInfoForNumbers)
497	catch FormatException
498	assert false, 'not expecting to get here given regex'
499	catch OverflowException
500	.recordError('Range overflow for float literal "[tok.text]".')
501	tok.which = 'FLOAT_LIT'
502	tok.info = size
503	return tok
504
505	def onFLOAT_LIT_2(tok as IToken) as IToken?
506	ensure
507	result.which == 'FLOAT_LIT'
508	(result.info to int) in [32, 64] # CC: axe cast
509	body
510	return .onFLOAT_LIT_1(tok)
511
512	def onDECIMAL_LIT(tok as IToken) as IToken?
513	s = tok.text
514	assert s.endsWith('d')
515	s = s[:-1]
516	s = s.replace('_', '')
517	try
518	tok.value = decimal.parse(s, Utils.cultureInfoForNumbers)
519	catch FormatException
520	assert false, 'not expecting to get here given regex'
521	catch OverflowException
522	.recordError('Range overflow for decimal literal "[tok.text]".')
523	return tok
524
525	def onNUMBER_LIT(tok as IToken) as IToken?
526	require tok.text.endsWith('n')
527	tok.which = 'FRACTIONAL_LIT'
528	tok.text = tok.text[:-1]
529	return .onFRACTIONAL_LIT(tok)
530
531	def onFRACTIONAL_LIT(tok as IToken) as IToken?
532	s = tok.text.replace('_', '')
533	try
534	assert _typeProvider
535	numberType = if(.typeProvider, .typeProvider.numberType, DecimalType())
536	# parse literal to same type as numberType
537	if numberType inherits DecimalType
538	tok.value = decimal.parse(s, Utils.cultureInfoForNumbers)
539	else if numberType inherits FloatType
540	tok.value = float.parse(s, Utils.cultureInfoForNumbers)
541	tok.which = 'FLOAT_LIT'
542	tok.info = numberType.size
543	else
544	throw FallThroughException(numberType)
545	catch FormatException
546	assert false, 'not expecting to get here given regex'
547	catch OverflowException
548	.recordError('[numberType.name.capitalized] range overflow for fractional literal "[tok.text]".')
549	return tok
550
551	def onINTEGER_LIT(tok as IToken) as IToken?
552	try
553	tok.value = int.parse(tok.text.replace('_', ''), Utils.cultureInfoForNumbers)
554	catch FormatException
555	assert false, 'not expecting to get here given regex'
556	catch OverflowException
557	.recordError('Range overflow for integer literal "[tok.text]".')
558	return tok
559
560	def onINTEGER_LIT_EXPLICIT(tok as IToken) as IToken?
561	require
562	'i' in tok.text or 'u' in tok.text
563	ensure
564	tok.which == 'INTEGER_LIT'
565	(tok.info to int) in [-8, 8, -16, 16, -32, 32, -64, 64] # CC: axe cast
566	body
567	s = tok.text.replace('_', '')
568	for c in s, if not c.isDigit, break
569	# c will be 'i' or 'u'
570	if s[s.length-1] == c
571	size = 32
572	s = s[:-1]
573	else if s.endsWith('32')
574	size = 32
575	s = s[:-3]
576	else if s.endsWith('64')
577	size = 64
578	s = s[:-3]
579	else if s.endsWith('16')
580	size = 16
581	s = s[:-3]
582	else if s.endsWith('8')
583	size = 8
584	s = s[:-2]
585	else
586	# cannot have other size specs given regex
587	size = 32
588	try
589	# TODO: The use of int.parse, as opposed to say int64.parse, uint32.parse, etc. probably
590	# means that legit int lits outside the signed 32 bit range will not work in Cobra
591	tok.value = int.parse(s, Utils.cultureInfoForNumbers)
592	catch FormatException
593	assert false, 'not expecting to get here given regex'
594	catch OverflowException
595	.recordError('Range overflow for integer literal "[tok.text]".')
596	tok.which = 'INTEGER_LIT'
597	tok.info = if(c == c'i', -1, +1) * size
598	return tok
599
600	def onHEX_LIT_UNSIGN(tok as IToken) as IToken?
601	require
602	'u' in tok.text
603	body
604	return .onHEX_LIT_EXPLICIT(tok)
605
606	def onHEX_LIT_EXPLICIT(tok as IToken) as IToken?
607	ensure
608	tok.which == 'INTEGER_LIT'
609	(tok.info to int) in [8, 16, 32, 64] # CC: axe cast
610	body
611	size = 32
612	h = tok.text
613	s = tok.text
614	if s.endsWith('32')
615	size = 32
616	s = s[:-2]
617	else if s.endsWith('64')
618	size = 64
619	s = s[:-2]
620	else if s.endsWith('16')
621	size = 16
622	s = s[:-2]
623	else if s.endsWith('8')
624	size = 8
625	s = s[:-1]
626	if s.endsWith('u')
627	s = s[:-1]
628	s = s.replace('_', '')
629	tok.text = s
630	tok = .onHEX_LIT(tok) to !
631	tok.info = size # unsigned
632	tok.text = h
633	return tok
634
635	def onHEX_LIT(tok as IToken) as IToken?
636	ensure
637	tok.which == 'INTEGER_LIT'
638	tok.info == 32
639	body
640	try
641	tok.value = int.parse(tok.text[2:], System.Globalization.NumberStyles.HexNumber)
642	catch FormatException
643	assert false, 'not expecting to get here given regex'
644	catch OverflowException
645	.recordError('Range overflow for hex literal "[tok.text]".')
646	tok.which = 'INTEGER_LIT'
647	tok.info = 32 # unsigned
648	return tok
649
650	def onINT_SIZE(tok as IToken) as IToken?
651	size = int.parse(tok.text[3:])
652	tok.value = size
653	return tok
654
655	def onUINT_SIZE(tok as IToken) as IToken?
656	size = int.parse(tok.text[4:])
657	tok.value = size
658	return tok
659
660	def onFLOAT_SIZE(tok as IToken) as IToken?
661	size = int.parse(tok.text[5:])
662	tok.value = size
663	return tok
664
665	def onCHAR_LIT_SINGLE(tok as IToken) as IToken?
666	return _onCharLit(tok)
667
668	def onCHAR_LIT_DOUBLE(tok as IToken) as IToken?
669	return _onCharLit(tok)
670
671	def _onCharLit(tok as IToken) as IToken?
672	require tok.text.startsWith('c')
673	s = tok.text[2:-1]
674	assert s.length==1 or s.length==2
675	tok.value = s
676	return tok
677
678
679	##
680	## String substitution handling
681	##
682
683	def tokValueForString(s as String) as String
684	"""
685	Utility class for onSTRING_START\|PART\|STOP_SINGLE\|DOUBLE.
686	"""
687	require
688	s.length >= 2 # CC: #s[0] in [c'"', c"'"] #s[s.length-1] in [c'"', c"'"]
689	body
690	s = s.substring(1, s.length-2)
691	chars = StringBuilder(s.length)
692	last = c'\0'
693	next as char?
694	for c in s
695	next = nil
696	if last == c'\\'
697	branch c
698	on c'a', next = c'\a'
699	on c'b', next = c'\b'
700	on c'f', next = c'\f'
701	on c'n', next = c'\n'
702	on c'r', next = c'\r'
703	on c't', next = c'\t'
704	on c'v', next = c'\v'
705	on c"'", next = c"'"
706	on c'"', next = c'"'
707	on c'?', next = c'?'
708	on c'0', next = c'\0'
709	on c'\\'
710	chars.append(c'\\')
711	# cannot have `last` being a backslash anymore--it's considered consumed now
712	last = c'\0'
713	continue
714	else, next = c # TODO: should probably be error: Invalid escape sequence
715	else if c <> '\\'
716	next = c
717	if next is not nil
718	chars.append(next)
719	last = c
720	return chars.toString
721
722	def onSTRING_START_SINGLE(tok as IToken) as IToken
723	require not _inSubstStringSingle
724	_inSubstStringSingle = true
725	tok.value = .tokValueForString(tok.text)
726	_tokenDefsByWhich['STRING_PART_SINGLE'].isActive = true
727	_tokenDefsByWhich['STRING_STOP_SINGLE'].isActive = true
728	_tokenDefsByWhich['STRING_PART_FORMAT'].isActive = true
729	return tok
730
731	def onSTRING_PART_SINGLE(tok as IToken) as IToken
732	require _inSubstStringSingle
733	tok.value = .tokValueForString(tok.text)
734	return tok
735
736	def onSTRING_STOP_SINGLE(tok as IToken) as IToken
737	require _inSubstStringSingle
738	_inSubstStringSingle = false
739	tok.value = .tokValueForString(tok.text)
740	_tokenDefsByWhich['STRING_PART_SINGLE'].isActive = false
741	_tokenDefsByWhich['STRING_STOP_SINGLE'].isActive = false
742	_tokenDefsByWhich['STRING_PART_FORMAT'].isActive = false
743	return tok
744
745
746	def onSTRING_START_DOUBLE(tok as IToken) as IToken
747	require not _inSubstStringDouble
748	_inSubstStringDouble = true
749	tok.value = .tokValueForString(tok.text)
750	_tokenDefsByWhich['STRING_PART_DOUBLE'].isActive = true
751	_tokenDefsByWhich['STRING_STOP_DOUBLE'].isActive = true
752	_tokenDefsByWhich['STRING_PART_FORMAT'].isActive = true
753	return tok
754
755	def onSTRING_PART_DOUBLE(tok as IToken) as IToken
756	require _inSubstStringDouble
757	tok.value = .tokValueForString(tok.text)
758	return tok
759
760	def onSTRING_STOP_DOUBLE(tok as IToken) as IToken
761	require _inSubstStringDouble
762	_inSubstStringDouble = false
763	tok.value = .tokValueForString(tok.text)
764	_tokenDefsByWhich['STRING_PART_DOUBLE'].isActive = false
765	_tokenDefsByWhich['STRING_STOP_DOUBLE'].isActive = false
766	_tokenDefsByWhich['STRING_PART_FORMAT'].isActive = false
767	return tok
768
769
770	def onLBRACKET(tok as IToken) as IToken
771	if _inSubstStringSingle or _inSubstStringDouble
772	_substLBracketCount += 1
773	if _substLBracketCount==1
774	_tokenDefsByWhich['RBRACKET_SPECIAL'].isActive = true
775	assert _tokenDefsByWhich['STRING_PART_FORMAT'].isActive
776	_tokenDefsByWhich['STRING_PART_FORMAT'].isActive = false
777	return tok
778
779	def onRBRACKET_SPECIAL(tok as IToken) as IToken
780	require
781	_inSubstStringSingle or _inSubstStringDouble
782	_substLBracketCount
783	body
784	_substLBracketCount -= 1
785	if _substLBracketCount == 0
786	_tokenDefsByWhich['RBRACKET_SPECIAL'].isActive = false
787	assert not _tokenDefsByWhich['STRING_PART_FORMAT'].isActive
788	_tokenDefsByWhich['STRING_PART_FORMAT'].isActive = true
789	tok.which = 'RBRACKET' # tricky, tricky. the parser never sees an RBRACKET_SPECIAL
790	return tok
791
792
793	##
794	## Doc Strings
795	##
796
797	def onDOC_STRING_START(tok as IToken) as IToken
798	assert not _inDocString
799	# narrow the tokenizer's token defs to a new shorter set
800	# TODO: cache the tokens below
801	t = List<of TokenDef>()
802	t.add(TokenRegexDef('DOC_STRING_STOP', r'[ \t]"""[ \t]\n'))
803	t.add(TokenRegexDef('DOC_STRING_BODY_TEXT', '.*\n'))
804	.pushTokenDefs(t)
805	_inDocString = true
806	return tok
807
808	def onDOC_STRING_STOP(tok as IToken) as IToken
809	assert _inDocString, tok
810	_inDocString = false
811	.popTokenDefs
812	return tok
813
814	def onDOC_STRING_BODY_TEXT(tok as IToken) as IToken
815	assert _inDocString, tok
816	return tok
817
818	def onDOC_STRING_LINE(tok as IToken) as IToken
819	tok.value = tok.text.trim[3:-3].trim
820	return tok
821
822	##
823	## Simple string literals
824	##
825
826	def onSTRING_RAW_SINGLE(tok as IToken) as IToken
827	require tok.text.startsWith('r')
828	tok.value = tok.text.substring(2, tok.text.length-3)
829	tok.which = 'STRING_SINGLE'
830	return tok
831
832	def onSTRING_RAW_DOUBLE(tok as IToken) as IToken
833	require tok.text.startsWith('r')
834	tok.value = tok.text.substring(2, tok.text.length-3)
835	tok.which = 'STRING_DOUBLE'
836	return tok
837
838	def onSTRING_NOSUB_SINGLE(tok as IToken) as IToken
839	require tok.text.startsWith('ns')
840	tok.value = .tokValueForString(tok.text.substring(2))
841	tok.which = 'STRING_SINGLE'
842	return tok
843
844	def onSTRING_NOSUB_DOUBLE(tok as IToken) as IToken
845	require tok.text.startsWith('ns')
846	tok.value = .tokValueForString(tok.text.substring(2))
847	tok.which = 'STRING_DOUBLE'
848	return tok
849
850	def onSTRING_SINGLE(tok as IToken) as IToken
851	tok.value = .tokValueForString(tok.text)
852	return tok
853
854	def onSTRING_DOUBLE(tok as IToken) as IToken
855	tok.value = .tokValueForString(tok.text)
856	return tok
857
858
859	##
860	## Self util
861	##
862
863	var _directiveRE = Regex(r'#\s?\.([\w\-]+)\.($\|\s)', RegexOptions.Compiled)
864
865	def checkForCommentDirective(tok as IToken) as bool
866	# check for .no-warnings.
867	reMatch = _directiveRE.match(tok.text)
868	if reMatch.success
869	tok.which = 'DIRECTIVE'
870	name = reMatch.groups[1].value
871	branch name
872	on 'no-warnings', .addNoWarning(tok)
873	# for testify
874	on 'args', pass # TODO: actually this could be worth implementing outside of testify
875	on 'compile-only', pass
876	on 'error', pass
877	on 'multi', pass
878	on 'multipart', pass
879	on 'require', pass
880	on 'skip', pass
881	on 'warning', pass
882	on 'warning-lax', pass
883	else, .throwError('Unrecognized compiler directive "[name]".')
884	return true
885	return false
886
887	def commentToken(tok as IToken) as IToken?
888	if .willReturnComments
889	tok.which = 'COMMENT'
890	return tok
891	else
892	return nil
893
894	def directiveToken(tok as IToken) as IToken?
895	return if(.willReturnDirectives, tok, nil)
896
897
898	class SpaceTokenDef inherits TokenDef
899	"""
900	This is just an idea for speeding up the tokenizer...
901	implement some of the token defs by hand in the hope that they will be faster than the regex.
902	"""
903
904	test
905	# .timeIt
906	pass
907
908	def timeIt is shared
909	input1, input2 = '\t\tx = 5', '# foo'
910	reps = 10_000_000
911
912	re = Regex(r'[ \t]+', RegexOptions.Compiled)
913	sw = System.Diagnostics.Stopwatch()
914	sw.start
915	for i in reps
916	re.match(input1)
917	re.match(input2)
918	sw.stop
919	timeRE = sw.elapsedMilliseconds
920
921	td = SpaceTokenDef('SPACE', r'[ \t]+')
922	sw = System.Diagnostics.Stopwatch()
923	sw.start
924	for i in reps
925	td.match(input1)
926	td.match(input2)
927	sw.stop
928	timeTD = sw.elapsedMilliseconds
929
930	ratio = timeRE / timeTD
931	trace timeRE, timeTD, ratio
932
933	# trace: timeRE=6875 (Int64); timeTD=462 (Int64); ratio=14.88 (Decimal);
934	# so at least for SpaceTokenDef, its .match is more than 14 X faster than a compiled regex!
935
936	cue init(which as String, definition as String)
937	base.init(which)
938	assert definition == r'[ \t]+'
939
940	get length as int is override
941	throw Exception('ordered token')
942
943	get firstChars as List<of char> is override
944	return [c' ', c'\t']
945
946	def match(input as String) as TokenMatch? is override
947	for i in input.length
948	c = input[i]
949	if c <> c' ' and c <> c'\t', break
950	return if(i==0, nil, TokenMatch(input.substring(0, i)))
951
952
953	/#
954	TODO
955	class OpenGenericTokenDef inherits TokenDef
956	""" [A-Za-z_][A-Za-z0-9_]*<of[ \n\r\t] """
957
958	cue init(which as String, definition as String)
959	base.init(which)
960
961	get length as int is override
962	throw Exception('ordered token')
963
964	get firstChars as List<of char> is override
965	t = List<of char>()
966	for c in 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz_'
967	t.add(c)
968	return t
969
970	def match(input as String) as TokenMatch? is override
971	state = 0
972	for i in input.length
973	...
974	#/
975
976	class StringTokenDef inherits TokenDef is abstract
977	"""
978	Regexes were:
979	r"STRING_START_SINGLE '[^'\n\[]*\[",
980	r"STRING_PART_SINGLE \][^'\n\[]*\[",
981	r"STRING_STOP_SINGLE \][^'\n\[]*'",
982
983	r'STRING_START_DOUBLE "[^"\n\[]*\[',
984	r'STRING_PART_DOUBLE \][^"\n\[]*\[',
985	r'STRING_STOP_DOUBLE \][^"\n\[]*"',
986
987	But there is also the need to escape left brackets which turned out difficult to express in
988	regexes. I did manage to do it, but then the regexes became extremely slow.
989	"""
990
991	var _quote as char
992
993	cue init(which as String, quote as char)
994	base.init(which)
995	_quote = quote
996
997	get length as int is override
998	throw Exception('ordered token')
999
1000	def match(input as String) as TokenMatch? is override
1001	s = _match(input)
1002	if s, return TokenMatch(s)
1003	else, return nil
1004
1005	def _match(input as String) as String? is abstract
1006
1007	def _matchBetween(input as String, startch as char, stopch as char, breakch as char) as String?
1008	"""
1009	Matches between a range of characters.
1010	"""
1011	if input[0] <> startch, return nil
1012	sb = StringBuilder(input[0].toString)
1013	isEscaped = false
1014	for i in 1 : input.length
1015	c = input[i]
1016	if c == breakch and not isEscaped, return nil
1017	if c == c'\n', return nil
1018	sb.append(c)
1019	if isEscaped
1020	isEscaped = false
1021	else
1022	if c == stopch, return sb.toString
1023	if c == c'\\', isEscaped = true
1024	return nil
1025
1026	def toString as String is override
1027	return '[.getType.name]([CobraCore.toTechString(_quote)])'
1028
1029
1030	class StringStartTokenDef inherits StringTokenDef
1031
1032	cue init(which as String, quote as char)
1033	base.init(which, quote)
1034
1035	get firstChars as List<of char> is override
1036	return [_quote]
1037
1038	def _match(input as String) as String? is override
1039	test
1040	x = StringStartTokenDef('foo', c"\'")
1041	# normal:
1042	assert x._match('aoeu') is nil
1043	assert x._match(r"'foo [bar]") == r"'foo ["
1044	# odd:
1045	assert x._match(r"'foo[") == r"'foo["
1046	# escaped:
1047	assert x._match(r"'foo\[ foo[") == r"'foo\[ foo["
1048	assert x._match(r"'foo\\[foo]") == r"'foo\\["
1049	assert x._match(r"'foo\\\[ foo[") == r"'foo\\\[ foo["
1050	# not this token def:
1051	assert x._match(r"]foo [bar]") is nil
1052	assert x._match(r"]foo[") is nil
1053	assert x._match(r"]foo[") is nil
1054	assert x._match(r"]foo' + ") is nil
1055	assert x._match(r"]foo ' bah blah") is nil
1056	assert x._match(r"]foo\[ foo'") is nil
1057	body
1058	return _matchBetween(input, _quote, c'[', _quote)
1059
1060
1061	class StringPartTokenDef inherits StringTokenDef
1062
1063	cue init(which as String, quote as char)
1064	base.init(which, quote)
1065
1066	get firstChars as List<of char> is override
1067	return [c']']
1068
1069	def _match(input as String) as String? is override
1070	test
1071	x = StringPartTokenDef('foo', c"\'")
1072	# normal:
1073	assert x._match('aoeu') is nil
1074	assert x._match(r"]foo [bar]") == r"]foo ["
1075	# odd:
1076	assert x._match(r"]foo[") == r"]foo["
1077	# escaped:
1078	assert x._match(r"]foo\[ foo[") == r"]foo\[ foo["
1079	assert x._match(r"]foo\\[foo]") == r"]foo\\["
1080	assert x._match(r"]foo\\\[ foo[") == r"]foo\\\[ foo["
1081	# not this token def:
1082	assert x._match(r"'foo [bar]") is nil
1083	assert x._match(r"'foo[") is nil
1084	assert x._match(r"'foo\[ foo[") is nil
1085	assert x._match(r"]foo' + ") is nil
1086	assert x._match(r"]foo ' bah blah") is nil
1087	assert x._match(r"]foo\[ foo'") is nil
1088	body
1089	return _matchBetween(input, c']', c'[', _quote)
1090
1091
1092	class StringStopTokenDef inherits StringTokenDef
1093
1094	cue init(which as String, quote as char)
1095	base.init(which, quote)
1096
1097	get firstChars as List<of char> is override
1098	return [c']']
1099
1100	def _match(input as String) as String? is override
1101	test
1102	x = StringStopTokenDef('foo', c"\'")
1103	# normal:
1104	assert x._match('aoeu') is nil
1105	assert x._match(r"]foo' + ") == r"]foo'"
1106	assert x._match(r"] '") == r"] '"
1107	# odd:
1108	assert x._match(r"]foo ' bah blah") == r"]foo '"
1109	# escaped:
1110	assert x._match(r"]foo\[ foo'") == r"]foo\[ foo'"
1111	assert x._match(r"]foo\\\[ foo'") == r"]foo\\\[ foo'"
1112	# not this token def:
1113	assert x._match(r"'foo [bar]") is nil
1114	assert x._match(r"'foo[") is nil
1115	assert x._match(r"'foo\[ foo[") is nil
1116	assert x._match(r"]foo[") is nil
1117	assert x._match(r"'foo [bar]") is nil
1118	assert x._match(r"'foo[") is nil
1119	assert x._match(r"]foo\\[foo]") is nil
1120	body
1121	return _matchBetween(input, c']', _quote, c'[' )

Note: See TracBrowser for help on using the browser.

root/cobra/trunk/Source/CobraTokenizer.cobra

Download in other formats: