Wiki

SyntaxHighlighter.cobra

Revision 2488, 9.9 KB (checked in by Charles.Esterbrook, 15 months ago)
Fixed: The Cobra syntax highlighter throws an exception when it encounters namespaces. reported-by:RIGHT_THEN
Property svn:eol-style set to `native`

Line
1	"""
2	The main test case for the syntax highlighter is the Cobra compiler itself:
3
4	cobra -highlight -files:files-to-compile.text
5
6	TODO:
7	[ ] The stylesheet path should be computed rather than hardcoded as ..\
8	[ ] Files that are in a deeper subdirectory, like gen-html\BackEndClr\, don't have a correct path to the stylesheet
9	[ ] Partial classes are not getting the 'tdn' style because they are merged into their classes. Likewise, their members are not getting 'mdn' style.
10	[ ] String substitution should not color the brackets [] the same color as the string text.
11	[ ] Negative constants like "-1" don't get highlighted correctly. The - is treated as an operator.
12	[ ] Right now "bool" is a bolded keyword and types like String and Shape are normal. Maybe there should be a "type" highlight.
13	[ ] Use what's found in OperatorSpecs instead of duplicating. This also requires completing a TODO item in OperatorSpecs
14	# TODO: move the binary op and unary op specs here
15	"""
16
17
18	class Compiler is partial
19
20	def highlightFiles
21	require
22	.modules.count
23	body
24	Directory.createDirectory(.targetDirectory)
25	for mod in .modules
26	if mod inherits CobraModule
27	if not mod.isImplicit
28	mod.writeHtmlHighlightedFileTo(.targetDirectory)
29
30	get targetDirectory as String
31	return 'gen-html'
32
33
34	class CobraModule is partial
35
36	def writeHtmlHighlightedFileTo(targetDirectory as String)
37	htmlFileName = Path.combine(targetDirectory, .fileName + '.html')
38	dir = Path.getDirectoryName(htmlFileName)
39	Directory.createDirectory(dir) # the .fileName may have relative subdirs in it
40	print 'Writing [htmlFileName]'
41	using tw = File.createText(htmlFileName)
42	.writeHtmlHighlightedFileTo(tw)
43
44	def writeHtmlHighlightedFileTo(tw as TextWriter)
45	tw.writeLine('<html>')
46	tw.writeLine('<head>')
47	tw.writeLine('<meta http-equiv="Content-Type" content="text/html; charset=utf-8">')
48	tw.writeLine('<title>[.fileName]</title>')
49	tw.writeLine('<link rel="stylesheet" href="../styles-cobra-doc.css" type="text/css">')
50	tw.writeLine('<link rel="stylesheet" href="../styles-cobra-shl.css" type="text/css">')
51	tw.writeLine('</head>')
52	tw.writeLine('<body>')
53	tw.write(.htmlSource)
54	tw.writeLine('</body>')
55	tw.writeLine('</html>')
56
57	def htmlSource as String
58	require .typeProvider
59	return .htmlSource(nil)
60
61	def htmlSource(typeProvider as ITypeProvider?) as String
62	require .typeProvider or typeProvider
63	if typeProvider
64	saveProvider = Node.typeProvider
65	Node.typeProvider = typeProvider
66	try
67	source = File.readAllText(.fileName)
68	marks = Marks(.fileName)
69
70	# first do token stream which covers all tokens including comments
71	# not expecting any errors here since we got past this phase before
72	tokenizer = CobraTokenizer(typeProvider=.typeProvider, willReturnComments=true)
73	tokens = tokenizer.startSource(.fileName, source).allTokens
74	marks.add(tokens)
75
76	# now do nodes -- gives better semantic highlighting than just tokens
77	.topNameSpace.highlight(marks)
78
79	sb = StringBuilder('<pre class="shl">')
80	marks.combine(source, sb)
81	sb.append('</pre>')
82	return sb.toString
83	finally
84	if typeProvider
85	Node.typeProvider = saveProvider
86
87
88	class Marks
89	"""
90	A collection of marks as in "mark up".
91	Used for syntax highlighting of source files.
92	For example, source code could be rewritten as HTML with <span> tags for each mark.
93	"""
94
95	shared
96
97	var _ignoreTokens = Set<of String>()
98	var _invisibleTokens = {'DEDENT', 'EOL', 'INDENT'}
99	var _normalTokens = {
100	'BANG', 'BANG_EQUALS', 'CARET', 'COLON', 'COMMA', 'DOT', 'DOTDOT', 'LCURLY', 'LPAREN', 'PERCENTPERCENT', 'QUESTION', 'QUESTION_EQUALS', 'RCURLY', 'RPAREN', 'INT_SIZE', 'UINT_SIZE',
101	}
102
103	get ignoreTokens as Set<of String>
104	if _ignoreTokens.count == 0
105	# 2009-03-13 Mono bug re: method overload resolution. https://bugzilla.novell.com/show_bug.cgi?id=485378
106	# _ignoreTokens.addRange(_invisibleTokens)
107	# _ignoreTokens.addRange(_normalTokens)
108	for token in _invisibleTokens, _ignoreTokens.add(token)
109	for token in _normalTokens, _ignoreTokens.add(token)
110	return _ignoreTokens
111
112	var _tokenToStyleSpecs = [
113	'TRUE kw-s',
114	'FALSE kw-s',
115	'NIL kw-s',
116	'BASE kw-b',
117	'THIS kw-t',
118
119	'CHAR_LIT_SINGLE lc',
120	'CHAR_LIT_DOUBLE lc',
121	'DECIMAL_LIT ld',
122	'AT_ID di',
123	'DIRECTIVE di',
124	'COMMA no',
125	'COMMENT c',
126	'DOC_STRING_START ds',
127	'DOC_STRING_BODY_TEXT ds',
128	'DOC_STRING_STOP ds',
129	'DOC_STRING_LINE ds',
130	'ID i',
131	'FLOAT_LIT lf',
132	'FRACTIONAL_LIT lfr',
133	'INTEGER_LIT li',
134	'STRING_DOUBLE ls',
135	'STRING_SINGLE ls',
136	'STRING_PART_FORMAT spf',
137	'TOQ kw',
138	]
139
140	var _ops = 'ARRAY_OPEN ASSIGN EQ NE GT LT GE LE DOUBLE_GT LBRACKET RBRACKET PLUS MINUS STAR STARSTAR SLASH SLASHSLASH PERCENT MINUS_EQUALS PLUS_EQUALS'
141
142	var _tokenToStyle = Dictionary<of String, String>()
143
144	get tokenToStyle as Dictionary<of String, String>
145	if _tokenToStyle.count == 0
146	for spec in _tokenToStyleSpecs
147	parts = spec.split
148	_tokenToStyle[parts[0]] = parts[1]
149	for op in _ops.split
150	_tokenToStyle[op] = 'op'
151	return _tokenToStyle
152
153	var _marks = Dictionary<of int, Mark>()
154
155	cue init(fileName as String)
156	base.init
157	_fileName = fileName
158
159	get fileName from var as String
160
161	def add(tokens as IToken*)
162	ignoreTokens, tokenToStyle = .ignoreTokens, .tokenToStyle
163	for token in tokens
164	# print token.which, token.text, token.charNum, token.isKeyword
165	if token.which in ignoreTokens
166	pass
167	else if tokenToStyle.containsKey(token.which)
168	.add(token, tokenToStyle[token.which])
169	else if token.isKeyword
170	.add(token, 'kw')
171	else
172	charNum = token.charNum - 1
173	len = token.text.length
174	text = token.text
175	branch token.which
176	on 'OPEN_CALL'
177	.add(token, 'i')
178	on 'OPEN_DO'
179	.add(charNum, 2, 'kw', 'do')
180	on 'OPEN_GENERIC'
181	.add(token, 'i')
182	on 'OPEN_IF'
183	.add(charNum, len-1, 'kw', text[:-1])
184	on 'SHARP_SINGLE' or 'SHARP_DOUBLE' # sharp'...'
185	.add(charNum, 5, 'kw', 'sharp')
186	.add(charNum+5, len-5, 'ls', text[5:])
187	on 'SHARP_OPEN'
188	.add(charNum, 6, 'kw', '$sharp') # deprecated
189	on 'STRING_START_SINGLE' or 'STRING_START_DOUBLE'
190	.add(charNum, len-1, 'ls', text[:-1])
191	.add(charNum+len-1, 1, 'lslb', '\[')
192	on 'STRING_PART_SINGLE' or 'STRING_PART_DOUBLE'
193	.add(charNum, 1, 'lsrb', ']')
194	if len > 2, .add(charNum+1, len-2, 'ls', text[1:-1]) # else text is ']['
195	.add(charNum+len-1, 1, 'lslb', '\[')
196	on 'STRING_STOP_SINGLE' or 'STRING_STOP_DOUBLE'
197	.add(charNum, 1, 'lsrb', ']')
198	.add(charNum+1, len-1, 'ls', text[1:])
199	else, print '*** unknown token type: [token.which]; [token]'
200
201	def add(token as IToken, kind as String)
202	# second check below guards against partial classes split across files
203	if not token.isEmpty and token.fileName.endsWith(.fileName)
204	# The OPEN_CALL and OPEN_GENERIC tokens come from multiple sources such as declarations
205	# and expressions. So we check for them here in one place.
206	charNum = token.charNum - 1
207	branch token.which
208	on 'OPEN_CALL'
209	.add(charNum, token.text.length-1, kind, token.text[:-1])
210	on 'OPEN_GENERIC'
211	.add(charNum, token.text.length-3, kind, token.text[:-3])
212	.add(charNum+token.text.length-3, 2, 'kw', 'of')
213	else, .add(charNum, token.text.length, kind, token.text)
214
215	def add(charNum as int, length as int, kind as String)
216	.add(charNum, length, kind, '')
217
218	def add(charNum as int, length as int, kind as String, text as String)
219	_marks[charNum] = Mark(charNum, length, kind, text)
220
221	def combine(plainSource as String, sb as StringBuilder)
222	# CC: TODO: next line causes exception in the compiler
223	# marks = _marks.values.toList.sorted
224	marks = List<of Mark>(_marks.values).sorted
225	closeSpansAt = Set<of int>()
226	ci = mi = 0
227	for c in plainSource
228	if ci in closeSpansAt
229	sb.append('</span>')
230	closeSpansAt.remove(ci)
231	if mi < marks.count
232	mark = marks[mi]
233	if mark.charNum == ci
234	sb.append('<span class="shl-[mark.kind]">')
235	# use next statement in place of above for debugging:
236	# sb.append('<span class="shl-[mark.kind]" what="[mark.text]">')
237	closeSpansAt.add(mark.charNum + mark.length)
238	while mi < marks.count and marks[mi].charNum <= ci
239	mi += 1
240	branch c
241	on c'\t', sb.append(' ')
242	on c'<', sb.append('<')
243	on c'>', sb.append('>')
244	on c'&', sb.append('&')
245	on c'"', sb.append('"')
246	else, sb.append(c)
247	if c <> c'\r', ci += 1
248
249
250	class Mark implements IComparable<of Mark>
251
252	cue init(charNum as int, length as int, kind as String, text as String)
253	require
254	charNum >= 0
255	length > 0
256	kind <> ''
257	body
258	base.init
259	_charNum, _length, _kind, _text = charNum, length, kind, text
260
261	get charNum from var as int
262
263	get length from var as int
264
265	get kind from var as String
266
267	get text from var as String
268
269	def toString as String is override
270	return '[.getType.name]([.charNum], [.length], [.kind], [.text])'
271
272	def compareTo(other as Mark) as int
273	diff = .charNum - other.charNum
274	if diff == 0
275	diff = .length - other.length
276	if diff == 0
277	diff = .kind.compareTo(other.kind)
278	return diff
279
280
281	class Container<of TMember> is partial
282
283	def highlight(marks as Marks)
284	h = Highlighter(marks)
285	for decl as dynamic in .declsInOrder
286	h.dispatch(decl)
287
288
289	class Highlighter inherits Visitor
290
291	cue init(marks as Marks)
292	base.init
293	_marks = marks
294
295	get methodName as String is override
296	return 'highlight'
297
298	get marks from var as Marks
299
300	def mark(token as IToken, kind as String)
301	.marks.add(token, kind)
302
303	def highlight(ns as NameSpace)
304	.mark(ns.token, 'tdn')
305	.dispatch(ns.declsInOrder)
306
307	def highlight(box as Box)
308	.mark(box.idToken, 'tdn')
309	.dispatch(box.declsInOrder)
310
311	def highlight(enumDecl as EnumDecl)
312	.mark(enumDecl.idToken, 'tdn')
313
314	def highlight(member as BoxMember)
315	.mark(member.token, 'kw-md')
316	.mark(member.idToken, 'mdn')
317
318	def highlight(method as AbstractMethod)
319	.mark(method.token, 'kw-md')
320	.mark(method.idToken, 'mdn')
321	.dispatch(method.statements)
322
323	def highlight(stmt as Stmt)
324	# nothing for now
325	pass

Note: See TracBrowser for help on using the browser.

root/cobra/trunk/Source/SyntaxHighlighter.cobra

Download in other formats: