Wiki

TestifyRunner.cobra

Revision 2640, 28.0 KB (checked in by Charles.Esterbrook, 3 weeks ago)
Minor. Fix something broken in a previous patch.
Property svn:eol-style set to `native`

Line
1	use System.Diagnostics
2	use System.Reflection
3	use System.Text.RegularExpressions
4	use System.Threading
5
6
7	class TestifyRunner
8	"""
9	Implements the -testify option of the command line.
10	"""
11
12	var _startTime as DateTime
13	var _cl as CommandLine
14	var _pathList as List<of String>
15	var _extraTestifyArgs as String?
16
17	var _testifyCount as int
18	var _failureCount as int
19
20	# comes in handy when profiling:
21	var _maxCount = 0
22	var _willRunExes = true
23
24	var _firstAttempt as bool
25
26	var _cachedTestifyModules as IList<of Module>?
27
28	# two writers. status goes to console and results goes to a file.
29	var _statusWriter as IndentedWriter?
30	var _statusCount as int
31	var _resultsWriter as IndentedWriter?
32
33	const bar = '----------------------------------------------------------------------------------------------------'
34
35	## Init
36
37	cue init(startTime as DateTime, cl as CommandLine, paths as List<of String>)
38	base.init
39	_startTime = startTime
40	_cl = cl
41	_pathList = paths
42
43
44	## Properties
45
46	get pathList from var
47
48	get options as OptionValues
49	return _cl.options
50
51	get verbosity as int
52	return _cl.verbosity
53
54
55	## Run
56
57	def run
58	paths = _pathList
59	if paths.count == 0
60	paths = .cobraTestPaths + [Path.getFullPath(Path.combine('..', 'HowTo'))]
61	numThreads = .options.getDefault('testify-threads', 1) to int
62	if numThreads > 1
63	.runThreaded(numThreads, paths)
64	else
65	.runNonThreaded(paths)
66
67	# these vars support running sub-processes, managed by multiple threads in the original cobra executable
68	var _subDirQueue = Queue<of String>()
69	var _subCobraExe = ''
70	var _subCommandLineArgs = List<of String>()
71	var _subResultsFileNames = List<of String>()
72
73	def runThreaded(numThreads as int, paths as List<of String>)
74	require numThreads > 1
75
76	_statusWriter = IndentedWriter(AutoFlushWriter(Console.out), indentString=' ')
77
78	args = CobraCore.commandLineArgs
79	_subCobraExe = args[0]
80	_subCommandLineArgs = for arg in args[1:] where arg.startsWith('-') and not '-testify-threads:' in arg
81
82	_statusWriter.writeLine('Queueing for threads:')
83	for path in paths
84	_statusWriter.writeLine(' [path]')
85	_subDirQueue.enqueue(path)
86	_subResultsFileNames.add('') # empty placeholder. these values are set in order
87	assert _subDirQueue.count > 0
88
89	threads = List<of Thread>()
90	for i in numThreads, threads.add(Thread(ref .runTestifyProcessInThread))
91	# CC: threads = for i in numThreads get Thread(ref .runTestifyProcessInThread)
92	for t in threads, t.start
93	for t in threads, t.join
94
95	_concatSubResults
96	_printTotals
97
98	def _concatSubResults
99	sep, sepNl = '', CobraCore.newLine + CobraCore.newLine + CobraCore.newLine
100	concat = StringBuilder()
101	for fileName in _subResultsFileNames.reversed # they are set in a reversed order
102	concat.append(sep)
103	concat.append(File.readAllText(fileName))
104	sep = sepNl
105	File.writeAllText(.options['testify-results'] to String, concat.toString)
106	for fileName in _subResultsFileNames
107	try
108	File.delete(fileName)
109	catch exc as Exception
110	_statusWriter.writeLine('warning: Cannot delete "[fileName]" due to: [exc]')
111
112	def _printTotals
113	resultsFileName = .options['testify-results'] to String
114	using resultsWriter = File.appendText(resultsFileName)
115	__printTotals(resultsWriter to !)
116	__printTotals(_statusWriter to !)
117
118	def __printTotals(writer as TextWriter)
119	writer.writeLine
120	writer.writeLine('Final multithreaded results:')
121	writer.writeLine('[_testifyCount] Tests')
122	if _failureCount > 0
123	writer.writeLine('[_failureCount] Failures')
124	else
125	writer.writeLine('Success.')
126
127	def runTestifyProcessInThread
128	tid = Thread.currentThread.getHashCode
129	while true # keep getting a work queue item
130	lock _subDirQueue
131	if _subDirQueue.count == 0, break # ...until they run out
132	path = _subDirQueue.dequeue
133	pathIndex = _subDirQueue.count
134	resultsFileName = 'r-testify-[pathIndex]'
135	lock _subResultsFileNames, _subResultsFileNames[pathIndex] = resultsFileName
136	args = _subCommandLineArgs + ['-testify-results:[resultsFileName]', '-testify-threads:1', '"[path]"']
137	lock _statusWriter, _statusWriter.writeLine('Thread [tid] start: [args.join(" ")]')
138	p = Process()
139	p.startInfo.useShellExecute = false
140	p.startInfo.redirectStandardOutput = true
141	p.startInfo.redirectStandardError = true
142	p.startInfo.fileName = _subCobraExe
143	p.startInfo.arguments = args.join(' ')
144	p.start
145	# TODO: get the process output line by line and display as we go
146	output = p.standardOutput.readToEnd
147	output += p.standardError.readToEnd
148	p.waitForExit
149	lock _statusWriter
150	_statusWriter.writeLine
151	_statusWriter.writeLine('Thread [tid] output:')
152	for line in output.splitLines
153	if .verbosity >= 2
154	_statusWriter.writeLine(' t[tid]\|[line]')
155	else
156	if line.startsWith('Finished at') or line.startsWith('timeit =')
157	continue
158	_statusWriter.writeLine(' ' + line)
159	m = Regex.match(line, r'(\d+) Tests? in ')
160	if m.success
161	nTests = _parseIntGroup(m)
162	_testifyCount += nTests
163	m = Regex.match(line, r'(\d+) Failure')
164	if m.success
165	nTests = _parseIntGroup(m)
166	_failureCount += nTests
167	_statusWriter.writeLine
168
169	def _parseIntGroup(m as Match) as int
170	""" Return first capture of first group of a Regexp.Match parsed to an integer. """
171	cap = m.groups[1].captures[0].value
172	n as int
173	int.tryParse(cap, out n)
174	return n
175
176	def runNonThreaded(paths as List<of String>)
177	_statusWriter = IndentedWriter(AutoFlushWriter(Console.out))
178	_statusWriter.indentString = ' '
179	try
180	resultsFileName = .options['testify-results'] to String
181	using resultsWriter = File.createText(resultsFileName)
182	_resultsWriter = IndentedWriter(AutoFlushWriter(resultsWriter))
183	print to _resultsWriter to !
184	print 'Cobra: Testify'
185	print 'Started at', DateTime.now
186	print
187	_innerRun(paths)
188	finally
189	if .verbosity >= 2
190	_statusWriter.writeLine('Results in [resultsFileName]')
191	_statusWriter = nil
192
193	def _innerRun(paths as List<of String>)
194	# TODO Console.error = Console.out
195	_testifyCount = 0
196	for pathName in paths
197	if Directory.exists(pathName)
198	.testifyDir(pathName)
199	else if File.exists(pathName)
200	_testifyCount += .testifyFile(pathName)
201	else
202	.error('No such directory or file named "[pathName]".')
203	if _maxCount > 0 and _testifyCount >= _maxCount
204	break
205	.testifyFinish(if(_failureCount, 'Failure.', 'Success.'))
206
207
208	## The Rest
209
210	def error(msg as String)
211	_cl.error(msg)
212
213	def parseArgs(args as String, options as out OptionValues?, paths as out List<of String>?)
214	"""
215	This overload is primarily for -testify.
216	Arguments that come in from the system are already divided up into a list.
217	"""
218	ensure
219	options
220	paths
221	body
222	args = args.trim
223	argsList = if('\|' in args, args.split(c'\|'), args.split)
224	_cl.parseArgs(argsList to passthrough, out options, out paths)
225
226	get cobraTestPaths as List<of String>
227	"""
228	Only used when -testify is passed with no path.
229	--testify is an "internal" feature of the cobra command line front end.
230	"""
231	# -testify is often invoked out of the next-door directory "Source"
232	# so check next door, first:
233	slash = Path.directorySeparatorChar
234	path = "..[slash]Tests"
235	if Directory.exists(path)
236	paths = List<of String>()
237	for subdir in Directory.getDirectories(path)
238	fileName = Path.getFileName(subdir)
239	if not fileName.startsWith('.') and not fileName.startsWith('_')
240	paths.add(Path.getFullPath(subdir))
241	return paths
242	throw Exception('Cannot find Tests directory next door.')
243
244	def _testifyFlush
245	_statusWriter.flush
246	_resultsWriter.flush
247
248	def testifyOptions as OptionValues
249	options as OptionValues?
250	paths as List<of String>?
251	.parseArgs(_extraTestifyArgs ? '', out options, out paths) # to get the default options
252	# remember that you cannot use synonyms below. you must use the canonical name of the option
253	options['debug'] = '+'
254	options['debugging-tips'] = false
255	options['embed-run-time'] = false
256	options['verbosity'] = 2
257	options['back-end'] = .options['back-end']
258	#trace options
259	return options to !
260
261	def testifyFinish(message as String)
262	_testifyFinish(message, _statusWriter to !)
263	_testifyFinish(message, _resultsWriter to !)
264
265	def _testifyFinish(message as String, writer as TextWriter)
266	print to writer
267	duration = DateTime.now.subtract(_startTime)
268	print
269	print 'Finished at', DateTime.now
270	print '[_testifyCount] Tests in [duration].'
271	if _failureCount
272	print '[_failureCount] Failures.'
273	print
274	print message
275
276	def testifyDir(dirName as String)
277	"""
278	Returns the number tests that passed.
279	"""
280	baseName = Path.getFileName(dirName) to ! # gets rid of "." and ".." as prefix for relative dirs
281	if baseName.startsWith('.') or baseName.startsWith('_')
282	# examples: .svn, _svn. Also, _ is a nice way to temporarily exclude a directory if possible
283	return
284	configPath = Path.combine(dirName, 'testify.kv')
285	if File.exists(configPath)
286	dirOptions = Utils.readKeyValues(configPath)
287	if dirOptions.containsKey('args')
288	argSets = dirOptions['args'].split(@[c'\|'])
289	for argSet in argSets
290	_testifyDir(dirName, argSet.trim)
291	return
292	_testifyDir(dirName, nil)
293
294	def _testifyDir(dirName as String, args as String?)
295	willCopyLibs = true # not CobraCore.isRunningMono # On Mono, we use MONO_PATH instead of coping the Cobra.Lang.dll around
296	libNames = ['Cobra.Lang.dll']
297	_extraTestifyArgs = args
298	_statusWriter.writeLine(if(args, '[dirName] with args: [args]', '[dirName]'))
299	_statusWriter.indent
300	try
301	print 'Running tests in [dirName]'
302	saveDir = Environment.currentDirectory
303	if willCopyLibs
304	# _cleanDir will remove these later
305	for libName in libNames
306	# Ideally, the file operations below would not be wrapped with try...catch and a
307	# 'warning' because if the current library files cannot be copied into the test
308	# directory, then the test is not valid. However, Windows has problems with
309	# deleting and/or copying over Cobra.Lang.dll right after it has been used. This
310	# comes up on Tests\340-contracts whose testify.kv config file specifies two
311	# runs of the directory.
312	targetPath = Path.combine(dirName, libName)
313	if File.exists(targetPath)
314	if Utils.isRunningOnUnix
315	File.delete(targetPath)
316	else
317	# Windows is just too damn picky...
318	try
319	File.delete(targetPath)
320	catch UnauthorizedAccessException
321	print 'testify warning: cannot delete:', targetPath
322	if libName == 'Cobra.Lang.dll' or File.exists(libName)
323	if Utils.isRunningOnUnix
324	File.copy(Path.combine(saveDir, libName), targetPath)
325	else
326	try
327	File.copy(Path.combine(saveDir, libName), targetPath, true)
328	catch IOException
329	print 'testify warning: cannot copy or copy over:', targetPath
330	Directory.setCurrentDirectory(dirName)
331	try
332	setUpScript = if(Utils.isRunningOnUnix, 'testify-set-up', 'testify-set-up.bat')
333	if File.exists(setUpScript)
334	print '* * * * Running [setUpScript]'
335	process = Process()
336	process.startInfo.fileName = setUpScript
337	setUpOutput = CobraCore.runAndCaptureAllOutput(process)
338	if process.exitCode <> 0
339	print 'TESTIFY WARNING: Script [setUpScript] exited with error code [process.exitCode]'
340	print 'begin output'
341	print setUpOutput.trim
342	print 'end output'
343	paths = List<of String>(Directory.getFiles('.'))
344	paths.addRange(Directory.getDirectories('.') to passthrough)
345	paths.sort
346	for baseName in paths
347	baseName = Utils.normalizePath(baseName)
348	if baseName.startsWith('_'), continue
349	if baseName.endsWith('.cobra') or baseName.endsWith('.COBRA')
350	_testifyCount += .testifyFile(baseName)
351	else if Directory.exists(baseName)
352	.testifyDir(baseName)
353	_testifyFlush
354	if _maxCount > 0 and _testifyCount >= _maxCount
355	break
356	_cleanDir(dirName)
357	finally
358	Directory.setCurrentDirectory(saveDir)
359	finally
360	_statusWriter.dedent
361	_testifyFlush
362
363	def testifyFile(baseName as String) as int
364	save = _failureCount
365	_firstAttempt = true
366	try
367	result1 = _testifyFile(baseName)
368	if _failureCount > save
369	print
370	print
371	print 'DUE TO FAILURE, RERUNNING WITH MORE OUTPUT:'
372	_firstAttempt = false
373	result2 = _testifyFile(baseName)
374	if result1 <> result2
375	print 'ERROR: Result mismatch. result1=[result1], result2=[result2]'
376	finally
377	_testifyFlush
378	return result1
379
380	def _testifyFile(baseName as String) as int
381	Node.setCompiler(nil)
382	verbose = not _firstAttempt
383	compilerVerbosity = if(.verbosity, .verbosity, if(verbose, 1, 0))
384	if Path.pathSeparator in baseName
385	return .testifyFilePath(baseName)
386
387	_statusWriter.writeLine('([_statusCount]) [baseName]')
388	_statusCount += 1
389
390	assert File.exists(baseName)
391	source = File.readAllText(baseName)
392
393	print
394	print
395	print
396	print 'RUN [baseName]'
397	print ' [Utils.combinePaths(Environment.currentDirectory, baseName)]'
398	print ' Test #[_testifyCount+1]'
399	print .bar
400	print .bar
401	if verbose
402	Utils.printSource(source)
403	print .bar
404	lines = List<of String>(source.split(c'\n')) # CC: axe list wrapper
405
406	fileNames = [baseName]
407	options = .testifyOptions
408	options['willRunExe'] = _willRunExes # internal, specific to testify
409
410	firstLine = ''
411	count0 = lines.count
412	rc = _processFirstlineDirectives(baseName, fileNames, inout lines, inout options, out firstLine)
413	if rc == 0, return 0
414	firstLineInsensitive = firstLine.trim.replace(' ', '')
415	nRemovedLines = count0 - lines.count
416	# Check for inline warning and error messages that are expected.
417	# (Starting in 2007-12 this is now the preferred way to specify these messages--
418	# with the actual line of code that generates them.
419	# The old method of specifying at the top will still be needed for errors
420	# and warnings that have no associated line number.)
421	expectingError = false
422	inLineMessages = _getInlineMessages(lines, nRemovedLines, out expectingError)
423	if inLineMessages.count > 0 # hasInlineMessages
424	return _testifyInlineMessages(inLineMessages, expectingError,
425	compilerVerbosity, [baseName], options, verbose )
426
427	if firstLineInsensitive.startsWith('#.error.')
428	# deprecated: put errors on the lines where they occur. the "hasInlineMessages" code above will detect them.
429	# Note that errors that are only detected by the backend C# compiler are not detectable by testify
430	# CC: support split with a String extension method
431	# error = firstLine.split('.error.',1)[1].trim.toLower
432	index = firstLine.indexOf('.error.')
433	error = firstLine.substring(index+7).trim.toLower
434	return _testifyHeadError(error, compilerVerbosity, fileNames, options, verbose)
435
436	if firstLineInsensitive.startsWith('#.warning.')
437	# deprecated: put warnings on the lines where they occur. the "hasInlineMessages" code above will detect them.
438	index = firstLine.indexOf('.warning.')
439	warning = firstLine.substring(index+9).trim.toLower
440	return _testifyHeadWarning(warning, false, compilerVerbosity, fileNames, options, verbose)
441
442	if firstLineInsensitive.startsWith('#.warning-lax.')
443	# deprecated: put warnings on the lines where they occur. the "hasInlineMessages" code above will detect them.
444	index = firstLine.indexOf('.warning-lax.')
445	warning = firstLine.substring(index+13).trim.toLower
446	return _testifyHeadWarning(warning, true, compilerVerbosity, fileNames, options, verbose)
447
448	return _testifyStd(compilerVerbosity, fileNames, options, verbose)
449
450
451	def _processFirstlineDirectives(baseName as String, fileNames as List<of String>, lines as inout List<of String>,
452	options as inout OptionValues, firstLine as out String) as int
453	"""
454	Check first few lines for Testify directives (start with '#.') and process or setup
455	for later handling.
456	"""
457	firstLine = lines[0]
458	firstLineInsensitive = firstLine.trim.replace(' ', '')
459	while firstLineInsensitive.startsWith('#.')
460
461	if firstLineInsensitive.startsWith('#.multi.')
462	print 'Running multiple files.'
463	for fileName in firstLine.substring(firstLine.indexOf('.multi.')+8).split
464	if fileName.length
465	fileNames.add(Utils.combinePaths(Path.getDirectoryName(baseName) to !, fileName))
466	print 'Multiple filenames:', fileNames.join(', ')
467	# enable having another directive on the next line, such as .error.
468	lines = lines[1:]
469	firstLine = lines[0]
470	firstLineInsensitive = firstLine.trim.replace(' ', '')
471	continue
472
473	if firstLineInsensitive.startsWith('#.require.')
474	rtPlatform = if(options['back-end']=='none', CobraCore.runtimePlatform, options['back-end'])
475	what = firstLineInsensitive[10:]
476	branch what
477	on 'mono'
478	if rtPlatform <> 'clr' or not CobraCore.isRunningOnMono
479	print 'Skipping test because requirement for "mono" is not met.'
480	return 0
481	on 'dotnet'
482	if rtPlatform <> 'clr' or CobraCore.isRunningOnMono
483	print 'Skipping test because requirement for "dotnet" is not met.'
484	return 0
485	on 'clr' # mono or dotNet
486	if rtPlatform <> 'clr'
487	print 'Skipping test because requirement for "clr" is not met.'
488	return 0
489	on 'jvm'
490	if rtPlatform <> 'jvm'
491	print 'Skipping test because requirement for "jvm" is not met.'
492	return 0
493	else
494	if what.endsWith('.dll')
495	try
496	loadAssemblyResult = Utils.loadWithPartialName(what)
497	catch assExc as Exception
498	print 'Skipping test because DLL requirement "[what]" failing with exception: [assExc]'
499	return 0
500	if not loadAssemblyResult
501	print 'Skipping test because DLL requirement "[what]" not found.'
502	return 0
503	else
504	.error('Unrecognized requirement: "[what]"')
505	lines = lines[1:]
506	firstLine = lines[0]
507	firstLineInsensitive = firstLine.trim.replace(' ', '')
508	continue
509
510	if firstLineInsensitive.startsWith('#.compile-only.') # also meaning don't run the .exe
511	options['willRunExe'] = false
512	lines = lines[1:]
513	firstLine = lines[0]
514	firstLineInsensitive = firstLine.trim.replace(' ', '')
515	continue
516
517	if firstLineInsensitive.startsWith('#.args.')
518	i = firstLine.indexOf('.args.')
519	args as OptionValues?
520	.parseArgs(firstLine[i+'.args.'.length:], out args, out _pathList)
521	options = .testifyOptions
522	options.combine(args to !)
523	# enable having another directive on the next line, such as .error.
524	lines = lines[1:]
525	firstLine = lines[0]
526	firstLineInsensitive = firstLine.trim.replace(' ', '')
527	continue
528
529	if firstLineInsensitive.startsWith('#.skip.')
530	comment = firstLine[firstLine.indexOf('.skip.')+6:].trim
531	if comment.length, comment = '"' + comment + '"'
532	msg = 'Skipping test because of directive. [comment]'.trim
533	_statusWriter.indent
534	_statusWriter.writeLine(msg)
535	_statusWriter.dedent
536	print msg
537	return 0
538
539	if firstLineInsensitive.startsWith('#.multipart.')
540	# .multi. is the one that gets run along with its associated files
541	# the associated files then specify .multipart. and get skipped when encountered
542	print 'Skipping test because multipart.'
543	return 0
544
545	if firstLineInsensitive.startsWith('#.error.') or firstLineInsensitive.startsWith('#.warning.') or firstLineInsensitive.startsWith('#.warning-lax.')
546	# these are handled below
547	break
548
549	throw Exception('Bad first line: [lines[0]]')
550
551	return 1 # continue processing in caller
552
553	def _getInlineMessages(lines as List<of String>, offset as int, expectingError as out bool) as Dictionary<of int, String>
554	""" Walk lines and accumulate inline warnings and error messages. """
555	inLineMessages = Dictionary<of int, String>()
556	firstLine = 1 + offset
557	lineNum = firstLine
558	expectingError = false
559	for line in lines
560	if lineNum > firstLine and ('.warning.' in line or '.error.' in line)
561	if '.warning.' in line
562	message = line[line.indexOf('.warning.') + '.warning.'.length:]
563	messageType = 'w'
564	else if '.error.' in line
565	message = line[line.indexOf('.error.') + '.error.'.length:]
566	messageType = 'e'
567	expectingError = true
568	else
569	throw FallThroughException(line)
570	inLineMessages[lineNum] = messageType + message
571	lineNum += 1
572	return inLineMessages
573
574	def _testifyInlineMessages(inLineMessages as Dictionary<of int, String>,
575	expectingError as bool,
576	compilerVerbosity as int,
577	fileNames as IList<of String>,
578	options as OptionValues, verbose as bool) as int
579	"""Testify on files that have inline checks for compiler errors and warnings"""
580	try
581	c = Compiler(compilerVerbosity, _cachedTestifyModules, commandLineArgParser=_cl.argParser)
582	c.testifyFilesNamed(fileNames, options, _resultsWriter to !, verbose)
583	catch StopCompilation
584	pass
585	catch exc as Exception
586	print 'Internal exception: [exc]'
587	.failed
588	return 0
589	for msg in c.messages
590	if not msg.hasSourceSite or msg.lineNum == 0
591	print 'Not expecting messages without any line number information:'
592	print msg
593	bad = true
594	continue
595	if not inLineMessages.containsKey(msg.lineNum)
596	print 'Encountered unexpected message:'
597	print msg
598	bad = true
599	continue
600	expected = inLineMessages[msg.lineNum]
601	branch expected[0]
602	on c'w'
603	if msg.isError
604	print 'Expecting warning on line [msg.lineNum], but got error instead.'
605	bad = true
606	on c'e'
607	if not msg.isError
608	print 'Expecting error on line [msg.lineNum], but got warning instead.'
609	bad = true
610	else
611	throw FallThroughException(expected)
612	if bad, continue
613	expected = expected[1:].trim
614	if msg.message.trim.toLower.indexOf(expected.toLower) == -1
615	print 'Expecting message :', expected
616	print 'But got :', msg.message.trim
617	print 'At line :', msg.lineNum
618	bad = true
619	continue
620	# we made it! same type of message and text
621	print 'Message for line [msg.lineNum] was expected.'
622	inLineMessages.remove(msg.lineNum)
623
624	# check for expected messages that never occurred
625	for key in inLineMessages.keys
626	bad = true
627	print 'Expecting message on line [key]:', inLineMessages[key][1:].trim
628	if bad
629	.failed
630	return 0
631	else if expectingError
632	return 1
633	else if options.boolValue('willRunExe') # a test case with nothing but warnings is still executed
634	return _testifyRun(c)
635	return 1
636
637	def _testifyHeadError(error as String, compilerVerbosity as int, fileNames as IList<of String>,
638	options as OptionValues, verbose as bool) as int
639	try
640	c = Compiler(compilerVerbosity, _cachedTestifyModules, commandLineArgParser=_cl.argParser)
641	c.testifyFilesNamed(fileNames, options, _resultsWriter to !, verbose)
642	print 'Expecting error(s): [error]'
643	print 'No error at all.'
644	if c.errors.count > 0
645	print 'warning: error count > 0 but StopCompilation was not thrown'
646	.failed
647	return 0
648	catch StopCompilation
649	assert c.errors.count
650	expectedErrors = error.split(c'&')
651	for i in 0 : expectedErrors.length
652	expectedError = expectedErrors[i].trim
653	print 'Expecting error substring [i+1] of [expectedErrors.length]: [expectedError]'
654	if i >= c.errors.count
655	print 'Ran out of real errors.'
656	.failed
657	return 0
658	actualError = c.errors[i]
659	if actualError.message.toLower.indexOf(expectedError) == -1
660	print 'Actual error is: [actualError.message]'
661	.failed
662	return 0
663	print 'Matches: "[actualError.message]"'
664	if c.errors.count > expectedErrors.length
665	print 'There are more actual errors than expected errors:'
666	for i in expectedErrors.length : c.errors.count
667	print 'Another actual error: [c.errors[i].message]'
668	.failed
669	return 0
670	catch exc as Exception
671	print 'Internal exception: [exc]'
672	.failed
673	return 0
674	return 1
675
676	def _testifyHeadWarning(warning as String, lax as bool, compilerVerbosity as int, fileNames as IList<of String>,
677	options as OptionValues, verbose as bool) as int
678	# TODO: the following code both checks for warnings to be thrown as well as going through a list of warnings. Seems like it should just need to do one or the other.
679	try
680	c = Compiler(compilerVerbosity, _cachedTestifyModules, commandLineArgParser=_cl.argParser)
681	c.testifyFilesNamed(fileNames, options, _resultsWriter to !, verbose)
682	catch StopCompilation
683	if not lax
684	print 'Expecting warning substring: "[warning]"'
685	print 'But got errors.'
686	.failed
687	return 0
688	catch exc as Exception
689	print 'Internal exception: [exc]'
690	.failed
691	return 0
692	expectedWarnings = warning.split(c'&')
693	for i in 0 : expectedWarnings.length
694	expectedWarning = expectedWarnings[i].trim
695	print 'Expecting warning substring [i+1] of [expectedWarnings.length]: [expectedWarning]'
696	if i >= c.warnings.count
697	print 'Ran out of real warnings.'
698	.failed
699	return 0
700	actualWarning = c.warnings[i]
701	if actualWarning.message.toLower.indexOf(expectedWarning)==-1
702	print 'Actual warning is: [actualWarning.message]'
703	.failed
704	return 0
705	else
706	print 'Matches: "[actualWarning.message]"'
707	if c.warnings.count > expectedWarnings.length
708	print 'There are more actual warnings than expected warnings:'
709	for i in expectedWarnings.length : c.warnings.count
710	print 'Another actual warning: [c.warnings[i].message]'
711	.failed
712	return 0
713	return 1
714
715	def _testifyStd(compilerVerbosity as int, fileNames as IList<of String>,
716	options as OptionValues, verbose as bool) as int
717	c = Compiler(compilerVerbosity, _cachedTestifyModules, commandLineArgParser=_cl.argParser)
718	try
719	c.testifyFilesNamed(fileNames, options, _resultsWriter to !, verbose)
720	catch StopCompilation
721	.failed
722	return 0
723	catch exc as Exception
724	print 'Internal exception: [exc]'
725	.failed
726	return 0
727
728	if options.boolValue('willRunExe') and options.boolValue('compile') # maybe changed by compiler directive
729	options['willRunExe'] = false
730
731	if c.messages.count
732	# can't be errors or StopCompilation would have been caught above
733	print 'Unexpected warnings in test.'
734	.failed
735	return 0
736
737	if options.boolValue('willRunExe')
738	return _testifyRun(c)
739	return 1
740
741	def _testifyRun(c as Compiler) as int
742	if not c.backEnd.isRunnableFile(c.fullExeFileName)
743	# below assumes created file placed in cwd
744	if File.exists(c.fullExeFileName)
745	print 'Produced file "[c.fullExeFileName]" as expected.'
746	return 1
747	else
748	print 'Did not produce file "[c.fullExeFileName]".'
749	.failed
750	return 0
751	else
752	print 'Run:'
753	if .verbosity >= 1, print 'c.fullExeFileName = "[c.fullExeFileName]"'
754	p = c.runProcess
755	if .verbosity >= 2, print '[p.startInfo.fileName] [p.startInfo.arguments]'
756	output = CobraCore.runAndCaptureAllOutput(p).trim
757
758	print 'Output:'
759	if output.length
760	print output
761	if p.exitCode <> 0
762	print 'Exit code = [p.exitCode]'
763	.failed
764	return 0
765	if output.toLower.indexOf('unhandled exception') <> -1
766	.failed
767	return 0
768
769	print .bar
770
771	_cachedTestifyModules = for mod in c.modules where mod inherits AssemblyModule get mod
772
773	return 1
774
775	def failed
776	"""
777	Produces output and increments the failure count,
778	but does not throw an exception or exit.
779	"""
780	if _firstAttempt
781	_statusWriter.writeLine('FAILURE ----------------------------------------------------------------------')
782	_failureCount += 1
783	print
784	print
785	print 'TEST FAILURE. SEE BELOW FOR VERBOSE RERUN.'
786
787	def testifyFilePath(pathName as String) as int
788	dirName = Path.getDirectoryName(pathName)
789	baseName = Path.getFileName(pathName) to !
790	assert dirName and dirName.length
791	assert baseName.length
792	saveDir = Environment.currentDirectory
793	Directory.setCurrentDirectory(dirName)
794	try
795	return .testifyFile(baseName)
796	finally
797	Directory.setCurrentDirectory(saveDir)
798
799	def _cleanDir(dirName as String)
800	curDir = Environment.currentDirectory
801	if dirName.endsWith(Path.directorySeparatorChar.toString), dirName = dirName[:-1]
802	assert Path.getFileName(dirName) == Path.getFileName(curDir)
803	di = DirectoryInfo(curDir)
804	for fileInfo in di.getFiles
805	try
806	if fileInfo.extension in ['.exe', '.dll', '.mdb', '.pdb', '.class', '.tmp'] _
807	and fileInfo.name not in ['Nested.dll', 'Foo.iSeries.dll']
808	# TODO: perhaps the extensions for generated files should come from the back-end
809	# TODO: find a general way to determine if a binary file is part of the workspace/repository
810	fileInfo.delete
811	catch ex as Exception
812	if not fileInfo.name.startsWith('Cobra.') # this happens regularly and I don't feel like seeing it
813	print 'warning: cannot delete [fileInfo] due to: [ex.message]'

Note: See TracBrowser for help on using the browser.

root/cobra/trunk/Source/TestifyRunner.cobra

Download in other formats: