| 441 | | if t |
| 442 | | tokens.add(t) |
| 443 | | else |
| 444 | | break |
| | 446 | if t, tokens.add(t) |
| | 447 | else, break |
| | 448 | # determine timings for various Tokenizer lexing changes |
| | 449 | if false # CC: can use /# ... #/ in the future |
| | 450 | duration = DateTime.now.subtract(start) |
| | 451 | if tokens.count > 100 and duration > TimeSpan(0) |
| | 452 | ticksPerToken = (duration.ticks to int) / tokens.count |
| | 453 | ticksPerTokenMS = ticksPerToken / 10_000 |
| | 454 | fileName = _fileName |
| | 455 | if fileName.startsWith('('), fileName = '(None)' |
| | 456 | else, fileName = Path.getFileName(fileName) |
| | 457 | print '[fileName.padRight(20)]\t[duration]\t[tokens.count]\t' stop |
| | 458 | print ' [ticksPerTokenMS:N3] ms/Token av ([ticksPerTokenMS/1000:N3]sec)' |
| 594 | | if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow |
| 595 | | assert _tokenDefsByFirstChar |
| 596 | | assert _sourceLine.length |
| 597 | | if _tokenDefsByFirstChar.containsKey(_sourceLine[0]) |
| 598 | | # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]' |
| 599 | | tokenDefs = _tokenDefsByFirstChar[_sourceLine[0]] |
| 600 | | else |
| 601 | | tokenDefs = _tokenDefs to ! |
| 602 | | else |
| 603 | | tokenDefs = _tokenDefs to ! |
| | 608 | tokenDefs = _getCandidateTokenDefs(_sourceLine) |
| 647 | | try |
| 648 | | tok = meth.invoke(this, @[tok]) to IToken? |
| 649 | | catch tie as TargetInvocationException |
| 650 | | throw tie.innerException |
| 651 | | if tok is nil |
| 652 | | # skip token, so go to the next one |
| 653 | | reinvoke = true |
| 654 | | else |
| 655 | | # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method |
| 656 | | while tok |
| 657 | | _tokenQueue.enqueue(tok) |
| 658 | | tok = tok.nextToken |
| 659 | | reinvoke = true # to pick up the queue |
| | 637 | tok = _tokenPostProcess(meth, tok) |
| | 638 | # nil indicates skipped token - will re call this method |
| | 639 | if not tok |
| | 640 | reinvoke = true # to pick up next token after skipped |
| | 672 | |
| | 673 | def _getCandidateTokenDefs(sourceLine as String?) as List<of TokenDef> |
| | 674 | if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow |
| | 675 | assert _tokenDefsByFirstChar |
| | 676 | assert _sourceLine.length |
| | 677 | if _tokenDefsByFirstChar.containsKey(_sourceLine[0]) |
| | 678 | # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]' |
| | 679 | return _tokenDefsByFirstChar[_sourceLine[0]] |
| | 680 | return _tokenDefs to ! |
| | 681 | |
| | 682 | def _skipMatchAttempt(tokenDef as TokenDef, sourceLineIndex as int) as bool |
| | 683 | if tokenDef.ignoreCount |
| | 684 | tokenDef.ignoreCount -= 1 |
| | 685 | return true |
| | 686 | if not tokenDef.isActive |
| | 687 | return true |
| | 688 | if sourceLineIndex>0 and tokenDef.requiresBOL |
| | 689 | return true |
| | 690 | if not .isActiveCall(tokenDef) |
| | 691 | return true |
| | 692 | return false |
| | 693 | |
| | 694 | def _getTokenMethod(which as String) as MethodInfo? |
| | 695 | """ |
| | 696 | Get method for token (On<WHICH>) using reflection. |
| | 697 | Caching is used to boost performance. |
| | 698 | """ |
| | 699 | if not _onTokenTypeCache.containsKey(which) # not already in methods cache |
| | 700 | methName = 'On' + which |
| | 701 | meth = .getType.getMethod(methName) |
| | 702 | _onTokenTypeCache[which] = meth |
| | 703 | else |
| | 704 | meth = _onTokenTypeCache[which] |
| | 705 | return meth |
| | 706 | |
| | 707 | def _tokenPostProcess(meth as MethodInfo, tok as IToken?) as IToken? |
| | 708 | """ |
| | 709 | Invoke any tokenMethod and return the result which could the same token, a replacement |
| | 710 | token or nil to skip the token. Any token returned could be the start of a token chain |
| | 711 | via token.nextToken. |
| | 712 | """ |
| | 713 | try |
| | 714 | tok = meth.invoke(this, @[tok]) to IToken? |
| | 715 | catch tie as TargetInvocationException |
| | 716 | throw tie.innerException |
| | 717 | if not tok |
| | 718 | return nil # token is to be skipped |
| | 719 | retTok = tok |
| | 720 | tok = tok.nextToken |
| | 721 | while tok |
| | 722 | # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method |
| | 723 | _tokenQueue.enqueue(tok) # store any token chain returned by method call |
| | 724 | tok = tok.nextToken |
| | 725 | return retTok |
