| | 446 | # Determine timings for various Tokeniser lexing changes |
| | 447 | #duration = DateTime.now.subtract(st) |
| | 448 | #if tokens.count > 100 and duration > TimeSpan(0) |
| | 449 | # avTicksPerToken = (duration.ticks / (tokens.count to int64)) to int |
| | 450 | # msecs = avTicksPerToken/10000 |
| | 451 | # fileNm = _fileName |
| | 452 | # fileNm = if( fileNm.startsWith("("), '(None)', Path.getFileName(fileNm) ) |
| | 453 | # print "[fileNm.padRight(20)]\t[duration]\t[tokens.count]\t" stop |
| | 454 | # print " [msecs:N3] ms/Token av ([msecs/1000:N3]sec)" |
| 594 | | if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow |
| 595 | | assert _tokenDefsByFirstChar |
| 596 | | assert _sourceLine.length |
| 597 | | if _tokenDefsByFirstChar.containsKey(_sourceLine[0]) |
| 598 | | # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]' |
| 599 | | tokenDefs = _tokenDefsByFirstChar[_sourceLine[0]] |
| 600 | | else |
| 601 | | tokenDefs = _tokenDefs to ! |
| 602 | | else |
| 603 | | tokenDefs = _tokenDefs to ! |
| | 604 | tokenDefs = _getCandidateTokenDefs(_sourceLine) |
| 647 | | try |
| 648 | | tok = meth.invoke(this, @[tok]) to IToken? |
| 649 | | catch tie as TargetInvocationException |
| 650 | | throw tie.innerException |
| 651 | | if tok is nil |
| 652 | | # skip token, so go to the next one |
| 653 | | reinvoke = true |
| 654 | | else |
| 655 | | # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method |
| 656 | | while tok |
| 657 | | _tokenQueue.enqueue(tok) |
| 658 | | tok = tok.nextToken |
| 659 | | reinvoke = true # to pick up the queue |
| | 633 | tok = _tokenPostProcess(meth, tok ) |
| | 634 | # nil indicates skipped token - will re call this method |
| | 635 | if not tok |
| | 636 | reinvoke = true # to pick up next token after skipped |
| | 669 | def _getCandidateTokenDefs(sourceLine as String?) as List<of TokenDef> |
| | 670 | if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow |
| | 671 | assert _tokenDefsByFirstChar |
| | 672 | assert _sourceLine.length |
| | 673 | if _tokenDefsByFirstChar.containsKey(_sourceLine[0]) |
| | 674 | # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]' |
| | 675 | return _tokenDefsByFirstChar[_sourceLine[0]] |
| | 676 | return _tokenDefs to ! |
| | 677 | |
| | 678 | def _skipMatchAttempt(tokenDef as TokenDef, sourceLineIndex as int) as bool |
| | 679 | if tokenDef.ignoreCount |
| | 680 | tokenDef.ignoreCount -= 1 |
| | 681 | return true |
| | 682 | if not tokenDef.isActive |
| | 683 | return true |
| | 684 | if sourceLineIndex>0 and tokenDef.requiresBOL |
| | 685 | return true |
| | 686 | if not .isActiveCall(tokenDef) |
| | 687 | return true |
| | 688 | return false |
| | 689 | |
| | 690 | def _getTokenMethod(which as String) as MethodInfo? |
| | 691 | """ |
| | 692 | Get the method for token (On<which>) from cache if there else |
| | 693 | reflect and cache it. |
| | 694 | """ |
| | 695 | if not _onTokenTypeCache.containsKey(which) # not already in methods cache |
| | 696 | methName = 'On' + which |
| | 697 | meth = .getType.getMethod(methName) |
| | 698 | _onTokenTypeCache[which] = meth |
| | 699 | else |
| | 700 | meth = _onTokenTypeCache[which] |
| | 701 | return meth |
| | 702 | |
| | 703 | def _tokenPostProcess(meth as MethodInfo, tok as IToken?) as IToken? |
| | 704 | """ |
| | 705 | Invoke any tokenMethod and return adjusted token or first token in any provided |
| | 706 | token chain (rest of chain queued). |
| | 707 | nil token returned indicates token is skipped. |
| | 708 | """ |
| | 709 | |
| | 710 | try |
| | 711 | tok = meth.invoke(this, @[tok]) to IToken? |
| | 712 | catch tie as TargetInvocationException |
| | 713 | throw tie.innerException |
| | 714 | if not tok |
| | 715 | return nil # token is to be skipped |
| | 716 | retTok = tok |
| | 717 | tok = tok.nextToken |
| | 718 | while tok |
| | 719 | # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method |
| | 720 | _tokenQueue.enqueue(tok) # store any token chain returned by method call |
| | 721 | tok = tok.nextToken |
| | 722 | return retTok |
| | 723 | |
