| | 446 | # determine timings for various Tokeniser lexing changes |
| | 447 | #duration = DateTime.now.subtract(st) |
| | 448 | #if tokens.count > 100 and duration > TimeSpan(0) |
| | 449 | # avt = (duration.ticks to int) / (tokens.count to int) |
| | 450 | # ms = avt/10000 |
| | 451 | # fnm = _fileName |
| | 452 | # if _fileName.startsWith("(") |
| | 453 | # fnm = '(None)' |
| | 454 | # lastsl =_fileName.lastIndexOf("\\") |
| | 455 | # if lastsl > 0 |
| | 456 | # fnm = _fileName[lastsl+1:] to ! |
| | 457 | # print "[fnm.padRight(20)]\t[duration]\t[tokens.count]\t" stop |
| | 458 | # print " [ms:N3] ms/Token av ([ms/1000:N3]sec)" |
| 594 | | if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow |
| 595 | | assert _tokenDefsByFirstChar |
| 596 | | assert _sourceLine.length |
| 597 | | if _tokenDefsByFirstChar.containsKey(_sourceLine[0]) |
| 598 | | # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]' |
| 599 | | tokenDefs = _tokenDefsByFirstChar[_sourceLine[0]] |
| 600 | | else |
| 601 | | tokenDefs = _tokenDefs to ! |
| 602 | | else |
| 603 | | tokenDefs = _tokenDefs to ! |
| | 608 | tokenDefs = _getCandidateTokenDefs(_sourceLine) |
| 647 | | try |
| 648 | | tok = meth.invoke(this, @[tok]) to IToken? |
| 649 | | catch tie as TargetInvocationException |
| 650 | | throw tie.innerException |
| 651 | | if tok is nil |
| 652 | | # skip token, so go to the next one |
| 653 | | reinvoke = true |
| 654 | | else |
| 655 | | # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method |
| 656 | | while tok |
| 657 | | _tokenQueue.enqueue(tok) |
| 658 | | tok = tok.nextToken |
| 659 | | reinvoke = true # to pick up the queue |
| | 637 | tok = _tokenPostProcess(meth, tok ) |
| | 638 | # nil indicates skipped token - will re call this method |
| | 639 | if not tok |
| | 640 | reinvoke = true # to pick up next token after skipped |
| | 673 | def _getCandidateTokenDefs(sourceLine as String?) as List<of TokenDef> |
| | 674 | if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow |
| | 675 | assert _tokenDefsByFirstChar |
| | 676 | assert _sourceLine.length |
| | 677 | if _tokenDefsByFirstChar.containsKey(_sourceLine[0]) |
| | 678 | # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]' |
| | 679 | return _tokenDefsByFirstChar[_sourceLine[0]] |
| | 680 | return _tokenDefs to ! |
| | 681 | |
| | 682 | def _skipMatchAttempt(tokenDef as TokenDef, sourceLineIndex as int) as bool |
| | 683 | if tokenDef.ignoreCount |
| | 684 | tokenDef.ignoreCount -= 1 |
| | 685 | return true |
| | 686 | if not tokenDef.isActive |
| | 687 | return true |
| | 688 | if sourceLineIndex>0 and tokenDef.requiresBOL |
| | 689 | return true |
| | 690 | if not .isActiveCall(tokenDef) |
| | 691 | return true |
| | 692 | return false |
| | 693 | |
| | 694 | # get method for token (On<which>) from cache if there else from reflection and cache it |
| | 695 | def _getTokenMethod(which as String) as MethodInfo? |
| | 696 | if not _onTokenTypeCache.containsKey(which) # not already in methods cache |
| | 697 | methName = 'On' + which |
| | 698 | meth = .getType.getMethod(methName) |
| | 699 | _onTokenTypeCache[which] = meth |
| | 700 | else |
| | 701 | meth = _onTokenTypeCache[which] |
| | 702 | return meth |
| | 703 | |
| | 704 | # invoke any tokenMethod and return adjusted token or first token in any returned |
| | 705 | # token chain (rest of chain queued) |
| | 706 | # nil token returned indicates token is skipped. |
| | 707 | def _tokenPostProcess(meth as MethodInfo, tok as IToken?) as IToken? |
| | 708 | try |
| | 709 | tok = meth.invoke(this, @[tok]) to IToken? |
| | 710 | catch tie as TargetInvocationException |
| | 711 | throw tie.innerException |
| | 712 | if not tok |
| | 713 | return nil # token is to be skipped |
| | 714 | retTok = tok |
| | 715 | tok = tok.nextToken |
| | 716 | while tok |
| | 717 | # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method |
| | 718 | _tokenQueue.enqueue(tok) # store any token chain returned by method call |
| | 719 | tok = tok.nextToken |
| | 720 | return retTok |
| | 721 | |
