Ticket #36: nextToken-cleanup.patch

Source/Tokenizer.cobra

@@ -436 +436 @@
         Returns all remaining tokens as a list.
         """
         tokens = List<of IToken>()
+        #st = DateTime.now
         while true
             t = .nextToken
             if t
                 tokens.add(t)
             else
                 break
+        # determine timings for various Tokeniser lexing changes
+        #duration = DateTime.now.subtract(st)
+        #if  tokens.count > 100 and duration > TimeSpan(0)
+        #   avt = (duration.ticks to int) / (tokens.count to int)
+        #   ms = avt/10000
+        #   fnm = _fileName
+        #   if _fileName.startsWith("(")
+        #       fnm = '(None)'
+        #   lastsl =_fileName.lastIndexOf("\\")
+        #   if lastsl > 0
+        #       fnm = _fileName[lastsl+1:] to !
+        #   print "[fnm.padRight(20)]\t[duration]\t[tokens.count]\t" stop
+        #   print " [ms:N3] ms/Token av ([ms/1000:N3]sec)"
         return tokens
 
     def restart
@@ -591 +605 @@
                     return nil
             try
                 assert _tokenDefs
-                if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow
-                    assert _tokenDefsByFirstChar
-                    assert _sourceLine.length
-                    if _tokenDefsByFirstChar.containsKey(_sourceLine[0])
-                        # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]'
-                        tokenDefs = _tokenDefsByFirstChar[_sourceLine[0]]
-                    else
-                        tokenDefs = _tokenDefs to !
-                else
-                    tokenDefs = _tokenDefs to !
+                tokenDefs = _getCandidateTokenDefs(_sourceLine)
                 for tokenDef in tokenDefs
                     count += 1
-                    if tokenDef.ignoreCount
-                        tokenDef.ignoreCount -= 1
+                    if _skipMatchAttempt(tokenDef, _sourceLineIndex)
                         continue
-                    if not tokenDef.isActive
-                        continue
-                    if _sourceLineIndex>0 and tokenDef.requiresBOL
-                        continue
-                    if not .isActiveCall(tokenDef)
-                        continue
                     #print '<> Trying to match [tokenDef]'
                     sourceLine = _sourceLine to !
                     #print '_sourceLineIndex=[_sourceLineIndex]'
@@ -634 +632 @@
 
                     # enable methods to customize handling of tokens
                     reinvoke = false
-
-                    which = tok.which
-                    if _onTokenTypeCache.containsKey(which)
-                        meth = _onTokenTypeCache[which]
-                    else
-                        methName = 'On' + which
-                        meth = .getType.getMethod(methName)
-                        _onTokenTypeCache[which] = meth
-                        
+                    meth = _getTokenMethod(tok.which)
                     if meth
-                        try
-                            tok = meth.invoke(this, @[tok]) to IToken?
-                        catch tie as TargetInvocationException
-                            throw tie.innerException
-                        if tok is nil
-                            # skip token, so go to the next one
-                            reinvoke = true
-                        else
-                            # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method
-                            while tok
-                                _tokenQueue.enqueue(tok)
-                                tok = tok.nextToken
-                            reinvoke = true  # to pick up the queue
+                        tok = _tokenPostProcess(meth, tok )
+                        # nil indicates skipped token - will re call this method
+                        if not tok 
+                            reinvoke = true  # to pick up next token after skipped
 
                     # finished with current line?
                     if _sourceLine.length==0
@@ -689 +670 @@
             _avgCount += count
             _avgCountNum += 1
 
+    def _getCandidateTokenDefs(sourceLine as String?) as List<of TokenDef>
+        if _narrowTokenDefs and _tokenDefs.count >= _minNumTokenDefsToNarrow
+            assert _tokenDefsByFirstChar
+            assert _sourceLine.length
+            if _tokenDefsByFirstChar.containsKey(_sourceLine[0])
+                # print 'Using short list for char: [_sourceLine[0]], [_sourceLine[0] to int]'
+                return _tokenDefsByFirstChar[_sourceLine[0]]
+        return _tokenDefs to !
+
+    def _skipMatchAttempt(tokenDef as TokenDef, sourceLineIndex as int) as bool
+        if tokenDef.ignoreCount
+            tokenDef.ignoreCount -= 1
+            return true
+        if not tokenDef.isActive
+            return true
+        if sourceLineIndex>0 and tokenDef.requiresBOL
+            return true
+        if not .isActiveCall(tokenDef)
+            return true
+        return false    
+
+    # get method for token (On<which>)  from cache if there else from reflection and cache it
+    def _getTokenMethod(which as String) as  MethodInfo? 
+        if not _onTokenTypeCache.containsKey(which) # not already in methods cache
+            methName = 'On' + which                 
+            meth = .getType.getMethod(methName)     
+            _onTokenTypeCache[which] = meth         
+        else
+            meth = _onTokenTypeCache[which]
+        return meth
+        
+    # invoke any tokenMethod and return adjusted token or first token in any returned
+    # token chain (rest of chain queued)
+    # nil token returned indicates token is skipped.
+    def _tokenPostProcess(meth as MethodInfo,  tok as IToken?) as IToken?
+        try
+            tok = meth.invoke(this, @[tok]) to IToken?
+        catch tie as TargetInvocationException
+            throw tie.innerException
+        if not tok
+            return nil  # token is to be skipped
+        retTok = tok
+        tok = tok.nextToken
+        while tok
+        # TODO: could probably make this more efficient by axing the queue and just checking for nextToken in this method
+            _tokenQueue.enqueue(tok)  # store any token chain returned by method call
+            tok = tok.nextToken
+        return retTok   
+
     def pushTokenDefs(defs as List<of TokenDef>)
         ensure
             _tokenDefs is defs