Changeset 1570 for cobra/trunk/Source/CobraTokenizer.cobra

Timestamp:

08/06/08 21:46:58 (5 months ago)

Author:

Chuck.Esterbrook

Message:

Added block comments /# ... #/
They don't nest yet. They can't be used inline.
credit:hopscc
ticket:20

Files:

• cobra/trunk/Source/CobraTokenizer.cobra (modified) (4 diffs)

cobra/trunk/Source/CobraTokenizer.cobra

@@ -51 +51 @@
         var _inSubstStringDouble = false
         var _inDocString = false
+        var _inCommentBlock = false
 
         def init
@@ -66 +67 @@
                 sb.append('_inSubstStringDouble=[_inSubstStringDouble], ')
                 sb.append('_inDocString=[_inDocString]')
+                sb.append('_inCommentBlock=[_inCommentBlock]')
 
         # Note: The Tokenizer class handles it's input one line at a time,
@@ -82 +84 @@
                         r'INDENT_ALL_TABS               ^[\t]+',
                         r'INDENT_ALL_SPACES             ^[ ]+',
-                        r'NO_INDENT                             ^(?=[^\t\n#])',
+                        r'NO_INDENT                             ^(?=[^\t\n#\/])',
                         r'EOL                                   \n',
+                        r'COMMENT_BLOCK_START   ^\/\#.*$',  
                         r'SINGLE_LINE_COMMENT   \#.*',
                         r'SPACE                                 [ \t]+',
@@ -381 +384 @@
                 return nil
 
+        ##
+        ## Comment out block
+        ##
+        
+        var _commentBlockStop as TokenRegexDef?
+        var _commentBlockLine as TokenRegexDef?
+
+        def onCOMMENT_BLOCK_START(tok as IToken) as IToken?
+                #print '<> onCOMMENT_BLOCK_START', tok
+                assert not _inCommentBlock
+                # narrow the tokenizer's token defs to a new shorter set
+                if _commentBlockStop is nil
+                        _commentBlockStop = TokenRegexDef('COMMENT_BLOCK_STOP', r'[^# \t]?\#\/.*$')
+                        _commentBlockLine = TokenRegexDef('COMMENT_BLOCK_LINE', '.*\n')
+                t = List<of TokenDef>()
+                t.add(_commentBlockStop)
+                t.add(_commentBlockLine)
+                .pushTokenDefs(t)
+                _inCommentBlock = true
+                return nil
+                
+        def onCOMMENT_BLOCK_LINE(tok as IToken) as IToken?
+                #print '<> onCOMMENT_BLOCK_LINE', tok.lineNum 
+                assert _inCommentBlock, tok
+                return nil
+                
+        def onCOMMENT_BLOCK_STOP(tok as IToken) as IToken?
+                #print '<> onCOMMENT_BLOCK_STOP', tok.lineNum
+                assert _inCommentBlock, tok
+                _inCommentBlock = false
+                .popTokenDefs
+                return nil
+
         def onINDENT_MIXED_TSNS(tok as IToken) as IToken?
                 # expecting tabs, spaces, non-whitespace