Ticket #211: implicit-line-cont-preparser.patch

Source/CobraParser.cobra

@@ -183 +183 @@
         The fileName is recorded, but the file is not physically accessed.
         After parsing, .references will hold a list of any @ref directive references that were parsed.
         """
-        if not _preParseSource(fileName, source)
+        if not _tokeniseSource(fileName, source)
             # Throw an exception to stop subsequent compilation phases.
             # Note that this doesn't get recorded as an error message.
             # That only happens through .throwError and .recordError
@@ -197 +197 @@
             throw
         if _tokenizer and _tokenizer.errors.count <> 0, _tokenizer = nil
 
-    def _preParseSource(fileName as String, source as String) as bool
+    def _tokeniseSource(fileName as String, source as String) as bool
         """
-        Sets up for parsing, but does not invoke `parseTokens`.
+        Sets up for parsing, 
+        Turns source text into a list of tokens in _tokens but does not invoke `parseTokens`.
         Used by `parseSource` and various test sections.
         Returns true if successful, false if there were one or more errors.
-        Upon success, you can use token methods lik .grab.
+        Upon success, you can use token methods like .grab.
         Does not .throwError but may .recordError.
         """
         _fileName = fileName
-        tokVerbosity = _verbosity - 4  # in order words, tokenizer does not spew unless our verbosity is 5 or greater
+        tokVerbosity = _verbosity - 4  # in other words, tokenizer does not spew unless our verbosity is 5 or greater
         if tokVerbosity < 0, tokVerbosity = 0
 
         _isNamesStack = Stack<of String>()  # for `shared` for example
@@ -222 +223 @@
         try
             tokens = tokenizer.startSource(_fileName, source).allTokens
             _tokens = List<of IToken>(tokens.count)
-            for i = 0 .. tokens.count
+            choppedIndents = 0  
+            for i in 0 : tokens.count
                 if tokens[i].text == '_' and tokens[i].which == 'ID'
                     if i < tokens.count-1 and tokens[i+1].which == 'EOL'
                         i += 1
                     else
                         .recordError(tokens[i], 'Unexpected line continuation character.')
-                else
-                    _tokens.add(tokens[i])
+                    continue
+                #handle implicit line continuation tokens
+                if  i < tokens.count-1 and tokens[i+1].which == 'EOL'  and _isImplicitLineContinuation(tokens, i)
+                    _tokens.add(tokens[i]) # add the operator
+                    i += 1  # eat the following EOL
+                    # remove any now hanging indents
+                    while i<tokens.count-1 and tokens[i+1].which == 'INDENT' 
+                        i += 1
+                        choppedIndents += 1
+                    continue
+                # correct following dedents for removed hanging indents
+                if choppedIndents >0 and tokens[i].which == 'DEDENT' 
+                    while choppedIndents >0 and tokens[i].which == 'DEDENT' 
+                        choppedIndents -= 1
+                        i += 1
+                    assert choppedIndents == 0
+                    i -= 1
+                    continue
+                    
+                #print i, tokens[i]
+                _tokens.add(tokens[i])
         catch te as TokenizerError
             .recordError(te.token, te.message)
             return false
@@ -248 +269 @@
 
         _nextTokenIndex = 0
         return true
+        
+    var _implicitContinuationOps = [
+        'ASSIGN', 'AND', 'OR', 'STAR', 'SLASH', 'PLUS',
+        'MINUS','EQ', 'NE', 'LT', 'GT', 'LE', 'GE', 'IN', 'NOTIN',
+        'PLUS_EQUALS', 'MINUS_EQUALS', 'STAR_EQUALS', 'SLASH_EQUALS', 'PERCENT_EQUALS',
+        'SLASHSLASH', 'PERCENT', 'AMPERSAND', 'VERTICAL_BAR', 'CARET', 'DOUBLE_LT',
+        'DOUBLE_GT', 'AMPERSAND_EQUALS', 'VERTICAL_BAR_EQUALS', 'CARET_EQUALS', 
+        'DOUBLE_LT_EQUALS', 'DOUBLE_GT_EQUALS', 'STARSTAR', 'STARSTAR_EQUALS'
+    ]
 
+    def _isImplicitLineContinuation(tokens as List<of IToken>, i as int) as bool
+        #binary ops that can act as implicit line continuation, order by freq of use
+        if tokens[i].which in _implicitContinuationOps 
+            if tokens[i].which == 'GT' or tokens[i].which == 'DOUBLE_GT'    # end Generic
+                for j in 1:10  # lookback 20 tokens
+                    if i-j == 0 or tokens[i-j].which == 'EOL', break
+                    if tokens[i-j].which == 'OPEN_GENERIC'
+                        return false
+            if tokens[i].which == 'STAR' # Stream dcl
+                for j in 1:20  # lookback 20 tokens
+                    if i-j == 0 or tokens[i-j].which == 'EOL', break
+                    if tokens[i-j].which == 'AS'  # as ...TYPE...*
+                        return false
+            return true
+        return false
+        
     def optionalStringLiteral as IToken?
         """
         Gets a token if it matches one of the string literals,

Tests/110-basics-two/150-line-continuation/120-implicit-line-cont-ops.cobra

@@ +1 @@
+# Test for implicit forms of line continuation: round brackets and trailing binary ops
+# ticket:211 (partially also addresses ticket 210)
+class Program
+
+    def main
+        assert 1 + 2 == 3       # ok
+            
+        assert (1 +     # braces ok
+            2) == 3
+        assert 1 *      # implicit trailing '*', 0 indent
+        2  == 2
+        
+        assert 1 +      # implicit trailing '+', 1 indent
+            2 == 3
+        #assert (1 +        # Ticket 210: Cannot mix tabs and spaces in indentation.
+        #      2)
+        #assert 1 +     # Both tickets.
+        #     2 == 3
+
+        assert (1 +     # braces, double indent
+                12) == 13
+        assert 1 +      # trailing '+', double indent after
+                2 == 3
+        assert 3 -      # trailing '-', double indent after
+                2 == 1
+                
+        assert 3 *      # ((3*1) + 4) -(8/2)  # multiple lines
+            1 +
+            4 -
+            8 /
+            2 == 3
+            
+        a = 1
+        b = 2
+        assert (a == 1 and  
+            b == 2)
+        assert a == 1 and   #implicit on 'and'
+            b == 2
+        assert a == 1 or    # implicit on or
+            b == 2
+
+        #assert a == 1 and  # implicit on and and tabs+spaces
+        #      b == 2
+
+