Ticket #120: bigraw.patch

Source/CobraTokenizer.cobra

@@ -54 +54 @@
     var _inSubstStringDouble = false
     var _inDocString = false
     var _inCommentBlock = 0
+    var _inBigRawString = false
+    var _bigRawIndent = -1
+    var _isBigRawBounded = false
 
     cue init
         base.init
@@ -70 +73 @@
         sb.append('_inSubstStringDouble=[_inSubstStringDouble], ')
         sb.append('_inDocString=[_inDocString]')
         sb.append('_inCommentBlock=[_inCommentBlock]')
+        sb.append('_inBigRawString=[_inBigRawString]')
 
     pro willReturnComments from var as bool
 
@@ -136 +140 @@
             r"STRING_RAW_SINGLE     r'(?:\\.?|[^'\n])*'",
             r'STRING_RAW_DOUBLE     r"(?:\\.?|[^"\n])*"',
 
+            # multiline raw strings
+            r"STRING_BIGRAW_SINGLE_ML R'(?:\\.?|[^'\n])*\n",
+            r'STRING_BIGRAW_DOUBLE_ML R"(?:\\.?|[^"\n])*\n',
+
+            r"STRING_BIGRAW_SINGLE  R'(?:\\.?|[^'\n])*'",
+            r'STRING_BIGRAW_DOUBLE  R"(?:\\.?|[^"\n])*"',
+            
             # substituted strings
             r'RBRACKET_SPECIAL      ]',
-            r"STRING_START_SINGLE   '",  # see "def make_STRING_FOO_BAR"
+            r"STRING_START_SINGLE   '",  # see "def makeSTRING_FOO_BAR"
             r"STRING_PART_SINGLE    '",
             r"STRING_STOP_SINGLE    '",
 
@@ -797 +808 @@
         t.add(TokenRegexDef('DOC_STRING_BODY_TEXT', '.*\n'))
         .pushTokenDefs(t)
         _inDocString = true
+        tok.value = tok.text.trim[3:].trim
         return tok
 
     def onDOC_STRING_STOP(tok as IToken) as IToken
@@ -841 +853 @@
         tok.which = 'STRING_DOUBLE'
         return tok
 
+    def onSTRING_BIGRAW_SINGLE_ML(tok as IToken) as IToken
+        return .doSTRING_BIGRAW_START(tok, 'SINGLE')
+        
+    def onSTRING_BIGRAW_DOUBLE_ML(tok as IToken) as IToken
+        return .doSTRING_BIGRAW_START(tok, 'DOUBLE')
+        
+    def doSTRING_BIGRAW_START(tok as IToken, which as String ) as IToken
+        require tok.text.startsWith('R')
+        assert not _inBigRawString
+        bigRawStopPtn = r"(?:\\.?|[^'\n])*'"
+        if which =='DOUBLE'
+            bigRawStopPtn = r'(?:\\.?|[^"\n])*"'
+        # narrow the tokenizer's token defs to a new shorter set
+        # TODO: cache the tokens below
+        t = List<of TokenDef>()
+        #t.add(TokenRegexDef('STRING_BIGRAW_STOP', r"(?:\\'?|[^'\n])*'"))
+        t.add(TokenRegexDef('STRING_BIGRAW_STOP', bigRawStopPtn))
+        t.add(TokenRegexDef('STRING_BIGRAW_TEXT', '.*\n'))
+        .pushTokenDefs(t)
+        _inBigRawString = true
+        tok.value = tok.text[2:]
+        tok.which = 'STRING_BIGRAW_TEXT'
+        if tok.value == '+|\n'
+            _isBigRawBounded = true
+            tok.value = ''
+        return tok
+
+    def onSTRING_BIGRAW_STOP(tok as IToken) as IToken
+        assert _inBigRawString, tok
+        text = tok.text[:-1]
+        if _bigRawIndent == -1   # first line after leadin of bigRaw String  
+            _bigRawIndent = _calcBigRawIndent(text)
+        tok.value = _bigRawFixup(text, _bigRawIndent, _isBigRawBounded)
+        _inBigRawString = false
+        _isBigRawBounded = false
+        _bigRawIndent = -1
+        .popTokenDefs
+        return tok
+
+    def onSTRING_BIGRAW_TEXT(tok as IToken) as IToken
+        assert _inBigRawString, tok
+        if _bigRawIndent == -1   # first full line after leadin of bigRaw String  
+            _bigRawIndent = _calcBigRawIndent(tok.text)
+        tok.value = _bigRawFixup(tok.text, _bigRawIndent, _isBigRawBounded)
+        return tok
+
+    def onSTRING_BIGRAW_SINGLE(tok as IToken) as IToken
+        require tok.text.startsWith('R')
+        tok.value = tok.text[2:-1]
+        tok.which = 'STRING_BIGRAW_LINE'
+        return tok
+        
+    def onSTRING_BIGRAW_DOUBLE(tok as IToken) as IToken
+        require tok.text.startsWith('R')
+        tok.value = tok.text[2:-1]
+        tok.which = 'STRING_BIGRAW_LINE'
+        return tok
+    
+    def _calcBigRawIndent(firstLine as String)  as int
+        """
+        Return a value for expected indentation level on BigRaw strings.  
+        Will be 
+        Current IndentLevel+1 if given string (assumed as first line after first \n) 
+            has sufficient whitespace to be trimmed,
+        0 otherwise.
+            Effect is to clear indentlevel if first line isnt sufficiently indented
+        """
+        curlvl = (_indentCount + 1) * 4
+        for i in 0 : firstLine.length
+            c = firstLine[i]
+            if c == ' ',        curlvl -= 1
+            else if c == '\t',  curlvl -= 4
+            else            # less wspace on first line than assumed indent level
+                return 0    # set indentlevel so no wspace pruning
+                break
+            if curlvl <= 0, break
+        return _indentCount + 1  # attempt LHS wspace pruning to this level     
+
+    # much code commonality with method above but purpose different
+    def _bigRawFixup(s as String, indentLevel as int, maybeBounded as bool) as String
+        """
+        Return string with whitespace trimmed to bound edge characters or
+         on LHS to indentLevel or to first non whitespace.
+        """
+        if maybeBounded
+            nl = '\n'  #Environment.newLine
+            hasEOL = s.endsWith('\n')
+            probe = s.trim
+            if probe[0] == '|' 
+                s = s.trimStart(nil) to !
+                s = s[1:]
+            if probe.endsWith('|')
+                s = s.trimEnd(nil) to !
+                s = s[:-1] + if(hasEOL, nl, '')
+            return s    
+
+        # This does the assumed indent level indentation removal        
+        if indentLevel <= 0
+            return s
+        curlvl = indentLevel * 4
+        for i in 0 : s.length
+            c = s[i]
+            if c == ' ',        curlvl -= 1
+            else if c == '\t',  curlvl -= 4
+            else 
+                break
+            if curlvl <= 0
+                i += 1
+                break
+        return s[i:]
+            
     def onSTRING_SINGLE(tok as IToken) as IToken
         tok.value = .tokValueForString(tok.text)
         return tok

Source/CobraParser.cobra

@@ -2782 +2782 @@
             return StringLit(.grab)
         else if peek=='STRING_DOUBLE'
             return StringLit(.grab)
+        else if peek == 'STRING_BIGRAW_TEXT' or peek == 'STRING_BIGRAW_LINE'
+            return .stringBigRaw
         else if peek=='INTEGER_LIT'
             return IntegerLit(.grab)
         else if peek=='DECIMAL_LIT'
@@ -3274 +3276 @@
                 .throwError('Expecting more string contents or the end of string after the bracketed expression.')
         return StringSubstLit(items)
 
+    def stringBigRaw as StringLit
+        token = .expect('STRING_BIGRAW_LINE', 'STRING_BIGRAW_TEXT')
+        if token.which == 'STRING_BIGRAW_LINE'
+            return StringLit(token)
+
+        sb = StringBuilder(token.value to String)
+        sbt = StringBuilder(token.text)
+        while true
+            next = .grab
+            if next.which == 'STRING_BIGRAW_TEXT'
+                sb.append(next.value)
+                sbt.append(next.text)
+            else if next.which == 'STRING_BIGRAW_STOP'
+                sb.append(next.value)
+                sbt.append(next.text)
+                break
+            else
+                if _verbosity>=4
+                    print '<> stringBigRaw, value=[sb.toString]'
+                .throwError('Expecting either another bigRaw string line or end of bigRaw string but got "[next.which]".')
+        token.text = sbt.toString           
+        token.value = sb.toString           
+        return StringLit(token)
+            
     def typeExpr as TypeExpr
         return TypeExpr(.typeId)
 

Tests/100-basics/064-2-string-bigraw-bounded.cobra

@@ +1 @@
+class BigRawTst2
+    """
+    Test of bigRaw string handling - '|' bounding
+
+    If first 3 chars of bigRaw string are '+|\n' then those 3 chars are removed.
+    Any indentation before a leading '|' and the leading and any trailing '|' are removed 
+    from each line of the string
+    this allows visual formatting of display text in code
+    Lines without leading or trailing '|' have no adjustments done to them at all
+    """
+    def main is shared
+        # alignment bars
+        a =  R'+|
+        | hello |' 
+        assert a  == ' hello '
+    
+        a = R'+|
+            | hello2'
+        assert a == ' hello2' 
+    
+        a = R'+|
+        hello3 |'
+        assert a == '\t\thello3 '   
+        
+        a = R'+|
+        | |'
+        assert a == ' ' 
+    
+        a = R'+|
+        ||'
+        assert a == ''  
+        
+        a = R'+|
+        |||'
+        assert a == '|' 
+        
+        a =  R'+|
+        | hello |
+        ||' 
+        assert a  == ' hello \n'
+    
+        # trailing spaces after '|' removed
+        a =  R'+|
+        | hello |   
+        ||  ' 
+        assert a  == ' hello \n'
+        
+        a =  R'+|
+        | 0 |
+        | say | 
+        | kan |
+        | U |
+        | C |'
+        assert a  == ' 0 \n say \n kan \n U \n C '
+        
+        # No indent removal if no bars
+        b = R'+|
+            xxx
+            yyy'
+        assert b == '\t\t\txxx\n\t\t\tyyy'
+        
+        c = R'+|
+            |align left       |
+            |  centered text  |
+            |      align right|'
+        assert c == 'align left       \n  centered text  \n      align right'
+        
+        
+        # " delimited
+        
+        a =  R"+|
+        | hello |
+        |.|" 
+        assert a  == ' hello \n.'
+        
+        b = R"+|
+            xxx
+            yyy"
+        assert b == '\t\t\txxx\n\t\t\tyyy'
+        
+        a = R"+|
+        hello3 |  "
+        assert a == '\t\thello3 '   
+    
+        a = R"+|
+        |||"
+        assert a == '|' 
+        
+        a = R"
+            |||
+        "
+        assert a == '\n|||\n'   
+        

Tests/100-basics/064-2-string-bigraw.cobra

@@ +1 @@
+class BigRaw
+    """
+    Test of bigRaw string handling 
+    
+    BigRaw string is multiline 'raw' string, No substitutions ([expr]),
+    and no escaped char (\n) processing 
+    All other standard control chars left in string as is. (WYDSIWYG)
+    
+    Convenience:
+    If the first line of the bigRaw string after the one starting the bigRaw string 
+    ( R'.....\n) starts one indent level in from the line containing the R'....\n then 
+    indentation to  that level on all lines with sufficient indentation is removed.
+    This allows alignment of lines for display viewability without losing code 
+    formatting
+    i.e
+        x=R'x
+            y
+            z' 
+        is same string as 
+        x=R'x
+y
+z'        
+        which is same as 'x\ny\nz'      
+
+    If the first 3 chars of bigRaw string are '+|\n' then those 3 chars are removed and
+    on subsequent lines any whitespace before a leading '|' and the leading '|' 
+    and any trailing '|' and whitespace are removed from each line of the bigRaw string.
+    Lines without leading or trailing '|' have no adjustments done to them at all
+    This allows visual formatting of display text in code with the bars acting as
+    a visual representation of the edges of the displayed text.
+    e.g.
+        x = R'+|
+        |align left       |
+        |  centered text  |
+        |      align right|'
+        
+        assert x == 'align left       \n  centered text  \n      align right'
+    """
+        
+    def main is shared
+        
+        .testRaw    # same tests as (little) raw strings
+        .testEscapes
+        .testIndentRemoval
+        .testNoIndentRemoval
+        .testMisc
+        .testDouble
+
+    def testRaw is shared
+        s as String = ''
+
+        s = R'aoeu'
+        assert s.length==4
+        assert s[3]==c'u'
+
+        s = R'\''
+        assert s.length==2
+        assert s[0]==c'\\'
+        assert s[1]==c"'"
+
+        s = R'\\'  # lexing fail w/o trailing ' but r'//' OK 
+        assert s.length==2
+        assert s[0]==c'\\'
+        assert s[1]==c'\\'
+
+        s = R'['
+        assert s.length==1
+        assert s[0]==c'['
+
+        s = R'[]'
+        assert s.length==2
+        assert s[0]==c'['
+        assert s[1]==c']'
+
+        assert R'\"' == r"\""
+        assert R'cobra\language' == "cobra\\language"
+        assert R'[]' == r"[]"       
+        assert R'[]' == r'[]'       
+            
+    def testEscapesOld is shared
+        # escaped single quote becomes quote 
+        assert R'ab\n\'c' == r"ab\n'c"
+        assert R'xx\'' == "xx'"
+        assert R'\'xx' == "'xx"
+
+        assert R'\n' == "\\n"
+        assert R'hello\'
+noddy' == "hello\\'\nnoddy"
+
+    def testEscapes is shared
+        # escaped single quote becomes quote 
+        assert R'ab\n\'c' == r"ab\n\'c"
+        assert R'xx\'' == "xx\\'"
+        assert R'\'xx' == "\\'xx"
+
+        assert R'\n' == "\\n"
+        assert R'hello\'
+noddy' == "hello\\'\nnoddy"
+
+        assert R'hello\'
+            noddy' == "hello\\'\nnoddy"
+
+
+    def testIndentRemoval is shared
+        a = R'aaa
+            bbb
+            cc'
+        assert a == 'aaa\nbbb\ncc'
+        
+        a = R'aaa\n
+            bbb
+            ..cc'
+        assert a == 'aaa\\n\nbbb\n..cc'
+        
+        assert R'hello
+            noddy' == "hello\nnoddy"
+        
+        # first line one indent level in, remove indentation
+        b = R'{
+            "profile_color":"9ae4e8",
+            "url":null,
+            "statuses_count":1,
+            "protected":false}'
+        assert b.startsWith('{\n"profile_color"')
+        lines = b.splitLines
+        assert lines[0] == '{'
+        assert lines[1] =='"profile_color":"9ae4e8",'
+        assert lines[2] =='"url":null,'
+        assert lines[3] == '"statuses_count":1,'
+        assert lines[4] == '"protected":false}'
+    
+        # first line > one indent level in, remove indentation upto current
+        b = R'{
+                "color":"9ae4e8",
+                "status_count":1,
+                "prot":false}'
+        assert b.startsWith('{\n\t"color"')
+        lines = b.splitLines
+        assert lines[0] == '{'
+        assert lines[1] =='\t"color":"9ae4e8",'
+        assert lines[2] == '\t"status_count":1,'
+        assert lines[3] == '\t"prot":false}'
+    
+        assert R'
+            ' == '\n'
+        
+        
+    def testNoIndentRemoval is shared
+        # indent removal suppresssed
+        assert R'hello
+        no noddy' == 'hello\n\t\tno noddy'
+    
+        # same with spaces not tabs
+        assert R'hello
+        no noddy' == 'hello\n        no noddy'
+
+        # first line not at one indent level in, leaves indentation as displayed
+        c = R'{
+        "profile_color":"9ae4e8",
+        "description":null,
+        "utc_offset":null,
+        "text_color":"000000",
+        "followers_count":0,
+        "following":null}'
+        assert c.startsWith('{\n\t\t"profile_color"')
+        lines = c.splitLines
+        assert lines[2] == '\t\t"description":null,'
+        assert lines[3] == '\t\t"utc_offset":null,'
+        assert lines[4] == '\t\t"text_color":"000000",'
+        assert lines[5] == '\t\t"followers_count":0,'
+        assert lines[6] == '\t\t"following":null}'
+
+    
+        # mixed spaces and tabs, no indentlevel trim
+        assert R'
+    xyzzy
+    foo
+    bar
+    bat:
+        cat
+        mat   ' == '\n\txyzzy\n    foo\n    bar\n\tbat:\n\t\tcat\n\t    mat   '
+
+    def testMisc is shared
+        # Chk auto indent removal at default indent same as slam left
+        assert R'x
+            y
+            z' == R'x
+y
+z'
+
+    def testDouble is shared
+        # selected tests above with "" delimiters
+        s = R"\""
+        assert s == r'\"'
+
+        s = R"\\" #     weird - fail lexing w/o another " trailing
+        assert s == r'\\'
+    
+        s = R"\\
+            " 
+        assert s == '\\\\\n'
+        
+        s = R"["
+        assert s.length==1
+        assert s[0]==c'['
+
+        s = R"[]"
+        assert s == r'[]'
+        
+        assert R"\n\"\c" == r'\n\"\c'
+        
+        a = R"aaa
+            bbb
+            cc"
+        assert a == "aaa\nbbb\ncc"
+        
+        assert R"hello
+        no noddy" == 'hello\n\t\tno noddy'
+        

Developer/IntermediateReleaseNotes.text

@@ -48 +48 @@
 
 * Sets with duplicate members such as `{1, 0, 1}` now generate a warning.
 
+* added big raw string support (multiline raw strings ) Ticket 142
+e.g.
+    s = R'line1
+        line2'
+    assert s == 'line1\nline2'    
+ 
 ================================================================================
 Library
 ================================================================================