Ticket #25: extIndexing.patch

Source/CobraLang.cs

@@ -1087 +1087 @@
         }
     }
 
+    static private int ProcessIndexArgs(int count, int? idx) {
+        if (idx==null)
+            idx = 0;
+        else if (idx < 0) {
+            idx += count;
+            //run off front should throw bounds exc
+            //if (idx < 0) {
+            //    idx = 0;
+            //}
+        }
+        return (int) idx;
+    }
+    
+    static public char GetExtIndexed(string s, int? idx) {
+        //Console.WriteLine("GetIndex String");
+        if (s==null)
+            throw new NullReferenceException("Cannot index null.");
+            int i = ProcessIndexArgs(s.Length, idx);
+        return s[i];
+    }
+
+    static public T GetExtIndexed<T>(T[] array, int? idx) {
+        //Console.WriteLine("GetIndex Array");
+        if (array==null)
+            throw new NullReferenceException("Cannot index null array.");
+        int i = ProcessIndexArgs(array.Length, idx);
+        return array[i];  // array.GetValue(i);
+    }
+    
+    static public object GetExtIndexed(System.Collections.IList list, int? idx) {
+        if (list==null)
+            throw new NullReferenceException("Cannot slice null.");
+        int i = ProcessIndexArgs(list.Count, idx);
+        return list[i];
+    }
+
+    static public T GetExtIndexed<T>(IList<T> list, int? idx) {
+        //Console.WriteLine("GetIndex IList<T>");
+        if (list==null)
+            throw new NullReferenceException("Cannot index null IList<T>.");
+        int i = ProcessIndexArgs(list.Count, idx);
+        return list[i];
+    }
+
+    static public T GetExtIndexed<T>(List<T> list, int? idx) {
+        //Console.WriteLine("GetIndex List<T>");
+        if (list==null)
+            throw new NullReferenceException("Cannot slice null List.");
+        int i = ProcessIndexArgs(list.Count, idx);
+        return list[i];
+    }
+
+/*  hops: Is this needed (for dynamic sequences) ??
+        static public object GetExtIndexed(Object obj, int? idx) {
+        // this is for when obj is type `dynamic`
+        if (obj==null)
+            throw new NullReferenceException("Cannot Index null.");
+        if (obj is String)
+                return GetExtIndexed((String)obj, idx);
+        else if (obj is ArrayList)
+            return GetExtIndexed((ArrayList)obj, idx);
+        else if (obj is IList)
+            return GetExtIndexed((IList)obj, idx);
+        else
+            throw new Exception(String.Format("GetExtIndexed unhandled Type {0}", obj));
+    }
+*/        
+
     static public object GetIndexerValue(Object obj, params object[] args) {
         if (obj==null)
             throw new ArgumentNullException("obj");

Source/Expr.cobra

@@ -1393 +1393 @@
             _type = _definition.resultType
             assert _type
 
-    def toCobraSource as String is override
+    def toCobraSource(startDelim as String, stopDelim as String) as String
         sb = StringBuilder()
         sb.append(_target.toCobraSource)
-        sb.append(r'[')
+        sb.append(startDelim)
         sep = ''
         for arg in _args
             sb.append(sep)
             sb.append(arg.toCobraSource)
             sep = ', '
-        sb.append(']')
+        sb.append(stopDelim)
         return sb.toString
+        
+    def toCobraSource as String is override
+        return .toCobraSource(r'[', ']')
 
     def writeSharpDef(sw as SharpWriter, parens as bool) is override
         if _target.type.isDynamic
@@ -1414 +1417 @@
                 expr.writeSharpDef(sw, false)
             sw.write(')')
             return
+            
         if parens
             sw.write('(')
-        if _target inherits IdentifierExpr
-            if _target.isKindOf(.compiler.typeType)
-                # here we're favoring "Foo[]" being an array type rather than a shared indexer
-                sw.write(_target.name)
-                handled = true
-        if not handled
-            _target.writeSharpDef(sw)
+        .writeTarget(sw)
         sw.write(r'[')
         sep = ''
         for expr in _args
@@ -1441 +1439 @@
             expr.writeSharpBreakdownItems(sw)
         sw.write(', -1')
 
+    def writeTarget(sw as SharpWriter)
+        if _target inherits IdentifierExpr
+            if _target.isKindOf(.compiler.typeType)
+                # here we're favoring "Foo[]" being an array type rather than a shared indexer
+                sw.write(_target.name)
+                handled = true
+        if not handled
+            _target.writeSharpDef(sw)
+        
+class ExtIndexExpr
+    inherits IndexExpr
+    """
+    Extended Index Expression - Maps index onto function supporting
+    conversion of single -ve arg to offset from end of sequence
+    """
 
+    def init(token as IToken, target as Expr, args as List<of Expr>)
+        base.init(token, target, args)
+        
+    def toCobraSource as String is override
+        return .toCobraSource(r'[[', ']]')
+
+    def writeSharpDef(sw as SharpWriter, parens as bool) is override
+        if _target.type.isDynamic  # TODO: should this also be overloaded ?
+            base.writeSharpDef(sw, parens)
+            return
+            
+        if not _isSequence(_target.type to !) or _args.count <> 1 
+            base.writeSharpDef(sw, parens)
+            return
+            
+        if parens
+            sw.write('(')
+        # GetExtIndexed(target, idx)
+        sw.write('CobraImp.GetExtIndexed(')
+        _target.writeSharpDef(sw, false)
+        sw.write(', ')
+        _args[0].writeSharpDefInContext(sw)
+        sw.write(')')
+        if parens
+            sw.write(')')
+            
+            
 class IsNilExpr
     inherits Expr
 

Source/CobraParser.cobra

@@ -1087 +1087 @@
         isNames = List<of String>(_isNamesStack)
         if .peek.which=='IS'
             isNames = .isDeclNames
-            attribs = .hasAttribs
+            #attribs = .hasAttribs
             assert .last.which=='EOL'
             .undo  # need the EOL
             if .optionalIndent
@@ -1096 +1096 @@
         else
             if .optionalIndent
                 isNames = .isDeclNames
-                attribs = .hasAttribs
+                #attribs = .hasAttribs
                 docString = .docString
                 .dedent
         return BoxVar(tok, .curBox, identifier, type, isNames, initExpr, docString)
@@ -2279 +2279 @@
         var _binaryOpPrec = {
             # CANNOT USE 0 AS A VALUE IN THIS DICTIONARY
             'DOT':              80,
+            'LBRACKET_DBL':     80,
             'LBRACKET':         80,
             'LPAREN':           80,
             'ARRAY_OPEN':       80,
@@ -2400 +2401 @@
                 token = .grab
                 exprs = .commaSepExprs('RPAREN')
                 return .expression(precedence, PostCallExpr(token, left, exprs))
+            else if peek=='LBRACKET_DBL'
+                # requires special handling - extIndexExpr
+                return .expression(precedence, .extIndexExpr(left to !))
             else
                 # most operators are one-word affairs
                 if op is nil
@@ -2767 +2771 @@
             for expr in exprs
                 assert expr, exprs
             return IndexExpr(token, left, exprs)
-
+            
+    def extIndexExpr(left as Expr) as Expr
+        token = .expect('LBRACKET_DBL')
+        exprs = .commaSepExprs(['RBRACKET_DBL'], false)
+        return ExtIndexExpr(token, left, exprs)
+            
     def literalList as ListLit
         token = .expect('LBRACKET')
         exprs = .commaSepExprs(['RBRACKET'], true)

Source/CobraTokenizer.cobra

@@ -50 +50 @@
     var _inSubstStringSingle = false
     var _inSubstStringDouble = false
     var _inDocString = false
+    var _lBracketDblCount as int
+    var _rBSpecStack as Collections.Stack = Collections.Stack()
 
     def init
         base.init
@@ -62 +64 @@
         base.addInfo(sb)
         sb.append('_indentCount=[_indentCount], ')
         sb.append('_substLBracketCount=[_substLBracketCount], ')
+        sb.append('_lBracketDblCount=[_lBracketDblCount], ')
         sb.append('_inSubstStringSingle=[_inSubstStringSingle], ')
         sb.append('_inDocString=[_inDocString]')
 
@@ -116 +119 @@
             r'STRING_RAW_DOUBLE     r"[^"\n]*"',
 
             # substituted strings
+            r'RBRACKET_DBL_SPECIAL  ]]',
             r'RBRACKET_SPECIAL      ]',
             r"STRING_START_SINGLE   '",  # see "def make_STRING_FOO_BAR"
             r"STRING_PART_SINGLE    '",
@@ -159 +163 @@
             r'ASSIGN            =',
             r'LPAREN            \(',
             r'RPAREN            \)',
+            r'LBRACKET_DBL      \[\[',
             r'LBRACKET          \[',
+            r'RBRACKET_DBL      \]\]',
             r'RBRACKET          \]',
             r'LCURLY            \{',
             r'RCURLY            \}',
@@ -210 +216 @@
         base._reset
         _indentCount = 0
         _substLBracketCount = 0
+        _lBracketDblCount = 0
 
     def makeSTRING_START_SINGLE(definition as String) as TokenDef?
         return StringStartTokenDef('STRING_START_SINGLE', definition[0])
@@ -228 +235 @@
         
     def makeSTRING_STOP_DOUBLE(definition as String) as TokenDef?
         return StringStopTokenDef('STRING_STOP_DOUBLE', definition[0])
-
+            
+    def makeLBRACKET_DBL(definition as String) as TokenDef?
+        return LBracketDblTokenDef('LBRACKET_DBL', definition, this)
+            
     def afterStart is override
         base.afterStart
         # CC:
         # _tokenDefsByWhich['STRING_PART_SINGLE'].isActiveCall = def(tokenizer)=tokenizer.inSubstStringSingle
         # _tokenDefsByWhich['STRING_STOP_SINGLE'].isActiveCall = def(tokenizer)=tokenizer.inSubstStringSingle
-        for which in 'RBRACKET_SPECIAL STRING_PART_SINGLE STRING_STOP_SINGLE STRING_PART_DOUBLE STRING_STOP_DOUBLE STRING_PART_FORMAT'.split(c' ')
+        for which in 'RBRACKET_SPECIAL RBRACKET_DBL_SPECIAL RBRACKET_DBL STRING_PART_SINGLE STRING_STOP_SINGLE STRING_PART_DOUBLE STRING_STOP_DOUBLE STRING_PART_FORMAT'.split(c' ')
             _tokenDefsByWhich[which].isActive = false
 
     def isActiveCall(tok as TokenDef) as bool is override
@@ -610 +620 @@
         _tokenDefsByWhich['STRING_STOP_DOUBLE'].isActive = false
         _tokenDefsByWhich['STRING_PART_FORMAT'].isActive = false
         return tok
+        
+    # Swap out and deactivate a (possibly inactive) lookout Token for 
+    # another one.
+    # pair to unSwapActive()    
+    def _swapActive(currTokName as String, nextTokName as String)
+        _rBSpecStack.push(_tokenDefsByWhich[currTokName].isActive)
+        _tokenDefsByWhich[currTokName].isActive = false 
+        _tokenDefsByWhich[nextTokName].isActive = true
+        
+    # Deactivate and swapout a lookout Token for a (possibly inactive)
+    # previous paired one
+    # pair to swapActive()  
+    def _unSwapActive(currTokName as String, prevTokName as String)
+        require
+            _rBSpecStack.count 
+        body
+            _tokenDefsByWhich[currTokName].isActive = false
+            _tokenDefsByWhich[prevTokName].isActive = _rBSpecStack.pop to bool
+        
+    def onLBRACKET_DBL(tok as IToken) as IToken
+        _lBracketDblCount += 1
+        if _inSubstStringSingle or _inSubstStringDouble
+            # change to lookout for ']]' rather than ']' closing subst in String. 
+            ._swapActive('RBRACKET_SPECIAL', 'RBRACKET_DBL_SPECIAL')
+        else
+            if _lBracketDblCount==1
+                _tokenDefsByWhich['RBRACKET_DBL'].isActive = true
+        return tok
+        
+    def onRBRACKET_DBL(tok as IToken) as IToken
+        _lBracketDblCount -= 1
+        if _lBracketDblCount == 0
+            _tokenDefsByWhich['RBRACKET_DBL'].isActive = false
+        return tok
+        
+    def onRBRACKET_DBL_SPECIAL(tok as IToken) as IToken
+        require
+            _inSubstStringSingle or _inSubstStringDouble
+            _lBracketDblCount
+        body
+            _lBracketDblCount -= 1
+            if _lBracketDblCount == 0
+                # revert back to any previous RBRACKET_SPECIAL lookout
+                ._unSwapActive('RBRACKET_DBL_SPECIAL', 'RBRACKET_SPECIAL')
+            tok.which = 'RBRACKET_DBL'
+            return tok
 
-
     def onLBRACKET(tok as IToken) as IToken
         if _inSubstStringSingle or _inSubstStringDouble
             _substLBracketCount += 1
             if _substLBracketCount==1
-                _tokenDefsByWhich['RBRACKET_SPECIAL'].isActive = true
+                ._swapActive('RBRACKET_DBL_SPECIAL', 'RBRACKET_SPECIAL') 
                 assert _tokenDefsByWhich['STRING_PART_FORMAT'].isActive
                 _tokenDefsByWhich['STRING_PART_FORMAT'].isActive = false
         return tok
@@ -628 +683 @@
         body
             _substLBracketCount -= 1
             if _substLBracketCount == 0
-                _tokenDefsByWhich['RBRACKET_SPECIAL'].isActive = false
+                ._unSwapActive('RBRACKET_SPECIAL', 'RBRACKET_DBL_SPECIAL')
                 assert not _tokenDefsByWhich['STRING_PART_FORMAT'].isActive
                 _tokenDefsByWhich['STRING_PART_FORMAT'].isActive = true
             tok.which = 'RBRACKET'  # tricky, tricky. the parser never sees an RBRACKET_SPECIAL
@@ -886 +941 @@
                     if c == quote, return sb.toString
                     if c == c'\\', isEscaped = true
             return nil
+            
+class LBracketDblTokenDef 
+    inherits TokenRegexDef
+    """
+    This would work using default TokenRegexDef except that matches
+        '[[' in literal lists of lists or arrays of lists   
+        e.g. @[[1,2],[3,4]].  [['a',1],['b',2]]
+    Thats avoidable by inserting whitespace into literal to disambiguate 
+       the [[ token         e.g.   @[ [1,2], [3,4] ]  [ ['a',1], ['b',2] ]
+    But it's nicer if we handle it regardless of ws
+    """
+
+    var _tknizr as Tokenizer
+    
+    def init(which as String, regExSource  as String, tknizer as Tokenizer)
+        base.init(which, regExSource)
+        _tknizr = tknizer
+    
+    # do a regex match and succeed if match and lastToken is an id  
+    def match(input as String) as TokenMatch? is override
+        reMatch = _re.match(input)
+        if reMatch.success and _
+                _tknizr.lastToken and _tknizr.lastToken.which == 'ID'
+            assert reMatch.index == 0
+            assert reMatch.value
+#           print "LBracketDbl match and ID"
+            return TokenRegexMatch(reMatch)
+        else
+            return nil

Tests/100-basics/067-extended-indexing.cobra

@@ +1 @@
+namespace Test
+    
+    class ExtIdx
+        shared
+            var _a= @['abc', 'def', 'ghi', 'jkl', 'mno', 'pqr', 'stu', 
+                'vw', 'xyz',
+            ]
+            var _b = @[1,2,3,4,5]
+            var _list = ['abc', 'def', 'ghi', 'jkl', 'mno', 'pqr', 
+                'stu', 'vw', 'xyz',
+            ]
+
+        pro [i as int] as int
+            get
+                return i*i
+            set
+                pass
+
+        def checkPropIdx is shared
+            x=ExtIdx()
+            assert x[1]==1
+            assert x[2]==4
+            assert x[3]==9
+            assert x[-3]==9
+            assert x[[-3]]==9
+            x[1] = 5
+            x[2] = 6
+                
+        def extListIdx is shared
+            l = _list
+            assert l[0] == 'abc'
+            assert l[l.count-1] == 'xyz'
+            t as String = l[1]
+            s = t
+            assert s==t
+            assert t==l[1]
+            assert l[0] == 'abc'
+            assert l[1] == 'def'
+            assert l[2] == 'ghi'
+            assert l[3] == 'jkl'
+            assert l[4] == 'mno'
+            assert l[5] == 'pqr'
+            assert l[6] == 'stu'
+            assert l[7] == 'vw'
+            assert l[8] == 'xyz'
+            assert l[0] == 'abc'
+        
+            # List Ext Indexing positive indexes
+
+            l = _list
+            assert l[[0]] == 'abc'
+            assert l[[l.count-1]] == 'xyz'
+            t as String = l[[1]]
+            s = t
+            assert s==t
+            assert t==l[[1]]
+            assert l[[0]] == 'abc'
+            assert l[[1]] == 'def'
+            assert l[[2]] == 'ghi'
+            assert l[[3]] == 'jkl'
+            assert l[[4]] == 'mno'
+            assert l[[5]] == 'pqr'
+            assert l[[6]] == 'stu'
+            assert l[[7]] == 'vw'
+            assert l[[8]] == 'xyz'
+            assert l[[0]] == 'abc'
+            
+            # List Ext Indexing negative values
+
+            l = _list
+            x = l[[-1]] 
+            assert x == 'xyz'
+            assert l[[l.count-1]] == 'xyz'
+            y = l[[-1]]
+            assert l[[l.count-1]] == y
+            t as String = l[[-2]]
+            s = t
+            assert s==t
+            assert t==l[[-2]]
+            assert s==l[[-2]]
+            assert l[[-9]] == 'abc'
+            assert l[[-8]] == 'def'
+            assert l[[-7]] == 'ghi'
+            assert l[[-6]] == 'jkl'
+            assert l[[-5]] == 'mno'
+            assert l[[-4]] == 'pqr'
+            assert l[[-3]] == 'stu'
+            assert l[[-2]] == 'vw'
+            assert l[[-1]] == 'xyz'
+            expect ArgumentOutOfRangeException
+                assert l[[-19]] == 'abc'
+            
+            
+        def extStrIdx is shared
+
+            s as String = 'aoeu'
+            c as char = s[0]
+            assert c==c'a'
+
+            assert '[s][s]'=='aoeuaoeu'
+
+            # now the real test:
+            assert '[s[0]]'=='a'
+
+            c = s[[s.length-1]]
+            assert c==c'u'
+            c=s[0]
+            assert c==c'a'
+            c=s[1]
+            assert c==c'o'
+            c=s[2]
+            assert c==c'e'
+            c=s[3]
+            assert c==c'u'
+
+            # String ext indexing positive indexes
+            s = 'aoeu'
+            c = s[0]
+            assert c==c'a'
+
+            assert '[s][s]'=='aoeuaoeu'
+
+            c = s[[s.length-1]]
+            assert c==c'u'
+            c=s[[0]]
+            assert c==c'a'
+            c=s[[1]]
+            assert c==c'o'
+            c=s[[2]]
+            assert c==c'e'
+            c=s[[3]]
+            assert c==c'u'
+            
+            # String ext indexing negative indexes
+
+            s = 'aoeu'
+            c = s[[-1]]
+            assert c==c'u'
+
+            assert '[s][s]'=='aoeuaoeu'
+
+            # now the real test:
+#           assert '[s[-1] ]'=='u'
+            assert '[s[[-1]]]'=='u'  
+## breaks here          a = [[1,2],[3,4]]   # treat as [ [1,2], [3,4] ] 
+
+            c = s[s.length-1]
+            assert c==c'u'
+            c=s[[-1]]
+            assert c==c'u'
+            c=s[[-2]]
+            assert c==c'e'
+            c=s[[-3]]
+            assert c==c'o'
+            c=s[[-4]]
+            assert c==c'a'
+            expect IndexOutOfRangeException
+                c=s[[-5]]
+                assert c==c'a'
+            expect IndexOutOfRangeException
+                c=s[[-6]]
+                assert c==c'a'
+        
+        def extArrIdx is shared
+            a = _a
+            assert a[0] == 'abc'
+            assert a[a.length-1] == 'xyz'
+            t as String = a[1]
+            s = t
+            assert s==t
+            assert s==a[1]
+        
+            b = _b
+            assert b[0] == 1
+            assert b[b.length-1] == 5
+            bt = b[1]
+            bs = bt
+            assert bs==bt
+            assert bs==b[1]
+            assert b[0] == 1
+            assert b[1] == 2
+            assert b[2] == 3
+            assert b[3] == 4
+            assert b[4] == 5
+        
+            # Array ext indexing positive indexes
+            a = _a
+            assert a[[0]] == 'abc'
+            assert a[[a.length-1]] == 'xyz'
+            t as String = a[[1]]
+            s = t
+            assert s==t
+            assert s==a[[1]]
+        
+            b = _b
+            assert b[[0]] == 1
+            assert b[[b.length-1]] == 5
+            bt = b[[1]]
+            bs = bt
+            assert bs==bt
+            assert bs==b[[1]]
+            assert b[[0]] == 1
+            assert b[[1]] == 2
+            assert b[[2]] == 3
+            assert b[[3]] == 4
+            assert b[[4]] == 5
+            
+            # Array ext Indexing - negative indexes
+            a = _a 
+            assert a[[-1]] == 'xyz'
+            expect IndexOutOfRangeException
+                assert a[[-10]] == 'abc'
+            t as String = a[[-2]]
+            s = t
+            assert s==t
+            assert s==a[[-2]]
+        
+            #b = @[1,2,3,4,5]
+            b = _b
+            assert b[[-1]] == 5
+            expect IndexOutOfRangeException
+                assert b[[-10]] == 1
+            bt = b[[-3]]
+            bs = bt
+            assert bs==bt
+            assert bs==b[[-3]]
+            assert b[[0]] == 1
+            assert b[[-1]] == 5
+            assert b[[-2]] == 4
+            assert b[[-3]] == 3
+            assert b[[-4]] == 2
+            assert b[[-5]] == 1
+            expect IndexOutOfRangeException
+                assert b[[-6]] == 1
+        
+            assert b[[-1]] == b[b.length-1]
+            assert b[[-2]] == 4
+            assert b[[-3]] == 3
+            assert b[[-4]] == 2
+            assert b[[-5]] == 1
+            expect IndexOutOfRangeException
+                assert b[[-6]] == 1
+            
+        def arrNegIdx1 is shared
+            a = _a #@['abc', 'def', 'ghi', 'jkl', 'mno', 'pqr', 'stu', 'vw', 'xyz']
+            i=-1
+            assert a[[i]] == 'xyz'
+            i=-10
+            expect IndexOutOfRangeException
+                assert a[[i]] == 'abc'
+            i=-2
+            t as String = a[[i]]
+            s = t
+            assert s==t
+            assert s==a[[i]]
+        
+            #b = @[1,2,3,4,5]
+            b = _b
+            i=-1
+            assert b[[i]] == 5
+            i=-10
+            expect IndexOutOfRangeException
+                assert b[[i]] == 1
+            i=-3
+            bt = b[[i]]
+            bs = bt
+            assert bs==bt
+            assert bs==b[[i]]
+            i=0
+            assert b[[i]] == 1
+            j=5
+            for i in -1:-5:-1
+                assert b[[i]] == j
+                if j>1
+                    j -= 1
+                
+        # Extended indexing in String substitutions
+        def extIdxInStrSubst is shared
+
+            s as String = 'aoeu'
+            c as char = s[0]
+            assert c==c'a'
+            ia = @[1,2,-1,3]
+
+            c = s[[-1]]
+            assert c==c'u'
+
+            # now the real tests:
+            assert '[s[[-1]]]'=='u'
+            
+            assert '[s[ia[1]]]!'=='e!'
+            assert '[s[[ia[1]]]]!' == 'e!' 
+            assert '[s[[ia[1] ]] ]!' == 'e!' 
+            assert '[s[[ia[[1]]]]]!' == 'e!' 
+            
+            assert '[s[[ia[[-1]]]]]!' == 'u!' 
+            assert '[ s[[ ia[[-1]] ]] ]!' == 'u!' 
+            assert '-[s[[ia[[-2]]]]]!' == '-u!' 
+            
+        def tokenCompileTest is shared
+            # these will fail to compile with recognition of dbl [
+            # as LBRACKET_DBL if tokeniser not clever
+            a = [[1,2],[3,4]]   # treat as [ [1,2], [3,4] ] 
+            b = @[[1,1],[2,2]]  # treat as @[ [1,1],[2,2] ] 
+            c = [1, [2,3], [[4,5],[6,7]]]
+            assert a.count == 2
+            assert b.length == 2
+            assert c.count == 3
+            
+        def main is shared
+            .extListIdx
+            .extArrIdx
+            .extStrIdx
+            .extIdxInStrSubst
+            
+            .checkPropIdx
+