Implement new Var capturing rules

doc/todo-list.txt

@@ -35,11 +35,6 @@
 
 - GG: consider allowing to debug an eval expression (debug eval)
 
-- lambda capture of a non-effectively final var as a Ref or Var needs to be explicitly allowed
-  by the code; for example: "@AllowCapture String name = ..."
-  Alternatively, we could have @CaptureByValue that would be equivalent of creating a synthetic
-  effectively final value to capture.
-
 - GG: consider changing Service.callLater() API to
     <Result> Future<Result> callLater(function Result doLater());
 

javatools/src/main/java/org/xvm/asm/ConstantPool.java

@@ -24,21 +24,16 @@
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Set;
-import java.util.Stack;
 import java.util.Vector;
 
 import java.util.concurrent.ConcurrentHashMap;
 
 import org.xvm.asm.Constant.Format;
 
-import org.xvm.asm.ast.RegisterAST;
-
 import org.xvm.asm.constants.*;
 import org.xvm.asm.constants.TypeConstant.Relation;
 import org.xvm.asm.constants.TypeInfo.Progress;
 
-import org.xvm.asm.ast.BinaryAST.ConstantResolver;
-
 import org.xvm.compiler.Parser;
 import org.xvm.compiler.Source;
 
@@ -2329,6 +2324,7 @@ public ByteConstant ensureNibbleConstant(int n)
     public ClassConstant     clzTest()           {ClassConstant     c = m_clzTest;           if (c == null) {m_clzTest           = c = (ClassConstant) getImplicitlyImportedIdentity("Test"            );} return c;}
     public ClassConstant     clzTransient()      {ClassConstant     c = m_clzTransient;      if (c == null) {m_clzTransient      = c = (ClassConstant) getImplicitlyImportedIdentity("Transient"       );} return c;}
     public ClassConstant     clzUnassigned()     {ClassConstant     c = m_clzUnassigned;     if (c == null) {m_clzUnassigned     = c = (ClassConstant) getImplicitlyImportedIdentity("Unassigned"      );} return c;}
+    public ClassConstant     clzVolatile()       {ClassConstant     c = m_clzVolatile;       if (c == null) {m_clzVolatile       = c = (ClassConstant) getImplicitlyImportedIdentity("Volatile"        );} return c;}
     public TypeConstant      typeObject()        {TypeConstant      c = m_typeObject;        if (c == null) {m_typeObject        = c = ensureTerminalTypeConstant(clzObject()                          );} return c;}
     public TypeConstant      typeInner()         {TypeConstant      c = m_typeInner;         if (c == null) {m_typeInner         = c = ensureVirtualChildTypeConstant(typeOuter(), "Inner"             );} return c;}
     public TypeConstant      typeOuter()         {TypeConstant      c = m_typeOuter;         if (c == null) {m_typeOuter         = c = ensureTerminalTypeConstant(clzOuter()                           );} return c;}
@@ -3928,6 +3924,7 @@ public static Auto withPool(ConstantPool pool) {
     private transient ClassConstant     m_clzTest;
     private transient ClassConstant     m_clzTransient;
     private transient ClassConstant     m_clzUnassigned;
+    private transient ClassConstant     m_clzVolatile;
     private transient TypeConstant      m_typeObject;
     private transient TypeConstant      m_typeInner;
     private transient TypeConstant      m_typeOuter;
@@ -4072,6 +4069,7 @@ private void optimize()
         m_clzTest           = null;
         m_clzTransient      = null;
         m_clzUnassigned     = null;
+        m_clzVolatile       = null;
         m_typeObject        = null;
         m_typeInner         = null;
         m_typeOuter         = null;

javatools/src/main/java/org/xvm/compiler/Compiler.java

@@ -1396,6 +1396,14 @@ public static Stage valueOf(int i)
      * A "try" must have either "catch" or "finally" block.
      */
     public static final String TRY_WITHOUT_CATCH                  = "COMPILER-196";
+    /**
+     * Attempt to mutate the captured variable {0}. (If intended, add the "@Volatile" annotation.)
+     */
+    public static final String WRITEABLE_CAPTURE                  = "COMPILER-197";
+    /**
+     * Invalid annotation combination: {0} and {1} are incompatible.
+     */
+    public static final String INVALID_ANNOTATIONS_COMBO          = "COMPILER-198";
     /**
      * {0} is not yet implemented.
      */

javatools/src/main/java/org/xvm/compiler/ast/LambdaExpression.java

@@ -13,7 +13,6 @@
 
 import org.xvm.asm.Argument;
 import org.xvm.asm.Component;
-import org.xvm.asm.Constant;
 import org.xvm.asm.ConstantPool;
 import org.xvm.asm.ErrorListener;
 import org.xvm.asm.GenericTypeResolver;
@@ -1180,9 +1179,16 @@ protected Argument[] calculateBindings(Context ctx, Code code, ErrorListener err
                         }
 
                     regCapture = regCapture.getOriginalRegister(); // remove any inferences
+
+                    boolean fAllowRefCapture = regCapture.isVar() &&
+                            regCapture.ensureRegType(true).isA(pool.clzVolatile().getType());
                     if (entry.getValue())
                         {
                         // it's a read/write capture; capture the Var
+                        if (!fAllowRefCapture)
+                            {
+                            log(errs, Severity.ERROR, Compiler.WRITEABLE_CAPTURE, sCapture);
+                            }
                         typeCapture = regCapture.isVar()
                                 ? regCapture.ensureRegType(true)
                                 : pool.ensureParameterizedTypeConstant(pool.typeVar(), typeCapture);
@@ -1195,7 +1201,7 @@ protected Argument[] calculateBindings(Context ctx, Code code, ErrorListener err
                         aAstBind[iParam] = new UnaryOpExprAST(
                                 regVal.getRegisterAST(), Operator.Var, typeCapture);
                         }
-                    else if (!regCapture.isEffectivelyFinal())
+                    else if (fAllowRefCapture && !regCapture.isEffectivelyFinal())
                         {
                         // it's a read-only capture, but since we were unable to prove that the
                         // register was effectively final, we need to capture the Ref

javatools/src/main/java/org/xvm/compiler/ast/VariableDeclarationStatement.java

@@ -24,8 +24,11 @@
 import org.xvm.asm.op.Var_DN;
 import org.xvm.asm.op.Var_N;
 
+import org.xvm.compiler.Compiler;
 import org.xvm.compiler.Token;
 
+import org.xvm.util.Severity;
+
 
 /**
  * A variable declaration statement specifies a type and a simple name for a variable.
@@ -176,8 +179,8 @@ protected Statement validateImpl(Context ctx, ErrorListener errs)
         // create the register
         TypeConstant typeVar = exprNew.ensureTypeConstant(ctx, errs).
                                     removeAutoNarrowing().normalizeParameters();
-        m_reg = ctx.createRegister(typeVar, getName());
-        ctx.registerVar(name, m_reg, errs);
+        Register reg = m_reg = ctx.createRegister(typeVar, getName());
+        ctx.registerVar(name, reg, errs);
 
         if (exprNew instanceof AnnotatedTypeExpression exprAnnoType)
             {
@@ -190,7 +193,7 @@ protected Statement validateImpl(Context ctx, ErrorListener errs)
                 if (exprAnnoType.isInjected())
                     {
                     ctx.markVarWrite(name, false, errs);
-                    m_reg.markEffectivelyFinal();
+                    reg.markEffectivelyFinal();
                     }
 
                 boolean      fVar        = exprAnnoType.isVar();
@@ -199,6 +202,9 @@ protected Statement validateImpl(Context ctx, ErrorListener errs)
                 boolean      fInflate    = false;
                 TypeConstant typeReg     = pool.ensureParameterizedTypeConstant(
                         fVar ? pool.typeVar() : pool.typeRef(), typeVar);
+                int          ixFinal     = -1; // for error reporting only
+                int          ixVolatile  = -1;
+
                 for (int i = cRefAnnos - 1; i >= 0; --i)
                     {
                     AnnotationExpression exprAnno = listRefAnnos.get(i);
@@ -209,8 +215,8 @@ protected Statement validateImpl(Context ctx, ErrorListener errs)
                     // don't inflate @Final or @Unassigned
                     if (clzAnno.equals(pool.clzFinal()))
                         {
-                        m_reg.markFinal();
-                        fInflate = true;
+                        reg.markFinal();
+                        ixFinal = i;
                         }
                     else if (clzAnno.equals(pool.clzUnassigned()))
                         {
@@ -230,15 +236,26 @@ else if (clzAnno.equals(pool.clzFuture()))
                         }
                     typeReg = pool.ensureAnnotatedTypeConstant(typeReg, anno);
                     fConst &= exprAnno.isConstant();
+
+                    if (ixVolatile < 0 && typeReg.isA(pool.clzVolatile().getType()))
+                        {
+                        ixVolatile = i;
+                        }
                     }
 
                 if (fUnassigned)
                     {
-                    m_reg.markAllowUnassigned();
+                    reg.markAllowUnassigned();
                     }
                 if (fInflate)
                     {
-                    m_reg.specifyRegType(typeReg);
+                    reg.specifyRegType(typeReg);
+
+                    if (ixFinal >= 0 && ixVolatile >= 0)
+                        {
+                        log(errs, Severity.ERROR, Compiler.INVALID_ANNOTATIONS_COMBO,
+                            listRefAnnos.get(ixFinal), listRefAnnos.get(ixVolatile));
+                        }
                     }
                 m_fConstAnno = fConst;
                 }

javatools/src/main/resources/errors.properties

@@ -240,6 +240,8 @@ COMPILER-193 = The "using" clause must specify an implementation of the Resource
 COMPILER-194 = The delegation target must be a property.
 COMPILER-195 = A super method indicated by @Override cannot be found.
 COMPILER-196 = A "try" must have either "catch" or "finally" block.
+COMPILER-197 = Attempt to mutate the captured variable "{0}". (If intended, add the "@Volatile" annotation.)
+COMPILER-198 = Invalid annotation combination: "{0}" and "{1}" are incompatible.
 COMPILER-NI  = "{0}" is not yet implemented.
 
 VERIFY-01 = Unknown fatal verifier error: "{0}".

javatools_bridge/src/main/x/_native/io/RTChannel.x

@@ -243,8 +243,8 @@ service RTChannel(RawChannel rawChannel)
             // buffers to carry the data-to-write, so that it can bulk up the writes as much as
             // it can
             @Future Int pendingResult;
-            Int remain = size;
-            Int total  = 0;
+            Int           remain = size;
+            @Volatile Int total  = 0;
             do {
                 Byte[]|Int result = rawChannel.allocate(internal=True);
                 if (result.is(Int)) {

lib_ecstasy/src/main/resources/implicit.x

@@ -60,6 +60,7 @@ import ecstasy.annotations.SoftVar          as Soft;
 import ecstasy.annotations.Synchronized;
 import ecstasy.annotations.Test;
 import ecstasy.annotations.Transient;
+import ecstasy.annotations.VolatileVar      as Volatile;
 import ecstasy.annotations.WatchVar         as Watch;
 import ecstasy.annotations.WeakVar          as Weak;
 

lib_ecstasy/src/main/x/ecstasy/Appender.x

@@ -39,7 +39,7 @@ interface Appender<Element> {
      */
     @Concurrent
     Appender addAll(Iterator<Element> iter) {
-        var result = this;
+        @Volatile Appender result = this;
         iter.forEach(e -> {result = result.add(e);});
         return result;
     }
@@ -60,4 +60,4 @@ interface Appender<Element> {
     Appender ensureCapacity(Int count) {
         return this;
     }
-}
+}

lib_ecstasy/src/main/x/ecstasy/annotations/AtomicVar.x

@@ -60,7 +60,7 @@
  *       }
  */
 mixin AtomicVar<Referent>
-        into Var<Referent>
+        extends VolatileVar<Referent>
         incorporates conditional AtomicIntNumber<Referent extends IntNumber> {
     /**
      * Atomically replace the referent for this variable reference.
@@ -109,4 +109,4 @@ mixin AtomicVar<Referent>
             return True, curValue;
         }
     }
-}
+}

lib_ecstasy/src/main/x/ecstasy/annotations/FutureVar.x

@@ -46,7 +46,7 @@
  *   be modified once it is set.
  */
 mixin FutureVar<Referent>
-        into Var<Referent>
+        extends VolatileVar<Referent>
         implements Closeable {
     /**
      * Future completion status:
@@ -263,7 +263,7 @@ mixin FutureVar<Referent>
      *   exception.
      */
     FutureVar!<Referent> orAny(FutureVar!<Referent>[] others = []) {
-        FutureVar<Referent> result = this;
+        @Volatile FutureVar<Referent> result = this;
         others.iterator().forEach(other -> {result = result.or(other);});
         return result;
     }
@@ -810,4 +810,4 @@ mixin FutureVar<Referent>
             }
         }
     }
-}
+}

lib_ecstasy/src/main/x/ecstasy/annotations/LazyVar.x

@@ -30,7 +30,7 @@
  * provided.)
  */
 mixin LazyVar<Referent>(function Referent ()? calculate = Null)
-        into Var<Referent> {
+        extends VolatileVar<Referent> {
 
     private function Referent ()? calculate;
 

lib_ecstasy/src/main/x/ecstasy/annotations/VolatileVar.x

@@ -0,0 +1,6 @@
+/**
+ * TODO
+ */
+mixin VolatileVar<Referent>
+        into Var<Referent> {
+}

lib_ecstasy/src/main/x/ecstasy/collections/Collection.x

@@ -555,7 +555,7 @@ interface Collection<Element>
     @Concurrent
     <Result> Result reduce(Result                           initial,
                            function Result(Result, Element) accumulate) {
-        Result result = initial;
+        @Volatile Result result = initial;
         forEach(e -> {
             result = accumulate(result, e);
         });
@@ -884,7 +884,7 @@ interface Collection<Element>
     @Override
     @Concurrent
     Collection addAll(Iterator<Element> iter) {
-        Collection collection = this;
+        @Volatile Collection collection = this;
         iter.forEach(value -> {
             collection = collection.add(value);
         });
@@ -950,7 +950,7 @@ interface Collection<Element>
         // this naive implementation is likely to be overridden in cases where optimizations can be
         // made with knowledge of either this collection and/or the passed in values, for example
         // if both are ordered; it must obviously be overridden for non-mutable collections
-        Collection result = this;
+        @Volatile Collection result = this;
         values.iterator().forEach(value -> {
             result = result.remove(value);
         });
@@ -997,7 +997,7 @@ interface Collection<Element>
      */
     @Concurrent
     (Collection, Int) removeAll(function Boolean (Element) shouldRemove) {
-        Element[]? values = Null;
+        @Volatile Element[]? values = Null;
         forEach(value -> {
             if (shouldRemove(value)) {
                 values = (values ?: new Element[]) + value;

lib_ecstasy/src/main/x/ecstasy/collections/Map.x

@@ -324,7 +324,7 @@ interface Map<Key, Value>
      */
     @Concurrent
     Map putAll(Map! that) {
-        Map result = this;
+        @Volatile Map result = this;
         that.entries.forEach(entry -> {
             result = result.put(entry.key, entry.value);
         });
@@ -426,7 +426,7 @@ interface Map<Key, Value>
     Map clear() {
         // this method should be overridden by any class that has a more efficient implementation
         // available
-        Map result = this;
+        @Volatile Map result = this;
         if (inPlace) {
             keys.clear();
         } else {

lib_ecstasy/src/main/x/ecstasy/collections/deferred/DeferredCollection.x

@@ -289,7 +289,7 @@
         // this is the default implementation from Collection; if we super(), we would instead go
         // go via delegation to reified, and we are trying not to reify (at least not the first time
         // through)
-        Result result = initial;
+        @Volatile Result result = initial;
         forEach(e -> {
             result = accumulate(result, e);
         });

lib_json/src/main/x/json/Parser.x

@@ -557,7 +557,7 @@ class Parser
         ListMap<String, Doc> map = new ListMap();
         expect(ObjectEnter);
         if (!match(ObjectExit)) {
-            Set<String>? dups = Null;
+            @Volatile Set<String>? dups = Null;
             do {
                 String name = expect(StrVal).value.as(String);
                 Token  sep  = expect(Colon);

lib_net/src/main/x/net/UriTemplate.x

@@ -324,7 +324,7 @@ const UriTemplate {
 
         @Override
         Appender<Char> expand(Appender<Char> buf, Lookup values) {
-            Boolean first = True;
+            @Volatile Boolean first = True;
             function void() checkComma = () -> {
                 // multiple defined variables are comma delimited (i.e. ignore undefined)
                 if (first) {

lib_xenia/src/main/x/xenia/Catalog.x

@@ -309,8 +309,8 @@ const Catalog(WebApp webApp, String systemPath, WebServiceInfo[] services, Class
                 classInfos    += new ClassInfo(child.as(Class<WebService>), path);
 
                 // scan classes inside the WebService class
-                Collection<Type> childTypes   = child.PrivateType.childTypes.values;
-                Class[]          childClasses = new Class[];
+                Collection<Type>  childTypes   = child.PrivateType.childTypes.values;
+                @Volatile Class[] childClasses = new Class[];
                 childTypes.forEach(t -> {
                     if (Class c := t.fromClass()) {
                         childClasses += c;