add delimited continuations

compiler/ast/ast_types.nim

@@ -820,6 +820,7 @@ type
     mSymIsInstantiationOf, mNodeId, mPrivateAccess
 
     mEvalToAst
+    mSuspend
 
     # magics only used internally:
     mStrToCStr

compiler/backend/cgirgen.nim

@@ -620,7 +620,7 @@ proc stmtToIr(tree: MirBody, env: MirEnv, cl: var TranslateCl,
     scopeToIr(tree, env, cl, cr, stmts)
   of mnkDestroy:
     unreachable("a 'destroy' that wasn't lowered")
-  of AllNodeKinds - StmtNodes + {mnkEndScope}:
+  of AllNodeKinds - StmtNodes + {mnkEndScope, mnkFork, mnkLand}:
     unreachable(n.kind)
 
 proc setElementToIr(tree: MirBody, cl: var TranslateCl,

compiler/front/condsyms.nim

@@ -78,3 +78,4 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimskullNoNkStmtListTypeAndNkBlockType")
   defineSymbol("nimskullNoNkNone")
   defineSymbol("nimskullHasSupportsZeroMem")
+  defineSymbol("nimskullHasSuspend")

compiler/mir/continuations.nim

@@ -0,0 +1,43 @@
+## Implements the lowering of `fork`/`land` pairs. For every static `fork`, the
+## continuation of the fork (i.e., the code following the land) is reified
+## into a standalone procedure -- the context (i.e., all active locals) are
+## saved into a separate context object, which must be passed to the reified
+## procedure.
+##
+## A rough summary of the reification process is that the procedure body is
+## duplicated and all basic-blocks not reachable from the resumption point
+## are removed.
+##
+## Forks in loops make this a bit trickier in practice. Consider:
+##
+##   loop:
+##     def _1 = ...
+##     if cond:
+##       goto L2
+##     fork a, b, L1
+##     ...
+##     land L1:
+##     ...
+##   L2:
+##   ...
+##
+## Here, the loop must also be part of the reified procedure, but a naive
+## removal of all unused basic blocks would leave it at the start, which is
+## wrong.
+##
+## The solution is to split the continuation into multiple subroutines, one
+## for each such problematic join point.
+##
+## There are two possible strategies to implement subroutines:
+## 1. via separate procedure that tail call each other, passing along extra
+##    locals crossing the border as parameter
+## 2. or, use a case statement dispatcher in a loop, with each target
+##    corresponding to a subroutine. Invoking a subroutine means changing the
+##    selector value and jumping back to the loop start -- local variables
+##    living across subroutine boundaries are lifted to the top-level scope
+##
+## Number 2 is chosen because it's slightly simpler to implement and doesn't
+## put pressure on the available tailcall argument sizes (and thus the size
+## of the context object).
+##
+## A reified continuation is created for each fork/land pair.

compiler/mir/mirgen.nim

@@ -1027,6 +1027,53 @@ proc genMagic(c: var TCtx, n: PNode; m: TMagic) =
     c.buildMagicCall m, rtyp:
       # skip the surrounding typedesc
       c.emitByVal typeLit(c.typeToMir(n[1].typ.skipTypes({tyTypeDesc})))
+  of mSuspend:
+    let label = c.allocLabel()
+    # emit a definition of the local storing the continuation:
+    discard c.addLocal(n[2].sym)
+    let tmp = c.nameNode(n[2].sym)
+
+    # treat the code in the suspend context as if was at the top level of the
+    # procedure
+    let saved = c.blocks.saveContext()
+    withFront c.builder:
+      c.buildStmt mnkScope: discard
+      discard c.blocks.startScope()
+
+      c.buildStmt mnkDef:
+        c.add tmp
+        c.add MirNode(kind: mnkNone)
+      c.buildStmt mnkFork:
+        c.add tmp
+        c.add labelNode(label)
+
+      if c.owner.typ[0].isEmptyType() or n[3].typ == c.graph.noreturnType:
+        c.genCall(n[3])
+      else:
+        let v = c.wrapTemp c.typeToMir(n.typ):
+          c.genCall(n[3])
+        c.buildStmt mnkAsgn:
+          c.add nameNode(c, c.owner.ast[resultPos].sym)
+          c.use v
+
+      # emit a return, close the scope, and restore the original context
+      blockExit(c.blocks, c.graph, c.env, c.builder, 0)
+      c.blocks.closeScope(c.builder, 0, false)
+      c.buildStmt mnkEndScope: discard
+      c.blocks.restoreContext(saved)
+
+    if rtyp == VoidType:
+      # the land receives nothing
+      c.buildStmt mnkLand:
+        c.add labelNode(label)
+    else:
+      # the land receives an owning value
+      let res = c.allocTemp(rtyp)
+      c.buildStmt mnkLand:
+        c.add labelNode(label)
+        c.use res
+      c.buildTree mnkMove, rtyp:
+        c.use res
 
   # arithmetic operations:
   of mAddI, mSubI, mMulI, mDivI, mModI, mPred, mSucc:

compiler/mir/mirgen_blocks.nim

@@ -184,6 +184,20 @@ proc tailExit*(c; bu) =
   bu.subTree mnkGoto:
     bu.add labelNode(bu.requestLabel(c.blocks[0]))
 
+proc saveContext*(c): BlockCtx =
+  ## Saves the current context and replaces it with one that only contains the
+  ## top-level block.
+  result = BlockCtx(blocks: @[c.blocks[0]])
+  swap(c, result)
+
+proc restoreContext*(c; with: sink BlockCtx) =
+  ## Restores the block context `with`. If the top-level block had a label
+  ## registered since the `saveContext` `with` was saved with, the label is
+  ## kept.
+  swap(c, with)
+  if c.blocks[0].id.isNone and with.blocks[0].id.isSome:
+    c.blocks[0].id = with.blocks[0].id
+
 template add*(c: var BlockCtx; b: Block) =
   c.blocks.add b
 

compiler/mir/mirtrees.nim

@@ -164,6 +164,12 @@ type
               ## kind is deliberately named "tail call" for the sake of
               ## discoverability
 
+    mnkFork ## saves the current context and creates a delimited continuation
+            ## starting at the specified `mnkLand`
+    mnkLand ## a special join that marks the start of a continuation. May only
+            ## be targeted by a `mnkFork`
+            # TODO: rename to resume
+
     # unary arithmetic operations:
     mnkNeg ## signed integer and float negation (for ints, overflow is UB)
     # binary arithmetic operations:
@@ -346,7 +352,7 @@ const
   StmtNodes* = {mnkScope, mnkGoto, mnkIf, mnkCase, mnkLoop, mnkJoin,
                 mnkLoopJoin, mnkExcept, mnkFinally, mnkContinue, mnkEndStruct,
                 mnkInit, mnkAsgn, mnkSwitch, mnkVoid, mnkRaise, mnkDestroy,
-                mnkEmit, mnkAsm, mnkEndScope} + DefNodes
+                mnkEmit, mnkAsm, mnkEndScope, mnkFork, mnkLand} + DefNodes
     ## Nodes that are treated like statements, in terms of syntax.
 
   # --- semantics-focused sets:

compiler/mir/utils.nim

@@ -633,6 +633,19 @@ proc stmtToStr(nodes: MirTree, i: var int, indent: var int, result: var string,
     tree "":
       labelToStr(nodes, i, result)
       result.add ":\n"
+  of mnkFork:
+    tree "fork ":
+      valueToStr()
+      result.add " "
+      targetToStr()
+      result.add "\n"
+  of mnkLand:
+    tree "land ":
+      labelToStr(nodes, i, result)
+      if n.len == 2:
+        result.add " "
+        valueToStr()
+      result.add ":\n"
   of mnkContinue:
     tree "continue ":
       targetToStr(nodes, i, result)

compiler/sem/mirexec.nim

@@ -439,6 +439,12 @@ func computeDfg*(tree: MirTree): DataFlowGraph =
       # emit a join at the end of an 'if'
       if ifs.len > 0 and tree[i, 0].label == ifs[^1]:
         join i, ifs.pop()
+    of mnkFork:
+      fork i, tree[i, 1].label
+      emitLvalueOp(env, opMutate, tree, i, OpValue tree.child(i, 0))
+    of mnkLand:
+      join i, tree[i, 0].label
+      emitLvalueOp(env, opDef, tree, i, OpValue tree.child(i, 1))
 
     of mnkDef, mnkDefCursor, mnkAsgn, mnkInit:
       emitForDef(env, tree, i)

compiler/sem/semexprs.nim

@@ -2898,6 +2898,8 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
   of mSizeOf:
     markUsed(c, n.info, s)
     result = semSizeOf(c, setMs(n, s))
+  of mSuspend:
+    result = semSuspend(c, n, s, flags)
   else:
     result = semDirectOp(c, n, flags)
 

compiler/sem/semmagic.nim

@@ -410,6 +410,100 @@ proc semPrivateAccess(c: PContext, n: PNode): PNode =
   c.currentScope.allowPrivateAccess.add t.sym
   result = newNodeIT(nkEmpty, n.info, getSysType(c.graph, n.info, tyVoid))
 
+proc semSuspend(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
+  ## Analyzes a 'suspend' magic call, producing a typed AST or an error. If
+  ## the call doesn't have the right shape, analysis fall back to overload
+  ## resolution.
+  addInNimDebugUtils(c.config, "semSuspend", n, result)
+  if n.len != 4:
+    # could be some other call
+    return semDirectOp(c, n, flags)
+
+  result = shallowCopy(n)
+  result[0] = newSymNode(s, n[0].info)
+  result[1] = semExprWithType(c, n[1])
+
+  var paramType = result[1].typ
+  if paramType.kind != tyError:
+    if paramType.kind == tyTypeDesc:
+      paramType = paramType.lastSon
+    else:
+      result[1] = c.config.newError(result[1], PAstDiag(kind: adSemTypeExpected))
+      paramType = result[1].typ
+
+  let hasResult = paramType.skipTypes({tyAlias}).kind != tyVoid
+
+  # create an new object for the context. It's populated at a (much) later stage
+  let objSym = newSym(skType, c.cache.getIdent("Ctx"), nextSymId(c.idgen),
+                      getCurrOwner(c), n.info)
+  # enable special name mangling:
+  objSym.flags.incl sfFromGeneric
+
+  let obj = newTypeS(tyObject, c)
+  obj.rawAddSon(nil) # the base type
+  obj.size = szUnknownSize
+  obj.align = szUnknownSize
+  obj.n = newTree(nkRecList)
+  obj.flags.incl tfHasAsgn # the object has custom copy logic
+  objSym.linkTo(obj)
+
+  proc addParam(prc: PType, name: string, typ: PType, info: TLineInfo,
+                c: PContext) =
+    let p = newSym(skParam, c.cache.getIdent(name), nextSymId(c.idgen),
+                    getCurrOwner(c), info)
+    p.typ = typ
+    prc.rawAddSon(typ, propagateHasAsgn=false)
+    prc.n.add newSymNode(p)
+
+  # create the type of the continuation procedure:
+  let prc = newProcType(n.info, nextTypeId(c.idgen), getCurrOwner(c))
+  prc.callConv = ccNimCall # TODO: use tailcall
+  # TODO: handle the "unresolved auto return type" case. The easiest solution
+  #       is just reporting an error
+  prc[0] = c.p.owner.typ[0] # use the enclosing routine's return type
+  if hasResult:
+    prc.addParam("arg", newTypeWithSons(c, tySink, @[paramType]), n.info, c)
+  prc.addParam("c", newTypeWithSons(c, tySink, @[obj]), n.info, c)
+
+  # set up the type to use for the local:
+  let tup = newTypeS(tyTuple, c)
+  tup.rawAddSon(obj)
+  tup.rawAddSon(prc)
+
+  c.openScope()
+  # create a let section and type that. This makes sure the symbol is properly
+  # registered everywhere, and retyping is also taken care. The initializer
+  # needs to be some well-formed, non empty expression for the analysis to
+  # succeed -- we use a correctly typed but gramatically incorrect node as
+  # the expression
+  let cons = newNodeIT(nkType, n.info, tup)
+  cons.flags.incl nfSem # prevent the expression from being analyzed
+
+  let
+    ls = nkLetSection.newTree(
+      nkIdentDefs.newTree(n[2], newNodeIT(nkType, n.info, tup), cons))
+    tmp = semNormalizedLetOrVar(c, ls, skLet)
+  if tmp.kind == nkError:
+    # place the erroneous identifier node back into the call
+    result[2] = tmp.diag.wrongNode[0][0]
+  else:
+    result[2] = tmp[0][0]
+
+  var call = semExprWithType(c, n[3])
+  # TODO: noreturn handling...
+  call = fitNode(c, c.p.owner.typ[0], call, n[3].info)
+  c.closeScope()
+
+  result[3] = call
+  if hasResult:
+    result.typ = paramType
+
+  if nkError in {result[1].kind, result[2].kind, result[3].kind}:
+    result = c.config.wrapError(result)
+  elif ecfStatic in c.executionCons[^1].flags:
+    # TODO: report an error
+    discard
+
 proc magicsAfterOverloadResolution(c: PContext, n: PNode,
                                    flags: TExprFlags): PNode =
   ## This is the preferred code point to implement magics.

compiler/sem/tailcall_analysis.nim

@@ -97,6 +97,10 @@ proc verifyTailCalls(g: ModuleGraph, owner: PSym, n, next, problem: PNode) =
       # the body was not typed properly, nor is it relevant for the
       # analysis; skip
       return
+    elif n[0].kind == nkSym and n[0].sym.magic == mSuspend:
+      # the suspended-to-expression appears in a tailing position
+      recurse(n[3], nil, nil)
+      return
 
     for it in n.items:
       recurse(it) # arguments are not tailing expressions