Merge pull request #52 from llllllllll/test-coverage

Test coverage

.coveragerc

@@ -0,0 +1,3 @@
+[run]
+omit =
+    codetransformer/_version.py

codetransformer/__init__.py

@@ -1,4 +1,4 @@
-from .code import Code
+from .code import Code, Flag
 from .core import CodeTransformer
 from . patterns import (
     matchany,
@@ -20,7 +20,7 @@
 del get_versions
 
 
-def load_ipython_extension(ipython):
+def load_ipython_extension(ipython):  # pragma: no cover
 
     def dis_magic(line, cell=None):
         if cell is None:
@@ -41,6 +41,7 @@ def ast_magic(line, cell=None):
     'display',
     'Code',
     'CodeTransformer',
+    'Flag',
     'instructions',
     'matchany',
     'not_',

codetransformer/code.py

@@ -199,6 +199,46 @@ def _freevar_argname(arg, cellvars, freevars):
     return freevars[arg - len_cellvars]
 
 
+def pycode(argcount,
+           kwonlyargcount,
+           nlocals,
+           stacksize,
+           flags,
+           codestring,
+           constants,
+           names,
+           varnames,
+           filename,
+           name,
+           firstlineno,
+           lnotab,
+           freevars=(),
+           cellvars=()):
+    """types.CodeType constructor that accepts keyword arguments.
+
+    See Also
+    --------
+    types.CodeType
+    """
+    return CodeType(
+        argcount,
+        kwonlyargcount,
+        nlocals,
+        stacksize,
+        flags,
+        codestring,
+        constants,
+        names,
+        varnames,
+        filename,
+        name,
+        firstlineno,
+        lnotab,
+        freevars,
+        cellvars,
+    )
+
+
 class Code:
     """A higher abstraction over python's CodeType.
 
@@ -218,14 +258,8 @@ class Code:
         The first line number of the code in this code object.
     lnotab : dict[Instruction -> int], optional
         The mapping from instruction to the line that it starts.
-    nested : bool, optional
-        Is this code object nested in another code object?
-    coroutine : bool, optional
-        Is this code object a coroutine (async def)?
-    iterable_coroutine : bool, optional
-        Is this code object a coroutine iterator?
-    new_locals : bool, optional
-        Should this code object construct new locals?
+    flags : dict[str -> bool], optional
+        Any flags to set. This updates the default flag set.
 
     Attributes
     ----------
@@ -276,10 +310,7 @@ def __init__(self,
                  filename='<code>',
                  firstlineno=1,
                  lnotab=None,
-                 nested=False,
-                 coroutine=False,
-                 iterable_coroutine=False,
-                 new_locals=False):
+                 flags=None):
 
         instrs = tuple(instrs)  # strictly evaluate any generators.
 
@@ -335,27 +366,30 @@ def __init__(self,
         self._filename = filename
         self._firstlineno = firstlineno
         self._lnotab = lnotab or {}
-        self._flags = Flag.pack(
-            CO_OPTIMIZED=True,
-            CO_NEWLOCALS=new_locals,
-            CO_VARARGS=varg is not None,
-            CO_VARKEYWORDS=kwarg is not None,
-            CO_NESTED=nested,
-            CO_GENERATOR=any(
-                isinstance(instr, (YIELD_VALUE, YIELD_FROM))
-                for instr in instrs
+        self._flags = Flag.pack(**dict(
+            dict(
+                CO_OPTIMIZED=True,
+                CO_NEWLOCALS=True,
+                CO_VARARGS=varg is not None,
+                CO_VARKEYWORDS=kwarg is not None,
+                CO_NESTED=False,
+                CO_GENERATOR=any(
+                    isinstance(instr, (YIELD_VALUE, YIELD_FROM))
+                    for instr in instrs
+                ),
+                CO_NOFREE=not any(map(op.attrgetter('uses_free'), instrs)),
+                CO_COROUTINE=False,
+                CO_ITERABLE_COROUTINE=False,
+                CO_FUTURE_DIVISION=False,
+                CO_FUTURE_ABSOLUTE_IMPORT=False,
+                CO_FUTURE_WITH_STATEMENT=False,
+                CO_FUTURE_PRINT_FUNCTION=False,
+                CO_FUTURE_UNICODE_LITERALS=False,
+                CO_FUTURE_BARRY_AS_BDFL=False,
+                CO_FUTURE_GENERATOR_STOP=False,
             ),
-            CO_NOFREE=not any(map(op.attrgetter('uses_free'), instrs)),
-            CO_COROUTINE=coroutine,
-            CO_ITERABLE_COROUTINE=iterable_coroutine,
-            CO_FUTURE_DIVISION=False,
-            CO_FUTURE_ABSOLUTE_IMPORT=False,
-            CO_FUTURE_WITH_STATEMENT=False,
-            CO_FUTURE_PRINT_FUNCTION=False,
-            CO_FUTURE_UNICODE_LITERALS=False,
-            CO_FUTURE_BARRY_AS_BDFL=False,
-            CO_FUTURE_GENERATOR_STOP=False,
-        )
+            **flags or {}
+        ))
 
     @classmethod
     def from_pyfunc(cls, f):
@@ -454,10 +488,7 @@ def from_pycode(cls, co):
             lnotab={
                 lno: sparse_instrs[off] for off, lno in findlinestarts(co)
             },
-            nested=flags['CO_NESTED'],
-            coroutine=flags['CO_COROUTINE'],
-            iterable_coroutine=flags['CO_ITERABLE_COROUTINE'],
-            new_locals=flags['CO_NEWLOCALS'],
+            flags=flags,
         )
 
     def to_pycode(self):

codetransformer/core.py

@@ -3,6 +3,12 @@
 from ctypes import py_object, pythonapi
 from itertools import chain
 from types import CodeType, FunctionType
+from weakref import WeakKeyDictionary
+
+try:
+    import threading
+except ImportError:
+    import dummy_threading as threading
 
 from .code import Code
 from .instructions import LOAD_CONST, STORE_FAST, LOAD_FAST
@@ -11,6 +17,7 @@
     patterndispatcher,
     DEFAULT_STARTCODE,
 )
+from .utils.instance import instance
 
 
 _cell_new = pythonapi.PyCell_New
@@ -48,7 +55,18 @@ class NoContext(Exception):
     attribute was accessed outside of a code context.
     """
     def __init__(self):
-        return super().__init__('no context')
+        return super().__init__('no active transformation context')
+
+
+class Context:
+    """Empty object for holding the transformation context.
+    """
+    def __init__(self, code):
+        self.code = code
+        self.startcode = DEFAULT_STARTCODE
+
+    def __repr__(self):  # pragma: no cover
+        return '<%s: %r>' % (type(self).__name__, self.__dict__)
 
 
 class CodeTransformerMeta(type):
@@ -81,11 +99,7 @@ class CodeTransformer(metaclass=CodeTransformerMeta):
     ----------
     code
     """
-    __slots__ = '_code_stack', '_startcode_stack'
-
-    def __init__(self):
-        self._code_stack = []
-        self._startcode_stack = []
+    __slots__ = '__weakref__',
 
     def transform_consts(self, consts):
         """transformer for the co_consts field.
@@ -186,9 +200,7 @@ def transform(self, code, *, name=None, filename=None):
                 filename=filename if filename is not None else code.filename,
                 firstlineno=code.firstlineno,
                 lnotab=_new_lnotab(post_transform, code.lnotab),
-                nested=code.is_nested,
-                coroutine=code.is_coroutine,
-                iterable_coroutine=code.is_iterable_coroutine,
+                flags=code.flags,
             )
 
     def __call__(self, f, *,
@@ -207,33 +219,67 @@ def __call__(self, f, *,
             closure,
         )
 
+    @instance
+    class _context_stack(threading.local):
+        """Thread safe transformation context stack.
+
+        Each thread will get it's own ``WeakKeyDictionary`` that maps
+        instances to a stack of ``Context`` objects. When this descriptor
+        is looked up we first try to get the weakkeydict off of the thread
+        local storage. If it doesn't exist we make a new map. Then we lookup
+        our instance in this map. If it doesn't exist yet create a new stack
+        (as an empty list).
+
+        This allows a single instance of ``CodeTransformer`` to be used
+        recursively to transform code objects in a thread safe way while
+        still being able to use a stateful context.
+        """
+        def __get__(self, instance, owner):
+            try:
+                stacks = self._context_stacks
+            except AttributeError:
+                stacks = self._context_stacks = WeakKeyDictionary()
+
+            if instance is None:
+                # when looked up off the class return the current threads
+                # context stacks map
+                return stacks
+
+            return stacks.setdefault(instance, [])
+
     @contextmanager
     def _new_context(self, code):
-        self._code_stack.append(code)
-        self._startcode_stack.append(DEFAULT_STARTCODE)
+        self._context_stack.append(Context(code))
         try:
             yield
         finally:
-            self._code_stack.pop()
-            self._startcode_stack.pop()
+            self._context_stack.pop()
 
     @property
-    def code(self):
-        """The code object we are currently manipulating.
+    def context(self):
+        """Lookup the current transformation context.
+
+        Raises
+        ------
+        NoContext
+            Raised when there is no active transformation context.
         """
         try:
-            return self._code_stack[-1]
+            return self._context_stack[-1]
         except IndexError:
             raise NoContext()
 
+    @property
+    def code(self):
+        """The code object we are currently manipulating.
+        """
+        return self.context.code
+
     @property
     def startcode(self):
         """The startcode we are currently in.
         """
-        try:
-            return self._startcode_stack[-1]
-        except IndexError:
-            raise NoContext()
+        return self.context.startcode
 
     def begin(self, startcode):
         """Begin a new startcode.
@@ -243,9 +289,4 @@ def begin(self, startcode):
         startcode : any
             The startcode to begin.
         """
-        try:
-            # "beginning" a new startcode changes the current startcode.
-            # Here we are mutating the current context's startcode.
-            self._startcode_stack[-1] = startcode
-        except IndexError:
-            raise NoContext()
+        self.context.startcode = startcode

codetransformer/instructions.py

@@ -4,6 +4,7 @@
     IntEnum,
     unique,
 )
+from operator import attrgetter
 from re import escape
 
 from .patterns import matchable
@@ -13,7 +14,7 @@
 
 __all__ = ['Instruction'] + sorted(list(opmap))
 
-# The opcodes that use the co_names tuple.
+# The instructions that use the co_names tuple.
 _uses_name = frozenset({
     'DELETE_ATTR',
     'DELETE_GLOBAL',
@@ -27,13 +28,13 @@
     'STORE_GLOBAL',
     'STORE_NAME',
 })
-# The opcodes that use the co_varnames tuple.
+# The instructions that use the co_varnames tuple.
 _uses_varname = frozenset({
     'LOAD_FAST',
     'STORE_FAST',
     'DELETE_FAST',
 })
-# The opcodes that use the free vars.
+# The instructions that use the co_freevars tuple.
 _uses_free = frozenset({
     'DELETE_DEREF',
     'LOAD_CLASSDEREF',
@@ -430,3 +431,16 @@ def __get__(self, instance, owner):
 del _check_jmp_arg
 del _call_repr
 del _mk_call_init
+
+# The instructions that use the co_names tuple.
+uses_name = frozenset(
+    filter(attrgetter('uses_name'), Instruction.__subclasses__()),
+)
+# The instructions that use the co_varnames tuple.
+uses_varname = frozenset(
+    filter(attrgetter('uses_varname'), Instruction.__subclasses__()),
+)
+# The instructions that use the co_freevars tuple.
+uses_free = frozenset(
+    filter(attrgetter('uses_free'), Instruction.__subclasses__()),
+)