Add static opcode effects

Summary: This adds the static opcode effects and the support for consuming them and generating the final byte code using them. The instr size calculation changes to include the size of `EXTENDED_OPCODE` and we emit the `EXTENDED_OPCODE` before emitting the rest of the opcode.

Reviewed By: jbower-fb

Differential Revision: D83679256

fbshipit-source-id: 79e5c31abc25bdd6c5aa47b390646aa8817987d7

Author: DinoV

cinderx/PythonLib/cinderx/compiler/debug.py

@@ -60,7 +60,7 @@ def dump_block(
     print(str_of_block_header(block))
     for instr in block.getInstructions():
         print("    ", str_of_block_instr(instr, pc, stack_effect))
-        pc += graph.instrsize(instr.opname, instr.ioparg) * opcode.CODEUNIT_SIZE
+        pc += graph.instrsize(instr, instr.ioparg) * opcode.CODEUNIT_SIZE
     return pc
 
 

cinderx/PythonLib/cinderx/compiler/opcodebase.py

@@ -24,6 +24,7 @@ class Opcode:
 
     HAVE_ARGUMENT = 90  # Opcodes from here have an argument:
     EXTENDED_ARG = 144
+    EXTENDED_OPCODE = 126
     CODEUNIT_SIZE = 2
 
     def __init__(self) -> None:

cinderx/PythonLib/cinderx/compiler/opcodes.py

@@ -205,9 +205,9 @@
     FORMAT_VALUE=lambda oparg, jmp=0: -1 if (oparg & FVS_MASK) == FVS_HAVE_SPEC else 0,
     BUILD_CONST_KEY_MAP=lambda oparg, jmp=0: -oparg,
     BUILD_STRING=lambda oparg, jmp=0: 1 - oparg,
-    INVOKE_METHOD=lambda oparg, jmp: -(oparg[1] + 1),
+    INVOKE_METHOD=lambda oparg, jmp=0: -(oparg[1] + 1),
     LOAD_METHOD=1,
-    CALL_METHOD=lambda oparg, jmp: -oparg - 1,
+    CALL_METHOD=lambda oparg, jmp=0: -oparg - 1,
     LIST_EXTEND=-1,
     SET_UPDATE=-1,
     DICT_MERGE=-1,
@@ -467,6 +467,41 @@
         UNPACK_SEQUENCE_TWO_TUPLE=1,
         WITH_EXCEPT_START=5,
         YIELD_VALUE=1,
+        # Static opcodes
+        EXTENDED_OPCODE=0,
+        LOAD_FIELD=1,
+        STORE_FIELD=2,
+        CAST=1,
+        LOAD_LOCAL=0,
+        STORE_LOCAL=1,
+        PRIMITIVE_BOX=1,
+        POP_JUMP_IF_ZERO=1,
+        POP_JUMP_IF_NONZERO=1,
+        PRIMITIVE_UNBOX=1,
+        PRIMITIVE_BINARY_OP=2,
+        PRIMITIVE_UNARY_OP=1,
+        PRIMITIVE_COMPARE_OP=2,
+        LOAD_ITERABLE_ARG=1,
+        LOAD_MAPPING_ARG=lambda oparg, jmp=0: 3 if oparg == 3 else 2,
+        INVOKE_METHOD=lambda oparg, jmp=0: (oparg[1] + 2),
+        INVOKE_FUNCTION=lambda oparg, jmp=0: (oparg[1]),
+        INVOKE_NATIVE=lambda oparg, jmp=0: (len(oparg[1]) - 1),
+        JUMP_IF_ZERO_OR_POP=lambda oparg, jmp=0: 0 if jmp else -1,
+        JUMP_IF_NONZERO_OR_POP=lambda oparg, jmp=0: 0 if jmp else -1,
+        FAST_LEN=1,
+        CONVERT_PRIMITIVE=1,
+        LOAD_CLASS=0,
+        BUILD_CHECKED_MAP=lambda oparg, jmp=0: 2 * oparg[1],
+        SEQUENCE_GET=2,
+        SEQUENCE_SET=3,
+        LIST_DEL=2,
+        REFINE_TYPE=1,
+        PRIMITIVE_LOAD_CONST=0,
+        RETURN_PRIMITIVE=1,
+        TP_ALLOC=0,
+        BUILD_CHECKED_LIST=lambda oparg, jmp=0: oparg[1],
+        LOAD_TYPE=1,
+        LOAD_METHOD_STATIC=1,
     )
 
     opcode.pushed.update(
@@ -711,6 +746,41 @@
         UNPACK_SEQUENCE_TWO_TUPLE=2,
         WITH_EXCEPT_START=6,
         YIELD_VALUE=1,
+        # Static opcodes
+        EXTENDED_OPCODE=0,
+        LOAD_FIELD=1,
+        STORE_FIELD=0,
+        CAST=1,
+        LOAD_LOCAL=1,
+        STORE_LOCAL=0,
+        PRIMITIVE_BOX=1,
+        POP_JUMP_IF_ZERO=0,
+        POP_JUMP_IF_NONZERO=0,
+        PRIMITIVE_UNBOX=1,
+        PRIMITIVE_BINARY_OP=1,
+        PRIMITIVE_UNARY_OP=1,
+        PRIMITIVE_COMPARE_OP=1,
+        LOAD_ITERABLE_ARG=2,
+        LOAD_MAPPING_ARG=1,
+        INVOKE_METHOD=1,
+        INVOKE_FUNCTION=1,
+        INVOKE_NATIVE=1,
+        JUMP_IF_ZERO_OR_POP=0,
+        JUMP_IF_NONZERO_OR_POP=0,
+        FAST_LEN=1,
+        CONVERT_PRIMITIVE=1,
+        LOAD_CLASS=1,
+        BUILD_CHECKED_MAP=1,
+        SEQUENCE_GET=1,
+        SEQUENCE_SET=0,
+        LIST_DEL=0,
+        REFINE_TYPE=1,
+        PRIMITIVE_LOAD_CONST=1,
+        RETURN_PRIMITIVE=0,
+        TP_ALLOC=1,
+        BUILD_CHECKED_LIST=1,
+        LOAD_TYPE=1,
+        LOAD_METHOD_STATIC=2,
     )
     for opname, popped in opcode.popped.items():
         pushed = opcode.pushed[opname]
@@ -846,3 +916,40 @@ def find_op_idx(opname: str) -> int:
             return i
 
     return -1
+
+
+STATIC_OPCODES = {
+    "LOAD_FIELD",
+    "STORE_FIELD",
+    "CAST",
+    "LOAD_LOCAL",
+    "STORE_LOCAL",
+    "PRIMITIVE_BOX",
+    "POP_JUMP_IF_ZERO",
+    "POP_JUMP_IF_NONZERO",
+    "PRIMITIVE_UNBOX",
+    "PRIMITIVE_BINARY_OP",
+    "PRIMITIVE_UNARY_OP",
+    "PRIMITIVE_COMPARE_OP",
+    "LOAD_ITERABLE_ARG",
+    "LOAD_MAPPING_ARG",
+    "INVOKE_FUNCTION",
+    "INVOKE_METHOD",
+    "INVOKE_NATIVE",
+    "JUMP_IF_ZERO_OR_POP",
+    "JUMP_IF_NONZERO_OR_POP",
+    "FAST_LEN",
+    "CONVERT_PRIMITIVE",
+    "LOAD_CLASS",
+    "BUILD_CHECKED_MAP",
+    "SEQUENCE_GET",
+    "SEQUENCE_SET",
+    "LIST_DEL",
+    "REFINE_TYPE",
+    "PRIMITIVE_LOAD_CONST",
+    "RETURN_PRIMITIVE",
+    "TP_ALLOC",
+    "BUILD_CHECKED_LIST",
+    "LOAD_TYPE",
+    "LOAD_METHOD_STATIC",
+}

cinderx/PythonLib/cinderx/compiler/pyassem.py

@@ -48,7 +48,7 @@
 )
 from .opcode_cinder import opcode as cinder_opcode
 from .opcodebase import Opcode
-from .opcodes import opcode as opcodes_opcode
+from .opcodes import opcode as opcodes_opcode, STATIC_OPCODES
 from .symbols import ClassScope, Scope
 
 
@@ -832,7 +832,7 @@ def compute_stack_depth(self) -> None:
                 self.stacksize = self.stackdepth_walk(block)
                 break
 
-    def instrsize(self, opname: str, oparg: int) -> int:
+    def instrsize(self, instr: Instruction, oparg: int) -> int:
         if oparg <= 0xFF:
             return 1
         elif oparg <= 0xFFFF:
@@ -878,17 +878,17 @@ def flatten_graph(self) -> None:
 
                 for inst in b.getInstructions():
                     insts.append(inst)
-                    pc += self.instrsize(inst.opname, inst.ioparg)
+                    pc += self.instrsize(inst, inst.ioparg)
 
             pc = 0
             for inst in insts:
-                pc += self.instrsize(inst.opname, inst.ioparg)
+                pc += self.instrsize(inst, inst.ioparg)
                 op = self.opcode.opmap[inst.opname]
                 if self.opcode.has_jump(op):
                     offset = self.flatten_jump(inst, pc)
 
-                    if self.instrsize(inst.opname, inst.ioparg) != self.instrsize(
-                        inst.opname, offset
+                    if self.instrsize(inst, inst.ioparg) != self.instrsize(
+                        inst, offset
                     ):
                         extended_arg_recompile = True
 
@@ -1092,7 +1092,7 @@ def make_line_table(self) -> bytes:
                 lnotab.nextLine(t.lineno, prev_offset, offset)
                 prev_offset = offset
 
-            offset += self.instrsize(t.opname, t.ioparg) * self.opcode.CODEUNIT_SIZE
+            offset += self.instrsize(t, t.ioparg) * self.opcode.CODEUNIT_SIZE
 
         # Since the linetable format writes the end offset of bytecodes, we can't commit the
         # last write until all the instructions are iterated over.
@@ -2022,7 +2022,8 @@ def prepare_localsplus(self) -> int:
         assert nlocalsplus >= 0
         return nlocalsplus
 
-    def instrsize(self, opname: str, oparg: int) -> int:
+    def instrsize(self, instr: Instruction, oparg: int) -> int:
+        opname = instr.opname
         opcode_index = opcodes_opcode.opmap[opname]
         if opcode_index >= len(_inline_cache_entries):
             # T190611021: This should never happen as we should remove pseudo
@@ -2133,7 +2134,7 @@ def make_line_table(self) -> bytes:
                 size = 0
 
             # The size is in terms of code units
-            size += self.instrsize(t.opname, t.ioparg)
+            size += self.instrsize(t, t.ioparg)
 
         # Since the linetable format writes the end offset of bytecodes, we can't commit the
         # last write until all the instructions are iterated over.
@@ -2152,7 +2153,7 @@ def make_exception_table(self) -> bytes:
                     exception_table.emit_entry(start, ioffset, handler)
                 start = ioffset
                 handler = instr.exc_handler
-            ioffset += self.instrsize(instr.opname, instr.ioparg)
+            ioffset += self.instrsize(instr, instr.ioparg)
         if handler:
             exception_table.emit_entry(start, ioffset, handler)
         return exception_table.getTable()
@@ -2356,6 +2357,9 @@ def flatten_jump(self, inst: Instruction, pc: int) -> int:
             # sys.monitoring needs to be able to find the matching END_SEND
             # but the target is the SEND, so we adjust it here.
             res -= self.END_SEND_OFFSET
+        elif inst.opname in STATIC_OPCODES:
+            # Account for EXTENDED_OPCODE
+            res += 2
 
         return res
 
@@ -2470,7 +2474,7 @@ def flatten_graph(self) -> None:
         for b in self.getBlocksInOrder():
             for inst in b.getInstructions():
                 if inst.is_jump(self.opcode):
-                    assert inst.target is not None
+                    assert inst.target is not None, inst
                     inst.ioparg = label_map[inst.target]
 
         super().flatten_graph()
@@ -2543,8 +2547,23 @@ def _convert_compare_op(self: PyFlowGraph, arg: object) -> int:
         "COMPARE_OP": _convert_compare_op,
     }
 
-    def instrsize(self, opname: str, oparg: int) -> int:
+    def get_ext_oparg(self, inst: Instruction) -> int:
+        pushed = self.opcode.get_num_pushed(inst.opname, inst.oparg)
+        popped = self.opcode.get_num_popped(inst.opname, inst.oparg)
+        assert pushed < 4, pushed
+        return popped << 2 | pushed
+
+    def instrsize(self, instr: Instruction, oparg: int) -> int:
+        opname = instr.opname
         base_size = _inline_cache_entries.get(opname, 0)
+        if opname in STATIC_OPCODES:
+            # extended opcode
+            base_size += 1
+            extoparg = self.get_ext_oparg(instr)
+            while extoparg >= 256:
+                extoparg >>= 8
+                base_size += 1
+
         if oparg <= 0xFF:
             return 1 + base_size
         elif oparg <= 0xFFFF:
@@ -2554,6 +2573,43 @@ def instrsize(self, opname: str, oparg: int) -> int:
         else:
             return 4 + base_size
 
+    def make_byte_code(self) -> bytes:
+        assert self.stage == FLAT, self.stage
+
+        code: bytearray = bytearray()
+
+        def addCode(opcode: int, oparg: int) -> None:
+            assert opcode < 256, opcode
+            assert oparg < 256, oparg
+            code.append(opcode)
+            code.append(oparg)
+
+        for t in self.insts:
+            if t.opname in STATIC_OPCODES:
+                extoparg = self.get_ext_oparg(t)
+
+                if extoparg > 0xFFFFFF:
+                    addCode(self.opcode.EXTENDED_ARG, (extoparg >> 24) & 0xFF)
+                if extoparg > 0xFFFF:
+                    addCode(self.opcode.EXTENDED_ARG, (extoparg >> 16) & 0xFF)
+                if extoparg > 0xFF:
+                    addCode(self.opcode.EXTENDED_ARG, (extoparg >> 8) & 0xFF)
+                addCode(self.opcode.EXTENDED_OPCODE, extoparg & 0xFF)
+
+            oparg = t.ioparg
+            assert 0 <= oparg <= 0xFFFFFFFF, oparg
+            if oparg > 0xFFFFFF:
+                addCode(self.opcode.EXTENDED_ARG, (oparg >> 24) & 0xFF)
+            if oparg > 0xFFFF:
+                addCode(self.opcode.EXTENDED_ARG, (oparg >> 16) & 0xFF)
+            if oparg > 0xFF:
+                addCode(self.opcode.EXTENDED_ARG, (oparg >> 8) & 0xFF)
+            addCode(self.opcode.opmap[t.opname], oparg & 0xFF)
+            self.emit_inline_cache(t.opname, addCode)
+
+        self.stage = DONE
+        return bytes(code)
+
     def emit_inline_cache(
         self, opcode: str, addCode: Callable[[int, int], None]
     ) -> None:

cinderx/TestScripts/3.14-opt-failures.txt

@@ -18,7 +18,6 @@ test_cinderx.test_compiler.test_static.context_decorator
 test_cinderx.test_compiler.test_static.crange
 test_cinderx.test_compiler.test_static.dataclass
 test_cinderx.test_compiler.test_static.decl_visitor
-test_cinderx.test_compiler.test_static.dependencies
 test_cinderx.test_compiler.test_static.double
 test_cinderx.test_compiler.test_static.dynamic_return
 test_cinderx.test_compiler.test_static.elide_type_checks
@@ -36,11 +35,9 @@ test_cinderx.test_compiler.test_static.non_static_inheritance
 test_cinderx.test_compiler.test_static.obj_creation
 test_cinderx.test_compiler.test_static.overrides
 test_cinderx.test_compiler.test_static.patch
-test_cinderx.test_compiler.test_static.perf_lint
 test_cinderx.test_compiler.test_static.primitives
 test_cinderx.test_compiler.test_static.property
 test_cinderx.test_compiler.test_static.refine_fields
-test_cinderx.test_compiler.test_static.reflection
 test_cinderx.test_compiler.test_static.return_cast_insertion
 test_cinderx.test_compiler.test_static.runtime
 test_cinderx.test_compiler.test_static.sequence
@@ -49,9 +46,7 @@ test_cinderx.test_compiler.test_static.subclass
 test_cinderx.test_compiler.test_static.super
 test_cinderx.test_compiler.test_static.sys_hexversion
 test_cinderx.test_compiler.test_static.type_alias
-test_cinderx.test_compiler.test_static.type_params
 test_cinderx.test_compiler.test_static.union
-test_cinderx.test_compiler.test_static.unknown_names
 test_cinderx.test_compiler.test_static.variadic_arg
 test_cinderx.test_compiler.test_static.walrus_operator
 test_cinderx.test_compiler.test_strict