Changeset 105

Timestamp:

07/29/07 02:46:39

Author:

fumanchu

Message:

Hackish first attempt at a GenExpDecompiler?.

Files:

• trunk/geniusql/codewalk.py (modified) (11 diffs)
• trunk/geniusql/test/test_codewalk.py (modified) (1 diff)

trunk/geniusql/codewalk.py

@@ -4 +4 @@
-and related data, is placed into the public domain.
-
-, Amor Ministries
+
-
-THIS SOFTWARE IS PROVIDED AS-IS, WITHOUT WARRANTY
@@ -20 +20 @@
-import operator
-import sets
-
+
-
-from compiler.consts import *
@@ -53 +53 @@
-#          names, varnames, filename, name, firstlineno,
-#          lnotab[, freevars[, cellvars]])
-
+types.
-                       (None,), ('cell',), (), '', '', 2, '', ('cell',))
-def deref_cell(cell):
-    """Return the value of 'cell' (an object from a func_closure)."""
-    # FunctionType(code, globals[, name[, argdefs[, closure]]])
-
+types.
-
-def make_closure(*args):
@@ -133 +133 @@
-        
-        # Distill supplied 'obj' arg to a code block string.
-
+types.
-            obj = obj.im_func
-
+types.
-            self._func = obj
-            obj = obj.func_code
@@ -297 +297 @@
-        self.walk()
-        codestr = ''.join(map(chr, self.newcode))
-
+types.
-                        # Notice co_consts should *not* be safe_tupled.
-                        self.co_flags, codestr, tuple(self.co_consts),
@@ -313 +313 @@
-                newname = ''
-            co = self.code_object()
-
+types.
-        else:
-            if newname is None:
-                newname = f.func_name
-            co = self.code_object()
-
+types.
-                                f.func_defaults, f.func_closure)
-    
@@ -509 +509 @@
-        # Assert CO_NOFREE, since all free vars should have been made constant.
-        self.co_flags |= CO_NOFREE
-
+types.
-                      self.co_flags, codestr, tuple(self.co_consts),
-                      safe_tuple(self.co_names), safe_tuple(self.co_varnames),
@@ -524 +524 @@
-                newname = ''
-            co = self.code_object()
-
+types.
-        else:
-            if newname is None:
@@ -530 +530 @@
-            co = self.code_object()
-            # All cells should be dereferenced, so force func_closure to None.
-
+types.
-    
-    def reduce(self, number_of_terms, transform=None, overwrite_length=None):
@@ -865 +865 @@
-        val = self.co_consts[lo + (hi << 8)]
-        mod = getattr(val, "__module__", None)
-
+types.
-            # The const in question is a factory function, like int or date.
-            name = val.__name__
@@ -963 +963 @@
-
-
+class GenexpDecompiler(Visitor):
+    """GenexpDecompiler(obj=generator expression).
+    
+    Produce decompiled Python code (a string) from a generator expression."""
+    
+    def __init__(self, obj, env=None):
+        if isinstance(obj, types.GeneratorType):
+            frame = obj.gi_frame
+            fcode = frame.f_code
+        elif isinstance(obj, types.FrameType):
+            frame = obj
+            fcode = frame.f_code
+        
+        Visitor.__init__(self, fcode)
+        
+        if env is None:
+            self.env = {}
+        else:
+            self.env = env.copy()
+        import __builtin__
+        self.env.update(vars(__builtin__))
+        self.env.update(frame.f_globals)
+        self.source = frame.f_locals['[outmost-iterable]']
+    
+    def code(self):
+        self.stage = 0
+        self.ifexpr = ""
+        self.attrs = ""
+        
+        self.walk()
+        
+        names = list(self.co_varnames)[1:]
+        names.reverse()
+        names = ', '.join(names)
+        
+        if isinstance(self.source, type(iter([]))):
+            self.source = "[%s]" % ", ".join([repr(x) for x in self.source])
+        return ("(%s for %s in %s if %s)" %
+                (self.attrs, names, self.source, self.ifexpr))
+    
+    def walk(self):
+        self.stack = []
+        self.newcode = []
+        self.targets = {}
+        
+        Visitor.walk(self)
+        
+        if self.verbose:
+            self.debug("stack:", self.stack)
+    
+    def visit_instruction(self, op, lo=None, hi=None):
+        # Get the instruction pointer for the current instruction.
+        ip = self.cursor - 3
+        if hi is None:
+            ip += 1
+            if lo is None:
+                ip += 1
+        
+        # This is where we do folding of logical AND and OR operators.
+        # The Python code just writes "a AND b", but the VM (bytecode)
+        # acts more like assembly, using conditional JUMP instructions to
+        # implement logical operators. The map stored in self.targets is
+        # of the form:
+        #     {JUMP target: [(self.stack[-1], 'and'), ...]}
+        # where "JUMP target" is the instruction number of the bytecode
+        # which is the target of the JUMP, and each item in the value list
+        # is a tuple of (top of the calling stack, operation).
+        # It's a list because a single bytecode may be the target of
+        # multiple JUMP instructions.
+        # See visit_JUMP_IF_FALSE / TRUE.
+        terms = self.targets.get(ip)
+        if terms:
+            if self.stage == 3:
+                # 'terms' is storing the complete 'if' portion of the genexp.
+                clause, unnecessary_oper = terms.pop()
+                while terms:
+                    term, oper = terms.pop()
+                    clause = "(%s) %s (%s)" % (term, oper, clause)
+                self.ifexpr = clause
+            else:
+                clause = self.stack[-1]
+                while terms:
+                    term, oper = terms.pop()
+                    clause = "(%s) %s (%s)" % (term, oper, clause)
+                
+                # Replace TOS with the new clause, so that further
+                # combinations have access to it.
+                self.stack[-1] = clause
+                if self.verbose:
+                    self.debug("clause:", clause, "\n")
+                
+                if op == 1:
+                    # Py2.4: The current instruction is POP_TOP, which means
+                    # the previous is probably JUMP_*. If so, we don't want to
+                    # pop the value we just placed on the stack and lose it.
+                    # We need to replace the entry that the JUMP_* made in
+                    # self.targets with our new TOS.
+                    target = self.targets[self.last_target_ip]
+                    target[-1] = ((clause, target[-1][1]))
+                    if self.verbose:
+                        self.debug("newtarget:", self.last_target_ip, target)
+    
+    def visit_BUILD_LIST(self, lo, hi):
+        terms = [str(self.stack.pop()) for i in range(lo + (hi << 8))]
+        terms.reverse()
+        self.stack.append("[%s]" % ", ".join(terms))
+    
+    def visit_BUILD_MAP(self, lo, hi):
+        # We're actually going to put a non-string object on the stack here,
+        # with the expectation that the next bytecodes will populate it.
+        self.stack.append(MapStackObject())
+    
+    def visit_BUILD_TUPLE(self, lo, hi):
+        terms = [str(self.stack.pop()) for i in range(lo + (hi << 8))]
+        terms.reverse()
+        self.stack.append("(%s)" % ", ".join(terms))
+    
+    def visit_CALL_FUNCTION(self, lo, hi):
+        kwargs = {}
+        for i in range(hi):
+            val = self.stack.pop()
+            key = self.stack.pop()
+            kwargs[key] = val
+        kwargs = ", ".join(["%s=%s" % (k, v) for k, v in kwargs.iteritems()])
+        
+        args = []
+        for i in xrange(lo):
+            arg = self.stack.pop()
+            args.append(arg)
+        args.reverse()
+        args = ", ".join([str(x) for x in args])
+        
+        if kwargs:
+            args += ", " + kwargs
+        
+        func = self.stack.pop()
+        self.stack.append("%s(%s)" % (func, args))
+    
+    def visit_COMPARE_OP(self, lo, hi):
+        term2, term1 = self.stack.pop(), self.stack.pop()
+        op = cmp_op[lo + (hi << 8)]
+        self.stack.append(term1 + " " + op + " " + term2)
+        if self.verbose:
+            self.debug(op)
+    
+    def visit_DUP_TOP(self):
+        self.stack.append(self.stack[-1])
+    
+    def visit_JUMP_IF_FALSE(self, lo, hi):
+        # Note that self.cursor has already advanced to the next instruction.
+        target = self.cursor + (lo + (hi << 8))
+        bucket = self.targets.setdefault(target, [])
+        bucket.append((self.stack[-1], 'and'))
+        if self.verbose:
+            self.debug("target:", target, bucket)
+        # Store target ip for the special code in visit_instruction
+        self.last_target_ip = target
+    
+    def visit_JUMP_IF_TRUE(self, lo, hi):
+        # Note that self.cursor has already advanced to the next instruction.
+        target = self.cursor + (lo + (hi << 8))
+        bucket = self.targets.setdefault(target, [])
+        bucket.append((self.stack[-1], 'or'))
+        if self.verbose:
+            self.debug("target:", target, bucket)
+        # Store target ip for the special code in visit_instruction
+        self.last_target_ip = target
+    
+    def visit_LOAD_ATTR(self, lo, hi):
+        term = self.co_names[lo + (hi << 8)]
+        self.stack[-1] += ("." + term)
+        if self.verbose:
+            self.debug(term)
+    
+    def visit_LOAD_CONST(self, lo, hi):
+        val = self.co_consts[lo + (hi << 8)]
+        mod = getattr(val, "__module__", None)
+        if isinstance(val, (types.FunctionType, type)):
+            # The const in question is a factory function, like int or date.
+            name = val.__name__
+            if name in self.env:
+                term = name
+            else:
+                term = mod + "." + name
+        else:
+            term = repr(val)
+            if mod and not mod.startswith("__"):
+                if not term.startswith(mod + "."):
+                    term = mod + "." + term
+        self.stack.append(term)
+        if self.verbose:
+            self.debug(term)
+    
+    def visit_LOAD_FAST(self, lo, hi):
+        if self.stage == 1:
+            return
+        
+        term = self.co_varnames[lo + (hi << 8)]
+        self.stack.append(term)
+        if self.verbose:
+            self.debug(term)
+    
+    def visit_LOAD_GLOBAL(self, lo, hi):
+        self.stack.append(self.co_names[lo + (hi << 8)])
+    
+    def visit_POP_TOP(self):
+        if self.stage < 3:
+            self.stack.pop()
+    
+    def visit_ROT_TWO(self):
+        v = self.stack.pop()
+        k = self.stack.pop()
+        self.stack.extend([v, k])
+    
+    def visit_ROT_THREE(self):
+        v = self.stack.pop()
+        k = self.stack.pop()
+        x = self.stack.pop()
+        self.stack.extend([v, x, k])
+    
+    def visit_SLICE_PLUS_0(self):
+        arg = self.stack.pop()
+        self.stack.append("%s[:]" % arg)
+    
+    def visit_SLICE_PLUS_1(self):
+        args = tuple(self.stack[-2:])
+        del self.stack[-2:]
+        self.stack.append("%s[%s:]" % args)
+    
+    def visit_SLICE_PLUS_2(self):
+        args = tuple(self.stack[-2:])
+        del self.stack[-2:]
+        self.stack.append("%s[:%s]" % args)
+    
+    def visit_SLICE_PLUS_3(self):
+        args = tuple(self.stack[-3:])
+        del self.stack[-3:]
+        self.stack.append("%s[%s:%s]" % args)
+    
+    def visit_STORE_SUBSCR(self):
+        k = self.stack.pop()
+        x = self.stack.pop()
+        v = self.stack.pop()
+        x[k] = v
+    
+    def visit_UNARY_CONVERT(self):
+        term = self.stack.pop()
+        self.stack.append("`(" + term + ")`")
+    
+    def visit_UNARY_INVERT(self):
+        term = self.stack.pop()
+        self.stack.append("~(" + term + ")")
+    
+    def visit_UNARY_NEGATIVE(self):
+        term = self.stack.pop()
+        self.stack.append("-(" + term + ")")
+    
+    def visit_UNARY_NOT(self):
+        term = self.stack.pop()
+        self.stack.append("not (" + term + ")")
+    
+    def visit_UNARY_POSITIVE(self):
+        term = self.stack.pop()
+        self.stack.append("+(" + term + ")")
+    
+    def binary_op(self, op):
+        op2, op1 = self.stack.pop(), self.stack.pop()
+        self.stack.append(op1 + " " + op + " " + op2)
+    
+    def visit_BINARY_SUBSCR(self):
+        op2, op1 = self.stack.pop(), self.stack.pop()
+        self.stack.append(op1 + "[" + op2 + "]")
+    
+    def visit_SETUP_LOOP(self, lo, hi):
+        self.stage = 1
+    
+    def visit_FOR_ITER(self, lo, hi):
+        self.stage = 2
+        self.for_loop_address = self.cursor - 3
+    
+    def visit_UNPACK_SEQUENCE(self, lo, hi):
+        # Skip all the STORE_FAST opcodes that follow.
+        numvars = lo + (hi << 8)
+        self.cursor += (3 * numvars)
+    
+    def visit_YIELD_VALUE(self):
+        self.stage = 3
+    
+    def visit_POP_BLOCK(self):
+        self.attrs = self.stack.pop()
+        self.stage = 4
+
+
+# Add visit_BINARY methods to GenexpDecompiler.
+for k, v in binary_repr.iteritems():
+    setattr(GenexpDecompiler, "visit_" + k,
+            lambda self, op=v: self.binary_op(op))
+
+
+
-class BranchTracker(Visitor):
-    """BranchTracker(obj=[func|co|str|list]).

trunk/geniusql/test/test_codewalk.py

@@ -253 +253 @@
-        e = lambda x, **kw: x.Date > x.newdate(kw['Year'], 1, 1)
-        self.assertEqual(codewalk.KeywordInspector(e).kwargs(), ["Year"])
+    
+    def test_GenexpDecompiler(self):
+        t = ([([t1.a, t2.b + 3] for t1, t2 in [1, 2, 3] if t1.x == 0),
+              '([t1.a, t2.b + 3] for t1, t2 in [1, 2, 3] if t1.x == 0)'],
+             )
+        for g, s in t:
+            g = codewalk.GenexpDecompiler(g)
+##            g.verbose = True
+            r = g.code()
+            self.assertEqual(r, s)
-
-