Merge pull request #150 from ponder-lab/42-supporting-python-3-causes-tensor-dimensions-not-to-be-calculated

Progress on tensor shape resolution for Python 3, wala#42.

Author: khatchad

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3Loader.java

@@ -10,6 +10,8 @@
  *****************************************************************************/
 package com.ibm.wala.cast.python.loader;
 
+import static java.util.logging.Level.WARNING;
+
 import com.ibm.wala.cast.ir.translator.ConstantFoldingRewriter;
 import com.ibm.wala.cast.ir.translator.RewritingTranslatorToCAst;
 import com.ibm.wala.cast.ir.translator.TranslatorToCAst;
@@ -32,10 +34,15 @@
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
+import java.util.logging.Logger;
 import org.python.core.PyObject;
+import org.python.core.PySyntaxError;
+import org.python.core.PyUnicode;
 
 public class Python3Loader extends PythonLoader {
 
+  private static final Logger logger = Logger.getLogger(Python3Loader.class.getName());
+
   public Python3Loader(IClassHierarchy cha, IClassLoader parent, List<File> pythonPath) {
     super(cha, parent, pythonPath);
   }
@@ -94,15 +101,25 @@ public ConstantFoldingRewriter createCAstRewriter(CAst ast) {
             return new ConstantFoldingRewriter(ast) {
               @Override
               protected Object eval(CAstOperator op, Object lhs, Object rhs) {
+                String s = lhs + " " + op.getValue() + " " + rhs;
+                logger.info(() -> "Evaluating: " + s);
+
+                // Use the Python interpreter to evaluate the expression.
+                PyUnicode unicode = new PyUnicode(s);
+                PyObject x;
+
                 try {
-                  PyObject x =
-                      Python3Interpreter.getInterp().eval(lhs + " " + op.getValue() + " " + rhs);
-                  if (x.isNumberType()) {
-                    System.err.println(lhs + " " + op.getValue() + " " + rhs + " -> " + x.asInt());
-                    return x.asInt();
-                  }
-                } catch (Exception e) {
-                  // interpreter died for some reason, so no information.
+                  x = Python3Interpreter.getInterp().eval(unicode);
+                } catch (PySyntaxError e) {
+                  // Handle syntax errors gracefully.
+                  logger.log(WARNING, e, () -> "Syntax error in expression: " + unicode);
+                  return null;
+                }
+
+                if (x.isNumberType()) {
+                  // If the result is a number, return its integer value.
+                  logger.info(() -> s + " -> " + x.asInt());
+                  return x.asInt();
                 }
                 return null;
               }

com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/util/Python3Interpreter.java

@@ -7,13 +7,9 @@ public class Python3Interpreter extends com.ibm.wala.cast.python.util.PythonInte
   private static PythonInterpreter interp;
 
   public static PythonInterpreter getInterp() {
-    try {
-      if (interp == null) {
-        //			PySystemState.initialize(  );
-        interp = new PythonInterpreter();
-      }
-    } catch (Throwable e) {
-
+    if (interp == null) {
+      //			PySystemState.initialize(  );
+      interp = new PythonInterpreter();
     }
     return interp;
   }

com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestNeuroImageExamples.java

@@ -21,14 +21,14 @@ public void testEx1CG()
         Ex1URL,
         (PropagationCallGraphBuilder cgBuilder, CallGraph CG, TensorTypeAnalysis result) -> {
           String in = "[{[D:Constant,64000] of pixel}]";
-          String out = "[{[D:Constant,40, D:Constant,40, D:Constant,40, D:Constant,1] of pixel}]";
-          // NOTE: Change last two arguments to `in`, `out` once
-          // https://github.com/wala/ML/issues/42 is fixed.
-          checkTensorOp(cgBuilder, CG, result, "reshape", null, null);
+          // NOTE: Change to "[{[D:Constant,40, D:Constant,40, D:Constant,40, D:Constant,1] of
+          // pixel}]" once https://github.com/wala/ML/issues/195 is fixed.
+          String out = "[{[D:Symbolic,n, D:Compound,[D:Constant,28, D:Constant,28]] of pixel}]";
+          checkTensorOp(cgBuilder, CG, result, "reshape", in, out);
 
           in = "[{[D:Constant,40, D:Constant,40, D:Constant,40, D:Constant,1] of pixel}]";
-          // NOTE: Change next to last argument to `in` once https://github.com/wala/ML/issues/42 is
-          // fixed.
+          // NOTE: Change next to last argument to `in` once https://github.com/wala/ML/issues/195
+          // is fixed.
           checkTensorOp(cgBuilder, CG, result, "conv3d", null, null);
         });
   }

com.ibm.wala.cast.python.test/pom.xml

@@ -51,6 +51,14 @@
       <groupId>junit</groupId>
       <artifactId>junit</artifactId>
     </dependency>
+    <dependency>
+      <groupId>org.ow2.asm</groupId>
+      <artifactId>asm-all</artifactId>
+    </dependency>
+    <dependency>
+      <groupId>com.github.jnr</groupId>
+      <artifactId>jnr-constants</artifactId>
+    </dependency>
   </dependencies>
   <build>
     <resources>