Skip to content

Commit cea5223

Browse files
khatchadkhatchad
authored
Get up-to-date (#132)
- Investigate decorator issue on tutorial code (#68) - Fix comment. - Add interprocedural dataset support (#69) - Change log from warning to info. - Add support for non-scalar datasets (#70) - Use interprocedural analysis for property read case. - Add logging. - Fix non-scalar determination. - Add logger. - Fix logger. - Add logging around callable workaround. - Handle callables with lexical scoping. - Formatting. - Handle possible null receivers in callable lookup. - Change log message. - Add log. - Add the ability to process datasets coming out of enumerate() (#71) - Fix tests for input files with multiple functions. - Enhance TF2 tests (#73) - Separate tests by calling context (#74) - Add tf.keras.layers.Dense. - Move comment. - Additional check. - New API. - Add autoencoder test. - Move the value number check outside the context loop. - More the sigmoid function to math. - Format. - Add API tf.add(). - Check the context/funciton PK mappings. (#77) - Add URL. - Add dataset generator and test. - tf.nn.softmax isn't a tensor generator. - Add URL. - Apply spotless. - Let's make the two tests only diff in the invocation. - Prevent infinite recursion when processing instructions when finding tensor data flow sources. - When we look for dataset tensor sources, we recurse up to the original defining instruction. However, there's a possibility that instructions have mutually recursive definitions due to the phi instruction. Let's prevent against that.
1 parent adf1894 commit cea5223

File tree

26 files changed

+1881
-201
lines changed

26 files changed

+1881
-201
lines changed

‎com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflowModel.java‎

Lines changed: 226 additions & 170 deletions
Large diffs are not rendered by default.

‎com.ibm.wala.cast.python.ml/data/tensorflow.xml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@
3232
<putfield class="LRoot" field="distribute" fieldType="LRoot" ref="x" value="distribute" />
3333
<new def="nn" class="Lobject" />
3434
<putfield class="LRoot" field="nn" fieldType="LRoot" ref="x" value="nn" />
35+
<new def="math" class="Lobject" />
36+
<putfield class="LRoot" field="math" fieldType="LRoot" ref="x" value="math" />
3537
<new def="random" class="Lobject" />
3638
<putfield class="LRoot" field="random" fieldType="LRoot" ref="x" value="random" />
3739
<new def="sparse" class="Lobject" />
@@ -59,6 +61,8 @@
5961
<putfield class="LRoot" field="numpy_input_fn" fieldType="LRoot" ref="inputs" value="numpy_input_fn" />
6062
<new def="from_tensor_slices" class="Ltensorflow/data/Dataset/from_tensor_slices" />
6163
<putfield class="LRoot" field="from_tensor_slices" fieldType="LRoot" ref="Dataset" value="from_tensor_slices" />
64+
<new def="from_generator" class="Ltensorflow/data/Dataset/from_generator" />
65+
<putfield class="LRoot" field="from_generator" fieldType="LRoot" ref="Dataset" value="from_generator" />
6266
<new def="reshape" class="Ltensorflow/functions/reshape" />
6367
<putfield class="LRoot" field="reshape" fieldType="LRoot" ref="x" value="reshape" />
6468
<new def="conv2d" class="Ltensorflow/functions/conv2d" />
@@ -67,6 +71,14 @@
6771
<putfield class="LRoot" field="conv2d" fieldType="LRoot" ref="layers" value="conv2d" />
6872
<new def="conv3d" class="Ltensorflow/functions/conv3d" />
6973
<putfield class="LRoot" field="conv3d" fieldType="LRoot" ref="nn" value="conv3d" />
74+
<new def="softmax" class="Ltensorflow/functions/softmax" />
75+
<putfield class="LRoot" field="softmax" fieldType="LRoot" ref="nn" value="softmax" />
76+
<new def="sigmoid" class="Ltensorflow/math/sigmoid" />
77+
<putfield class="LRoot" field="sigmoid" fieldType="LRoot" ref="nn" value="sigmoid" />
78+
<putfield class="LRoot" field="sigmoid" fieldType="LRoot" ref="math" value="sigmoid" />
79+
<new def="add" class="Ltensorflow/math/add" />
80+
<putfield class="LRoot" field="add" fieldType="LRoot" ref="x" value="add" />
81+
<putfield class="LRoot" field="add" fieldType="LRoot" ref="math" value="add" />
7082
<new def="placeholder" class="Ltensorflow/functions/placeholder" />
7183
<putfield class="LRoot" field="placeholder" fieldType="LRoot" ref="x" value="placeholder" />
7284
<new def="examples" class="Lobject" />
@@ -120,6 +132,8 @@
120132
<new def="Input" class="Ltensorflow/functions/Input" />
121133
<putfield class="LRoot" field="Input" fieldType="LRoot" ref="keras" value="Input" />
122134
<putfield class="LRoot" field="Input" fieldType="LRoot" ref="layers" value="Input" />
135+
<new def="Dense" class="Ltensorflow/keras/layers/Dense" />
136+
<putfield class="LRoot" field="Dense" fieldType="LRoot" ref="layers" value="Dense" />
123137
<new def="Variable" class="Ltensorflow/functions/Variable" />
124138
<putfield class="LRoot" field="Variable" fieldType="LRoot" ref="x" value="Variable" />
125139
<putfield class="LRoot" field="Variable" fieldType="LRoot" ref="variables" value="Variable" />
@@ -247,6 +261,26 @@
247261
<package name="keras/objects">
248262
<class name="feature" allocatable="true" />
249263
</package>
264+
<package name="tensorflow/math">
265+
<class name="sigmoid" allocatable="true">
266+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/nn/sigmoid -->
267+
<method name="do" descriptor="()LRoot;" numArgs="3" paramNames="self x name">
268+
<return value="x" />
269+
</method>
270+
</class>
271+
<class name="add" allocatable="true">
272+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/add -->
273+
<method name="read_data" descriptor="()LRoot;">
274+
<new def="x" class="Ltensorflow/math/add" />
275+
<return value="x" />
276+
</method>
277+
<method name="do" descriptor="()LRoot;" numArgs="4" paramNames="self x y name">
278+
<!-- Even though tf.add() isn't a tensor "generator," it can convert its non-tensor arguments to tensors. -->
279+
<call class="LRoot" name="read_data" descriptor="()LRoot;" type="virtual" arg0="arg0" def="xx" />
280+
<return value="xx" />
281+
</method>
282+
</class>
283+
</package>
250284
<package name="tensorflow/functions">
251285
<class name="AdamOptimizer" allocatable="true">
252286
<method name="do" descriptor="()LRoot;">
@@ -631,6 +665,12 @@
631665
<return value="x" />
632666
</method>
633667
</class>
668+
<class name="softmax" allocatable="true">
669+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/nn/softmax -->
670+
<method name="do" descriptor="()LRoot;" numArgs="4" paramNames="self logits axis name">
671+
<return value="logits" />
672+
</method>
673+
</class>
634674
</package>
635675
<package name="tensorflow/estimator">
636676
<class name="Estimator" allocatable="true">
@@ -650,6 +690,32 @@
650690
</method>
651691
</class>
652692
</package>
693+
<package name="tensorflow/keras/layers">
694+
<class name="Dense" allocatable="true">
695+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/keras/layers/Dense -->
696+
<method name="do" descriptor="()LRoot;" numArgs="11" paramNames="self units activation use_bias kernel_initializer bias_initializer kernel_regularizer bias_regularizer activity_regularizer kernel_constraint bias_constraint">
697+
<new def="__call__" class="Ltensorflow/keras/layers/__call__" />
698+
<putfield class="LRoot" field="__call__" fieldType="LRoot" ref="arg0" value="__call__" />
699+
<new def="call" class="Ltensorflow/keras/layers/call" />
700+
<putfield class="LRoot" field="call" fieldType="LRoot" ref="arg0" value="call" />
701+
<return value="arg0" />
702+
</method>
703+
</class>
704+
<!-- FIXME: These methods must be called explicitly. The implicit cases blocked on https://github.com/wala/ML/issues/127. -->
705+
<class name="__call__" allocatable="true">
706+
<!-- https://github.com/keras-team/keras/blob/07e13740fd181fc3ddec7d9a594d8a08666645f6/keras/layers/core/dense.py#L166-L240 -->
707+
<method name="do" descriptor="()LRoot;" numArgs="2" paramNames="self inputs">
708+
<return value="inputs" />
709+
</method>
710+
</class>
711+
<!-- FIXME: Workaround for https://github.com/wala/ML/issues/106. -->
712+
<class name="call" allocatable="true">
713+
<!-- https://github.com/keras-team/keras/blob/07e13740fd181fc3ddec7d9a594d8a08666645f6/keras/layers/core/dense.py#L166-L240 -->
714+
<method name="do" descriptor="()LRoot;" numArgs="2" paramNames="self inputs">
715+
<return value="inputs" />
716+
</method>
717+
</class>
718+
</package>
653719
<package name="tensorflow/data">
654720
<class name="Dataset" allocatable="true">
655721
<!-- "read_dataset" means that this function reads a tensor iterable. -->
@@ -658,6 +724,12 @@
658724
<putfield class="LRoot" field="shuffle" fieldType="LRoot" ref="arg0" value="shuffle" />
659725
<new def="batch" class="Ltensorflow/data/batch" />
660726
<putfield class="LRoot" field="batch" fieldType="LRoot" ref="arg0" value="batch" />
727+
<new def="repeat" class="Ltensorflow/data/repeat" />
728+
<putfield class="LRoot" field="repeat" fieldType="LRoot" ref="arg0" value="repeat" />
729+
<new def="prefetch" class="Ltensorflow/data/prefetch" />
730+
<putfield class="LRoot" field="prefetch" fieldType="LRoot" ref="arg0" value="prefetch" />
731+
<new def="take" class="Ltensorflow/data/take" />
732+
<putfield class="LRoot" field="take" fieldType="LRoot" ref="arg0" value="take" />
661733
<return value="arg0" />
662734
</method>
663735
<method name="do" descriptor="()LRoot;" numArgs="2" paramNames="self variant_tensor">
@@ -683,15 +755,51 @@
683755
<return value="xx" />
684756
</method>
685757
</class>
758+
<class name="repeat" allocatable="true">
759+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/data/Dataset#repeat -->
760+
<method name="do" descriptor="()LRoot;" numArgs="6" paramNames="self count name">
761+
<!-- FIXME: Workaround for https://github.com/wala/ML/issues/127. -->
762+
<new def="x" class="Ltensorflow/data/Dataset" />
763+
<call class="Ltensorflow/data/Dataset" name="read_dataset" descriptor="()LRoot;" type="virtual" arg0="x" def="xx" />
764+
<return value="xx" />
765+
</method>
766+
</class>
767+
<class name="prefetch" allocatable="true">
768+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/data/Dataset#prefetch -->
769+
<method name="do" descriptor="()LRoot;" numArgs="3" paramNames="self buffer_size name">
770+
<!-- FIXME: Workaround for https://github.com/wala/ML/issues/127. -->
771+
<new def="x" class="Ltensorflow/data/Dataset" />
772+
<call class="Ltensorflow/data/Dataset" name="read_dataset" descriptor="()LRoot;" type="virtual" arg0="x" def="xx" />
773+
<return value="xx" />
774+
</method>
775+
</class>
776+
<class name="take" allocatable="true">
777+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/data/Dataset#take -->
778+
<method name="do" descriptor="()LRoot;" numArgs="3" paramNames="self count name">
779+
<!-- FIXME: Workaround for https://github.com/wala/ML/issues/127. -->
780+
<new def="x" class="Ltensorflow/data/Dataset" />
781+
<call class="Ltensorflow/data/Dataset" name="read_dataset" descriptor="()LRoot;" type="virtual" arg0="x" def="xx" />
782+
<return value="xx" />
783+
</method>
784+
</class>
686785
</package>
687786
<package name="tensorflow/data/Dataset">
688787
<class name="from_tensor_slices" allocatable="true">
788+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/data/Dataset#from_tensor_slices -->
689789
<method name="do" descriptor="()LRoot;" numArgs="2" paramNames="tensors name">
690790
<new def="x" class="Ltensorflow/data/Dataset" />
691791
<call class="Ltensorflow/data/Dataset" name="read_dataset" descriptor="()LRoot;" type="virtual" arg0="x" def="xx" />
692792
<return value="xx" />
693793
</method>
694794
</class>
795+
<class name="from_generator" allocatable="true">
796+
<!-- https://www.tensorflow.org/versions/r2.9/api_docs/python/tf/data/Dataset#from_generator -->
797+
<method name="do" descriptor="()LRoot;" numArgs="6" paramNames="generator output_types output_shapes args output_signature name">
798+
<new def="x" class="Ltensorflow/data/Dataset" />
799+
<call class="Ltensorflow/data/Dataset" name="read_dataset" descriptor="()LRoot;" type="virtual" arg0="x" def="xx" />
800+
<return value="xx" />
801+
</method>
802+
</class>
695803
</package>
696804
<package name="tensorflow/estimator/train">
697805
<class name="train" allocatable="true">

0 commit comments

Comments
 (0)