Merge branch 'main' of https://github.com/microsoft/DFOL-VQA into main

Author: amizadeh

.DS_Store

@@ -384,8 +384,8 @@ def _parse_compare(self, op_tokens, arg_tokens):
     gqap.preprocess(args['input_file'], join(output_path, 'p_' + input_file + '.json'), True, args['length_segregation'], discard_global=args['discard_global'])
 
     if args['h5']:
-        ATTRIBUTE_PATH = "./nsvqa/data/metadata/gqa_attribute.json"
-        CLASS_PATH = "./nsvqa/data/metadata/gqa_class_clean.json"
+        ATTRIBUTE_PATH = "./nsvqa/data/metadata/gqa_all_attribute.json"
+        CLASS_PATH = "./nsvqa/data/metadata/gqa_all_class.json"
         VOCAB_PATH = "./nsvqa/data/metadata/gqa_vocab.json"
 
         ontology = GQAOntology(ATTRIBUTE_PATH, CLASS_PATH, VOCAB_PATH)

src/.DS_Store

@@ -162,9 +162,11 @@ def collate_meta_data(self, questions):
             relations = np.zeros((len(image_idx), self._max_relation_per_image, self._relation_feature_dim))
             pair_indices = np.zeros((len(image_idx), self._max_relation_per_image, 2))
 
+            object_nums_cum_sum = np.array(object_nums, dtype=np.int64).cumsum()
+
             for j, (chunck_id, offset) in enumerate(zip(image_chk, image_idx)):
                 relations[j, :, :] = self._file_handles[chunck_id]['relation_features'][offset, :, :]
-                pair_indices[j, :, :] = self._file_handles[chunck_id]['relation_indices'][offset, :, :] + (object_nums[j - 1] if j > 0 else 0)
+                pair_indices[j, :, :] = self._file_handles[chunck_id]['relation_indices'][offset, :, :] + (object_nums_cum_sum[j - 1] if j > 0 else 0)
 
             relations = relations.reshape((-1, self._relation_feature_dim))
             pair_indices = pair_indices.reshape((-1, 2))

src/gqa_preprocess.py

@@ -583,7 +583,7 @@ def forward(self, op_id, world, subject_variable_set, object_variable_set, relat
             object_variable_set.object_num(), predicate_num, quantifiers=quantifier, 
             log_attention=log_attention_posterior[:, 1, :], batch_object_map=object_variable_set._batch_object_map,
             predicate_question_map=predicate_question_map, base_cumulative_loss=subject_variable_set.cumulative_loss() + object_variable_set.cumulative_loss(),
-            prev_variable_sets_num=subject_variable_set._prev_variable_sets_num + object_variable_set._prev_variable_sets_num + 1)
+            prev_variable_sets_num=subject_variable_set._prev_variable_sets_num + object_variable_set._prev_variable_sets_num + 1).to(object_variable_set.dtype)
 
         if op_id in self._subject_modulations:
             new_subject_set = new_subject_set.apply_modulations(self._subject_modulations[op_id], subject_variable_set, predicate_question_map).to(subject_variable_set.dtype)

src/nsvqa/.DS_Store

@@ -143,7 +143,14 @@ def compute_all_log_likelihood(self, object_features, pair_object_features):
         return attr_output, rel_output
 
     def compute_all_log_likelihood_2(self, object_features, pair_object_features):
-        attr_output = self._embedding_network(self._attribute_network(object_features))
-        rel_output = self._embedding_network(self._relation_network(pair_object_features))[:, self._ontology._relation_index]
+        if self._embedding_network is None or self._attribute_network is None:
+            attr_output = object_features
+        else:
+            attr_output = self._embedding_network(self._attribute_network(object_features))
+
+        if self._embedding_network is None or self._relation_network is None:
+            rel_output = pair_object_features
+        else:
+            rel_output = self._embedding_network(self._relation_network(pair_object_features))[:, self._ontology._relation_index]
 
         return attr_output, rel_output