Making ConcurrentContainers require UnorderedIterable

Summary:
Look at ConcurrentContainers.h to see how all concurrent container types are adapted to pull out the iterable functionality.

This also adapts *all* uses in the codebase.

Reviewed By: agampe

Differential Revision: D72023602

fbshipit-source-id: 2630aa312138ee5183c6ed3401cfc495416b098f

Author: Nikolai Tillmann

analysis/ip-reflection-analysis/IPReflectionAnalysis.cpp

@@ -237,7 +237,7 @@ void IPReflectionAnalysisPass::run_pass(DexStoresVector& stores,
   analysis.run();
   const auto& summaries = analysis.registry.get_map();
   m_result = std::make_shared<Result>();
-  for (const auto& entry : summaries) {
+  for (const auto& entry : UnorderedIterable(summaries)) {
     (*m_result)[entry.first] = entry.second.get_reflection_sites();
   }
   if (m_export_results) {

analysis/max-depth/MaxDepthAnalysis.cpp

@@ -191,7 +191,7 @@ void MaxDepthAnalysisPass::run_pass(DexStoresVector& stores,
   analysis.run();
   m_result = std::make_shared<UnorderedMap<const DexMethod*, int>>();
 
-  for (const auto& entry : analysis.registry.get_map()) {
+  for (const auto& entry : UnorderedIterable(analysis.registry.get_map())) {
     if (entry.second.is_value()) {
       (*m_result)[entry.first] = entry.second.depth();
     }

libredex/BinarySerialization.h

@@ -80,7 +80,7 @@ class GraphWriter {
   template <class NodeContainer>
   void write(std::ostream& os, const NodeContainer& nodes) {
     // Give each node a unique ID.
-    for (const auto& node : nodes) {
+    for (const auto& node : UnorderedIterable(nodes)) {
       number_node_recursive(node);
     }
     // Emit the node label and adjacency list.

libredex/CallGraph.cpp

@@ -161,7 +161,7 @@ RootAndDynamic MultipleCalleeBaseStrategy::get_roots() const {
   });
   // Gather methods that override or implement external or native methods
   // as well.
-  for (auto& pair : m_method_override_graph.nodes()) {
+  for (auto& pair : UnorderedIterable(m_method_override_graph.nodes())) {
     auto method = pair.first;
     if (method->is_external()) {
       dynamic_methods.emplace(method);
@@ -254,7 +254,7 @@ RootAndDynamic CompleteCallGraphStrategy::get_roots() const {
     }
   });
   // Gather methods that override or implement external methods
-  for (auto& pair : m_method_override_graph.nodes()) {
+  for (auto& pair : UnorderedIterable(m_method_override_graph.nodes())) {
     auto method = pair.first;
     if (method->is_external()) {
       const auto& overriding_methods =
@@ -353,9 +353,8 @@ RootAndDynamic MultipleCalleeStrategy::get_roots() const {
       root_and_dynamic.roots.insert(method);
     }
   };
-  std::for_each(m_big_virtuals.begin(), m_big_virtuals.end(), add_root);
-  std::for_each(m_big_virtual_overrides.begin(), m_big_virtual_overrides.end(),
-                add_root);
+  unordered_for_each(m_big_virtuals, add_root);
+  unordered_for_each(m_big_virtual_overrides, add_root);
   return root_and_dynamic;
 }
 

libredex/ClassChecker.cpp

@@ -9,6 +9,7 @@
 
 #include "ClassChecker.h"
 #include "ClassHierarchy.h"
+#include "DeterministicContainers.h"
 #include "DexClass.h"
 #include "Show.h"
 #include "Timer.h"
@@ -21,7 +22,7 @@ void print_failed_things(const Collection& items,
                          const size_t print_limit,
                          std::ostringstream* oss) {
   size_t counter = 0;
-  for (auto& fail : items) {
+  for (auto& fail : UnorderedIterable(items)) {
     *oss << show(fail)
          << " (deobfuscated: " << fail->get_deobfuscated_name_or_empty_copy()
          << ")\n";

libredex/ConcurrentContainers.h

@@ -20,6 +20,7 @@
 #include <utility>
 
 #include "Debug.h"
+#include "DeterministicContainers.h"
 #include "Timer.h"
 
 namespace cc_impl {
@@ -985,6 +986,12 @@ class ConcurrentContainer {
   using const_iterator =
       cc_impl::ConcurrentContainerIterator<const ConcurrentHashtable, n_slots>;
 
+  using key_type = typename Container::key_type;
+  using value_type = typename iterator::value_type;
+  using hasher = typename Container::hasher;
+  using key_equal = typename Container::key_equal;
+  using difference_type = typename Container::difference_type;
+
   virtual ~ConcurrentContainer() {
     auto timer_scope = cc_impl::s_destructor.scope();
     if (!cc_impl::is_thread_pool_active() ||
@@ -998,41 +1005,180 @@ class ConcurrentContainer {
         0, n_slots, [this](size_t slot) { m_slots[slot].destroy(); });
   }
 
+  class FixedIterator;
+
+  class ConstFixedIterator {
+    const_iterator m_entry;
+    friend class FixedIterator;
+
+   public:
+    const value_type* operator->() const { return &*m_entry; }
+
+    const value_type& operator*() const { return *m_entry; }
+
+    bool operator==(const FixedIterator& other) const {
+      return m_entry == other.m_entry;
+    }
+
+    bool operator!=(const FixedIterator& other) const {
+      return m_entry != other.m_entry;
+    }
+
+    bool operator==(const ConstFixedIterator& other) const {
+      return m_entry == other.m_entry;
+    }
+
+    bool operator!=(const ConstFixedIterator& other) const {
+      return m_entry != other.m_entry;
+    }
+
+    explicit ConstFixedIterator(const_iterator entry) : m_entry(entry) {}
+
+    const_iterator _internal_unsafe_unwrap() { return m_entry; }
+  };
+
+  class FixedIterator {
+    iterator m_entry;
+    friend class ConstFixedIterator;
+
+   public:
+    value_type* operator->() { return &*m_entry; }
+
+    value_type& operator*() { return *m_entry; }
+
+    bool operator==(const FixedIterator& other) const {
+      return m_entry == other.m_entry;
+    }
+
+    bool operator!=(const FixedIterator& other) const {
+      return m_entry != other.m_entry;
+    }
+
+    bool operator==(const ConstFixedIterator& other) const {
+      return m_entry == other.m_entry;
+    }
+
+    bool operator!=(const ConstFixedIterator& other) const {
+      return m_entry != other.m_entry;
+    }
+
+    explicit FixedIterator(iterator entry) : m_entry(entry) {}
+
+    iterator _internal_unsafe_unwrap() { return m_entry; }
+  };
+
+  // TODO: Make extra non-deterministic in debug builds
+  class UnorderedIterable {
+    ConcurrentHashtable* m_slots;
+
+   public:
+    explicit UnorderedIterable(ConcurrentHashtable* slots) : m_slots(slots) {}
+
+    iterator begin() { return iterator(&m_slots[0], 0, m_slots[0].begin()); }
+
+    iterator end() { return iterator(&m_slots[0]); }
+
+    const_iterator begin() const {
+      const auto* cslots = m_slots;
+      return const_iterator(&cslots[0], 0, cslots[0].begin());
+    }
+
+    const_iterator end() const {
+      const auto* cslots = m_slots;
+      return const_iterator(&cslots[0]);
+    }
+
+    const_iterator cbegin() const { return begin(); }
+
+    const_iterator cend() const { return end(); }
+
+    iterator find(const Key& key) {
+      size_t slot = Hash()(key) % n_slots;
+      const auto& it = m_slots[slot].find(key);
+      if (it == m_slots[slot].end()) {
+        return end();
+      }
+      return iterator(&m_slots[0], slot, it);
+    }
+
+    const_iterator find(const Key& key) const {
+      size_t slot = Hash()(key) % n_slots;
+      const auto* cslots = m_slots;
+      const auto& it = cslots[slot].find(key);
+      if (it == cslots[slot].end()) {
+        return end();
+      }
+      return const_iterator(&cslots[0], slot, it);
+    }
+  };
+
+  // TODO: Make extra non-deterministic in debug builds
+  class ConstUnorderedIterable {
+    const ConcurrentHashtable* m_slots;
+
+   public:
+    explicit ConstUnorderedIterable(const ConcurrentHashtable* slots)
+        : m_slots(slots) {}
+
+    const_iterator begin() const {
+      return const_iterator(&m_slots[0], 0, m_slots[0].begin());
+    }
+
+    const_iterator end() const { return const_iterator(&m_slots[0]); }
+
+    const_iterator cbegin() const { return begin(); }
+
+    const_iterator cend() const { return end(); }
+
+    const_iterator find(const Key& key) const {
+      size_t slot = Hash()(key) % n_slots;
+      const auto& it = m_slots[slot].find(key);
+      if (it == m_slots[slot].end()) {
+        return end();
+      }
+      return const_iterator(&m_slots[0], slot, it);
+    }
+  };
+
   /*
    * Using iterators or accessor functions while the container is concurrently
    * modified will result in undefined behavior.
    */
 
-  iterator begin() { return iterator(&m_slots[0], 0, m_slots[0].begin()); }
+  UnorderedIterable _internal_unordered_iterable() {
+    return UnorderedIterable(m_slots);
+  }
 
-  iterator end() { return iterator(&m_slots[0]); }
+  ConstUnorderedIterable _internal_unordered_iterable() const {
+    return ConstUnorderedIterable(m_slots);
+  }
 
-  const_iterator begin() const {
-    return const_iterator(&m_slots[0], 0, m_slots[0].begin());
+  FixedIterator _internal_unordered_any() {
+    return FixedIterator(_internal_unordered_iterable().begin());
+  }
+
+  ConstFixedIterator _internal_unordered_any() const {
+    return ConstFixedIterator(_internal_unordered_iterable().begin());
   }
 
-  const_iterator end() const { return const_iterator(&m_slots[0]); }
+  FixedIterator find(const Key& key) {
+    return FixedIterator(_internal_unordered_iterable().find(key));
+  }
 
-  const_iterator cbegin() const { return begin(); }
+  ConstFixedIterator find(const Key& key) const {
+    return ConstFixedIterator(_internal_unordered_iterable().find(key));
+  }
 
-  const_iterator cend() const { return end(); }
+  ConstFixedIterator end() const {
+    return ConstFixedIterator(_internal_unordered_iterable().cend());
+  }
 
-  iterator find(const Key& key) {
-    size_t slot = Hash()(key) % n_slots;
-    const auto& it = m_slots[slot].find(key);
-    if (it == m_slots[slot].end()) {
-      return end();
-    }
-    return iterator(&m_slots[0], slot, it);
+  FixedIterator end() {
+    return FixedIterator(_internal_unordered_iterable().end());
   }
 
-  const_iterator find(const Key& key) const {
-    size_t slot = Hash()(key) % n_slots;
-    const auto& it = m_slots[slot].find(key);
-    if (it == m_slots[slot].end()) {
-      return end();
-    }
-    return const_iterator(&m_slots[0], slot, it);
+  ConstFixedIterator cend() const {
+    return ConstFixedIterator(_internal_unordered_iterable().cend());
   }
 
   /*
@@ -1167,7 +1313,8 @@ template <typename Key,
           size_t n_slots = cc_impl::kDefaultSlots>
 class ConcurrentMap final
     : public ConcurrentContainer<std::unordered_map<Key, Value, Hash, KeyEqual>,
-                                 n_slots> {
+                                 n_slots>,
+      public UnorderedBase<ConcurrentMap<Key, Value, Hash, KeyEqual, n_slots>> {
  public:
   using Base =
       ConcurrentContainer<std::unordered_map<Key, Value, Hash, KeyEqual>,
@@ -1181,6 +1328,8 @@ class ConcurrentMap final
 
   using KeyValuePair = typename Base::Value;
 
+  using mapped_type = Value;
+
   ConcurrentMap() = default;
 
   ConcurrentMap(const ConcurrentMap& container) noexcept : Base(container) {}
@@ -1482,7 +1631,9 @@ template <typename Key,
           size_t n_slots = cc_impl::kDefaultSlots>
 class InsertOnlyConcurrentMap final
     : public ConcurrentContainer<std::unordered_map<Key, Value, Hash, KeyEqual>,
-                                 n_slots> {
+                                 n_slots>,
+      public UnorderedBase<
+          InsertOnlyConcurrentMap<Key, Value, Hash, KeyEqual, n_slots>> {
  public:
   using Base =
       ConcurrentContainer<std::unordered_map<Key, Value, Hash, KeyEqual>,
@@ -1495,6 +1646,8 @@ class InsertOnlyConcurrentMap final
 
   using KeyValuePair = typename Base::Value;
 
+  using mapped_type = Value;
+
   InsertOnlyConcurrentMap() = default;
 
   InsertOnlyConcurrentMap(const InsertOnlyConcurrentMap& container) noexcept
@@ -1782,7 +1935,8 @@ template <typename Key,
 class AtomicMap final
     : public ConcurrentContainer<
           std::unordered_map<Key, std::atomic<Value>, Hash, KeyEqual>,
-          n_slots> {
+          n_slots>,
+      public UnorderedBase<AtomicMap<Key, Value, Hash, KeyEqual, n_slots>> {
  public:
   using Base = ConcurrentContainer<
       std::unordered_map<Key, std::atomic<Value>, Hash, KeyEqual>,
@@ -1975,7 +2129,8 @@ template <typename Key,
           size_t n_slots = cc_impl::kDefaultSlots>
 class ConcurrentSet final
     : public ConcurrentContainer<std::unordered_set<Key, Hash, KeyEqual>,
-                                 n_slots> {
+                                 n_slots>,
+      public UnorderedBase<ConcurrentSet<Key, Hash, KeyEqual, n_slots>> {
  public:
   using Base =
       ConcurrentContainer<std::unordered_set<Key, Hash, KeyEqual>, n_slots>;
@@ -2056,7 +2211,9 @@ template <typename Key,
           size_t n_slots = cc_impl::kDefaultSlots>
 class InsertOnlyConcurrentSet final
     : public ConcurrentContainer<std::unordered_set<Key, Hash, KeyEqual>,
-                                 n_slots> {
+                                 n_slots>,
+      public UnorderedBase<
+          InsertOnlyConcurrentSet<Key, Hash, KeyEqual, n_slots>> {
  public:
   using Base =
       ConcurrentContainer<std::unordered_set<Key, Hash, KeyEqual>, n_slots>;

libredex/DeterministicContainers.h

@@ -41,6 +41,9 @@
  * - `unordered_copy(collection, ...)` / `unordered_copy_if(collection, ...)`
  *   provides the same functionality as `std::copy` / `std::copy_if`, except
  *   that it works on (potentially) unordered collections.
+ * - `unordered_count(collection, ...)` / `unordered_count_if(collection, ...)`
+ *   provides the same functionality as `std::count` / `std::count_if`, except
+ *   that it works on (potentially) unordered collections.
  * - `unordered_erase_if(collection, ...)` provides the same functionality as
  *   `std::erase_if`, except that it works on (potentially) unordered
  *   collections.
@@ -897,6 +900,20 @@ OutputIt unordered_copy_if(const Collection& collection,
   return std::copy_if(ui.begin(), ui.end(), target, std::move(pred));
 }
 
+template <class Collection, class T>
+typename Collection::difference_type unordered_count(
+    const Collection& collection, const T& value) {
+  auto ui = UnorderedIterable(collection);
+  return std::count(ui.begin(), ui.end(), value);
+}
+
+template <class Collection, class UnaryPred>
+typename Collection::difference_type unordered_count_if(
+    const Collection& collection, UnaryPred pred) {
+  auto ui = UnorderedIterable(collection);
+  return std::count_if(ui.begin(), ui.end(), std::move(pred));
+}
+
 template <class Collection, typename Pred>
 size_t unordered_erase_if(Collection& collection, const Pred& pred) {
   size_t removed = 0;