[WIP] Switch to Internal Resource Identifier in the Index

Author: alexanderkiel

docs/implementation/database.md

@@ -97,7 +97,7 @@ The result is a dataset which consists only of the resource types Patient, Obser
 
 ## Controlling and Monitoring the Caches
 
-The size of the resource cache and the resource handle cache can be set by their respective environment variables `DB_RESOURCE_CACHE_SIZE` and `DB_RESOURCE_HANDLE_CACHE_SIZE`. The size denotes the number of resources / resource handles. Because one has to specify a number of resources / resource handles, it's important to know how many bytes a resource / resource handle allocates on the heap. For resource handles, it can be said that they allocate between 272 and 328 bytes depending on the size of the resource id. For resources, the size varies widely. Monitoring of the heap usage is critical.
+The size of the resource cache and the resource handle cache can be set by their respective environment variables `DB_RESOURCE_CACHE_SIZE` and `DB_RESOURCE_HANDLE_CACHE_SIZE`. The size denotes the number of resources / resource handles. Because one has to specify a number of resources / resource handles, it's important to know how many bytes a resource / resource handle allocates on the heap. For resource handles, it can be said that they allocate between 152 and 208 bytes depending on the size of the resource id. For resources, the size varies widely. Monitoring of the heap usage is critical.
 
 ### Monitoring 
 

docs/performance/fhir-search.md

@@ -78,6 +78,12 @@
   (.putInt ^ByteBuffer byte-buffer x))
 
 
+(defn put-5-byte-long!
+  [byte-buffer ^long x]
+  (put-byte! byte-buffer (bit-shift-right (unchecked-long x) 32))
+  (put-int! byte-buffer x))
+
+
 (defn put-long!
   {:inline
    (fn [byte-buffer x]
@@ -220,6 +226,11 @@
   (.getInt ^ByteBuffer byte-buffer))
 
 
+(defn get-5-byte-long! [byte-buffer]
+  (+ (bit-shift-left (bit-and (get-byte! byte-buffer) 0xFF) 32)
+     (bit-and (get-int! byte-buffer) 0xFFFFFFFF)))
+
+
 (defn get-long!
   {:inline
    (fn [byte-buffer]
@@ -243,24 +254,6 @@
    (.get ^ByteBuffer byte-buffer ^bytes byte-array offset length)))
 
 
-(defn size-up-to-null [byte-buffer]
-  (when (pos? (remaining byte-buffer))
-    (mark! byte-buffer)
-    (loop [byte (bit-and (long (get-byte! byte-buffer)) 0xFF)
-           size 0]
-      (cond
-        (zero? byte)
-        (do (reset! byte-buffer)
-            size)
-
-        (pos? (remaining byte-buffer))
-        (recur (bit-and (long (get-byte! byte-buffer)) 0xFF) (inc size))
-
-        :else
-        (do (reset! byte-buffer)
-            nil)))))
-
-
 (defn mismatch
   "Finds and returns the relative index of the first mismatch between `a` and
   `b`.

modules/byte-buffer/src/blaze/byte_buffer.clj

@@ -3,7 +3,7 @@
     [blaze.byte-buffer :as bb]
     [blaze.test-util :refer [satisfies-prop]]
     [clojure.spec.test.alpha :as st]
-    [clojure.test :as test :refer [deftest is testing]]
+    [clojure.test :as test :refer [are deftest]]
     [clojure.test.check.generators :as gen]
     [clojure.test.check.properties :as prop]))
 
@@ -26,41 +26,19 @@
       (= capacity (bb/limit (bb/allocate capacity))))))
 
 
-(deftest size-up-to-null-test
-  (testing "empty buffer"
-    (let [buf (bb/allocate 0)]
-      (is (nil? (bb/size-up-to-null buf)))))
+(defn- transcode-5-byte-long [x]
+  (let [buf (bb/allocate 5)]
+    (bb/put-5-byte-long! buf x)
+    (bb/flip! buf)
+    (= x (bb/get-5-byte-long! buf))))
 
-  (testing "buffer with only one null byte"
-    (let [buf (bb/allocate 1)]
-      (bb/put-byte! buf 0)
-      (bb/flip! buf)
-      (is (zero? (bb/size-up-to-null buf)))))
 
-  (testing "buffer with only one non-null byte"
-    (let [buf (bb/allocate 1)]
-      (bb/put-byte! buf 1)
-      (bb/flip! buf)
-      (is (nil? (bb/size-up-to-null buf)))))
+(deftest transcode-5-byte-long-test
+  (are [x] (transcode-5-byte-long x)
+    0
+    0xFFFFFFFF
+    0xFFFFFFFFFF)
 
-  (testing "buffer with one non-null and one null byte"
-    (let [buf (bb/allocate 2)]
-      (bb/put-byte! buf 1)
-      (bb/put-byte! buf 0)
-      (bb/flip! buf)
-      (is (= 1 (bb/size-up-to-null buf)))))
-
-  (testing "buffer with two null bytes"
-    (let [buf (bb/allocate 2)]
-      (bb/put-byte! buf 0)
-      (bb/put-byte! buf 0)
-      (bb/flip! buf)
-      (is (zero? (bb/size-up-to-null buf)))))
-
-  (testing "buffer with two non-null and one null byte"
-    (let [buf (bb/allocate 3)]
-      (bb/put-byte! buf 1)
-      (bb/put-byte! buf 2)
-      (bb/put-byte! buf 0)
-      (bb/flip! buf)
-      (is (= 2 (bb/size-up-to-null buf))))))
+  (satisfies-prop 100000
+    (prop/for-all [x (gen/choose 0 0xFFFFFFFFFF)]
+      (transcode-5-byte-long x))))

modules/byte-buffer/test/blaze/byte_buffer_test.clj

@@ -76,8 +76,8 @@
   (-compartment-keys [search-param context compartment tid compiled-value])
   (-matches? [search-param context resource-handle modifier compiled-values])
   (-compartment-ids [_ resolver resource])
-  (-index-values [_ resolver resource])
-  (-index-value-compiler [_]))
+  (-index-values [_ resource-id resolver resource])
+  (-index-value-compiler [_ resource-id]))
 
 
 (defprotocol Pull

modules/db-protocols/src/blaze/db/impl/protocols.clj

@@ -58,6 +58,7 @@
      :tx-success-index {:reverse-comparator? true}
      :tx-error-index nil
      :t-by-instant-index {:reverse-comparator? true}
+     :resource-id-index nil
      :resource-as-of-index nil
      :type-as-of-index nil
      :system-as-of-index nil

modules/db-stub/src/blaze/db/api_stub.clj

@@ -31,4 +31,5 @@
    {cloverage/cloverage
     {:mvn/version "1.2.4"}}
 
-   :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"]}}}
+   :main-opts ["-m" "cloverage.coverage" "--codecov" "-p" "src" "-s" "test"
+               "-e" ".+spec"]}}}

modules/db-tx-log/deps.edn

@@ -4,7 +4,8 @@
     [blaze.db.tx-log :as tx-log]
     [blaze.fhir.spec]
     [blaze.spec]
-    [clojure.spec.alpha :as s]))
+    [clojure.spec.alpha :as s]
+    [clojure.spec.gen.alpha :as gen]))
 
 
 (s/def :blaze.db/tx-log
@@ -27,8 +28,20 @@
   (s/coll-of :blaze.fhir/local-ref-tuple))
 
 
+(def ^:private ^:const ^long max-t 0xFFFFFFFFFF)
+
+
+;; With a 5 byte long `t`, we can create one transaction each millisecond
+;; for 34 years. Alone the t-values would need a storage of 5 TB.
+(comment
+  (let [millis-per-year (* 1000 3600 24 365)
+        five-bytes (long (Math/pow 2 40))]
+    (double (/ five-bytes millis-per-year))))
+
+
+;; The point in time `t` of a database value.
 (s/def :blaze.db/t
-  (s/and int? #(<= 0 % 0xFFFFFFFFFFFFFF)))
+  (s/with-gen (s/and int? #(<= 0 % max-t)) #(gen/choose 0 max-t)))
 
 
 (s/def :blaze.db.tx-cmd/if-none-exist

modules/db-tx-log/src/blaze/db/tx_log/spec.clj

@@ -27,7 +27,7 @@
   (are [x] (s/valid? :blaze.db/t x)
     0
     1
-    0xFFFFFFFFFFFFFF))
+    0xFFFFFFFFFF))
 
 
 (def patient-hash-0 (hash/generate {:fhir/type :fhir/Patient :id "0"}))