Merge pull request #33 from lpgauth/bench-pgo-glazer

PGO-build glazer in bench CI and bump to 0.5.11

Author: lpgauth

.github/workflows/bench.yml

@@ -82,6 +82,25 @@ jobs:
           "$profdata" merge -o pgo-data/merged.profdata "${raw[@]}"
           ls -la pgo-data
 
+      # PGO the comparison library too, so the benchmark is PGO-vs-PGO instead
+      # of PGO-torque-vs-plain-glazer. glazer's `make OPTIMIZE=1` target drives
+      # PGO from bin/pgo-profile.es, which the hex package doesn't ship, so we
+      # drive its c_src Makefile's PGO=generate/use directly with the same
+      # payloads the torque pass uses (bench/glazer_pgo_workload.exs). The
+      # runtime loads deps/glazer/priv/glazer.so (the _build app priv dir is a
+      # symlink to it), so rebuilding there is what the benchmark picks up.
+      # CI's g++ is GCC: it writes .gcda and reads them back from the profile
+      # dir on -fprofile-use with no separate merge step. A later
+      # `mix compile --force` only forces torque, not the glazer dep, so this
+      # optimised .so survives into the benchmark step.
+      - name: Build instrumented glazer & collect PGO profile
+        run: |
+          make -C deps/glazer/c_src PGO=generate clean all
+          mix run bench/glazer_pgo_workload.exs
+
+      - name: Rebuild glazer with PGO profile
+        run: make -C deps/glazer/c_src PGO=use all
+
       # PGO pass 2: rebuild torque with the collected profile, then benchmark.
       # --force is mandatory here: mix does not treat a changed RUSTFLAGS as a
       # reason to recompile, so a plain `mix compile` is a no-op and the

README.md

@@ -176,65 +176,65 @@ optimised (PGO) builds: **Torque PGO** (via `scripts/pgo-build.sh`) and
 
 | Library | ips | mean | median | p99 | memory |
 |---|---|---|---|---|---|
-| **torque** | **404.0K** | **2.48 μs** | **2.29 μs** | **5.75 μs** | 1.56 KB |
-| **glazer** | 378.7K | 2.64 μs | 2.38 μs | 7.08 μs | 1.56 KB |
-| **jiffy** | 201.3K | 4.97 μs | 4.25 μs | 13.21 μs | **1.55 KB** |
-| **simdjsone** | 180.8K | 5.53 μs | 5.21 μs | 12.50 μs | 1.59 KB |
-| **otp json** | 140.6K | 7.11 μs | 6.71 μs | 15.38 μs | 7.73 KB |
-| **jason** | 109.4K | 9.14 μs | 8.50 μs | 18.83 μs | 9.54 KB |
+| **torque** | **413.3K** | **2.42 μs** | **2.29 μs** | **4.04 μs** | 1.56 KB |
+| **glazer** | 401.8K | 2.49 μs | 2.38 μs | 4.58 μs | 1.56 KB |
+| **jiffy** | 202.5K | 4.94 μs | 4.50 μs | 9.75 μs | **1.55 KB** |
+| **simdjsone** | 178.2K | 5.61 μs | 5.25 μs | 12.63 μs | 1.59 KB |
+| **otp json** | 147.4K | 6.78 μs | 6.58 μs | 10.58 μs | 7.73 KB |
+| **jason** | 113.2K | 8.83 μs | 8.46 μs | 13.96 μs | 9.54 KB |
 
 ### Decode (750 KB Twitter)
 
 | Library | ips | mean | median | p99 | memory |
 |---|---|---|---|---|---|
-| **torque** | **648.3** | **1.54 ms** | **1.34 ms** | **2.12 ms** | **1.57 KB** |
-| **glazer** | 587.5 | 1.70 ms | 1.56 ms | 2.24 ms | 1.58 KB |
-| **simdjsone** | 437.9 | 2.28 ms | 1.95 ms | 3.37 ms | 1.59 KB |
-| **jiffy** | 278.5 | 3.59 ms | 3.69 ms | 4.00 ms | 2.30 MB |
-| **otp json** | 201.3 | 4.97 ms | 5.02 ms | 5.68 ms | 2.48 MB |
-| **jason** | 140.3 | 7.13 ms | 7.13 ms | 7.65 ms | 3.54 MB |
+| **torque** | **660.7** | **1.51 ms** | **1.33 ms** | 2.21 ms | **1.57 KB** |
+| **glazer** | 659.6 | 1.52 ms | 1.43 ms | **1.94 ms** | 1.58 KB |
+| **simdjsone** | 413.6 | 2.42 ms | 1.93 ms | 3.75 ms | 1.59 KB |
+| **jiffy** | 301.4 | 3.32 ms | 3.39 ms | 3.96 ms | 2.30 MB |
+| **otp json** | 205.3 | 4.87 ms | 4.80 ms | 7.52 ms | 2.48 MB |
+| **jason** | 151.7 | 6.59 ms | 6.58 ms | 6.97 ms | 3.52 MB |
 
 ### Encode (1.2 KB OpenRTB)
 
 | Library | ips | mean | median | p99 | memory |
 |---|---|---|---|---|---|
-| **torque** [proplist() :: iodata()] | **1360K** | **0.74 μs** | **0.67 μs** | **0.92 μs** | **64 B** |
-| **torque** [proplist() :: binary()] | 1340K | 0.75 μs | **0.67 μs** | **0.92 μs** | 88 B |
-| **glazer** [map() :: binary()] | 1230K | 0.81 μs | 0.75 μs | 1.00 μs | **64 B** |
-| **otp json** [map() :: iodata()] | 1190K | 0.84 μs | 0.79 μs | 1.21 μs | 3928 B |
-| **torque** [map() :: iodata()] | 1160K | 0.86 μs | 0.79 μs | 1.04 μs | **64 B** |
-| **torque** [map() :: binary()] | 1160K | 0.86 μs | 0.79 μs | 1.04 μs | 88 B |
-| **jiffy** [proplist() :: iodata()] | 720K | 1.39 μs | 1.17 μs | 1.79 μs | 120 B |
-| **jiffy** [map() :: iodata()] | 610K | 1.65 μs | 1.42 μs | 3.58 μs | 824 B |
-| **jason** [map() :: iodata()] | 600K | 1.68 μs | 1.50 μs | 4.21 μs | 3848 B |
-| **simdjsone** [proplist() :: iodata()] | 450K | 2.22 μs | 2.04 μs | 2.71 μs | 184 B |
-| **jason** [map() :: binary()] | 390K | 2.59 μs | 2.38 μs | 6.42 μs | 3912 B |
-| **simdjsone** [map() :: iodata()] | 380K | 2.62 μs | 2.42 μs | 3.33 μs | 888 B |
+| **otp json** [map() :: iodata()] | **1180K** | **0.85 μs** | **0.79 μs** | **1.08 μs** | 3928 B |
+| **torque** [proplist() :: binary()] | 1110K | 0.90 μs | 0.83 μs | **1.08 μs** | 88 B |
+| **torque** [proplist() :: iodata()] | 1080K | 0.92 μs | 0.83 μs | 1.83 μs | **64 B** |
+| **torque** [map() :: iodata()] | 1000K | 1.00 μs | 0.96 μs | 1.13 μs | **64 B** |
+| **glazer** [map() :: binary()] | 990K | 1.01 μs | 0.92 μs | 1.29 μs | **64 B** |
+| **torque** [map() :: binary()] | 980K | 1.02 μs | 0.96 μs | 1.17 μs | 88 B |
+| **jiffy** [proplist() :: iodata()] | 640K | 1.57 μs | 1.33 μs | 1.83 μs | 120 B |
+| **jason** [map() :: iodata()] | 620K | 1.62 μs | 1.54 μs | 2.63 μs | 3848 B |
+| **jiffy** [map() :: iodata()] | 520K | 1.91 μs | 1.75 μs | 2.25 μs | 824 B |
+| **simdjsone** [proplist() :: iodata()] | 410K | 2.42 μs | 2.29 μs | 3.00 μs | 184 B |
+| **jason** [map() :: binary()] | 380K | 2.63 μs | 2.38 μs | 6.42 μs | 3912 B |
+| **simdjsone** [map() :: iodata()] | 340K | 2.94 μs | 2.75 μs | 4.38 μs | 888 B |
 
 ### Encode (750 KB Twitter)
 
 | Library | ips | mean | median | p99 | memory |
 |---|---|---|---|---|---|
-| **torque** [proplist() :: binary()] | **1424.6** | **0.70 ms** | **0.68 ms** | **0.84 ms** | 88 B |
-| **torque** [proplist() :: iodata()] | 1416.7 | 0.71 ms | **0.68 ms** | 1.11 ms | **64 B** |
-| **torque** [map() :: binary()] | 1211.6 | 0.83 ms | 0.81 ms | 1.00 ms | 88 B |
-| **torque** [map() :: iodata()] | 1208.4 | 0.83 ms | 0.81 ms | 0.99 ms | **64 B** |
-| **glazer** [map() :: binary()] | 1082.5 | 0.92 ms | 0.91 ms | 1.06 ms | **64 B** |
-| **jiffy** [proplist() :: iodata()] | 537.1 | 1.86 ms | 1.84 ms | 2.11 ms | 37.7 KB |
-| **jiffy** [map() :: iodata()] | 409.0 | 2.45 ms | 2.33 ms | 2.99 ms | 1.06 MB |
-| **simdjsone** [proplist() :: iodata()] | 259.1 | 3.86 ms | 3.81 ms | 5.68 ms | 37.7 KB |
-| **jason** [map() :: iodata()] | 258.9 | 3.86 ms | 3.62 ms | 5.88 ms | 4.96 MB |
-| **otp json** [map() :: iodata()] | 252.7 | 3.96 ms | 4.14 ms | 6.63 ms | 5.40 MB |
-| **simdjsone** [map() :: iodata()] | 229.7 | 4.35 ms | 4.34 ms | 4.67 ms | 1.06 MB |
-| **jason** [map() :: binary()] | 130.4 | 7.67 ms | 7.70 ms | 8.35 ms | 4.96 MB |
+| **torque** [proplist() :: iodata()] | **1156.0** | **0.87 ms** | **0.85 ms** | **1.06 ms** | **64 B** |
+| **torque** [proplist() :: binary()] | 1154.1 | **0.87 ms** | **0.85 ms** | **1.06 ms** | 88 B |
+| **torque** [map() :: binary()] | 1045.4 | 0.96 ms | 0.95 ms | 1.13 ms | 88 B |
+| **torque** [map() :: iodata()] | 1040.4 | 0.96 ms | 0.96 ms | 1.13 ms | **64 B** |
+| **glazer** [map() :: binary()] | 1033.3 | 0.97 ms | 0.96 ms | 1.08 ms | **64 B** |
+| **jiffy** [proplist() :: iodata()] | 469.5 | 2.13 ms | 2.10 ms | 2.51 ms | 37.7 KB |
+| **jiffy** [map() :: iodata()] | 347.6 | 2.88 ms | 2.96 ms | 3.90 ms | 1.06 MB |
+| **simdjsone** [proplist() :: iodata()] | 254.7 | 3.93 ms | 3.87 ms | 5.44 ms | 37.7 KB |
+| **otp json** [map() :: iodata()] | 251.8 | 3.97 ms | 4.13 ms | 6.44 ms | 5.40 MB |
+| **jason** [map() :: iodata()] | 233.5 | 4.28 ms | 4.03 ms | 6.50 ms | 4.96 MB |
+| **simdjsone** [map() :: iodata()] | 215.1 | 4.65 ms | 4.76 ms | 5.22 ms | 1.06 MB |
+| **jason** [map() :: binary()] | 127.9 | 7.82 ms | 7.84 ms | 8.43 ms | 4.96 MB |
 
 ### Parse (1.2 KB OpenRTB)
 
 | Library | ips | mean | median | p99 |
 |---|---|---|---|---|
-| **torque** parse(unique_keys) | **572.6K** | **1.75 μs** | 1.46 μs | **4.83 μs** |
-| **torque** parse | 510.3K | 1.96 μs | 1.46 μs | 6.29 μs |
-| **simdjsone** parse | 304.0K | 3.29 μs | **1.21 μs** | 5.33 μs |
+| **torque** parse(unique_keys) | **542.0K** | **1.84 μs** | 1.46 μs | **5.63 μs** |
+| **torque** parse | 522.8K | 1.91 μs | 1.46 μs | 6.00 μs |
+| **simdjsone** parse | 306.3K | 3.27 μs | **1.21 μs** | 5.83 μs |
 
 ### Extract 5 fields from raw JSON (1.2 KB OpenRTB)
 
@@ -245,11 +245,11 @@ selective extraction skips materializing the whole document.
 
 | Library | ips | mean | median | p99 |
 |---|---|---|---|---|
-| **torque** parse + get_many | **476.1K** | **2.10 μs** | 1.75 μs | **4.75 μs** |
-| **torque** parse + get x5 | 434.6K | 2.30 μs | 1.92 μs | 5.96 μs |
-| **torque** parse(unique_keys) + get_many | 417.8K | 2.39 μs | 1.75 μs | 8.08 μs |
-| **simdjsone** parse + get x5 | 415.8K | 2.41 μs | **1.67 μs** | 6.50 μs |
-| **glazer** decode + find x5 | 317.7K | 3.15 μs | 2.79 μs | 8.54 μs |
+| **torque** parse + get_many | **436.6K** | **2.29 μs** | **1.79 μs** | **6.21 μs** |
+| **torque** parse(unique_keys) + get_many | 429.5K | 2.33 μs | **1.79 μs** | 6.67 μs |
+| **torque** parse + get x5 | 409.8K | 2.44 μs | 2.00 μs | 6.67 μs |
+| **simdjsone** parse + get x5 | 374.9K | 2.67 μs | **1.79 μs** | 7.21 μs |
+| **glazer** decode + find x5 | 344.9K | 2.90 μs | 2.75 μs | 6.71 μs |
 
 Run benchmarks locally:
 

bench/glazer_pgo_workload.exs

@@ -0,0 +1,61 @@
+# PGO training workload for the glazer NIF.
+#
+# Mirror of bench/pgo_workload.exs but driving glazer instead of torque, so a
+# glazer PGO build (deps/glazer c_src Makefile PGO=generate/use) collects
+# branch/call-frequency data over the same representative payloads the README
+# benchmark uses — keeping the glazer-vs-torque comparison apples-to-apples
+# (both libraries profiled against equivalent decode/encode/find workloads).
+#
+# Run with an instrumented glazer.so loaded: MIX_ENV=bench mix run this file.
+
+small_json =
+  ~s({"id":"req-0001","site":{"domain":"example.com","page":"https://example.com/articles/x","cat":["IAB1","IAB2-3"],"publisher":{"id":"pub-12345"}},"device":{"devicetype":2,"ua":"Mozilla/5.0 Macintosh; Intel Mac OS X 10_15_7 Chrome/120.0.0.0","ip":"203.0.113.42","geo":{"country":"US","lat":40.7128,"lon":-74.006,"zip":"10001"},"connectiontype":2},"user":{"id":"u-abcdef","name":"cafe resume"},"imp":[{"id":"imp-1","banner":{"w":300,"h":250},"bidfloor":0.5},{"id":"imp-2","video":{"mimes":["video/mp4"],"maxduration":30},"bidfloor":2.0}],"regs":{"coppa":0},"ext":null,"test":true})
+
+record = fn i ->
+  ~s({"metadata":{"result_type":"recent","iso_language_code":"en"},"id":#{505_874_924_000_000_000 + i},"id_str":"#{505_874_924_000_000_000 + i}","text":"Sample tweet #{i} lorem ipsum dolor sit amet consectetur adipiscing elit","truncated":false,"in_reply_to_status_id":null,"user":{"id":#{1_000_000 + i},"screen_name":"username_#{i}","location":"San Francisco, CA","url":null,"followers_count":#{rem(i * 1337, 100_000)},"verified":false,"lang":"en","profile_image_url":"http://pbs.twimg.com/profile_images/#{i}/photo.jpeg"},"geo":null,"retweet_count":#{rem(i * 3, 1000)},"favorite_count":#{rem(i * 7, 2000)},"entities":{"hashtags":[{"text":"elixir","indices":[15,22]}],"urls":[],"user_mentions":[{"screen_name":"user_#{i}","id":#{2_000_000 + i}}]},"favorited":false,"lang":"en"})
+end
+
+large_json =
+  ~s({"statuses":[) <>
+    Enum.map_join(1..200, ",", record) <>
+    ~s(],"search_metadata":{"count":200,"completed_in":0.035,"max_id":505874924095815681,"query":"%23elixir"}})
+
+small_term = :glazer_json.decode(small_json)
+large_term = :glazer_json.decode(large_json)
+
+paths =
+  Enum.map(
+    [
+      ".id",
+      ".site.domain",
+      ".site.page",
+      ".device.ip",
+      ".device.geo.country",
+      ".user.id",
+      ".regs.coppa"
+    ],
+    &:glazer.compile_path/1
+  )
+
+IO.puts("glazer PGO workload: small=#{byte_size(small_json)}B large=#{byte_size(large_json)}B")
+
+decode = fn ->
+  :glazer_json.decode(small_json)
+  :glazer_json.decode(large_json)
+end
+
+encode = fn ->
+  :glazer_json.encode(small_term)
+  :glazer_json.encode(large_term)
+end
+
+find = fn ->
+  d = :glazer_json.decode(small_json)
+  Enum.each(paths, &:glazer.find(d, &1))
+end
+
+Enum.each(1..5_000, fn _ -> decode.() end)
+Enum.each(1..5_000, fn _ -> encode.() end)
+Enum.each(1..10_000, fn _ -> find.() end)
+
+IO.puts("glazer PGO workload complete")

mix.lock

@@ -6,7 +6,7 @@
   "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"},
   "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"},
   "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"},
-  "glazer": {:hex, :glazer, "0.5.7", "ebf8dfc855d905110b84fd0e62f3fac28bfbf39152c09205974a141363644c79", [:rebar3], [], "hexpm", "fa78ffe5fd162a50a61bba21489aa78c148dbe07289a3765e21edc7ab7c3209a"},
+  "glazer": {:hex, :glazer, "0.5.11", "c6f4bf32c3da539351d4e3811484ffe0ba341a871dc2ce47ddb5284c78f5166f", [:rebar3], [], "hexpm", "f5abec34de2f4cec226632f0db25567d4c6ffc0fd2e7ed5d285df81afb5582f8"},
   "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"},
   "jiffy": {:hex, :jiffy, "2.0.0", "1d7605ee31c4fd9b454725d744716b59e2abd88297862c8931b605c83b076480", [:rebar3], [], "hexpm", "73f58877ccaae14e14601c95928e2913fd1c1ef28e6ef4f274a8d62819f9e04f"},
   "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"},