WIP trust without conflicts

prisma/migrations/20250214010811_arc_without_conflict/migration.sql

@@ -0,0 +1,25 @@
+/*
+  Warnings:
+
+  - The primary key for the `Arc` table will be changed. If it partially fails, the table could be left without primary key constraint.
+  - A unique constraint covering the columns `[fromId,toId,withoutConflict]` on the table `Arc` will be added. If there are existing duplicate values, this will fail.
+  - A unique constraint covering the columns `[toId,fromId,withoutConflict]` on the table `Arc` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- DropIndex
+DROP INDEX "Arc_toId_fromId_idx";
+
+-- AlterTable
+ALTER TABLE "Arc" DROP CONSTRAINT "Arc_pkey",
+ADD COLUMN     "id" SERIAL NOT NULL,
+ADD COLUMN     "withoutConflict" BOOLEAN NOT NULL DEFAULT false,
+ADD CONSTRAINT "Arc_pkey" PRIMARY KEY ("id");
+
+-- AlterTable
+ALTER TABLE "users" ADD COLUMN  "trustWithoutConflict" DOUBLE PRECISION NOT NULL DEFAULT 0;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Arc_fromId_toId_withoutConflict_key" ON "Arc"("fromId", "toId", "withoutConflict");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Arc_toId_fromId_withoutConflict_key" ON "Arc"("toId", "fromId", "withoutConflict");

prisma/migrations/20250214015347_zap_rank_personal_noconflicts/migration.sql

@@ -0,0 +1,69 @@
+DROP MATERIALIZED VIEW IF EXISTS zap_rank_personal_view;
+CREATE MATERIALIZED VIEW IF NOT EXISTS zap_rank_personal_view AS
+WITH item_votes AS (
+    SELECT "Item".id, "Item"."parentId", "Item".boost, "Item".created_at, "Item"."weightedComments", "ItemAct"."userId" AS "voterId",
+        LOG((SUM("ItemAct".msats) FILTER (WHERE "ItemAct".act IN ('TIP', 'FEE'))) / 1000.0) AS "vote",
+        GREATEST(LOG((SUM("ItemAct".msats) FILTER (WHERE "ItemAct".act = 'DONT_LIKE_THIS')) / 1000.0), 0) AS "downVote"
+    FROM "Item"
+    CROSS JOIN zap_rank_personal_constants
+    JOIN "ItemAct" ON "ItemAct"."itemId" = "Item".id
+    WHERE (
+        ("ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID')
+        AND
+        (
+            ("ItemAct"."userId" <> "Item"."userId" AND "ItemAct".act IN ('TIP', 'FEE', 'DONT_LIKE_THIS'))
+        OR
+            ("ItemAct".act = 'BOOST' AND "ItemAct"."userId" = "Item"."userId")
+        )
+    )
+    AND "Item".created_at >= now_utc() - item_age_bound
+    GROUP BY "Item".id, "Item"."parentId", "Item".boost, "Item".created_at, "Item"."weightedComments", "ItemAct"."userId"
+    HAVING SUM("ItemAct".msats) > 1000
+), viewer_votes AS (
+    SELECT item_votes.id, item_votes."parentId", item_votes.boost, item_votes.created_at,
+        item_votes."weightedComments", "Arc"."fromId" AS "viewerId",
+        GREATEST("Arc"."zapTrust", g."zapTrust", 0) * item_votes."vote" AS "weightedVote",
+        GREATEST("Arc"."zapTrust", g."zapTrust", 0) * item_votes."downVote" AS "weightedDownVote"
+    FROM item_votes
+    CROSS JOIN zap_rank_personal_constants
+    LEFT JOIN "Arc" ON "Arc"."toId" = item_votes."voterId"
+    LEFT JOIN "Arc" g ON g."fromId" = global_viewer_id AND g."toId" = item_votes."voterId"
+    AND ("Arc"."zapTrust" IS NOT NULL OR g."zapTrust" IS NOT NULL)
+), viewer_weighted_votes AS (
+    SELECT viewer_votes.id, viewer_votes."parentId", viewer_votes.boost, viewer_votes.created_at, viewer_votes."viewerId",
+        viewer_votes."weightedComments", SUM(viewer_votes."weightedVote") AS "weightedVotes",
+        SUM(viewer_votes."weightedDownVote") AS "weightedDownVotes"
+    FROM viewer_votes
+    GROUP BY viewer_votes.id, viewer_votes."parentId", viewer_votes.boost, viewer_votes.created_at, viewer_votes."viewerId", viewer_votes."weightedComments"
+), viewer_zaprank AS (
+    SELECT l.id, l."parentId", l.boost, l.created_at, l."viewerId", l."weightedComments",
+        GREATEST(l."weightedVotes", g."weightedVotes") AS "weightedVotes", GREATEST(l."weightedDownVotes", g."weightedDownVotes") AS "weightedDownVotes"
+    FROM viewer_weighted_votes l
+    CROSS JOIN zap_rank_personal_constants
+    JOIN users ON users.id = l."viewerId"
+    JOIN viewer_weighted_votes g ON l.id = g.id AND g."viewerId" = global_viewer_id
+    WHERE (l."weightedVotes" > min_viewer_votes
+        AND g."weightedVotes" / l."weightedVotes" <= max_personal_viewer_vote_ratio
+        AND users."lastSeenAt" >= now_utc() - user_last_seen_bound)
+    OR l."viewerId" = global_viewer_id
+    GROUP BY l.id, l."parentId", l.boost, l.created_at, l."viewerId", l."weightedVotes", l."weightedComments",
+        g."weightedVotes", l."weightedDownVotes", g."weightedDownVotes", min_viewer_votes
+    HAVING GREATEST(l."weightedVotes", g."weightedVotes") > min_viewer_votes OR l.boost > 0
+), viewer_fractions_zaprank AS (
+    SELECT z.*,
+        (CASE WHEN z."weightedVotes" - z."weightedDownVotes" > 0 THEN
+              GREATEST(z."weightedVotes" - z."weightedDownVotes", POWER(z."weightedVotes" - z."weightedDownVotes", vote_power))
+            ELSE
+                z."weightedVotes" - z."weightedDownVotes"
+            END + z."weightedComments" * CASE WHEN z."parentId" IS NULL THEN comment_scaler ELSE 0 END) AS tf_numerator,
+        POWER(GREATEST(age_wait_hours, EXTRACT(EPOCH FROM (now_utc() - z.created_at))/3600), vote_decay) AS decay_denominator,
+        (POWER(z.boost/boost_per_vote, boost_power)
+         /
+         POWER(GREATEST(age_wait_hours, EXTRACT(EPOCH FROM (now_utc() - z.created_at))/3600), boost_decay)) AS boost_addend
+    FROM viewer_zaprank z, zap_rank_personal_constants
+)
+SELECT z.id, z."parentId", z."viewerId",
+    COALESCE(tf_numerator, 0) / decay_denominator + boost_addend AS tf_hot_score,
+    COALESCE(tf_numerator, 0) AS tf_top_score
+FROM viewer_fractions_zaprank z
+WHERE tf_numerator > 0 OR boost_addend > 0;

prisma/schema.prisma

@@ -37,6 +37,7 @@ model User {
   tipPopover                Boolean              @default(false)
   upvotePopover             Boolean              @default(false)
   trust                     Float                @default(0)
+  trustWithoutConflict      Float                @default(0)
   lastSeenAt                DateTime?
   stackedMsats              BigInt               @default(0)
   stackedMcredits           BigInt               @default(0)
@@ -343,14 +344,18 @@ model Mute {
 }
 
 model Arc {
-  fromId   Int
-  fromUser User  @relation("fromUser", fields: [fromId], references: [id], onDelete: Cascade)
-  toId     Int
-  toUser   User  @relation("toUser", fields: [toId], references: [id], onDelete: Cascade)
-  zapTrust Float
-
-  @@id([fromId, toId])
-  @@index([toId, fromId])
+  id              Int     @id @default(autoincrement())
+  fromId          Int
+  fromUser        User    @relation("fromUser", fields: [fromId], references: [id], onDelete: Cascade)
+  toId            Int
+  toUser          User    @relation("toUser", fields: [toId], references: [id], onDelete: Cascade)
+  zapTrust        Float
+  // this is used to store trust without conflicts
+  // it's a temporary solution until we create trust graphs for each territory
+  withoutConflict Boolean @default(false)
+
+  @@unique([fromId, toId, withoutConflict])
+  @@unique([toId, fromId, withoutConflict])
 }
 
 enum StreakType {

worker/trust.js

@@ -1,17 +1,32 @@
 import * as math from 'mathjs'
 import { USER_ID, SN_ADMIN_IDS } from '@/lib/constants'
+import { Prisma } from '@prisma/client'
 
 export async function trust ({ boss, models }) {
+  // for simplicity, until this is completely rewritten, we want to do two trust runs
+  // one run not checking for conflicts, computing global AND personal trust
+  // one run removing conflicts, recording unconflicted arcs
   try {
+    // first run
     console.time('trust')
     console.timeLog('trust', 'getting graph')
     const graph = await getGraph(models)
     console.timeLog('trust', 'computing trust')
     const [vGlobal, mPersonal] = await trustGivenGraph(graph)
     console.timeLog('trust', 'storing trust')
     await storeTrust(models, graph, vGlobal, mPersonal)
+
+    // second run
+    console.time('trust without conflicts')
+    console.timeLog('trust without conflicts', 'getting graph')
+    const graphWithoutConflicts = await getGraph(models, true)
+    console.timeLog('trust without conflicts', 'computing trust')
+    const [vGlobalWithoutConflicts, mPersonalWithoutConflicts] = await trustGivenGraph(graphWithoutConflicts)
+    console.timeLog('trust without conflicts', 'storing trust')
+    await storeTrust(models, graphWithoutConflicts, vGlobalWithoutConflicts, mPersonalWithoutConflicts, true)
   } finally {
     console.timeEnd('trust')
+    console.timeEnd('trust without conflicts')
   }
 }
 
@@ -26,7 +41,7 @@ const Z_CONFIDENCE = 6.109410204869 // 99.9999999% confidence
 const GLOBAL_ROOT = 616
 const SEED_WEIGHT = 0.25
 const AGAINST_MSAT_MIN = 1000
-const MSAT_MIN = 1000
+const MSAT_MIN = 10001 // 10001 is the minimum for a tip to be counted in trust
 const SIG_DIFF = 0.1 // need to differ by at least 10 percent
 
 /*
@@ -114,7 +129,7 @@ function trustGivenGraph (graph) {
     ...
   ]
 */
-async function getGraph (models) {
+async function getGraph (models, withoutConflict = false) {
   return await models.$queryRaw`
     SELECT id, json_agg(json_build_object(
       'node', oid,
@@ -128,7 +143,11 @@ async function getGraph (models) {
         JOIN "Item" ON "Item".id = "ItemAct"."itemId" AND "ItemAct".act IN ('FEE', 'TIP', 'DONT_LIKE_THIS')
           AND "Item"."parentId" IS NULL AND NOT "Item".bio AND "Item"."userId" <> "ItemAct"."userId"
         JOIN users ON "ItemAct"."userId" = users.id AND users.id <> ${USER_ID.anon}
-        WHERE "ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID'
+        JOIN "Sub" ON "Sub".name = "Item"."subName"
+        WHERE ("ItemAct"."invoiceActionState" IS NULL OR "ItemAct"."invoiceActionState" = 'PAID')
+        ${withoutConflict
+          ? Prisma.sql` AND ("Sub"."userId" <> "ItemAct"."userId" OR "Sub"."userId" = ANY (${SN_ADMIN_IDS}))`
+          : Prisma.empty}
         GROUP BY user_id, name, item_id, user_at, against
         HAVING CASE WHEN
           "ItemAct".act = 'DONT_LIKE_THIS' THEN sum("ItemAct".msats) > ${AGAINST_MSAT_MIN}
@@ -167,7 +186,7 @@ async function getGraph (models) {
     ORDER BY id ASC`
 }
 
-async function storeTrust (models, graph, vGlobal, mPersonal) {
+async function storeTrust (models, graph, vGlobal, mPersonal, withoutConflict = false) {
   // convert nodeTrust into table literal string
   let globalValues = ''
   let personalValues = ''
@@ -176,29 +195,36 @@ async function storeTrust (models, graph, vGlobal, mPersonal) {
     if (globalValues) globalValues += ','
     globalValues += `(${graph[idx].id}, ${val}::FLOAT)`
     if (personalValues) personalValues += ','
-    personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT)`
+    personalValues += `(${GLOBAL_ROOT}, ${graph[idx].id}, ${val}::FLOAT, ${withoutConflict}::BOOLEAN)`
   })
 
   math.forEach(mPersonal, (val, [fromIdx, toIdx]) => {
     const globalVal = vGlobal.get([toIdx])
     if (isNaN(val) || val - globalVal <= SIG_DIFF) return
     if (personalValues) personalValues += ','
-    personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT)`
+    personalValues += `(${graph[fromIdx].id}, ${graph[toIdx].id}, ${val}::FLOAT, ${withoutConflict}::BOOLEAN)`
   })
 
   // update the trust of each user in graph
   await models.$transaction([
-    models.$executeRaw`UPDATE users SET trust = 0`,
+    models.$executeRaw`UPDATE users SET ${withoutConflict ? 'trustWithoutConflict' : 'trust'} = 0`,
     models.$executeRawUnsafe(
       `UPDATE users
-        SET trust = g.trust
+        SET ${withoutConflict ? 'trustWithoutConflict' : 'trust'} = g.trust
         FROM (values ${globalValues}) g(id, trust)
         WHERE users.id = g.id`),
     models.$executeRawUnsafe(
-      `INSERT INTO "Arc" ("fromId", "toId", "zapTrust")
-        SELECT id, oid, trust
-        FROM (values ${personalValues}) g(id, oid, trust)
-        ON CONFLICT ("fromId", "toId") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"`
+      `INSERT INTO "Arc" ("fromId", "toId", "zapTrust", "withoutConflict")
+        SELECT id, oid, trust, "withoutConflict"
+        FROM (values ${personalValues}) g(id, oid, trust, "withoutConflict")
+        ON CONFLICT ("fromId", "toId", "withoutConflict") DO UPDATE SET "zapTrust" = EXCLUDED."zapTrust"`
+    ),
+    // select all arcs that don't exist in personalValues and delete them
+    models.$executeRawUnsafe(
+      `DELETE FROM "Arc"
+        WHERE "Arc"."fromId" NOT IN (SELECT id FROM (values ${personalValues}) g(id, oid, trust, "withoutConflict"))
+        AND "Arc"."toId" NOT IN (SELECT oid FROM (values ${personalValues}) g(id, oid, trust, "withoutConflict"))
+        AND "Arc"."withoutConflict" = ${withoutConflict}::BOOLEAN`
     )
   ])
 }