feat: collect daily surface counts

Author: Julusian

usage-statistics/prisma/schema.prisma

@@ -58,6 +58,28 @@ model CompanionUsers {
   @@map("companion_users")
 }
 
+// Count number of unique user+surface pairs seen in the period
+model CompanionSurfaceDailyCounts {
+  id          Int               @id @default(autoincrement())
+  type        StatsSamplePeriod
+  description String            @db.VarChar(128)
+  count       Int
+  ts          DateTime          @default(now()) @db.DateTime(0)
+
+  @@index([ts], map: "ts")
+  @@index([type], map: "type")
+}
+
+// Count total number of seen surfaces
+model CompanionSurfaceTotalSeen {
+  id          Int      @id @default(autoincrement())
+  description String   @db.VarChar(128)
+  count       Int
+  ts          DateTime @default(now()) @db.DateTime(0)
+
+  @@index([ts], map: "ts")
+}
+
 enum StatsSamplePeriod {
   day1  @map("1day")
   day7  @map("7day")

usage-statistics/src/main.ts

@@ -6,6 +6,7 @@ import { runPlatforms } from './platforms.js'
 import { runPlatformStats } from './platform-stats.js'
 import type { AppStore } from './types.js'
 import mariadb from 'mariadb'
+import { runSurfaceCounts, runSurfaceTotals } from './surfaces.js'
 
 console.log('hello world')
 
@@ -45,6 +46,8 @@ try {
 		runModules(store),
 		runPlatforms(store),
 		runPlatformStats(store),
+		runSurfaceCounts(store),
+		runSurfaceTotals(store),
 	])
 
 	console.log('all done!')

usage-statistics/src/surfaces.ts

@@ -0,0 +1,76 @@
+import { DRY_RUN, runQuery } from './util.js'
+import { writeFile } from 'node:fs/promises'
+import path from 'node:path'
+import type { AppStore } from './types.js'
+import type { CompanionSurfaceDailyCounts, CompanionSurfaceTotalSeen, StatsSamplePeriod } from './prisma/client.js'
+
+export async function runSurfaceCounts(store: AppStore): Promise<void> {
+	async function writeData(stats: any[], type: StatsSamplePeriod) {
+		const data = stats.map(
+			(m) =>
+				({
+					type,
+					description: m.surface_description?.slice(0, 128), // Trim to fit DB field
+					count: Number(m.surfaces) || 0,
+				}) satisfies Omit<CompanionSurfaceDailyCounts, 'id' | 'ts'>
+		)
+
+		if (DRY_RUN) {
+			await writeFile(
+				path.join(import.meta.dirname, `../dry-run/surface-daily-counts-${type}.json`),
+				JSON.stringify(data, null, 2)
+			)
+		} else {
+			const res = await store.prismaDest.companionSurfaceDailyCounts.createMany({ data })
+			console.log(`Inserted ${res.count} records for surface daily counts ${type}`)
+		}
+	}
+
+	function formatQuery(interval: string) {
+		return `SELECT count(distinct id) surfaces, surface_description FROM \`SurfaceUserLastSeen\` where last_seen >= date_sub(CURRENT_DATE, interval ${interval}) group by surface_description;`
+	}
+
+	await Promise.all([
+		runQuery('Surfaces daily 30day', async () => {
+			const rows = await store.srcDb.query(formatQuery('30 day'))
+			await writeData(rows, 'day30' as '30day')
+		}),
+		runQuery('Surfaces daily 7day', async () => {
+			const rows = await store.srcDb.query(formatQuery('7 day'))
+			await writeData(rows, 'day7' as '7day')
+		}),
+		runQuery('Surfaces daily 1day', async () => {
+			const rows = await store.srcDb.query(formatQuery('24 hour'))
+			await writeData(rows, 'day1' as '1day')
+		}),
+	])
+}
+
+export async function runSurfaceTotals(store: AppStore): Promise<void> {
+	async function writeData(stats: any[]) {
+		const data = stats.map(
+			(m) =>
+				({
+					description: m.surface_description?.slice(0, 128), // Trim to fit DB field
+					count: Number(m.surfaces) || 0,
+				}) satisfies Omit<CompanionSurfaceTotalSeen, 'id' | 'ts'>
+		)
+
+		if (DRY_RUN) {
+			await writeFile(
+				path.join(import.meta.dirname, `../dry-run/surface-total-counts.json`),
+				JSON.stringify(data, null, 2)
+			)
+		} else {
+			const res = await store.prismaDest.companionSurfaceTotalSeen.createMany({ data })
+			console.log(`Inserted ${res.count} records for surface totals`)
+		}
+	}
+
+	const query = `SELECT count(distinct id) surfaces, surface_description FROM \`SurfaceUserLastSeen\` group by surface_description;`
+
+	await runQuery('Surfaces total seen', async () => {
+		const rows = await store.srcDb.query(query)
+		await writeData(rows)
+	})
+}