Skip to content

Commit 1eb708b

Browse files
nicobaonicobao
authored
refactor(api): move conversations to project ownership (#1087)
* refactor(api): move conversations to project ownership Replace direct conversation ownership with project-based ownership through organization memberships and capabilities. Changes: - add project, project ownership, organization membership, and capability schema - backfill existing conversations, memberships, projects, capabilities, and premium entitlements - route conversation create/edit/delete/export/moderation checks through project capabilities - move premium access to organization-owned entitlement checks - update import-worker queue payloads to carry authorized project IDs - update Python generated models and worker premium analysis ownership logic - add pure project access authorization tests Deploy: api, import-worker, math-updater, scoring-worker, agora perf(api): tune ownership indexes Adjust the org/project ownership migration indexes after reviewing the actual query shapes. The redundant project ownership project index is removed because the unique project/organization index already covers project lookups, while organization membership gets an organization lookup index for member fanout queries. Add a partial project timeline index for managed conversation listings that page by created_at and id after filtering to non-empty conversations. Deploy: api * refactor(api): move conversations to project ownership Replace direct conversation ownership with project-based ownership through organization memberships and capabilities. Changes: - add project, project ownership, organization membership, and capability schema - backfill existing conversations, memberships, projects, capabilities, and premium entitlements - route conversation create/edit/delete/export/moderation checks through project capabilities - move premium access to organization-owned entitlement checks - update import-worker queue payloads to carry authorized project IDs - update Python generated models and worker premium analysis ownership logic - add pure project access authorization tests Deploy: api, import-worker, math-updater, scoring-worker, agora perf(api): tune ownership indexes Adjust the org/project ownership migration indexes after reviewing the actual query shapes. The redundant project ownership project index is removed because the unique project/organization index already covers project lookups, while organization membership gets an organization lookup index for member fanout queries. Add a partial project timeline index for managed conversation listings that page by created_at and id after filtering to non-empty conversations. Deploy: api fix: temporarily remove schema change to rebase
1 parent 32629c5 commit 1eb708b

43 files changed

Lines changed: 27039 additions & 754 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎services/api/database/flyway/V0071.1__backfill_org_project_ownership.sql‎

Lines changed: 278 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,278 @@
1+
UPDATE "user"
2+
SET first_name = username
3+
WHERE first_name IS NULL;
4+
5+
WITH normalized AS (
6+
SELECT
7+
organization.id,
8+
organization.name AS display_name,
9+
lower(
10+
trim(
11+
both '-' from regexp_replace(
12+
regexp_replace(organization.name, '[^[:alnum:]]+', '-', 'g'),
13+
'-+',
14+
'-',
15+
'g'
16+
)
17+
)
18+
) AS raw_slug
19+
FROM organization
20+
WHERE organization.auto_provisioned_for_user_id IS NULL
21+
), slug_candidates AS (
22+
SELECT
23+
normalized.id,
24+
normalized.display_name,
25+
CASE
26+
WHEN normalized.raw_slug = '' THEN 'org-' || normalized.id::text
27+
ELSE left(normalized.raw_slug, 65)
28+
END AS base_slug
29+
FROM normalized
30+
), slugged AS (
31+
SELECT
32+
slug_candidates.id,
33+
slug_candidates.display_name,
34+
CASE
35+
WHEN row_number() OVER (
36+
PARTITION BY slug_candidates.base_slug
37+
ORDER BY slug_candidates.id
38+
) = 1 THEN slug_candidates.base_slug
39+
ELSE left(slug_candidates.base_slug, 58) || '-' || row_number() OVER (
40+
PARTITION BY slug_candidates.base_slug
41+
ORDER BY slug_candidates.id
42+
)::text
43+
END AS slug
44+
FROM slug_candidates
45+
)
46+
UPDATE organization
47+
SET
48+
slug = slugged.slug,
49+
display_name = slugged.display_name,
50+
directory_visibility = 'listed'::directory_visibility
51+
FROM slugged
52+
WHERE organization.id = slugged.id;
53+
54+
INSERT INTO organization (
55+
name,
56+
slug,
57+
display_name,
58+
directory_visibility,
59+
auto_provisioned_for_user_id,
60+
image_path,
61+
is_full_image_path,
62+
created_at,
63+
updated_at
64+
)
65+
SELECT
66+
'user-' || replace("user".id::text, '-', '') AS name,
67+
'user-' || replace("user".id::text, '-', '') AS slug,
68+
"user".first_name AS display_name,
69+
'unlisted'::directory_visibility AS directory_visibility,
70+
"user".id AS auto_provisioned_for_user_id,
71+
'' AS image_path,
72+
false AS is_full_image_path,
73+
now() AS created_at,
74+
now() AS updated_at
75+
FROM "user"
76+
WHERE NOT EXISTS (
77+
SELECT 1
78+
FROM organization existing
79+
WHERE existing.auto_provisioned_for_user_id = "user".id
80+
);
81+
82+
INSERT INTO organization_membership (
83+
user_id,
84+
organization_id,
85+
created_at,
86+
updated_at
87+
)
88+
SELECT
89+
user_organization_mapping.user_id,
90+
user_organization_mapping.organization_id,
91+
user_organization_mapping.created_at,
92+
user_organization_mapping.created_at AS updated_at
93+
FROM user_organization_mapping
94+
ON CONFLICT (user_id, organization_id) DO NOTHING;
95+
96+
INSERT INTO organization_membership (
97+
user_id,
98+
organization_id,
99+
created_at,
100+
updated_at
101+
)
102+
SELECT
103+
organization.auto_provisioned_for_user_id,
104+
organization.id,
105+
now(),
106+
now()
107+
FROM organization
108+
WHERE organization.auto_provisioned_for_user_id IS NOT NULL
109+
ON CONFLICT (user_id, organization_id) DO NOTHING;
110+
111+
INSERT INTO project (
112+
slug,
113+
display_name,
114+
directory_visibility,
115+
auto_provisioned_for_organization_id,
116+
created_at,
117+
updated_at
118+
)
119+
SELECT
120+
'org-' || organization.id::text || '-default' AS slug,
121+
organization.display_name,
122+
'unlisted'::directory_visibility AS directory_visibility,
123+
organization.id AS auto_provisioned_for_organization_id,
124+
now() AS created_at,
125+
now() AS updated_at
126+
FROM organization
127+
WHERE NOT EXISTS (
128+
SELECT 1
129+
FROM project existing
130+
WHERE existing.auto_provisioned_for_organization_id = organization.id
131+
);
132+
133+
INSERT INTO project_organization_ownership (
134+
project_id,
135+
organization_id,
136+
created_at
137+
)
138+
SELECT
139+
project.id,
140+
project.auto_provisioned_for_organization_id,
141+
now()
142+
FROM project
143+
WHERE project.auto_provisioned_for_organization_id IS NOT NULL
144+
ON CONFLICT (project_id, organization_id) DO NOTHING;
145+
146+
UPDATE conversation
147+
SET project_id = project.id
148+
FROM project
149+
WHERE conversation.organization_id IS NOT NULL
150+
AND project.auto_provisioned_for_organization_id = conversation.organization_id
151+
AND conversation.project_id IS NULL;
152+
153+
UPDATE conversation
154+
SET project_id = project.id
155+
FROM organization
156+
JOIN project
157+
ON project.auto_provisioned_for_organization_id = organization.id
158+
WHERE conversation.organization_id IS NULL
159+
AND conversation.author_id = organization.auto_provisioned_for_user_id
160+
AND conversation.project_id IS NULL;
161+
162+
INSERT INTO organization_membership_capability (
163+
organization_membership_id,
164+
capability,
165+
created_at
166+
)
167+
SELECT
168+
organization_membership.id,
169+
capability.capability,
170+
now()
171+
FROM organization_membership
172+
CROSS JOIN unnest(ARRAY[
173+
'organization_manage_members'::organization_membership_capability_enum,
174+
'organization_manage_profile'::organization_membership_capability_enum,
175+
'project_create'::organization_membership_capability_enum
176+
]) AS capability(capability)
177+
ON CONFLICT (organization_membership_id, capability) DO NOTHING;
178+
179+
INSERT INTO organization_membership_all_project_capability (
180+
organization_membership_id,
181+
capability,
182+
created_at
183+
)
184+
SELECT
185+
organization_membership.id,
186+
capability.capability,
187+
now()
188+
FROM organization_membership
189+
CROSS JOIN unnest(ARRAY[
190+
'project_update'::organization_membership_all_project_capability_enum,
191+
'project_delete'::organization_membership_all_project_capability_enum,
192+
'project_manage_owner_organizations'::organization_membership_all_project_capability_enum,
193+
'conversation_create'::organization_membership_all_project_capability_enum,
194+
'conversation_update'::organization_membership_all_project_capability_enum,
195+
'conversation_delete'::organization_membership_all_project_capability_enum,
196+
'conversation_view_private_results'::organization_membership_all_project_capability_enum,
197+
'conversation_export_owner_data'::organization_membership_all_project_capability_enum,
198+
'conversation_moderate'::organization_membership_all_project_capability_enum,
199+
'conversation_manage_integrations'::organization_membership_all_project_capability_enum
200+
]) AS capability(capability)
201+
ON CONFLICT (organization_membership_id, capability) DO NOTHING;
202+
203+
UPDATE premium_feature_entitlement
204+
SET
205+
organization_id = organization.id,
206+
user_id = NULL,
207+
updated_at = now()
208+
FROM organization
209+
WHERE premium_feature_entitlement.user_id = organization.auto_provisioned_for_user_id
210+
AND premium_feature_entitlement.organization_id IS NULL;
211+
212+
DO $$
213+
DECLARE
214+
missing_first_names integer;
215+
missing_organization_fields integer;
216+
missing_personal_memberships integer;
217+
missing_project_ownerships integer;
218+
missing_conversation_projects integer;
219+
missing_entitlement_organizations integer;
220+
BEGIN
221+
SELECT count(*) INTO missing_first_names
222+
FROM "user"
223+
WHERE first_name IS NULL;
224+
225+
IF missing_first_names > 0 THEN
226+
RAISE EXCEPTION 'org/project backfill failed: % users still have null first_name', missing_first_names;
227+
END IF;
228+
229+
SELECT count(*) INTO missing_organization_fields
230+
FROM organization
231+
WHERE slug IS NULL
232+
OR display_name IS NULL
233+
OR directory_visibility IS NULL;
234+
235+
IF missing_organization_fields > 0 THEN
236+
RAISE EXCEPTION 'org/project backfill failed: % organizations still have missing identity fields', missing_organization_fields;
237+
END IF;
238+
239+
SELECT count(*) INTO missing_personal_memberships
240+
FROM organization
241+
LEFT JOIN organization_membership
242+
ON organization_membership.organization_id = organization.id
243+
AND organization_membership.user_id = organization.auto_provisioned_for_user_id
244+
WHERE organization.auto_provisioned_for_user_id IS NOT NULL
245+
AND organization_membership.id IS NULL;
246+
247+
IF missing_personal_memberships > 0 THEN
248+
RAISE EXCEPTION 'org/project backfill failed: % personal organizations lack owner membership', missing_personal_memberships;
249+
END IF;
250+
251+
SELECT count(*) INTO missing_project_ownerships
252+
FROM project
253+
LEFT JOIN project_organization_ownership
254+
ON project_organization_ownership.project_id = project.id
255+
AND project_organization_ownership.organization_id = project.auto_provisioned_for_organization_id
256+
WHERE project.auto_provisioned_for_organization_id IS NOT NULL
257+
AND project_organization_ownership.id IS NULL;
258+
259+
IF missing_project_ownerships > 0 THEN
260+
RAISE EXCEPTION 'org/project backfill failed: % auto-provisioned projects lack ownership rows', missing_project_ownerships;
261+
END IF;
262+
263+
SELECT count(*) INTO missing_conversation_projects
264+
FROM conversation
265+
WHERE project_id IS NULL;
266+
267+
IF missing_conversation_projects > 0 THEN
268+
RAISE EXCEPTION 'org/project backfill failed: % conversations still have null project_id', missing_conversation_projects;
269+
END IF;
270+
271+
SELECT count(*) INTO missing_entitlement_organizations
272+
FROM premium_feature_entitlement
273+
WHERE organization_id IS NULL;
274+
275+
IF missing_entitlement_organizations > 0 THEN
276+
RAISE EXCEPTION 'org/project backfill failed: % premium entitlements still have null organization_id', missing_entitlement_organizations;
277+
END IF;
278+
END $$;

‎services/api/database/flyway/V0071__abnormal_dakota_north.sql‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
CREATE TYPE "public"."directory_visibility" AS ENUM('listed', 'unlisted');--> statement-breakpoint
2+
CREATE TYPE "public"."organization_membership_all_project_capability_enum" AS ENUM('project_update', 'project_delete', 'project_manage_owner_organizations', 'conversation_create', 'conversation_update', 'conversation_delete', 'conversation_view_private_results', 'conversation_export_owner_data', 'conversation_moderate', 'conversation_manage_integrations');--> statement-breakpoint
3+
CREATE TYPE "public"."organization_membership_capability_enum" AS ENUM('organization_manage_members', 'organization_manage_profile', 'project_create');--> statement-breakpoint
4+
CREATE TABLE "organization_membership_all_project_capability" (
5+
"id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "organization_membership_all_project_capability_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
6+
"organization_membership_id" integer NOT NULL,
7+
"capability" "organization_membership_all_project_capability_enum" NOT NULL,
8+
"created_at" timestamp (0) DEFAULT now() NOT NULL,
9+
CONSTRAINT "organization_membership_all_project_capability_unique" UNIQUE("organization_membership_id","capability")
10+
);
11+
--> statement-breakpoint
12+
CREATE TABLE "organization_membership_capability" (
13+
"id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "organization_membership_capability_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
14+
"organization_membership_id" integer NOT NULL,
15+
"capability" "organization_membership_capability_enum" NOT NULL,
16+
"created_at" timestamp (0) DEFAULT now() NOT NULL,
17+
CONSTRAINT "organization_membership_capability_unique" UNIQUE("organization_membership_id","capability")
18+
);
19+
--> statement-breakpoint
20+
CREATE TABLE "organization_membership" (
21+
"id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "organization_membership_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
22+
"user_id" uuid NOT NULL,
23+
"organization_id" integer NOT NULL,
24+
"created_at" timestamp (0) DEFAULT now() NOT NULL,
25+
"updated_at" timestamp (0) DEFAULT now() NOT NULL,
26+
CONSTRAINT "organization_membership_user_organization_unique" UNIQUE("user_id","organization_id")
27+
);
28+
--> statement-breakpoint
29+
CREATE TABLE "project_organization_ownership" (
30+
"id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "project_organization_ownership_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
31+
"project_id" integer NOT NULL,
32+
"organization_id" integer NOT NULL,
33+
"created_at" timestamp (0) DEFAULT now() NOT NULL,
34+
CONSTRAINT "project_organization_ownership_project_org_unique" UNIQUE("project_id","organization_id")
35+
);
36+
--> statement-breakpoint
37+
CREATE TABLE "project" (
38+
"id" integer PRIMARY KEY GENERATED ALWAYS AS IDENTITY (sequence name "project_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 2147483647 START WITH 1 CACHE 1),
39+
"slug" varchar(65) NOT NULL,
40+
"display_name" varchar(65) NOT NULL,
41+
"directory_visibility" "directory_visibility" NOT NULL,
42+
"auto_provisioned_for_organization_id" integer,
43+
"created_at" timestamp (0) DEFAULT now() NOT NULL,
44+
"updated_at" timestamp (0) DEFAULT now() NOT NULL,
45+
CONSTRAINT "project_slug_unique" UNIQUE("slug"),
46+
CONSTRAINT "project_auto_provisioned_for_organization_id_unique" UNIQUE("auto_provisioned_for_organization_id")
47+
);
48+
--> statement-breakpoint
49+
ALTER TABLE "conversation" ADD COLUMN "project_id" integer;--> statement-breakpoint
50+
ALTER TABLE "organization" ADD COLUMN "slug" varchar(65);--> statement-breakpoint
51+
ALTER TABLE "organization" ADD COLUMN "display_name" varchar(65);--> statement-breakpoint
52+
ALTER TABLE "organization" ADD COLUMN "directory_visibility" "directory_visibility";--> statement-breakpoint
53+
ALTER TABLE "organization" ADD COLUMN "auto_provisioned_for_user_id" uuid;--> statement-breakpoint
54+
ALTER TABLE "user" ADD COLUMN "first_name" varchar(65);--> statement-breakpoint
55+
ALTER TABLE "organization_membership_all_project_capability" ADD CONSTRAINT "organization_membership_all_project_capability_organization_membership_id_organization_membership_id_fk" FOREIGN KEY ("organization_membership_id") REFERENCES "public"."organization_membership"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
56+
ALTER TABLE "organization_membership_capability" ADD CONSTRAINT "organization_membership_capability_organization_membership_id_organization_membership_id_fk" FOREIGN KEY ("organization_membership_id") REFERENCES "public"."organization_membership"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
57+
ALTER TABLE "organization_membership" ADD CONSTRAINT "organization_membership_user_id_user_id_fk" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
58+
ALTER TABLE "organization_membership" ADD CONSTRAINT "organization_membership_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
59+
ALTER TABLE "project_organization_ownership" ADD CONSTRAINT "project_organization_ownership_project_id_project_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."project"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
60+
ALTER TABLE "project_organization_ownership" ADD CONSTRAINT "project_organization_ownership_organization_id_organization_id_fk" FOREIGN KEY ("organization_id") REFERENCES "public"."organization"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
61+
ALTER TABLE "project" ADD CONSTRAINT "project_auto_provisioned_for_organization_id_organization_id_fk" FOREIGN KEY ("auto_provisioned_for_organization_id") REFERENCES "public"."organization"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
62+
CREATE INDEX "organization_membership_organization_idx" ON "organization_membership" USING btree ("organization_id");--> statement-breakpoint
63+
CREATE INDEX "project_organization_ownership_organization_idx" ON "project_organization_ownership" USING btree ("organization_id");--> statement-breakpoint
64+
ALTER TABLE "conversation" ADD CONSTRAINT "conversation_project_id_project_id_fk" FOREIGN KEY ("project_id") REFERENCES "public"."project"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
65+
ALTER TABLE "organization" ADD CONSTRAINT "organization_auto_provisioned_for_user_id_user_id_fk" FOREIGN KEY ("auto_provisioned_for_user_id") REFERENCES "public"."user"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint
66+
CREATE INDEX "conversation_project_id_idx" ON "conversation" USING btree ("project_id");--> statement-breakpoint
67+
CREATE INDEX "conversation_project_timeline_idx" ON "conversation" USING btree ("project_id","is_importing","created_at" DESC,"id" DESC) WHERE "conversation"."current_content_id" is not null;--> statement-breakpoint
68+
ALTER TABLE "organization" ADD CONSTRAINT "organization_slug_unique" UNIQUE("slug");--> statement-breakpoint
69+
ALTER TABLE "organization" ADD CONSTRAINT "organization_auto_provisioned_for_user_id_unique" UNIQUE("auto_provisioned_for_user_id");

‎services/api/database/flyway/V0072__nervous_killraven.sql‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
ALTER TABLE "user_organization_mapping" DISABLE ROW LEVEL SECURITY;--> statement-breakpoint
2+
DROP TABLE "user_organization_mapping" CASCADE;--> statement-breakpoint
3+
ALTER TABLE "organization" DROP CONSTRAINT "organization_name_unique";--> statement-breakpoint
4+
ALTER TABLE "premium_feature_entitlement" DROP CONSTRAINT "premium_feature_entitlement_single_subject_check";--> statement-breakpoint
5+
ALTER TABLE "conversation" DROP CONSTRAINT "conversation_author_id_user_id_fk";
6+
--> statement-breakpoint
7+
ALTER TABLE "conversation" DROP CONSTRAINT "conversation_organization_id_organization_id_fk";
8+
--> statement-breakpoint
9+
ALTER TABLE "premium_feature_entitlement" DROP CONSTRAINT "premium_feature_entitlement_user_id_user_id_fk";
10+
--> statement-breakpoint
11+
DROP INDEX "conversation_author_timeline_idx";--> statement-breakpoint
12+
DROP INDEX "conversation_organization_timeline_idx";--> statement-breakpoint
13+
DROP INDEX "premium_feature_entitlement_user_idx";--> statement-breakpoint
14+
ALTER TABLE "conversation" ALTER COLUMN "project_id" SET NOT NULL;--> statement-breakpoint
15+
ALTER TABLE "organization" ALTER COLUMN "slug" SET NOT NULL;--> statement-breakpoint
16+
ALTER TABLE "organization" ALTER COLUMN "display_name" SET NOT NULL;--> statement-breakpoint
17+
ALTER TABLE "organization" ALTER COLUMN "directory_visibility" SET NOT NULL;--> statement-breakpoint
18+
ALTER TABLE "premium_feature_entitlement" ALTER COLUMN "organization_id" SET NOT NULL;--> statement-breakpoint
19+
ALTER TABLE "user" ALTER COLUMN "first_name" SET NOT NULL;--> statement-breakpoint
20+
ALTER TABLE "conversation" DROP COLUMN "author_id";--> statement-breakpoint
21+
ALTER TABLE "conversation" DROP COLUMN "organization_id";--> statement-breakpoint
22+
ALTER TABLE "organization" DROP COLUMN "name";--> statement-breakpoint
23+
ALTER TABLE "premium_feature_entitlement" DROP COLUMN "user_id";

0 commit comments

Comments
 (0)