fix(api): Audit and fix schema to reflect real db

I went through a copy of the prod db and found lots of optional fields that aren't marked as such, old documents missing fields, etc.

Introduces an IPostInput type that accounts for the mongoose schema doing defaults and fixes, while also allowing IPost to reflect the real db type.

Groundwork for a proper DTO structure.

Author: ashquarky

apps/miiverse-api/package.json

@@ -26,7 +26,7 @@
 		"express-subdomain": "^1.0.5",
 		"fs-extra": "^9.0.0",
 		"moment": "^2.24.0",
-		"mongoose": "^6.10.1",
+		"mongoose": "^8.15.1",
 		"multer": "^1.4.5-lts.1",
 		"nice-grpc": "^2.1.12",
 		"node-snowflake": "0.0.1",

apps/miiverse-api/src/database.ts

@@ -11,7 +11,7 @@ import type { HydratedConversationDocument } from '@/types/mongoose/conversation
 import type { HydratedContentDocument } from '@/types/mongoose/content';
 import type { HydratedSettingsDocument } from '@/types/mongoose/settings';
 import type { HydratedEndpointDocument } from '@/types/mongoose/endpoint';
-import type { HydratedPostDocument, IPost } from '@/types/mongoose/post';
+import type { HydratedPostDocument, IPostInput } from '@/types/mongoose/post';
 import type { HydratedCommunityDocument } from '@/types/mongoose/community';
 
 let connection: mongoose.Connection;
@@ -97,7 +97,7 @@ export async function getPostReplies(postID: string, limit: number): Promise<Hyd
 	}).limit(limit);
 }
 
-export async function getDuplicatePosts(pid: number, post: IPost): Promise<HydratedPostDocument | null> {
+export async function getDuplicatePosts(pid: number, post: IPostInput): Promise<HydratedPostDocument | null> {
 	verifyConnected();
 
 	return Post.findOne({

apps/miiverse-api/src/models/post.ts

@@ -7,29 +7,28 @@ import type { HydratedCommunityDocument } from '@/types/mongoose/community';
 import type { PostToJSONOptions } from '@/types/mongoose/post-to-json-options';
 import type { PostData, PostPainting, PostScreenshot, PostTopicTag } from '@/types/miiverse/post';
 
+/* Constraints here (default, required etc.) apply to new documents being added
+ * See IPost for expected shape of query results
+ * If you add default: or required:, please also update IPost and IPostInput!
+ */
 const PostSchema = new Schema<IPost, PostModel, IPostMethods>({
-	id: String,
-	title_id: String,
-	screen_name: String,
-	body: String,
-	app_data: String,
-	painting: String,
-	screenshot: String,
-	screenshot_length: Number,
-	search_key: {
-		type: [String],
-		default: undefined
-	},
-	topic_tag: {
-		type: String,
-		default: undefined
-	},
-	community_id: {
-		type: String,
-		default: undefined
-	},
-	created_at: Date,
-	feeling_id: Number,
+	id: { type: String, required: true },
+	title_id: { type: String },
+	screen_name: { type: String, required: true },
+	body: { type: String, required: true },
+	app_data: { type: String },
+
+	painting: { type: String },
+	screenshot: { type: String },
+	screenshot_length: { type: Number },
+
+	search_key: { type: [String] },
+	topic_tag: { type: String },
+
+	community_id: { type: String, required: true },
+	created_at: { type: Date, required: true },
+	feeling_id: { type: Number },
+
 	is_autopost: {
 		type: Number,
 		default: 0
@@ -46,6 +45,7 @@ const PostSchema = new Schema<IPost, PostModel, IPostMethods>({
 		type: Number,
 		default: 0
 	},
+
 	empathy_count: {
 		type: Number,
 		default: 0,
@@ -59,12 +59,15 @@ const PostSchema = new Schema<IPost, PostModel, IPostMethods>({
 		type: Number,
 		default: 1
 	},
-	mii: String,
-	mii_face_url: String,
-	pid: Number,
-	platform_id: Number,
-	region_id: Number,
-	parent: String,
+
+	mii: { type: String, required: true },
+	mii_face_url: { type: String, required: true },
+
+	pid: { type: Number, required: true },
+	platform_id: { type: Number },
+	region_id: { type: Number },
+	parent: { type: String },
+
 	reply_count: {
 		type: Number,
 		default: 0
@@ -73,17 +76,21 @@ const PostSchema = new Schema<IPost, PostModel, IPostMethods>({
 		type: Boolean,
 		default: false
 	},
+
 	message_to_pid: {
 		type: String,
 		default: null
 	},
+
 	removed: {
 		type: Boolean,
 		default: false
 	},
-	removed_reason: String,
-	yeahs: [Number],
-	number: Number
+	removed_reason: { type: String },
+	removed_by: { type: Number },
+	removed_at: { type: Date },
+
+	yeahs: { type: [Number], default: [] }
 }, {
 	id: false // * Disables the .id() getter used by Mongoose in TypeScript. Needed to have our own .id field
 });
@@ -114,12 +121,12 @@ PostSchema.method<HydratedPostDocument>('cleanedMiiData', function cleanedMiiDat
 	return this.mii.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
 });
 
-PostSchema.method<HydratedPostDocument>('cleanedPainting', function cleanedPainting(): string {
-	return this.painting.replace(/[\n\r]+/gm, '').trim();
+PostSchema.method<HydratedPostDocument>('cleanedPainting', function cleanedPainting(): string | undefined {
+	return this.painting?.replace(/[\n\r]+/gm, '').trim();
 });
 
-PostSchema.method<HydratedPostDocument>('cleanedAppData', function cleanedAppData(): string {
-	return this.app_data.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
+PostSchema.method<HydratedPostDocument>('cleanedAppData', function cleanedAppData(): string | undefined {
+	return this.app_data?.replace(/[^A-Za-z0-9+/=]/g, '').replace(/[\n\r]+/gm, '').trim();
 });
 
 PostSchema.method<HydratedPostDocument>('formatPainting', function formatPainting(): PostPainting | undefined {
@@ -146,7 +153,7 @@ PostSchema.method<HydratedPostDocument>('formatTopicTag', function formatTopicTa
 	if (this.topic_tag?.trim()) {
 		return {
 			name: this.topic_tag,
-			title_id: this.title_id
+			title_id: this.title_id ?? ''
 		};
 	}
 });
@@ -158,26 +165,26 @@ PostSchema.method<HydratedPostDocument>('json', function json(options: PostToJSO
 		community_id: this.community_id, // TODO - This sucks
 		country_id: this.country_id,
 		created_at: moment(this.created_at).format('YYYY-MM-DD HH:MM:SS'),
-		feeling_id: this.feeling_id,
+		feeling_id: this.feeling_id ?? 0,
 		id: this.id,
 		is_autopost: this.is_autopost ? 1 : 0,
 		is_community_private_autopost: this.is_community_private_autopost ? 1 : 0,
 		is_spoiler: this.is_spoiler ? 1 : 0,
 		is_app_jumpable: this.is_app_jumpable ? 1 : 0,
-		empathy_count: this.empathy_count || 0,
+		empathy_count: this.empathy_count,
 		language_id: this.language_id,
 		mii: undefined, // * Conditionally set later
 		mii_face_url: undefined, // * Conditionally set later
 		number: 0,
 		painting: this.formatPainting(),
 		pid: this.pid,
-		platform_id: this.platform_id,
-		region_id: this.region_id,
-		reply_count: this.reply_count || 0,
+		platform_id: this.platform_id ?? 1,
+		region_id: this.region_id ?? 0,
+		reply_count: this.reply_count,
 		screen_name: this.screen_name,
 		screenshot: this.formatScreenshot(),
 		topic_tag: undefined, // * Conditionally set later
-		title_id: this.title_id
+		title_id: this.title_id ?? ''
 	};
 
 	if (options.app_data) {

apps/miiverse-api/src/services/api/routes/communities.ts

@@ -298,7 +298,7 @@ router.post('/', multer().none(), async function (request: express.Request, resp
 		return badRequest(response, ApiErrorCode.CREATE_TOO_MANY_FAVORITES, 403);
 	}
 
-	const communitiesCount = await Community.count();
+	const communitiesCount = await Community.countDocuments();
 	const communityID = (parseInt(parentCommunity.community_id) + (5000 * communitiesCount)); // Change this to auto increment
 	const community = await Community.create({
 		platform_id: 0, // WiiU

apps/miiverse-api/src/services/api/routes/friend_messages.ts

@@ -245,20 +245,20 @@ router.get('/', async function (request: express.Request, response: express.Resp
 				is_app_jumpable: message.is_app_jumpable,
 				empathy_added: message.empathy_count,
 				language_id: message.language_id,
-				message_to_pid: message.message_to_pid,
+				message_to_pid: message.message_to_pid ?? undefined,
 				mii: message.mii,
 				mii_face_url: message.mii_face_url,
-				number: message.number || 0,
+				number: 0,
 				pid: message.pid,
 				platform_id: message.platform_id || 0,
 				region_id: message.region_id || 0,
 				reply_count: message.reply_count,
 				screen_name: message.screen_name,
 				topic_tag: {
-					name: message.topic_tag,
+					name: message.topic_tag ?? '',
 					title_id: 0
 				},
-				title_id: message.title_id
+				title_id: message.title_id ?? ''
 			}
 		});
 	}

apps/miiverse-api/src/services/api/routes/posts.ts

@@ -22,7 +22,7 @@ import { Community } from '@/models/community';
 import { config } from '@/config';
 import { ApiErrorCode, badRequest, serverError } from '@/errors';
 import type { PostRepliesResult } from '@/types/miiverse/post';
-import type { HydratedPostDocument } from '@/types/mongoose/post';
+import type { HydratedPostDocument, IPostInput } from '@/types/mongoose/post';
 
 const newPostSchema = z.object({
 	community_id: z.string().optional(),
@@ -312,7 +312,7 @@ async function newPost(request: express.Request, response: express.Response): Pr
 		searchKey = [searchKey];
 	}
 
-	const document = {
+	const document: IPostInput = {
 		id: '', // * This gets changed when saving the document for the first time
 		title_id: request.paramPack.title_id,
 		community_id: community.olive_community_id,

apps/miiverse-api/src/types/mongoose/post.ts

@@ -3,41 +3,73 @@ import type { HydratedCommunityDocument } from '@/types/mongoose/community';
 import type { PostToJSONOptions } from '@/types/mongoose/post-to-json-options';
 import type { PostData, PostPainting, PostScreenshot, PostTopicTag } from '@/types/miiverse/post';
 
+/* This type needs to reflect "reality" as it is in the DB
+ * Thus, all the optionals, since some legacy documents are missing many fields
+ */
 export interface IPost {
 	id: string;
-	title_id: string;
+	title_id?: string; // u64
 	screen_name: string;
 	body: string;
-	app_data: string;
-	painting: string;
-	screenshot: string;
-	screenshot_length: number;
-	search_key: string[];
-	topic_tag: string;
-	community_id: string;
+	app_data?: string; // nintendo base64
+
+	painting?: string; // base64, can be empty or undefined
+	screenshot?: string; // url fragment (leading /), can be empty or undefined
+	screenshot_length?: number;
+
+	search_key?: string[]; // can be empty or undefined
+	topic_tag?: string;
+
+	community_id: string; // actually a number
 	created_at: Date;
-	feeling_id: number;
-	is_autopost: number;
-	is_community_private_autopost?: number;
-	is_spoiler: number;
-	is_app_jumpable: number;
-	empathy_count?: number;
+	feeling_id?: number; // missing on PMs
+
+	is_autopost: number; // 0 | 1
+	is_community_private_autopost: number; // 0 | 1
+	is_spoiler: number; // 0 | 1
+	is_app_jumpable: number; // 0 | 1
+
+	empathy_count: number;
 	country_id: number;
 	language_id: number;
-	mii: string;
-	mii_face_url: string;
+
+	mii: string; // nintendo base64
+	mii_face_url: string; // fully qualified (usually cdn. or r2-cdn.)
+
 	pid: number;
-	platform_id: number;
-	region_id: number;
-	parent: string;
-	reply_count?: number;
+	platform_id?: number; // missing on PMs
+	region_id?: number; // missing on PMs
+	parent?: string | null; // both undef and null exist in db
+
+	reply_count: number;
 	verified: boolean;
-	message_to_pid?: string;
+
+	message_to_pid: string | null;
+
 	removed: boolean;
 	removed_reason?: string;
-	yeahs?: Types.Array<number>;
-	number?: number;
+	removed_by?: number;
+	removed_at?: Date;
+
+	yeahs: Types.Array<number>;
 }
+// Fields that have "default: " in the Mongoose schema should also be listed here to make them optional
+// on input but not output
+// We really need an ORM
+type PostDefaultedFields =
+	'is_autopost' |
+	'is_community_private_autopost' |
+	'is_spoiler' |
+	'is_app_jumpable' |
+	'empathy_count' |
+	'country_id' |
+	'language_id' |
+	'reply_count' |
+	'verified' |
+	'message_to_pid' |
+	'removed' |
+	'yeahs';
+export type IPostInput = Omit<IPost, PostDefaultedFields> & Partial<Pick<IPost, PostDefaultedFields>>;
 
 export interface IPostMethods {
 	del(reason: string): Promise<void>;
@@ -52,6 +84,6 @@ export interface IPostMethods {
 	json(options: PostToJSONOptions, community?: HydratedCommunityDocument): PostData;
 }
 
-export type PostModel = Model<IPost, object, IPostMethods>;
+export type PostModel = Model<IPost, {}, IPostMethods>;
 
 export type HydratedPostDocument = HydratedDocument<IPost, IPostMethods>;