test(tk:backend): defined profile parser e2e suite

Author: ascariandrea

platforms/tktrex/backend/__tests__/fixtures/profile/876a43574215629696bccb876a6f7660d3dc2dc9.json

@@ -0,0 +1,125 @@
+import {
+  readFixtureJSON,
+  readFixtureJSONPaths,
+  runParserTest,
+} from '@shared/test/utils/parser.utils';
+import { v4 as uuid } from 'uuid';
+import { TKMetadata } from '@tktrex/shared/models';
+import { parsers } from '@tktrex/shared/parser/parsers';
+import base58 from 'bs58';
+import { parseISO, subMinutes } from 'date-fns';
+import path from 'path';
+import nacl from 'tweetnacl';
+import { GetTest, Test } from '../test/Test';
+import {
+  addDom,
+  getLastHTMLs,
+  getMetadata,
+  getMetadataSchema,
+  getSourceSchema,
+  parserConfig,
+  updateMetadataAndMarkHTML,
+} from '../lib/parser';
+import { HTMLSource } from '@tktrex/shared/parser/source';
+import { toMetadata } from '@tktrex/shared/parser/metadata';
+
+describe('Parser: "profile"', () => {
+  let appTest: Test;
+  const newKeypair = nacl.sign.keyPair();
+  const publicKey = base58.encode(newKeypair.publicKey);
+
+  let db: any;
+  beforeAll(async () => {
+    appTest = await GetTest();
+
+    db = {
+      api: appTest.mongo3,
+      read: appTest.mongo,
+      write: appTest.mongo,
+    };
+  });
+
+  afterEach(async () => {
+    await appTest.mongo3.deleteMany(
+      appTest.mongo,
+      appTest.config.get('schema').htmls,
+      {
+        publicKey: {
+          $eq: publicKey,
+        },
+      }
+    );
+  });
+
+  describe('Nature Profile', () => {
+    jest.setTimeout(20 * 1000);
+
+    const history = readFixtureJSONPaths(
+      path.resolve(__dirname, 'fixtures/profile')
+    );
+
+    test.each(history)(
+      'Should correctly parse "profile" contribution from path %s',
+      async (fixturePath) => {
+        const { sources: _sources, metadata } = readFixtureJSON(
+          fixturePath,
+          publicKey
+        );
+        const sources = _sources.map((s: any) => ({
+          html: {
+            ...s,
+            id: uuid(),
+            clientTime: parseISO(s.clientTime ?? new Date().toISOString()),
+            savingTime: subMinutes(new Date(), 2),
+          },
+          supporter: { version: process.env.VERSION },
+        }));
+
+        await runParserTest({
+          name: 'native-parser',
+          log: appTest.logger,
+          parsers: parsers,
+          db,
+          codecs: {
+            contribution: HTMLSource,
+            metadata: TKMetadata.TKMetadata,
+          },
+          addDom,
+          sourceSchema: getSourceSchema(),
+          metadataSchema: getMetadataSchema(),
+          getEntryId: (e) => e.html.id,
+          getEntryDate: (e) => e.html.savingTime,
+          getEntryNatureType: (e) => e.html.type,
+          getContributions: getLastHTMLs(db),
+          getMetadata: getMetadata(db),
+          saveResults: updateMetadataAndMarkHTML(db),
+          buildMetadata: toMetadata,
+          config: parserConfig,
+          expectSources: (receivedSources) => {
+            receivedSources.forEach((s) => {
+              expect((s as any).processed).toBe(true);
+            });
+          },
+          expectMetadata: (receivedMetadata, expectedMetadata) => {
+            const {
+              _id: received_Id,
+              id: receivedId,
+              savingTime: savingTimeR,
+              ...receivedM
+            } = receivedMetadata as any;
+
+            const {
+              _id: _received_Id,
+              id: _receivedId,
+              clientTime: clientTimeExp,
+              savingTime: savingTimeExp,
+              ...expectedM
+            } = expectedMetadata as any;
+
+            expect(receivedM).toMatchObject(expectedM);
+          },
+        })({ sources, metadata });
+      }
+    );
+  });
+});

platforms/tktrex/backend/__tests__/profile.e2e.ts

@@ -62,7 +62,7 @@ describe('Metadata API', () => {
       const expectedMetadata = metadataWithResearchTag
         .sort((a, b) => b.savingTime.getTime() - a.savingTime.getTime())
         .slice(0, amount)
-        .map(({ _id, ...m }) => {
+        .map((m) => {
           return {
             ...m,
             clientTime: m.clientTime.toISOString(),

platforms/tktrex/backend/routes/__tests__/metadata.e2e.ts

@@ -12,9 +12,9 @@ import { ContributionEventArb } from '@tktrex/shared/arbitraries/ContributionEve
 import { TKMetadata } from '@tktrex/shared/models/Metadata';
 import { parsers } from '@tktrex/shared/parser/parsers';
 import { HTMLSource } from '@tktrex/shared/parser/source';
+import { toMetadata } from '@tktrex/shared/parser/metadata';
 import {
   addDom,
-  buildMetadata,
   getLastHTMLs,
   getMetadata,
   updateMetadataAndMarkHTML,
@@ -60,7 +60,7 @@ describe('/v2/personal', () => {
       getMetadata: getMetadata(db),
       getEntryDate: (e) => e.html.savingTime,
       getEntryNatureType: (e) => e.html.type,
-      buildMetadata: buildMetadata,
+      buildMetadata: toMetadata,
       saveResults: updateMetadataAndMarkHTML(db),
     });
   });

platforms/tktrex/backend/routes/__tests__/personal.e2e.ts

@@ -1,5 +1,5 @@
 import * as t from 'io-ts';
-import { FollowingN, ForYouN, SearchN, NativeVideoN } from './Nature';
+import { FollowingN, ForYouN, SearchN, NativeVideoN, ProfileN } from './Nature';
 
 const Music = t.type(
   {
@@ -54,6 +54,7 @@ export const MetadataBase = t.type(
     clientTime: t.string,
     savingTime: t.string,
     publicKey: t.string,
+    timelineId: t.string,
   },
   'MetadataBase',
 );
@@ -165,8 +166,26 @@ export const NativeMetadata = t.intersection(
 
 export type NativeMetadata = t.TypeOf<typeof NativeMetadata>;
 
+export const ProfileMetadata = t.intersection(
+  [
+    MetadataBase,
+    ProfileN,
+    t.type({
+      videos: t.array(t.any),
+    }),
+  ],
+  'ProfileMetadata',
+);
+export type ProfileMetadata = t.TypeOf<typeof ProfileMetadata>;
+
 export const TKMetadata = t.union(
-  [ForYouVideoMetadata, FollowingVideoMetadata, SearchMetadata, NativeMetadata],
+  [
+    ForYouVideoMetadata,
+    FollowingVideoMetadata,
+    SearchMetadata,
+    NativeMetadata,
+    ProfileMetadata,
+  ],
   'TKMetadata',
 );
 

platforms/tktrex/shared/src/models/Metadata.ts

@@ -25,14 +25,6 @@ export const FollowingN = t.type(
 
 export type FollowingN = t.TypeOf<typeof FollowingN>;
 
-export const CreatorN = t.type(
-  {
-    type: CreatorType,
-  },
-  'CreatorN',
-);
-export type CreatorN = t.TypeOf<typeof CreatorN>;
-
 export const VideoN = t.type(
   {
     type: VideoType,
@@ -65,7 +57,7 @@ export type NativeVideoN = t.TypeOf<typeof NativeVideoN>;
 
 export const ProfileN = t.type(
   {
-    type: t.literal('profile'),
+    type: t.literal('creator'),
     creatorName: t.string,
   },
   'ProfileN',
@@ -83,16 +75,7 @@ export const HashtagsN = t.type(
 export type HashtagsN = t.TypeOf<typeof HashtagsN>;
 
 export const Nature = t.union(
-  [
-    ForYouN,
-    FollowingN,
-    CreatorN,
-    VideoN,
-    SearchN,
-    ProfileN,
-    HashtagsN,
-    NativeVideoN,
-  ],
+  [ForYouN, FollowingN, VideoN, SearchN, ProfileN, HashtagsN, NativeVideoN],
   'Nature',
 );
 

platforms/tktrex/shared/src/models/Nature.ts

@@ -54,9 +54,10 @@ export const toMetadata: BuildMetadataFn<HTMLSource, TKMetadata, TKParsers> = (
     metadata.nature.query = metadata.query;
     break;
   }
-  case 'profile': {
+  case 'creator': {
     const { nature, profile, downloader } = entry.findings;
     metadata = {
+      videos: [],
       ...metadata,
       nature,
       ...nature,