feat: tiktok profile scraper (#531)

Co-authored-by: ascariandrea <[email protected]>

Author: vecna

Diff for: packages/shared/src/extension/app.ts

@@ -108,14 +108,11 @@ function setupObserver({
 
   const observer = new MutationObserver(
     debounce((mutations) => {
-      mutations.forEach(function (mutation) {
-        // appLog.debug('mutation (%s) %O', mutation.type, mutation.target);
-
-        if (window?.document) {
-          if (
-            oldHref !== window.location.href &&
-            platformMatch.test(window.location.href)
-          ) {
+      // appLog.debug('mutation (%s) %O', mutation.type, mutation.target);
+
+      if (window?.document) {
+        if (platformMatch.test(window.location.href)) {
+          if (oldHref !== window.location.href) {
             const newHref = window.location.href;
 
             appLog.debug(
@@ -126,25 +123,25 @@ function setupObserver({
 
             onLocationChange(oldHref, newHref);
 
-            const routeHandlerKey = handlersList.find((h) => {
-              const handler = handlers[h];
+            oldHref = newHref;
+          }
 
-              if (handler.match.type === 'route') {
-                return window.location.pathname.match(handler.match.location);
-              }
-              return false;
-            });
+          const routeHandlerKey = handlersList.find((h) => {
+            const handler = handlers[h];
 
-            if (routeHandlerKey) {
-              appLog.debug('Route handler key %s', routeHandlerKey);
-              const { handle, ...routeHandlerOpts } = handlers[routeHandlerKey];
-              handle(window.document.body, routeHandlerOpts, routeHandlerKey);
+            if (handler.match.type === 'route') {
+              return window.location.pathname.match(handler.match.location);
             }
+            return false;
+          });
 
-            oldHref = newHref;
+          if (routeHandlerKey) {
+            appLog.debug('Route handler key %s', routeHandlerKey);
+            const { handle, ...routeHandlerOpts } = handlers[routeHandlerKey];
+            handle(window.document.body, routeHandlerOpts, routeHandlerKey);
           }
         }
-      });
+      }
     }, 300)
   );
 

Diff for: platforms/tktrex/backend/lib/parserchain.js

@@ -14,6 +14,7 @@ const parserList = {
   stitch: require('../parsers/stitch'),
   author: require('../parsers/author'),
   search: require('../parsers/search'),
+  profile: require('../parsers/profile'),
   downloader: require('../parsers/downloader'),
 };
 
@@ -47,6 +48,18 @@ function buildMetadata(entry) {
     return metadata;
   }
 
+  if (entry.findings.nature.type === 'profile') {
+    const metadata = {
+      ...entry.findings.nature,
+      // ...entry.findings.downloader,
+      ...entry.findings.profile,
+    };
+    metadata.savingTime = new Date(entry.source.html.savingTime);
+    metadata.id = entry.source.html.id;
+    metadata.publicKey = entry.source.html.publicKey;
+    return metadata;
+  }
+
   /* else ... */
   const metadata = {
     ...entry.findings.nature,

Diff for: platforms/tktrex/backend/lib/utils.js

@@ -15,10 +15,11 @@ function hash(obj, fields) {
     memo += fname + '∴' + JSON.stringify(_.get(obj, fname, '…miss!')) + ',';
     return memo;
   }, '');
-  // debug("(note) hashing of %s", plaincnt);
   const sha1sum = crypto.createHash('sha1');
   sha1sum.update(plaincnt);
-  return sha1sum.digest('hex');
+  const retval = sha1sum.digest('hex');
+  // debug('(note) hashing of %s\n%s', plaincnt, retval);
+  return retval;
 }
 
 function verifyRequestSignature(req) {

Diff for: platforms/tktrex/backend/parsers/profile.js

@@ -0,0 +1,67 @@
+const _ = require('lodash');
+const debug = require('debug')('parser:profile');
+
+const getNatureByHref = require('./shared').getNatureByHref;
+
+function getFullProfileMetadata(renod, order) {
+  const vlink = renod.querySelector('a[href^="https://www.tiktok.com/@"]');
+  const vhref = vlink.getAttribute('href');
+  const vidnat = getNatureByHref(vhref);
+
+  const titleel = renod.querySelector('a[title]');
+  if(!titleel)
+    return null;
+  const title = titleel.getAttribute('title');
+  const viewsel = renod.querySelector('[data-e2e="video-views"]');
+  const views = viewsel.textContent;
+  const img = renod.querySelector('img[alt]');
+  const thumbnail = img.getAttribute('src');
+
+  return {
+    order,
+    video: vidnat,
+    title,
+    views,
+    thumbnail,
+  };
+}
+
+/* this is returning a bunch of native information,
+ * perhaps might be splitted in appropriate files.
+ * videoId, error messages, comment disabled, etc */
+function profile(envelop, previous) {
+
+  if (previous.nature.type !== 'profile') return false;
+
+  /* this piece of code return a list of videos, because
+       the search selector is not per video, but per 'body' */
+  const descs = envelop.jsdom.querySelectorAll('[data-e2e="user-post-item"]');
+  const results = _.compact(_.map(descs, function (elem, i) {
+    return getFullProfileMetadata(elem.parentNode, i + 1);
+  }));
+
+  const retval = {};
+
+  debug("Video Results found in profile %d", results.length);
+  if (results.length) {
+    retval.amount = results.length;
+    retval.results = results;
+  } else {
+    const errmsg = 'No results found';
+    const h2 = envelop.jsdom.querySelectorAll('h2');
+    // there are various 'h2' but only one can be an error
+    _.each(h2, function (h) {
+      if (errmsg === h.textContent) {
+        retval.error = errmsg;
+        retval.message = h.parentNode.querySelector('p')?.textContent;
+        // it can be 'hateful' or 'violate' but we don't know about other languages
+        debug('No results found: found this message: %s', retval.message);
+        retval.hatespeech = !!retval?.message?.match(/hateful/);
+      }
+    });
+  }
+
+  return retval;
+}
+
+module.exports = profile;

Diff for: platforms/tktrex/backend/parsers/shared.js

@@ -22,9 +22,9 @@ function getNatureByHref(href) {
       retval.type = 'video';
       retval.videoId = chunks[3];
       retval.authorId = chunks[1];
-    } else if (_.startsWith(urlO.pathname, '/@')) {
-      retval.type = 'creator';
-      retval.creatorName = urlO.pathname.substr(1);
+    } else if (_.startsWith(urlO.pathname, '/@') && chunks.length === 2) {
+      retval.type = 'profile';
+      retval.creatorName = chunks[1].substring(1);
     } else if (urlO.pathname === '/search') {
       retval.type = 'search';
       retval.query = urlO.searchParams.get('q');
@@ -51,6 +51,7 @@ function getUUID(url, type) {
   const fullname = type === 'video' ? `${fname}.mp4` : `${fname}.jpeg`;
   const cwd = process.cwd();
   if (!nconf.get('downloads')) {
+    /* eslint-disable no-console */
     console.log("WRONG CONFIGURATION SETTINGS!! missing 'downloads' from", cwd);
     process.exit(1);
   }

Diff for: platforms/tktrex/backend/routes/events.js

@@ -112,6 +112,7 @@ async function saveInDB(experinfo, objects, dbcollection) {
   }
 }
 
+/*
 function handleFullSave(body, headers) {
   // ["html","href","feedId","feedCounter", "reason",
   //  "videoCounter","rect","clientTime","type","incremental"]
@@ -132,7 +133,7 @@ function handleFullSave(body, headers) {
     geoip: geo(headers['x-forwarded-for']),
     researchTag: body.researchTag,
   };
-}
+} */
 
 async function processEvents(req) {
   const headers = processHeaders(_.get(req, 'headers'), mandatoryHeaders);
@@ -161,14 +162,13 @@ async function processEvents(req) {
       //  "videoCounter","rect","clientTime","type","incremental"]
       // 'type' can be ignored as it is always 'video' and doesn't reflect nature
 
-      console.log("--- feedId %s randomUUID",
-        body.feedId ?? 'x', body.randomUUID ?? 'x')
       const id = utils.hash({
         clientRGN: body.feedId
           ? body.feedId
           : body.href + new Date().toISOString(),
         serverPRGN: supporter.publicKey,
-        impressionNumber: body.videoCounter || Math.random(),
+        type: body.type,
+        impressionNumber: body.videoCounter,
       });
       const timelineIdHash = utils.hash({
         session: body.feedId
@@ -193,6 +193,7 @@ async function processEvents(req) {
       optionalNumbers.push(_.size(body.html));
       const html = {
         id,
+        type: body.type,
         rect: body.rect,
         href: body.href,
         timelineId: timelineWord + '-' + timelineIdHash.substr(0, 10),
@@ -208,9 +209,10 @@ async function processEvents(req) {
   );
 
   debug(
-    '[+] (p %s) from %s saving %s',
+    '[+] (p %s) from %s -- %s -- %s',
     supporter.p,
     JSON.stringify(_.map(req.body, 'type')),
+    JSON.stringify(_.map(req.body, 'href')),
     JSON.stringify(_.map(htmls, 'n'))
   );
 
@@ -233,7 +235,7 @@ async function processEvents(req) {
 }
 
 async function handshake(req) {
-  debug('Not implemented protocol (yet) [handshake API %j]', req.body);
+  // debug('Not implemented protocol (yet) [handshake API %j]', req.body);
   return {
     json: { ignored: true },
   };

Diff for: platforms/tktrex/backend/routes/personal.js

@@ -6,6 +6,7 @@ const utils = require('../lib/utils');
 const automo = require('../lib/automo');
 const CSV = require('../lib/CSV');
 const flattenSearch = require('./search').flattenSearch;
+const flattenProfile = require('./search').flattenProfile;
 
 function pickFeedFields(metae) {
   return {
@@ -33,7 +34,7 @@ async function getPersonal(req) {
   const amount = _.parseInt(req.query.amount) || 50;
   const skip = _.parseInt(req.query.skip) || 0;
   const what = req.params.what;
-  const allowed = ['summary', 'search', 'foryou', 'following'];
+  const allowed = ['summary', 'search', 'foryou', 'following', 'profile'];
 
   if (allowed.indexOf(what) === -1) {
     return {
@@ -47,7 +48,7 @@ async function getPersonal(req) {
   }
 
   debug(
-    'Asked to get data kind %s (%d-%d), preparing JSON',
+    'Requested data [nature %s] (%d-%d), preparing JSON',
     what,
     amount,
     skip
@@ -88,6 +89,24 @@ async function getPersonal(req) {
         counters: { metadata: avail.counters?.metadata },
         metadata,
       };
+    } else if (what === 'profile') {
+      /* this function access to 'search' results which is a
+       * bit different than the other. as in the collection
+       * there is not one entry for video, but one entry for search
+       * query --> hence, the _.map/_.pick
+       * note, this data should match
+       * packages/shared/src/models/contributor/ContributorPersonalSummary.ts
+       */
+      const avail = await automo.getPersonalTableData(
+        k,
+        { type: 'profile' },
+        { amount, skip }
+      );
+
+      retval = {
+        counters: { metadata: avail.counters?.metadata },
+        metadata: avail.metadata,
+      };
     } else if (what === 'foryou' || what === 'following') {
       retval = await automo.getMetadataByFilter(
         { type: what, publicKey: k },
@@ -111,26 +130,47 @@ async function getPersonal(req) {
 }
 
 async function getPersonalCSV(req) {
-  const CSV_MAX_SIZE = 1000;
+  const CSV_MAX_SIZE = 9000;
   const k = req.params.publicKey;
   const type = req.params.what;
-  if (['foryou', 'search', 'following'].indexOf(type) === -1)
-    return { text: 'Error, only foryou and search is supported ' };
+
+  if (['foryou', 'search', 'following', 'profile'].indexOf(type) === -1)
+    return { text: 'Error, nature not supported ' };
 
   const data = await automo.getMetadataByFilter(
     { publicKey: k, type },
     { amount: CSV_MAX_SIZE, skip: 0 }
   );
 
-  /* remind self, search has a different logic than for you,
+  if (!data.length) {
+    debug("getPersonalCSV didn't found DB entry matching %o", {
+      publicKey: k,
+      type,
+    });
+    return { text: 'No data not found in the DB' };
+  }
+
+  debug(
+    'type [%s] return %d with amount %d skip-zero',
+    type,
+    data.length,
+    CSV_MAX_SIZE
+  );
+
+  /* remind: search and profile have a different logic than
+     foryou and following.
      this is why is a reduce instead of map */
   let unrolledData = [];
   if (type === 'search') unrolledData = _.reduce(data, flattenSearch, []);
+  else if (type === 'profile')
+    unrolledData = _.reduce(data, flattenProfile, []);
   else unrolledData = _.map(data, pickFeedFields);
 
   if (!unrolledData.length) {
-    debug('getPersonalCSV return empty data');
-    return { text: 'Data not found: are you sure any search worked?' };
+    debug(
+      'getPersonalCSV produced empty data during transformation: investigate parsers and pipeline!'
+    );
+    return { text: 'Data not found, from metadata: ' + data.length };
   }
 
   /* XXX TMP FIXME (not if we pick the pseudo via mongodb)