fix(OrganicResults): Update desc selector and fix other minor issues (#62)

* fix(OrganicResults): Update desc selector and fix other minor issues

* chore: Update tests

* chore: Update package version

Author: LuanRT

lib/core/nodes/FeaturedSnippet.js

@@ -18,7 +18,7 @@ class FeaturedSnippet {
       $(Constants.SELECTORS.FEATURED_SNIPPET_TITLE[1]).text() ||
       $(Constants.SELECTORS.FEATURED_SNIPPET_TITLE[2]).text();
 
-    const featured_snippet_url = $(Constants.SELECTORS.FEATURED_SNIPPET_URL).map((i, el) => $(el).attr('href')).get()[0];
+    const featured_snippet_url = $(Constants.SELECTORS.FEATURED_SNIPPET_URL).map((_i, el) => $(el).attr('href')).get()[0];
 
     const featured_snippet = Constants.SELECTORS.FEATURED_SNIPPET_DESC.map((selector) => {
       if ($(selector)[0] && selector != Constants.SELECTORS.FEATURED_SNIPPET_DESC[2]) {

lib/core/nodes/OrganicResults.js

@@ -5,19 +5,19 @@ const Constants = require('../../utils/constants');
 class OrganicResult {
   /** @type {string} */
   title;
-  
+
   /** @type {string} */
   description;
-  
+
   /** @type {string} */
   url;
 
   /** @type {boolean} */
   is_sponsored;
-  
+
   /** @type {{ high_res: string; low_res: string; }} */
   favicons;
-  
+
   constructor(data) {
     this.title = data.title;
     this.description = data.description;
@@ -46,23 +46,23 @@ class OrganicResults {
 
     const titles = $(Constants.SELECTORS.TITLE)
       .map((_i, el) => {
-        const is_ad = 
-          el.parent.attribs.style == '-webkit-line-clamp:2' ||
+        const is_ad =
+          Reflect.has(el.parent?.parent?.parent?.attribs || {}, 'data-rw') ||
           (!is_mobile && el.parent.attribs.class.startsWith('vdQmEd'));
-        
+
         // Ignore ad titles if parse_ads is false
         if (!parse_ads && is_ad)
           return null;
 
-        return is_mobile ? 
+        return is_mobile ?
           $(el).text().trim() : $(el).find('h3').text().trim() || $(el).find('a > div > span').first().text().trim();
       }).get();
 
     const descriptions = $(Constants.SELECTORS.DESCRIPTION)
       .map((_i, el) => {
-        const is_ad = el.parent.attribs.class == 'w1C3Le' || 
+        const is_ad = Reflect.has(el.parent?.parent?.parent?.attribs || {}, 'data-text-ad') ||
           (!is_mobile && !Object.keys(el.parent.attribs).length);
-       
+
         // Ignore ad descriptions if parse_ads is false
         if (!parse_ads && is_ad) {
           return null;
@@ -75,31 +75,25 @@ class OrganicResults {
 
     const urls = $(is_mobile ? Constants.SELECTORS.URL : `${Constants.SELECTORS.TITLE} > a`)
       .map((_i, el) => {
-        const is_ad = el.parent?.parent?.attribs?.class?.startsWith('vdQmEd');
-        
-        /**
-         * Since the selector for URLs is the same as the one for titles on desktop,
-         * we need to check if the element is an ad. If we're parsing the mobile page,
-         * then ads are simply stripped out of the results.
-         */
-        if (!is_mobile && !parse_ads && is_ad) {
+        const is_ad = !!$(el).attr('data-rw') || el.parent?.parent?.attribs?.class?.startsWith('vdQmEd');
+
+        if (!parse_ads && is_ad)
           return null;
-        }
 
         return $(el).attr('href');
       }).get();
-    
+
     // Refine results
     if (titles.length < urls.length && titles.length < descriptions.length) {
       urls.shift();
     }
-    
+
     if (urls.length > titles.length) {
       urls.shift();
     }
-    
+
     const is_innacurate_data = descriptions.length < urls.slice(1).length;
-    
+
     urls.forEach((item, index) => {
       // Why YouTube? Because video results usually don't have a description.
       if (item.includes('m.youtube.com') && is_innacurate_data) {
@@ -108,23 +102,23 @@ class OrganicResults {
         index--;
       }
     });
-    
+
     const results = [];
-    
+
     for (let i = 0; i < titles.length; i++) {
       const title = titles[i];
       const description = descriptions[i];
-      
+
       let url = urls[i];
 
       // Some results have a different URL format (AMP and ad results).
       if (url?.startsWith('/aclk') || url?.startsWith('/amp/s')) {
         url = `${Constants.URLS.W_GOOGLE}${url.substring(1)}`;
       }
-      
+
       const high_res_favicon = `${Constants.URLS.FAVICONKIT}/${new URL(url || Constants.URLS.W_GOOGLE).hostname}/192`;
       const low_res_favicon = `${Constants.URLS.W_GOOGLE}s2/favicons?sz=64&domain_url=${new URL(url || Constants.URLS.W_GOOGLE).hostname}`;
-         
+
       if (titles[i] && descriptions[i] && urls[i]) {
         results.push(new OrganicResult({
           title,
@@ -138,7 +132,7 @@ class OrganicResults {
         }));
       }
     }
-    
+
     return results;
   }
 }

lib/utils/constants.js

@@ -38,7 +38,7 @@ module.exports = {
     // Featured Snippet 
     FEATURED_SNIPPET_TITLE: ['div[class="co8aDb gsrt"]', 'a[class="sXtWJb gsrt"]', 'div[class="Xv4xee"]'],
     FEATURED_SNIPPET_DESC: ['ol[class="X5LH0c"]', 'ul[class="i8Z77e"]', 'div[data-attrid="wa:/description"]'],
-    FEATURED_SNIPPET_URL: 'div > div > h3 > a',
+    FEATURED_SNIPPET_URL: 'div > div > div > div > div > h3 > div > span > a',
 
     // Unit converter
     UNIT_CONVERTER_INPUT: 'div.rpnBye > input',

lib/utils/utils.js

@@ -43,9 +43,9 @@ function getHeaders(options = { mobile: false }) {
 function refineData (data, parse_ads = false, is_mobile = true) {
   let result = data
     // Removes classes we don't need: 
-    .replace(/N6jJud MUxGbd lyLwlc/g, '')
-    .replace(/YjtGef ExmHv MUxGbd/g, '')
-    .replace(/MUxGbd lyLwlc aLF0Z/g, '')
+    .replace(/N6jJud VwiC3b lyLwlc/g, '')
+    .replace(/YjtGef ExmHv VwiC3b/g, '')
+    .replace(/VwiC3b lyLwlc aLF0Z/g, '')
 
     /*
      * Transforms all possible variations of some classes' name into a
@@ -54,10 +54,12 @@ function refineData (data, parse_ads = false, is_mobile = true) {
 
     // Descriptions: -> MUxGbd yDYNvb 
     .replace(/yDYNvb lEBKkf/g, 'yDYNvb')
+    .replace(/VwiC3b yDYNvb/g, 'MUxGbd yDYNvb')
     .replace(/VwiC3b MUxGbd yDYNvb/g, 'MUxGbd yDYNvb')
 
     // Urls: -> C8nzq BmP5tf 
     .replace(/cz3goc BmP5tf/g, 'C8nzq BmP5tf')
+    .replace(/cz3goc v5yQqb BmP5tf/g, 'C8nzq BmP5tf')
 
     // Titles: -> ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr
     .replace(/ynAwRc q8U8x MBeuO oewGkc LeUQr/g, 'ynAwRc q8U8x MBeuO gsrt oewGkc LeUQr')

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "googlethis",
-  "version": "1.7.1",
+  "version": "1.8.0",
   "description": "A simple yet powerful module to retrieve organic search results and much more from Google.",
   "main": "lib/index.js",
   "author": "LuanRT <luan.lrt4@gmail.com> (https://github.com/LuanRT)",

package.json

@@ -8,11 +8,6 @@ describe('GoogleThis Tests', () => {
     expect(search.results).not.toHaveLength(0);
   });
 
-  it('Should search using a desktop user agent', async () => {
-    const search = await google.search('Stephen Hawking', { use_mobile_ua: false });
-    expect(search.results).not.toHaveLength(0);
-  });
-
   it('Should search images', async () => {
     const search = await google.image('Supermassive Blackhole');
     expect(search).not.toHaveLength(0);