Detect page variants to increase reliability (#3)

Detect page variants to increase reliability

Author: joskfg

Diff for: README.md

@@ -63,7 +63,7 @@ Due to the pages usually have different structures depending on different variab
 trying to cover maximum page variations possible. The scraper WILL NOT BE ABLE to scrap page variations not incorporated
 in the dataset.
 
-Once we did the job, all is ready to work. You should not care about updates always you have enought data in the dataset
+Once we did the job, all is ready to work. You should not care about updates always you have enough data in the dataset
 to cover all the new modifications on the page, so the scraper will recalculate the modifications on the fly. You can 
 check [how it works](how-it-works.md) to know much about the internals.
 
@@ -74,6 +74,7 @@ We will check more deeply how we can create a new dataset and what options are a
 The dataset is composed by `url` and `data`. 
 * The `url` part is simple, you just need to indicate the url from where you obtained the data.
 * The `type` part gives a item name to the current dataset. This allows you to define multiple types.
+* The `variant` identifies the page variant. The identifier is a sha1 hash build based in the xpath used to get the data.
 * The `data` part is where you indicate what data and assign the label that you want to get. 
 The data could be a list of items or a single item.
 
@@ -83,9 +84,10 @@ A basic example could be:
 use Softonic\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset;
 
 ScrapedDataset::create([
-    'url'  => 'https://test.c/p/my-objective',
-    'type' => 'Item-definition-1',
-    'data' => [
+    'url'     => 'https://test.c/p/my-objective',
+    'type'    => 'Item-definition-1',
+    'variant' => '8ed10778a83f1266e7ffed90205f7fb61ddcdf78',
+    'data'    => [
         'title'     => 'My title',
         'body'      => 'This is the body content I want to get',
         'images'    => [
@@ -112,6 +114,7 @@ use Softonic\LaravelIntelligentScraper\Scraper\Models\ScrapedDataset;
 ScrapedDataset::create([
     'url'  => 'https://test.c/p/my-objective',
     'type' => 'Item-definition-1',
+    'variant' => '8ed10778a83f1266e7ffed90205f7fb61ddcdf78',
     'data' => [
         'title'     => 'My title',
         'body'      => regexp('/^Body starts here, but it is do long that.*$/si'),
@@ -160,7 +163,7 @@ After configure the scraper, you will be able to request an specific scrape usin
 ```php
 <?php 
 
-scrape('https://test.c/p/my-objective', 'Item-definition-1');
+scrape('https://test.c/p/my-objective', 'Item-definition-1', 'variant-sha1');
 ```
 
 The scrape will produce a `\Softonic\LaravelIntelligentScraper\Scraper\Events\Scraped` event if all worked as expected.
@@ -170,6 +173,7 @@ So attach a listener to that event to receive the data.
 $event->scrapeRequest->url  // Url scraped
 $event->scrapeRequest->type // Request type
 $event->data // Contains all the data in a [ 'fieldName' => 'value' ] format.
+$event->variant // Contains the page variation sha1 hash.
 ```
 
 All the output fields are arrays that can contain one or more results.
@@ -212,6 +216,7 @@ The scraper is auto configurable, but needs an initial dataset or add a configur
 The dataset tells the configurator which data do you want and how to label it.
 
 There are three services that have unique responsibilities and are connected using the event system.
+
 ### Scrape
 
 It is fired when the system receive a `\Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest` event. It

Diff for: src/Scraper/Application/Configurator.php

@@ -22,10 +22,16 @@ class Configurator
      */
     private $xpathBuilder;
 
-    public function __construct(Client $client, XpathBuilder $xpathBuilder)
+    /**
+     * @var VariantGenerator
+     */
+    private $variantGenerator;
+
+    public function __construct(Client $client, XpathBuilder $xpathBuilder, VariantGenerator $variantGenerator)
     {
-        $this->client       = $client;
-        $this->xpathBuilder = $xpathBuilder;
+        $this->client           = $client;
+        $this->xpathBuilder     = $xpathBuilder;
+        $this->variantGenerator = $variantGenerator;
     }
 
     /**
@@ -76,18 +82,23 @@ private function getCrawler($scrapedData)
     private function findConfigByScrapedData($scrapedData, $crawler)
     {
         $result = [];
+
         foreach ($scrapedData['data'] as $field => $value) {
             try {
                 $result[$field] = $this->xpathBuilder->find(
                     $crawler->getNode(0),
                     $value
                 );
+                $this->variantGenerator->addConfig($field, $result[$field]);
             } catch (\UnexpectedValueException $e) {
+                $this->variantGenerator->fieldNotFound();
                 $value = is_array($value) ? json_encode($value) : $value;
                 Log::warning("Field '{$field}' with value '{$value}' not found for '{$crawler->getUri()}'.");
             }
         }
 
+        $this->updateVariant($scrapedData);
+
         return $result;
     }
 
@@ -134,4 +145,10 @@ private function checkConfiguration($data, Collection $finalConfig)
             throw new ConfigurationException("Field(s) \"{$fieldsMissing}\" not found.", 0);
         }
     }
+
+    private function updateVariant($scrapedData): void
+    {
+        $scrapedData['variant'] = $this->variantGenerator->getId($scrapedData['type']);
+        $scrapedData->save();
+    }
 }

Diff for: src/Scraper/Application/VariantGenerator.php

@@ -0,0 +1,53 @@
+<?php
+
+namespace Softonic\LaravelIntelligentScraper\Scraper\Application;
+
+class VariantGenerator
+{
+    protected $type = null;
+
+    protected $configPerField = [];
+
+    protected $allFieldsFound = true;
+
+    public function setType($type): void
+    {
+        $this->type = $type;
+    }
+
+    public function addConfig($field, $xpath)
+    {
+        $this->configPerField[] = $field . $xpath;
+    }
+
+    public function fieldNotFound()
+    {
+        $this->allFieldsFound = false;
+    }
+
+    public function getId($type = null)
+    {
+        $type = $type ?? $this->type;
+        if (empty($type)) {
+            throw new \InvalidArgumentException('Type should be provided in the getVariantId call or setType');
+        }
+
+        if (empty($this->configPerField) || !$this->allFieldsFound) {
+            return null;
+        }
+
+        sort($this->configPerField);
+
+        $id = sha1($type . implode('', $this->configPerField));
+        $this->reset();
+
+        return $id;
+    }
+
+    public function reset()
+    {
+        $this->setType(null);
+        $this->configPerField = [];
+        $this->allFieldsFound = true;
+    }
+}

Diff for: src/Scraper/Application/XpathFinder.php

@@ -12,9 +12,15 @@ class XpathFinder
      */
     private $client;
 
-    public function __construct(GoutteClient $client)
+    /**
+     * @var VariantGenerator
+     */
+    private $variantGenerator;
+
+    public function __construct(GoutteClient $client, VariantGenerator $variantGenerator)
     {
-        $this->client = $client;
+        $this->client           = $client;
+        $this->variantGenerator = $variantGenerator;
     }
 
     public function extract(string $url, $configs): array
@@ -32,6 +38,7 @@ public function extract(string $url, $configs): array
                 $subcrawler = $crawler->filterXPath($xpath);
 
                 if ($subcrawler->count()) {
+                    $this->variantGenerator->addConfig($config['name'], $xpath);
                     break;
                 }
             }
@@ -43,11 +50,13 @@ public function extract(string $url, $configs): array
                 );
             }
 
-            $result[$config['name']] = $subcrawler->each(function ($node) {
+            $result['data'][$config['name']] = $subcrawler->each(function ($node) {
                 return $node->text();
             });
         }
 
+        $result['variant'] = $this->variantGenerator->getId($config['type']);
+
         return $result;
     }
 }

Diff for: src/Scraper/Events/Scraped.php

@@ -19,15 +19,22 @@ class Scraped
      */
     public $data;
 
+    /**
+     * @var string
+     */
+    public $variant;
+
     /**
      * Create a new event instance.
      *
      * @param ScrapeRequest $scrapeRequest
      * @param array         $data
+     * @param string        $variant
      */
-    public function __construct(ScrapeRequest $scrapeRequest, array $data)
+    public function __construct(ScrapeRequest $scrapeRequest, array $data, string $variant)
     {
         $this->scrapeRequest = $scrapeRequest;
         $this->data          = $data;
+        $this->variant       = $variant;
     }
 }

Diff for: src/Scraper/Listeners/ConfigureScraper.php

@@ -74,8 +74,8 @@ private function extractData(ScrapeRequest $scrapeRequest, $config): void
     {
         $this->logger->debug("Extracting data from $scrapeRequest->url for type '$scrapeRequest->type'");
 
-        $data = $this->xpathFinder->extract($scrapeRequest->url, $config);
-        event(new Scraped($scrapeRequest, $data));
+        list('data' => $data, 'variant' => $variant) = $this->xpathFinder->extract($scrapeRequest->url, $config);
+        event(new Scraped($scrapeRequest, $data, $variant));
     }
 
     /**

Diff for: src/Scraper/Listeners/Scrape.php

@@ -77,7 +77,7 @@ private function extractData(ScrapeRequest $scrapeRequest, $config): void
     {
         $this->logger->debug("Extracting data from $scrapeRequest->url for type '$scrapeRequest->type'");
 
-        $data = $this->xpathFinder->extract($scrapeRequest->url, $config);
-        event(new Scraped($scrapeRequest, $data));
+        list('data' => $data, 'variant' => $variant) = $this->xpathFinder->extract($scrapeRequest->url, $config);
+        event(new Scraped($scrapeRequest, $data, $variant));
     }
 }

Diff for: src/Scraper/Listeners/UpdateDataset.php

@@ -23,16 +23,19 @@ public function handle(Scraped $event)
 
     private function addDataset(Scraped $event)
     {
-        $scraperDatasets = ScrapedDataset::withType($event->scrapeRequest->type);
+        $scraperDatasets = ScrapedDataset::withType($event->scrapeRequest->type)
+            ->withVariant($event->variant);
+
         if (self::DATASET_AMOUNT_LIMIT <= $scraperDatasets->count()) {
             $scraperDatasets->orderBy('updated_at', 'desc')->first()->delete();
         }
 
         ScrapedDataset::create(
             [
-                'url'  => $event->scrapeRequest->url,
-                'type' => $event->scrapeRequest->type,
-                'data' => $event->data,
+                'url'     => $event->scrapeRequest->url,
+                'type'    => $event->scrapeRequest->type,
+                'variant' => $event->variant,
+                'data'    => $event->data,
             ]
         );
     }

Diff for: src/Scraper/Models/ScrapedDataset.php

@@ -35,11 +35,17 @@ class ScrapedDataset extends Model
     protected $fillable = [
         'url',
         'type',
+        'variant',
         'data',
     ];
 
     public function scopeWithType($query, string $type)
     {
         return $query->where('type', $type);
     }
+
+    public function scopeWithVariant($query, string $variant)
+    {
+        return $query->where('variant', $variant);
+    }
 }

Diff for: src/Scraper/Traits/VariantId.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Softonic\LaravelIntelligentScraper\Scraper\Traits;
+
+trait VariantId
+{
+    protected function getVariantId(string $type, array $variant)
+    {
+        if (empty($variant)) {
+            return null;
+        }
+
+        sort($variant);
+
+        return sha1($type . implode('', $variant));
+    }
+}

Diff for: src/Scraper/helpers.php

@@ -1,11 +1,15 @@
 <?php
 
-function regexp($regexp)
-{
-    return ['regexp' => $regexp];
+if (!function_exists('regexp')) {
+    function regexp($regexp)
+    {
+        return ['regexp' => $regexp];
+    }
 }
 
-function scrape($url, $type)
-{
-    event(new \Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest($url, $type));
+if (!function_exists('scrape')) {
+    function scrape($url, $type)
+    {
+        event(new \Softonic\LaravelIntelligentScraper\Scraper\Events\ScrapeRequest($url, $type));
+    }
 }

Diff for: src/database/factories/ScrapedDatasetFactory.php

@@ -16,9 +16,10 @@
 /* @var \Illuminate\Database\Eloquent\Factory $factory */
 $factory->define(ScrapedDataset::class, function (Faker\Generator $faker) {
     return [
-        'url'  => $faker->url . $faker->randomDigit,
-        'type' => 'post',
-        'data' => [
+        'url'     => $faker->url . $faker->randomDigit,
+        'type'    => 'post',
+        'variant' => $faker->sha1,
+        'data'    => [
             'title'  => $faker->word,
             'author' => $faker->word,
         ],

Diff for: src/database/migrations/2018_07_16_091154_create_scraped_datasets_table.php

@@ -15,6 +15,7 @@ public function up()
         Schema::create('scraped_datasets', function (Blueprint $table) {
             $table->string('url', 1024)->primary();
             $table->string('type');
+            $table->string('variant', 40)->index()->nullable();
             $table->json('data');
             $table->timestamps();
         });