Improved Resource and Cache Updater

app/Http/Controllers/V4DB/AnimeController.php

@@ -20,6 +20,8 @@
 use App\Http\Resources\V4\UserUpdatesResource;
 use App\Http\Resources\V4\AnimeVideosResource;
 use App\Http\Resources\V4\ForumResource;
+use App\Http\Resources\V4\NewsResource;
+use App\Jobs\UpdateResource;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\DB;
 use Jikan\Helper\Constants;
@@ -92,11 +94,11 @@ public function full(Request $request, int $id)
             if ($results->isEmpty()) {
                 $meta = [
                     'createdAt' => new UTCDateTime(),
-                    'modifiedAt' => new UTCDateTime(),
+                    'updated_at' => new UTCDateTime(),
                     'request_hash' => $this->fingerprint
                 ];
             }
-            $meta['modifiedAt'] = new UTCDateTime();
+            $meta['updated_at'] = new UTCDateTime();
 
             $response = $meta + $response;
 
@@ -165,55 +167,42 @@ public function main(Request $request, int $id)
             ->where('mal_id', $id)
             ->get();
 
-        if (
-            $results->isEmpty()
-            || $this->isExpired($request, $results)
-        ) {
+        if ($results->isEmpty()) {
             $response = Anime::scrape($id);
 
             if (HttpHelper::hasError($response)) {
                 return HttpResponse::notFound($request);
             }
 
-            if ($results->isEmpty()) {
-                $meta = [
-                    'createdAt' => new UTCDateTime(),
-                    'modifiedAt' => new UTCDateTime(),
-                    'request_hash' => $this->fingerprint
-                ];
-            }
-            $meta['modifiedAt'] = new UTCDateTime();
-
-            $response = $meta + $response;
-
-            if ($results->isEmpty()) {
-                Anime::create($response);
-            }
-
-            if ($this->isExpired($request, $results)) {
-                Anime::query()
-                    ->where('mal_id', $id)
-                    ->update($response);
-            }
-
-            $results = Anime::query()
-                ->where('mal_id', $id)
-                ->get();
+            Anime::query()
+                ->insert(
+                    [
+                        'created_at' => new UTCDateTime(),
+                        'updated_at' => new UTCDateTime(),
+                        'request_hash' => $this->fingerprint
+                    ] + $response
+                );
         }
 
-        if ($results->isEmpty()) {
-            return HttpResponse::notFound($request);
+        if ($this->isExpired($request, $results)) {
+            dispatch(new UpdateResource(
+                Anime::class, $id, $this->fingerprint
+            ));
         }
 
-        $response = (new \App\Http\Resources\V4\AnimeResource(
-            $results->first()
-        ))->response();
+        $results = Anime::query()
+            ->where('mal_id', $id)
+            ->get();
 
-        return $this->prepareResponse(
-            $response,
-            $results,
-            $request
-        );
+        return $results->isEmpty()
+            ? HttpResponse::notFound($request)
+            : $this->prepareResponse(
+                (new \App\Http\Resources\V4\AnimeResource(
+                    $results->first()
+                ))->response(),
+                $results,
+                $request
+            );
     }
 
     /**

app/Http/Controllers/V4DB/Controller.php

@@ -83,7 +83,7 @@ public function __construct(Request $request, MalClient $jikan)
         $this->fingerprint = HttpHelper::resolveRequestFingerprint($request);
     }
 
-    protected  function isExpired($request, $results) : bool
+    protected function isExpired($request, $results) : bool
     {
         $lastModified = $this->getLastModified($results);
 
@@ -118,11 +118,11 @@ protected function getTtl($results, $request)
     protected function getLastModified($results) : ?int
     {
         if (is_array($results->first())) {
-            return (int) $results->first()['modifiedAt']->toDateTime()->format('U');
+            return (int) $results->first()['updated_at']->toDateTime()->format('U');
         }
 
         if (is_object($results->first())) {
-            return (int) $results->first()->modifiedAt->toDateTime()->format('U');
+            return (int) $results->first()->updated_at->toDateTime()->format('U');
         }
 
         return null;
@@ -168,13 +168,13 @@ protected function updateCache($request, $results, $response)
         if ($results->isEmpty()) {
             $meta = [
                 'createdAt' => new UTCDateTime(),
-                'modifiedAt' => new UTCDateTime(),
+                'updated_at' => new UTCDateTime(),
                 'request_hash' => $this->fingerprint
             ];
         }
 
         // Update `modifiedAt` meta
-        $meta['modifiedAt'] = new UTCDateTime();
+        $meta['updated_at'] = new UTCDateTime();
         // join meta data with response
         $response = $meta + $response;
 

app/Jobs/UpdateResource.php

@@ -0,0 +1,115 @@
+<?php
+
+namespace App\Jobs;
+
+use Carbon\Carbon;
+use DateTime;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Queue\Middleware\ThrottlesExceptions;
+use MongoDB\BSON\UTCDateTime;
+
+/**
+ *
+ */
+class UpdateResource extends Job implements ShouldBeUnique
+{
+    /**
+     * @var string
+     */
+    public string $resource;
+
+    /**
+     * @var int
+     */
+    public int $malId;
+
+    /**
+     * @var string
+     */
+    public string $fingerprint;
+
+    /**
+     * @var int
+     */
+    public int $uniqueFor = 3600;
+
+    /**
+     * @var int
+     */
+    public int $maxExceptions = 5;
+
+    /**
+     * @var int
+     */
+    public int $timeout = 30;
+
+    /**
+     * @var int
+     */
+    public int $tries = 3;
+
+    /**
+     * @return int
+     */
+    public function uniqueId()
+    {
+        return $this->malId;
+    }
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(
+        string $resource,
+        int $malId,
+        string $fingerprint
+    )
+    {
+        $this->resource = $resource;
+        $this->malId = $malId;
+        $this->fingerprint = $fingerprint;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle(): void
+    {
+        $instance = new $this->resource;
+
+        // @todo handle exceptions
+        $response = $instance::scrape($this->malId);
+
+        $response = ['updated_at' => new UTCDateTime()]
+            + $response;
+
+        $instance::query()
+            ->where('mal_id', $this->malId)
+            ->update($response);
+    }
+
+    /**
+     * @return array
+     */
+    public function middleware() : array
+    {
+        return [
+            (new ThrottlesExceptions(5, 10))
+                ->backoff(60)
+                ->by("$this->resource:$this->malId")
+        ];
+    }
+
+    /**
+     * @return DateTime
+     */
+    public function retryUntil() : DateTime
+    {
+        return Carbon::now()
+            ->addMinutes(10);
+    }
+}

config/queue.php

@@ -36,34 +36,41 @@
 
         'database' => [
             'driver' => 'mongodb',
-            'connection' => 'mongodb',
-            'dsn'=> "mongodb+srv://".env('DB_USERNAME', 'jikan').":".env('DB_PASSWORD', '')."@".env('MONGODB_DSN', ''),
+            'dsn'=> "mongodb://".env('DB_USERNAME', 'admin').":".env('DB_PASSWORD', '')."@".env('DB_HOST', 'localhost').":".env('DB_PORT', 27017)."/".env('DB_ADMIN', 'admin'),
+            'database' => env('DB_DATABASE', 'jikan'),
             'table' => env('QUEUE_TABLE', 'jobs'),
-            'queue' => 'low',
+            'queue' => 'default',
 //            'retry_after' => 60,
+            'after_commit' => false,
         ],
 
         'beanstalkd' => [
             'driver' => 'beanstalkd',
             'host' => 'localhost',
             'queue' => 'default',
-            'retry_after' => 60,
+            'retry_after' => 90,
+            'block_for' => 0,
+            'after_commit' => false,
         ],
 
         'sqs' => [
             'driver' => 'sqs',
-            'key' => 'your-public-key',
-            'secret' => 'your-secret-key',
-            'queue' => 'your-queue-url',
-            'region' => 'us-east-1',
+            'key' => env('AWS_ACCESS_KEY_ID'),
+            'secret' => env('AWS_SECRET_ACCESS_KEY'),
+            'prefix' => env('SQS_PREFIX', 'https://sqs.us-east-1.amazonaws.com/your-account-id'),
+            'queue' => env('SQS_QUEUE', 'default'),
+            'suffix' => env('SQS_SUFFIX'),
+            'region' => env('AWS_DEFAULT_REGION', 'us-east-1'),
+            'after_commit' => false,
         ],
 
         'redis' => [
             'driver' => 'redis',
-            'connection' => env('QUEUE_REDIS_CONNECTION', 'default'),
-            'queue' => 'low',
-            'retry_after' => 60,
-//            'block_for' => 5,
+            'connection' => 'default',
+            'queue' => env('REDIS_QUEUE', 'default'),
+            'retry_after' => 90,
+            'block_for' => null,
+            'after_commit' => false,
         ],
 
     ],
@@ -80,9 +87,9 @@
     */
 
     'failed' => [
-        'driver' => env('QUEUE_CONNECTION', 'mongodb'),
-        'database' => env('DB_DATABASE', 'jikan'),
-        'table' => env('QUEUE_FAILED_TABLE', 'failed_jobs'),
+        'driver' => env('QUEUE_FAILED_DRIVER', 'database-uuids'),
+        'database' => env('DB_CONNECTION', 'mongodb'),
+        'table' => 'jobs_failed',
     ],
 
 ];

database/migrations/2020_05_21_214747_create_queue_index.php → database/migrations/2022_06_29_231146_create_jobs_table.php

@@ -4,7 +4,7 @@
 use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Support\Facades\Schema;
 
-class CreateQueueIndex extends Migration
+return new class extends Migration
 {
     /**
      * Run the migrations.
@@ -13,12 +13,11 @@ class CreateQueueIndex extends Migration
      */
     public function up()
     {
-        Schema::create(env('QUEUE_TABLE', 'jobs'), function (Blueprint $table) {
-            $table->index(['queue', 'reserved_at']);
+        Schema::create('jobs', function (Blueprint $table) {
             $table->bigIncrements('id');
-            $table->string('queue');
+            $table->string('queue')->index();
             $table->longText('payload');
-            $table->tinyInteger('attempts')->unsigned();
+            $table->unsignedTinyInteger('attempts');
             $table->unsignedInteger('reserved_at')->nullable();
             $table->unsignedInteger('available_at');
             $table->unsignedInteger('created_at');
@@ -32,6 +31,6 @@ public function up()
      */
     public function down()
     {
-        Schema::dropIfExists(env('QUEUE_TABLE', 'jobs'));
+        Schema::dropIfExists('jobs');
     }
-}
+};

database/migrations/2020_05_21_215040_create_queue_failed_index.php → database/migrations/2022_06_29_231217_create_jobs_failed_table.php

@@ -4,7 +4,7 @@
 use Illuminate\Database\Schema\Blueprint;
 use Illuminate\Support\Facades\Schema;
 
-class CreateQueueFailedIndex extends Migration
+return new class extends Migration
 {
     /**
      * Run the migrations.
@@ -13,8 +13,9 @@ class CreateQueueFailedIndex extends Migration
      */
     public function up()
     {
-        Schema::create(env('QUEUE_FAILED_TABLE', 'jobs_failed'), function (Blueprint $table) {
-            $table->increments('id');
+        Schema::create('jobs_failed', function (Blueprint $table) {
+            $table->id();
+            $table->string('uuid')->unique();
             $table->text('connection');
             $table->text('queue');
             $table->longText('payload');
@@ -30,6 +31,6 @@ public function up()
      */
     public function down()
     {
-        Schema::dropIfExists(env('QUEUE_FAILED_TABLE', 'jobs_failed'));
+        Schema::dropIfExists('jobs_failed');
     }
-}
+};