bug fixes

.env.dist

@@ -24,7 +24,7 @@ MONGODB_DSN=
 #
 # If MyAnimeList is down or unavailable,
 # Jikan will try to generate responses from internally
-# populated database caches
+# populated cache where available
 #
 # Note: DB_CACHING must be enabled
 ###

app/Http/Middleware/DatabaseResolver.php

@@ -53,7 +53,6 @@ class DatabaseResolver
 
     public function handle(Request $request, Closure $next)
     {
-
         if ($request->header('auth') === env('APP_KEY')) {
             return $next($request);
         }
@@ -84,13 +83,19 @@ class DatabaseResolver
         $this->requestCached = DB::table($this->table)->where('request_hash', $this->fingerprint)->exists();
 
         // Is the request queueable?
-        if (\in_array($this->route, self::NON_QUEUEABLE) || env('CACHE_METHOD', 'legacy') === 'legacy') {
+        if (\in_array($this->route, self::NON_QUEUEABLE) || env('DB_METHOD', 'legacy') === 'legacy') {
             $this->queueable = false;
         }
 
         // If cache does not exist
         if (!$this->requestCached) {
-            return $this->fetchFresh($request, $next);
+            $response = $next($request);
+
+            if (HttpHelper::hasError($response)) {
+                return $response;
+            }
+
+            $this->insertCache($response);
         }
 
         // Fetch Cache & Generate Meta
@@ -103,12 +108,17 @@ class DatabaseResolver
         // If cache is expired, handle it depending on whether it's queueable
         $expiresAt = $cacheMutable['expiresAt']['$date']['$numberLong']/1000;
 
-        if ($this->requestCached && $expiresAt > time() && !$this->queueable) {
+        if ($this->requestCached && $expiresAt < time() && !$this->queueable) {
-            return $this->fetchFresh($request, $next);
+            $response = $next($request);
+
+            if (HttpHelper::hasError($response)) {
+                return $response;
+            }
+
+            $this->insertCache($response);
         }
 
-        if ( $this->queueable && $expiresAt > time()) {
+        if ( $this->queueable && $expiresAt < time()) {
-            $queueFingerprint = "queue_update:{$this->fingerprint}";
             $queueHighPriority = \in_array($this->route, self::HIGH_PRIORITY_QUEUE);
 
             // Don't duplicate the job in the queue for same request
@@ -116,15 +126,14 @@ class DatabaseResolver
 
             if (!$job->exists()) {
                 dispatch(
-                    (new UpdateDatabaseJob($request))
+                    (new UpdateDatabaseJob($request, $this->table))
                         ->onQueue($queueHighPriority ? 'high' : 'low')
                 );
             }
         }
 
-
         $response = array_merge($meta, $cacheMutable);
-        unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id']);
+        unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id'], $response['expiresAt']);
 
         // Build and return response
         return response()
@@ -185,14 +194,8 @@ class DatabaseResolver
         return $data;
     }
 
-    public function fetchFresh($request, $next)
+    public function insertCache($response)
     {
-        $response = $next($request);
-
-        if (HttpHelper::hasError($response)) {
-            return $response;
-        }
-
         DB::table($this->table)->insert(array_merge(
             [
                 'expiresAt' => new UTCDateTime((time()+$this->requestCacheTtl)*1000),

app/Jobs/UpdateCacheJob.php

@@ -74,7 +74,6 @@ class UpdateCacheJob extends Job
         unset($cache['fingerprint'], $cache['request_cached'], $cache['request_cache_expiry']);
         $cache = json_encode($cache);
 
-
         Cache::forever($this->fingerprint, $cache);
         Cache::forever($this->cacheExpiryFingerprint, time() + $this->requestCacheTtl);
         app('redis')->del($queueFingerprint);

app/Jobs/UpdateDatabaseJob.php

@@ -8,6 +8,7 @@ use Illuminate\Http\Request;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Facades\Cache;
+use MongoDB\BSON\UTCDateTime;
 
 
 /**
@@ -22,51 +23,56 @@ class UpdateDatabaseJob extends Job
 
     public $retryAfter = 60;
 
-    /**
+    protected $request;
-     * @var string
-     */
     protected $requestUri;
     protected $requestType;
     protected $requestCacheTtl;
     protected $fingerprint;
-    protected $cacheExpiryFingerprint;
+    protected $table;
-    protected $requestCached;
 
     /**
      * Create a new job instance.
      *
-     * @return void
+     * @param Request $request
+     * @param $table
      */
-    public function __construct(Request $request)
+    public function __construct(Request $request, $table)
     {
+        $this->table = $table;
         $this->fingerprint = HttpHelper::resolveRequestFingerprint($request);
 
-        $this->requestCached = DB::table(env('QUEUE_TABLE', 'jobs'))->where('request_hash', $this->fingerprint);
+        $this->requestType = HttpHelper::requestType($request);
+        $this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
     }
 
-
-    /**
-     * @throws \GuzzleHttp\Exception\GuzzleException
-     */
     public function handle() : void
     {
-        $client = new Client();
 
-        $response = $client
+        $response = app('GuzzleClient')
             ->request(
                 'GET',
                 env('APP_URL') . $this->requestUri,
                 [
                     'headers' => [
-                        'auth' => env('APP_KEY') // skip middleware
+                        'auth' => env('APP_KEY') // skips middleware
                     ]
                 ]
             );
 
         $cache = json_decode($response->getBody()->getContents(), true);
-        unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
+        unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry'], $cache['DEVELOPMENT_NOTICE'], $cache['MIGRATION']);
         $cache = json_encode($cache);
 
+        DB::table($this->table)
+            ->where('request_hash', $this->fingerprint)
+            ->update(array_merge(
+            [
+                'expiresAt' => new UTCDateTime((time()+$this->requestCacheTtl)*1000),
+                'request_hash' => $this->fingerprint
+            ],
+            json_decode($cache, true)
+        ));
+
         sleep((int) env('QUEUE_DELAY_PER_JOB', 5));
     }
 

app/Listeners/SourceHealthListener.php

@@ -25,7 +25,7 @@ class SourceHealthListener
     public function __construct()
     {
         $this->logger = new Logger('source-health-monitor');
-        $this->logger->pushHandler(new StreamHandler(storage_path().'/logs/source-health-monitor.log'), Logger::DEBUG);
+        $this->logger->pushHandler(new StreamHandler(storage_path().'/logs/source-health-monitor.log'), env('APP_DEBUG') ? Logger::DEBUG : Logger::WARNING);
 
         if (SourceHealthServiceProvider::isFailoverEnabled()) {
             $lastFailoverLockTimestamp = $this->getLastFailoverLockTimestamp();

composer.lock

@@ -2210,16 +2210,16 @@
         },
         {
             "name": "jikan-me/jikan",
-            "version": "v3.0.0-alpha.4",
+            "version": "v3.0.0-alpha.5",
             "source": {
                 "type": "git",
                 "url": "https://github.com/jikan-me/jikan.git",
-                "reference": "c5f4052c40454bf3a7c5a092fed6ed24650426cb"
+                "reference": "e1d11d663669a6e6e5d7ac536138834846764596"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/jikan-me/jikan/zipball/c5f4052c40454bf3a7c5a092fed6ed24650426cb",
+                "url": "https://api.github.com/repos/jikan-me/jikan/zipball/e1d11d663669a6e6e5d7ac536138834846764596",
-                "reference": "c5f4052c40454bf3a7c5a092fed6ed24650426cb",
+                "reference": "e1d11d663669a6e6e5d7ac536138834846764596",
                 "shasum": ""
             },
             "require": {
@@ -2256,7 +2256,7 @@
                 }
             ],
             "description": "Jikan is an unofficial MyAnimeList API",
-            "time": "2020-05-20T07:44:20+00:00"
+            "time": "2020-05-26T18:52:10+00:00"
         },
         {
             "name": "jms/metadata",

config/queue.php

@@ -15,7 +15,7 @@ return [
     |
     */
 
-    'default' => env('QUEUE_CONNECTION', 'redis'),
+    'default' => env('QUEUE_CONNECTION', 'mongodb'),
 
     /*
     |--------------------------------------------------------------------------
@@ -37,6 +37,7 @@ return [
         'database' => [
             'driver' => 'mongodb',
             'connection' => 'mongodb',
+            'dsn'=> "mongodb+srv://".env('DB_USERNAME', 'jikan').":".env('DB_PASSWORD', '')."@".env('MONGODB_DSN', ''),
             'table' => env('QUEUE_TABLE', 'jobs'),
             'queue' => 'low',
 //            'retry_after' => 60,

database/migrations/2020_05_21_051725_create_index.php

@@ -22,7 +22,7 @@ class CreateIndex extends Migration
             }
 
             Schema::create($table, function (Blueprint $table) {
-                $table->index(['request_hash' => 1], 'request_hash');
+                $table->unique(['request_hash' => 1], 'request_hash');
             });
 
             $mapped[] = $table;

database/migrations/2020_05_21_214747_create_queue_index.php

@@ -13,8 +13,18 @@ class CreateQueueIndex extends Migration
      */
     public function up()
     {
-        Schema::create(env('QUEUE_TABLE', 'jobs_failed'), function (Blueprint $table) {
+        Schema::create(env('QUEUE_TABLE', 'jobs'), function (Blueprint $table) {
-            $table->index(['request_hash' => 1], 'request_hash');
+            $table->index(['queue', 'reserved_at']);
+            $table->bigIncrements('id');
+            $table->string('queue');
+            $table->longText('payload');
+            $table->tinyInteger('attempts')->unsigned();
+            $table->unsignedInteger('reserved_at')->nullable();
+            $table->unsignedInteger('available_at');
+            $table->unsignedInteger('created_at');
+        });
+
+        Schema::create(env('QUEUE_TABLE', 'jobs'), function (Blueprint $table) {
             $table->index(['queue', 'reserved_at']);
             $table->bigIncrements('id');
             $table->string('queue');
@@ -33,6 +43,6 @@ class CreateQueueIndex extends Migration
      */
     public function down()
     {
-        Schema::dropIfExists(env('QUEUE_TABLE', 'jobs_failed'));
+        Schema::dropIfExists(env('QUEUE_TABLE', 'jobs'));
     }
 }

database/migrations/2020_05_21_215040_create_queue_failed_index.php

@@ -14,7 +14,6 @@ class CreateQueueFailedIndex extends Migration
     public function up()
     {
         Schema::create(env('QUEUE_FAILED_TABLE', 'jobs_failed'), function (Blueprint $table) {
-            $table->index(['request_hash' => 1], 'request_hash');
             $table->increments('id');
             $table->text('connection');
             $table->text('queue');