add automatic MAL health checker/failover based on uncached response statuses

app/Events/SourceHealthEvent.php

@@ -0,0 +1,31 @@
+<?php
+
+namespace App\Events;
+
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Storage;
+
+/**
+ * Class SourceHealthEvent
+ * @package App\Events
+ */
+class SourceHealthEvent extends Event
+{
+    public const BAD_HEALTH = 1;
+    public const GOOD_HEALTH = 0;
+
+    public $health;
+    public $status;
+
+    /**
+     * SourceHealthEvent constructor.
+     * @param int $health
+     * @param int $status
+     */
+    public function __construct(int $health = self::BAD_HEALTH, ?int $status)
+    {
+        $this->health = $health;
+        $this->status = $status ?? 0;
+    }
+}

app/Exceptions/Handler.php

@@ -2,6 +2,7 @@
 
 namespace App\Exceptions;
 
+use App\Events\SourceHealthEvent;
 use App\Http\HttpHelper;
 use Exception;
 use GuzzleHttp\Exception\ClientException;
@@ -109,10 +110,17 @@ class Handler extends ExceptionHandler
                             'message' => 'Jikan is being rate limited by MyAnimeList',
                             'error' => $e->getMessage()
                         ], $e->getCode());
+                case 403:
                 case 500:
+                case 501:
                 case 502:
                 case 503:
                 case 504:
+                    // Dispatch Bad source health event to prompt database fallback if enabled
+                    if (env('SOURCE_BAD_HEALTH_FALLBACK') && env('DB_CACHING')) {
+                        event(new SourceHealthEvent(SourceHealthEvent::BAD_HEALTH, $e->getCode()));
+                    }
+
                     return response()
                         ->json([
                             'status' => $e->getCode(),

app/Http/Middleware/DatabaseResolver.php

@@ -3,8 +3,11 @@
 namespace App\Http\Middleware;
 
 use App\DatabaseHandler;
+use App\Events\SourceHealthEvent;
 use App\Http\HttpHelper;
 use App\Jobs\UpdateCacheJob;
+use App\Jobs\UpdateDatabaseJob;
+use App\Providers\SourceHealthServiceProvider;
 use Closure;
 use Flipbox\LumenGenerator\LumenGeneratorServiceProvider;
 use Illuminate\Http\Request;
@@ -12,6 +15,7 @@ use Illuminate\Support\Facades\Cache;
 use Illuminate\Support\Facades\Date;
 use Illuminate\Support\Facades\DB;
 use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\Storage;
 use Jenssegers\Mongodb\MongodbServiceProvider;
 use MongoDB\BSON\UTCDateTime;
 use MongoDB\Client;
@@ -33,6 +37,8 @@ class DatabaseResolver
 
     private $queueable = true;
 
+    private $table;
+
     private const NON_QUEUEABLE = [
         'UserController@profile',
         'UserController@history',
@@ -47,6 +53,7 @@ class DatabaseResolver
 
     public function handle(Request $request, Closure $next)
     {
+
         if ($request->header('auth') === env('APP_KEY')) {
             return $next($request);
         }
@@ -73,35 +80,49 @@ class DatabaseResolver
         $this->route = end($this->route);
 
         $db = new DatabaseHandler();
-        $table = $db::getMappedTableName($this->route);
+        $this->table = $db::getMappedTableName($this->route);
+        $this->requestCached = DB::table($this->table)->where('request_hash', $this->fingerprint)->exists();
 
-        $this->requestCached = DB::table($table)->where('request_hash', $this->fingerprint)->exists();
+        // Is the request queueable?
-
+        if (\in_array($this->route, self::NON_QUEUEABLE) || env('CACHE_METHOD', 'legacy') === 'legacy') {
-        // Cache if it doesn't exist
+            $this->queueable = false;
-        if (!$this->requestCached) {
-            $response = $next($request);
-
-            if (HttpHelper::hasError($response)) {
-                return $response;
-            }
-
-            DB::table($table)->insert(array_merge(
-                [
-                    'expireAfterSeconds' => $this->requestCacheTtl,
-                    'request_hash' => $this->fingerprint
-                ],
-                json_decode($response->original, true)
-            ));
         }
 
+        // If cache does not exist
+        if (!$this->requestCached) {
+            return $this->fetchFresh($request, $next);
+        }
 
-        // Return response
+        // Fetch Cache & Generate Meta
         $meta = $this->generateMeta($request);
 
-        $cache = DB::table($table)->where('request_hash', $this->fingerprint)->get();
+        $cache = DB::table($this->table)->where('request_hash', $this->fingerprint)->get();
         $cacheMutable = json_decode($cache, true)[0];
         $cacheMutable = $this->cacheMutation($cacheMutable);
 
+        // If cache is expired, handle it depending on whether it's queueable
+        $expiresAt = $cacheMutable['expiresAt']['$date']['$numberLong']/1000;
+
+        if ($this->requestCached && $expiresAt > time() && !$this->queueable) {
+            return $this->fetchFresh($request, $next);
+        }
+
+        if ( $this->queueable && $expiresAt > time()) {
+            $queueFingerprint = "queue_update:{$this->fingerprint}";
+            $queueHighPriority = \in_array($this->route, self::HIGH_PRIORITY_QUEUE);
+
+            // Don't duplicate the job in the queue for same request
+            $job = DB::table(env('QUEUE_TABLE', 'jobs'))->where('request_hash', $this->fingerprint);
+
+            if (!$job->exists()) {
+                dispatch(
+                    (new UpdateDatabaseJob($request))
+                        ->onQueue($queueHighPriority ? 'high' : 'low')
+                );
+            }
+        }
+
+
         $response = array_merge($meta, $cacheMutable);
         unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id']);
 
@@ -163,4 +184,21 @@ class DatabaseResolver
 
         return $data;
     }
+
+    public function fetchFresh($request, $next)
+    {
+        $response = $next($request);
+
+        if (HttpHelper::hasError($response)) {
+            return $response;
+        }
+
+        DB::table($this->table)->insert(array_merge(
+            [
+                'expiresAt' => new UTCDateTime((time()+$this->requestCacheTtl)*1000),
+                'request_hash' => $this->fingerprint
+            ],
+            json_decode($response->original, true)
+        ));
+    }
 }

app/Http/Middleware/SourceHealthMonitor.php

@@ -0,0 +1,25 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use Closure;
+use App\Events\SourceHealthEvent;
+
+class SourceHealthMonitor
+{
+    /**
+     * Handle an incoming request.
+     *
+     * @param \Illuminate\Http\Request $request
+     * @param \Closure $next
+     * @return mixed
+     */
+    public function handle($request, Closure $next)
+    {
+        if (env('SOURCE_BAD_HEALTH_FAILOVER') && env('DB_CACHING')) {
+            event(new SourceHealthEvent(SourceHealthEvent::GOOD_HEALTH, 200));
+        }
+
+        return $next($request);
+    }
+}

app/Jobs/UpdateDatabaseJob.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Http\HttpHelper;
+use GuzzleHttp\Client;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Log;
+use Illuminate\Support\Facades\Cache;
+
+
+/**
+ * Class UpdateDatabaseJob
+ * @package App\Jobs
+ */
+class UpdateDatabaseJob extends Job
+{
+
+
+    public $timeout = 60;
+
+    public $retryAfter = 60;
+
+    /**
+     * @var string
+     */
+    protected $requestUri;
+    protected $requestType;
+    protected $requestCacheTtl;
+    protected $fingerprint;
+    protected $cacheExpiryFingerprint;
+    protected $requestCached;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(Request $request)
+    {
+        $this->fingerprint = HttpHelper::resolveRequestFingerprint($request);
+
+        $this->requestCached = DB::table(env('QUEUE_TABLE', 'jobs'))->where('request_hash', $this->fingerprint);
+    }
+
+
+    /**
+     * @throws \GuzzleHttp\Exception\GuzzleException
+     */
+    public function handle() : void
+    {
+        $client = new Client();
+
+        $response = $client
+            ->request(
+                'GET',
+                env('APP_URL') . $this->requestUri,
+                [
+                    'headers' => [
+                        'auth' => env('APP_KEY') // skip middleware
+                    ]
+                ]
+            );
+
+        $cache = json_decode($response->getBody()->getContents(), true);
+        unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
+        $cache = json_encode($cache);
+
+        sleep((int) env('QUEUE_DELAY_PER_JOB', 5));
+    }
+
+    public function failed(\Exception $e)
+    {
+        Log::error($e->getMessage());
+    }
+}

app/Listeners/SourceHealthListener.php

@@ -0,0 +1,150 @@
+<?php
+
+namespace App\Listeners;
+
+use App\Events\ExampleEvent;
+use App\Events\SourceHealthEvent;
+use App\Providers\SourceHealthServiceProvider;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Support\Facades\Storage;
+use League\Flysystem\FileNotFoundException;
+use Monolog\Handler\StreamHandler;
+use Monolog\Logger;
+
+class SourceHealthListener
+{
+
+    private $logger;
+
+    /**
+     * Create the event listener.
+     *
+     * @return void
+     */
+    public function __construct()
+    {
+        $this->logger = new Logger('source-health-monitor');
+        $this->logger->pushHandler(new StreamHandler(storage_path().'/logs/source-health-monitor.log'), Logger::DEBUG);
+
+        if (SourceHealthServiceProvider::isFailoverEnabled()) {
+            $lastFailoverLockTimestamp = $this->getLastFailoverLockTimestamp();
+            $this->logger->debug('Failover is RUNNING');
+
+            // Disable failover if it has expired
+            if (time() > ($lastFailoverLockTimestamp + env('SOURCE_BAD_HEALTH_RECHECK'))) {
+                // Disable failover if successful requests score
+                $this->attemptDisableFailover();
+            }
+        }
+    }
+
+    /**
+     * Handle the event.
+     *
+     * @param  ExampleEvent  $event
+     * @return void
+     */
+    public function handle(SourceHealthEvent $event)
+    {
+        $eventCount = $this->insertFail($event);
+        $this->logger->debug('Event count: '.$eventCount);
+
+        if ($this->getSuccessfulRequestsScore() <= 0.25) {
+            $this->enableFailover();
+        }
+    }
+
+    private function insertFail(SourceHealthEvent $event) : int
+    {
+        $fails = $this->getRecentFails();
+        $fails[] = [time(), $event->status, $event->health];
+
+        $failsJson = json_encode($fails);
+        Storage::put('failovers.json', $failsJson);
+
+        return count($fails);
+    }
+
+    private function enableFailover()
+    {
+        // create lock file
+        Storage::put('source_failover.lock', '');
+        $this->logger->debug('Failover ENABLED');
+    }
+
+    private function disableFailover()
+    {
+        // delete lock file
+        Storage::delete('source_failover.lock');
+
+        // Delete meta
+        Storage::delete('failovers.json');
+    }
+
+    private function attemptDisableFailover()
+    {
+        $score = $this->getSuccessfulRequestsScore();
+
+        if ($score >= 0.9) {
+            $this->disableFailover();
+            $this->logger->debug('Failover disabled; Score: '.$score);
+            $this->logger->debug('Failover DISABLED');
+            return true;
+        }
+
+        return false;
+    }
+
+    private function getLastFailoverLockTimestamp()
+    {
+        try {
+            return Storage::lastModified('source_failover.lock');
+        } catch (\Exception $e) {
+            return 0;
+        }
+    }
+
+    private function getRecentFails()
+    {
+        try {
+            $failsJson = Storage::get('failovers.json');
+            $fails = json_decode($failsJson, true);
+        } catch (\Exception $e) {
+            $fails = [];
+        }
+
+        // remove any fails greater than SOURCE_BAD_HEALTH_RANGE
+        foreach ($fails as $fail) {
+
+            if ($fail[0] >= (time()-env('SOURCE_BAD_HEALTH_RANGE'))) {
+                unset($fail);
+            }
+        }
+
+        // slice
+        if (count($fails) > env('SOURCE_BAD_HEALTH_MAX_STORE')) {
+            $fails = array_slice($fails, 0 - env('SOURCE_BAD_HEALTH_MAX_STORE'));
+        }
+
+        return $fails;
+    }
+
+    private function getSuccessfulRequestsScore() : float
+    {
+        $fails = $this->getRecentFails();
+        $score = 0;
+        $totalFails = count($fails) - 1;
+
+        foreach ($fails as $fail) {
+            if ((int) $fail[2] === SourceHealthEvent::GOOD_HEALTH) {
+                $score++;
+            }
+        }
+
+        $scored = $score / max($totalFails, 1);
+        $this->logger->debug('Failover successful requests score: '.$scored);
+
+        return $scored;
+    }
+}

app/Providers/SearchQueryBuilder.php

@@ -235,9 +235,6 @@ class SearchQueryBuilder
             }
         }
 
-
-
-
         // Anime
         if ($request instanceof AnimeSearchRequest) {
             // Rating/Rated

app/Providers/SourceHealthServiceProvider.php

@@ -0,0 +1,28 @@
+<?php
+
+namespace App\Providers;
+
+use Illuminate\Support\Facades\Storage;
+use Laravel\Lumen\Providers\EventServiceProvider as ServiceProvider;
+
+class SourceHealthServiceProvider extends ServiceProvider
+{
+
+    const BAD_HEALTH_STATUSES = [403, 500, 501, 502, 503, 504, 505];
+
+    /**
+     * The event listener mappings for the application.
+     *
+     * @var array
+     */
+    protected $listen = [
+        'App\Events\SourceHealthEvent' => [
+            'App\Listeners\SourceHealthListener',
+        ],
+    ];
+
+    public static function isFailoverEnabled() : bool
+    {
+        return Storage::exists('source_failover.lock');
+    }
+}

bootstrap/app.php

@@ -77,7 +77,8 @@ $app->routeMiddleware([
     'throttle' => App\Http\Middleware\Throttle::class,
     'etag' => \App\Http\Middleware\EtagMiddleware::class,
     'microcaching' => \App\Http\Middleware\MicroCaching::class,
-    'database-resolver' => \App\Http\Middleware\DatabaseResolver::class
+    'database-resolver' => \App\Http\Middleware\DatabaseResolver::class,
+    'source-health-monitor' => \App\Http\Middleware\SourceHealthMonitor::class
 ]);
 
 /*
@@ -108,6 +109,10 @@ $app->instance('GuzzleClient', $guzzleClient);
 $jikan = new \Jikan\MyAnimeList\MalClient(app('GuzzleClient'));
 $app->instance('JikanParser', $jikan);
 
+if (env('SOURCE_BAD_HEALTH_FAILOVER') && env('DB_CACHING')) {
+    $app->register(\App\Providers\SourceHealthServiceProvider::class);
+}
+
 
 /**
  * Load Blacklist into Redis
@@ -134,6 +139,7 @@ $commonMiddleware = [
 //    'microcaching',
 //    'cache-resolver',
 //    'throttle'
+    'source-health-monitor'
 ];
 
 $app->router->group(

composer.json

@@ -17,6 +17,7 @@
         "jikan-me/jikan": "^3.0",
         "jms/serializer": "^1.13",
         "laravel/lumen-framework": "^7.0",
+        "league/flysystem": "^1.0",
         "ocramius/package-versions": "^1.4",
         "predis/predis": "^1.1",
         "symfony/yaml": "^4.1",

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
         "This file is @generated automatically"
     ],
-    "content-hash": "6d42be876cc0e4c456de620bd8aa6321",
+    "content-hash": "14c8c6eb422e03b6838bba397af565ff",
     "packages": [
         {
             "name": "brick/math",
@@ -2523,6 +2523,90 @@
             ],
             "time": "2020-05-05T18:17:08+00:00"
         },
+        {
+            "name": "league/flysystem",
+            "version": "1.0.69",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/thephpleague/flysystem.git",
+                "reference": "7106f78428a344bc4f643c233a94e48795f10967"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/7106f78428a344bc4f643c233a94e48795f10967",
+                "reference": "7106f78428a344bc4f643c233a94e48795f10967",
+                "shasum": ""
+            },
+            "require": {
+                "ext-fileinfo": "*",
+                "php": ">=5.5.9"
+            },
+            "conflict": {
+                "league/flysystem-sftp": "<1.0.6"
+            },
+            "require-dev": {
+                "phpspec/phpspec": "^3.4",
+                "phpunit/phpunit": "^5.7.26"
+            },
+            "suggest": {
+                "ext-fileinfo": "Required for MimeType",
+                "ext-ftp": "Allows you to use FTP server storage",
+                "ext-openssl": "Allows you to use FTPS server storage",
+                "league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2",
+                "league/flysystem-aws-s3-v3": "Allows you to use S3 storage with AWS SDK v3",
+                "league/flysystem-azure": "Allows you to use Windows Azure Blob storage",
+                "league/flysystem-cached-adapter": "Flysystem adapter decorator for metadata caching",
+                "league/flysystem-eventable-filesystem": "Allows you to use EventableFilesystem",
+                "league/flysystem-rackspace": "Allows you to use Rackspace Cloud Files",
+                "league/flysystem-sftp": "Allows you to use SFTP server storage via phpseclib",
+                "league/flysystem-webdav": "Allows you to use WebDAV storage",
+                "league/flysystem-ziparchive": "Allows you to use ZipArchive adapter",
+                "spatie/flysystem-dropbox": "Allows you to use Dropbox storage",
+                "srmklive/flysystem-dropbox-v2": "Allows you to use Dropbox storage for PHP 5 applications"
+            },
+            "type": "library",
+            "extra": {
+                "branch-alias": {
+                    "dev-master": "1.1-dev"
+                }
+            },
+            "autoload": {
+                "psr-4": {
+                    "League\\Flysystem\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "MIT"
+            ],
+            "authors": [
+                {
+                    "name": "Frank de Jonge",
+                    "email": "info@frenky.net"
+                }
+            ],
+            "description": "Filesystem abstraction: Many filesystems, one API.",
+            "keywords": [
+                "Cloud Files",
+                "WebDAV",
+                "abstraction",
+                "aws",
+                "cloud",
+                "copy.com",
+                "dropbox",
+                "file systems",
+                "files",
+                "filesystem",
+                "filesystems",
+                "ftp",
+                "rackspace",
+                "remote",
+                "s3",
+                "sftp",
+                "storage"
+            ],
+            "time": "2020-05-18T15:13:39+00:00"
+        },
         {
             "name": "mongodb/mongodb",
             "version": "1.6.0",

storage/app/.gitignore

@@ -0,0 +1,3 @@
+!.gitignore
+failovers.json
+source_failover.lock

storage/logs/.gitignore

@@ -1,2 +1,5 @@
 *
 !.gitignore
+source-health-monitor.log
+worker.error.log
+worker.log