malhuda/jikan-rest
1
0
Fork 0
mirror of https://github.com/jikan-me/jikan-rest.git synced 2025-02-20 11:23:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

add automatic MAL health checker/failover based on uncached response statuses

Browse Source
This commit is contained in:
Irfan 2020-05-26 19:10:12 +05:00
parent b542d100c7
commit 7d9d60833e
13 changed files with 476 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
app/Events/SourceHealthEvent.php Normal file
View File

@ -0,0 +1,31 @@
<?php
namespace App\Events;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Storage;
/**
* Class SourceHealthEvent
* @package App\Events
*/
class SourceHealthEvent extends Event
{
public const BAD_HEALTH = 1;
public const GOOD_HEALTH = 0;
public $health;
public $status;
/**
* SourceHealthEvent constructor.
* @param int $health
* @param int $status
*/
public function __construct(int $health = self::BAD_HEALTH, ?int $status)
{
$this->health = $health;
$this->status = $status ?? 0;
}
}

8
app/Exceptions/Handler.php
View File

@ -2,6 +2,7 @@
namespace App\Exceptions;
use App\Events\SourceHealthEvent;
use App\Http\HttpHelper;
use Exception;
use GuzzleHttp\Exception\ClientException;
@ -109,10 +110,17 @@ class Handler extends ExceptionHandler
'message' => 'Jikan is being rate limited by MyAnimeList',
'error' => $e->getMessage()
], $e->getCode());
case 403:
case 500:
case 501:
case 502:
case 503:
case 504:
// Dispatch Bad source health event to prompt database fallback if enabled
if (env('SOURCE_BAD_HEALTH_FALLBACK') && env('DB_CACHING')) {
event(new SourceHealthEvent(SourceHealthEvent::BAD_HEALTH, $e->getCode()));
}
return response()
->json([
'status' => $e->getCode(),

78
app/Http/Middleware/DatabaseResolver.php
View File

@ -3,8 +3,11 @@
namespace App\Http\Middleware;
use App\DatabaseHandler;
use App\Events\SourceHealthEvent;
use App\Http\HttpHelper;
use App\Jobs\UpdateCacheJob;
use App\Jobs\UpdateDatabaseJob;
use App\Providers\SourceHealthServiceProvider;
use Closure;
use Flipbox\LumenGenerator\LumenGeneratorServiceProvider;
use Illuminate\Http\Request;
@ -12,6 +15,7 @@ use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Date;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;
use Illuminate\Support\Facades\Storage;
use Jenssegers\Mongodb\MongodbServiceProvider;
use MongoDB\BSON\UTCDateTime;
use MongoDB\Client;
@ -33,6 +37,8 @@ class DatabaseResolver
private $queueable = true;
private $table;
private const NON_QUEUEABLE = [
'UserController@profile',
'UserController@history',
@ -47,6 +53,7 @@ class DatabaseResolver
public function handle(Request $request, Closure $next)
{
if ($request->header('auth') === env('APP_KEY')) {
return $next($request);
}
@ -73,35 +80,49 @@ class DatabaseResolver
$this->route = end($this->route);
$db = new DatabaseHandler();
$table = $db::getMappedTableName($this->route);
$this->table = $db::getMappedTableName($this->route);
$this->requestCached = DB::table($this->table)->where('request_hash', $this->fingerprint)->exists();
$this->requestCached = DB::table($table)->where('request_hash', $this->fingerprint)->exists();
// Cache if it doesn't exist
if (!$this->requestCached) {
$response = $next($request);
if (HttpHelper::hasError($response)) {
return $response;
}
DB::table($table)->insert(array_merge(
[
'expireAfterSeconds' => $this->requestCacheTtl,
'request_hash' => $this->fingerprint
],
json_decode($response->original, true)
));
// Is the request queueable?
if (\in_array($this->route, self::NON_QUEUEABLE) || env('CACHE_METHOD', 'legacy') === 'legacy') {
$this->queueable = false;
}
// If cache does not exist
if (!$this->requestCached) {
return $this->fetchFresh($request, $next);
}
// Return response
// Fetch Cache & Generate Meta
$meta = $this->generateMeta($request);
$cache = DB::table($table)->where('request_hash', $this->fingerprint)->get();
$cache = DB::table($this->table)->where('request_hash', $this->fingerprint)->get();
$cacheMutable = json_decode($cache, true)[0];
$cacheMutable = $this->cacheMutation($cacheMutable);
// If cache is expired, handle it depending on whether it's queueable
$expiresAt = $cacheMutable['expiresAt']['$date']['$numberLong']/1000;
if ($this->requestCached && $expiresAt > time() && !$this->queueable) {
return $this->fetchFresh($request, $next);
}
if ( $this->queueable && $expiresAt > time()) {
$queueFingerprint = "queue_update:{$this->fingerprint}";
$queueHighPriority = \in_array($this->route, self::HIGH_PRIORITY_QUEUE);
// Don't duplicate the job in the queue for same request
$job = DB::table(env('QUEUE_TABLE', 'jobs'))->where('request_hash', $this->fingerprint);
if (!$job->exists()) {
dispatch(
(new UpdateDatabaseJob($request))
->onQueue($queueHighPriority ? 'high' : 'low')
);
}
}
$response = array_merge($meta, $cacheMutable);
unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id']);
@ -163,4 +184,21 @@ class DatabaseResolver
return $data;
}
public function fetchFresh($request, $next)
{
$response = $next($request);
if (HttpHelper::hasError($response)) {
return $response;
}
DB::table($this->table)->insert(array_merge(
[
'expiresAt' => new UTCDateTime((time()+$this->requestCacheTtl)*1000),
'request_hash' => $this->fingerprint
],
json_decode($response->original, true)
));
}
}

25
app/Http/Middleware/SourceHealthMonitor.php Normal file
View File

@ -0,0 +1,25 @@
<?php
namespace App\Http\Middleware;
use Closure;
use App\Events\SourceHealthEvent;
class SourceHealthMonitor
{
/**
* Handle an incoming request.
*
* @param \Illuminate\Http\Request $request
* @param \Closure $next
* @return mixed
*/
public function handle($request, Closure $next)
{
if (env('SOURCE_BAD_HEALTH_FAILOVER') && env('DB_CACHING')) {
event(new SourceHealthEvent(SourceHealthEvent::GOOD_HEALTH, 200));
}
return $next($request);
}
}

77
app/Jobs/UpdateDatabaseJob.php Normal file
View File

@ -0,0 +1,77 @@
<?php
namespace App\Jobs;
use App\Http\HttpHelper;
use GuzzleHttp\Client;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Cache;
/**
* Class UpdateDatabaseJob
* @package App\Jobs
*/
class UpdateDatabaseJob extends Job
{
public $timeout = 60;
public $retryAfter = 60;
/**
* @var string
*/
protected $requestUri;
protected $requestType;
protected $requestCacheTtl;
protected $fingerprint;
protected $cacheExpiryFingerprint;
protected $requestCached;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct(Request $request)
{
$this->fingerprint = HttpHelper::resolveRequestFingerprint($request);
$this->requestCached = DB::table(env('QUEUE_TABLE', 'jobs'))->where('request_hash', $this->fingerprint);
}
/**
* @throws \GuzzleHttp\Exception\GuzzleException
*/
public function handle() : void
{
$client = new Client();
$response = $client
->request(
'GET',
env('APP_URL') . $this->requestUri,
[
'headers' => [
'auth' => env('APP_KEY') // skip middleware
]
]
);
$cache = json_decode($response->getBody()->getContents(), true);
unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
$cache = json_encode($cache);
sleep((int) env('QUEUE_DELAY_PER_JOB', 5));
}
public function failed(\Exception $e)
{
Log::error($e->getMessage());
}
}

150
app/Listeners/SourceHealthListener.php Normal file
View File

@ -0,0 +1,150 @@
<?php
namespace App\Listeners;
use App\Events\ExampleEvent;
use App\Events\SourceHealthEvent;
use App\Providers\SourceHealthServiceProvider;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Support\Facades\Storage;
use League\Flysystem\FileNotFoundException;
use Monolog\Handler\StreamHandler;
use Monolog\Logger;
class SourceHealthListener
{
private $logger;
/**
* Create the event listener.
*
* @return void
*/
public function __construct()
{
$this->logger = new Logger('source-health-monitor');
$this->logger->pushHandler(new StreamHandler(storage_path().'/logs/source-health-monitor.log'), Logger::DEBUG);
if (SourceHealthServiceProvider::isFailoverEnabled()) {
$lastFailoverLockTimestamp = $this->getLastFailoverLockTimestamp();
$this->logger->debug('Failover is RUNNING');
// Disable failover if it has expired
if (time() > ($lastFailoverLockTimestamp + env('SOURCE_BAD_HEALTH_RECHECK'))) {
// Disable failover if successful requests score
$this->attemptDisableFailover();
}
}
}
/**
* Handle the event.
*
* @param ExampleEvent $event
* @return void
*/
public function handle(SourceHealthEvent $event)
{
$eventCount = $this->insertFail($event);
$this->logger->debug('Event count: '.$eventCount);
if ($this->getSuccessfulRequestsScore() <= 0.25) {
$this->enableFailover();
}
}
private function insertFail(SourceHealthEvent $event) : int
{
$fails = $this->getRecentFails();
$fails[] = [time(), $event->status, $event->health];
$failsJson = json_encode($fails);
Storage::put('failovers.json', $failsJson);
return count($fails);
}
private function enableFailover()
{
// create lock file
Storage::put('source_failover.lock', '');
$this->logger->debug('Failover ENABLED');
}
private function disableFailover()
{
// delete lock file
Storage::delete('source_failover.lock');
// Delete meta
Storage::delete('failovers.json');
}
private function attemptDisableFailover()
{
$score = $this->getSuccessfulRequestsScore();
if ($score >= 0.9) {
$this->disableFailover();
$this->logger->debug('Failover disabled; Score: '.$score);
$this->logger->debug('Failover DISABLED');
return true;
}
return false;
}
private function getLastFailoverLockTimestamp()
{
try {
return Storage::lastModified('source_failover.lock');
} catch (\Exception $e) {
return 0;
}
}
private function getRecentFails()
{
try {
$failsJson = Storage::get('failovers.json');
$fails = json_decode($failsJson, true);
} catch (\Exception $e) {
$fails = [];
}
// remove any fails greater than SOURCE_BAD_HEALTH_RANGE
foreach ($fails as $fail) {
if ($fail[0] >= (time()-env('SOURCE_BAD_HEALTH_RANGE'))) {
unset($fail);
}
}
// slice
if (count($fails) > env('SOURCE_BAD_HEALTH_MAX_STORE')) {
$fails = array_slice($fails, 0 - env('SOURCE_BAD_HEALTH_MAX_STORE'));
}
return $fails;
}
private function getSuccessfulRequestsScore() : float
{
$fails = $this->getRecentFails();
$score = 0;
$totalFails = count($fails) - 1;
foreach ($fails as $fail) {
if ((int) $fail[2] === SourceHealthEvent::GOOD_HEALTH) {
$score++;
}
}
$scored = $score / max($totalFails, 1);
$this->logger->debug('Failover successful requests score: '.$scored);
return $scored;
}
}

3
app/Providers/SearchQueryBuilder.php
View File

@ -235,9 +235,6 @@ class SearchQueryBuilder
}
}
// Anime
if ($request instanceof AnimeSearchRequest) {
// Rating/Rated

28
app/Providers/SourceHealthServiceProvider.php Normal file
View File

@ -0,0 +1,28 @@
<?php
namespace App\Providers;
use Illuminate\Support\Facades\Storage;
use Laravel\Lumen\Providers\EventServiceProvider as ServiceProvider;
class SourceHealthServiceProvider extends ServiceProvider
{
const BAD_HEALTH_STATUSES = [403, 500, 501, 502, 503, 504, 505];
/**
* The event listener mappings for the application.
*
* @var array
*/
protected $listen = [
'App\Events\SourceHealthEvent' => [
'App\Listeners\SourceHealthListener',
],
];
public static function isFailoverEnabled() : bool
{
return Storage::exists('source_failover.lock');
}
}

8
bootstrap/app.php
View File

@ -77,7 +77,8 @@ $app->routeMiddleware([
'throttle' => App\Http\Middleware\Throttle::class,
'etag' => \App\Http\Middleware\EtagMiddleware::class,
'microcaching' => \App\Http\Middleware\MicroCaching::class,
'database-resolver' => \App\Http\Middleware\DatabaseResolver::class
'database-resolver' => \App\Http\Middleware\DatabaseResolver::class,
'source-health-monitor' => \App\Http\Middleware\SourceHealthMonitor::class
]);
/*
@ -108,6 +109,10 @@ $app->instance('GuzzleClient', $guzzleClient);
$jikan = new \Jikan\MyAnimeList\MalClient(app('GuzzleClient'));
$app->instance('JikanParser', $jikan);
if (env('SOURCE_BAD_HEALTH_FAILOVER') && env('DB_CACHING')) {
$app->register(\App\Providers\SourceHealthServiceProvider::class);
}
/**
* Load Blacklist into Redis
@ -134,6 +139,7 @@ $commonMiddleware = [
// 'microcaching',
// 'cache-resolver',
// 'throttle'
'source-health-monitor'
];
$app->router->group(

1
composer.json
View File

@ -17,6 +17,7 @@
"jikan-me/jikan": "^3.0",
"jms/serializer": "^1.13",
"laravel/lumen-framework": "^7.0",
"league/flysystem": "^1.0",
"ocramius/package-versions": "^1.4",
"predis/predis": "^1.1",
"symfony/yaml": "^4.1",

86
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file",
"This file is @generated automatically"
],
"content-hash": "6d42be876cc0e4c456de620bd8aa6321",
"content-hash": "14c8c6eb422e03b6838bba397af565ff",
"packages": [
{
"name": "brick/math",
@ -2523,6 +2523,90 @@
],
"time": "2020-05-05T18:17:08+00:00"
},
{
"name": "league/flysystem",
"version": "1.0.69",
"source": {
"type": "git",
"url": "https://github.com/thephpleague/flysystem.git",
"reference": "7106f78428a344bc4f643c233a94e48795f10967"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/thephpleague/flysystem/zipball/7106f78428a344bc4f643c233a94e48795f10967",
"reference": "7106f78428a344bc4f643c233a94e48795f10967",
"shasum": ""
},
"require": {
"ext-fileinfo": "*",
"php": ">=5.5.9"
},
"conflict": {
"league/flysystem-sftp": "<1.0.6"
},
"require-dev": {
"phpspec/phpspec": "^3.4",
"phpunit/phpunit": "^5.7.26"
},
"suggest": {
"ext-fileinfo": "Required for MimeType",
"ext-ftp": "Allows you to use FTP server storage",
"ext-openssl": "Allows you to use FTPS server storage",
"league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2",
"league/flysystem-aws-s3-v3": "Allows you to use S3 storage with AWS SDK v3",
"league/flysystem-azure": "Allows you to use Windows Azure Blob storage",
"league/flysystem-cached-adapter": "Flysystem adapter decorator for metadata caching",
"league/flysystem-eventable-filesystem": "Allows you to use EventableFilesystem",
"league/flysystem-rackspace": "Allows you to use Rackspace Cloud Files",
"league/flysystem-sftp": "Allows you to use SFTP server storage via phpseclib",
"league/flysystem-webdav": "Allows you to use WebDAV storage",
"league/flysystem-ziparchive": "Allows you to use ZipArchive adapter",
"spatie/flysystem-dropbox": "Allows you to use Dropbox storage",
"srmklive/flysystem-dropbox-v2": "Allows you to use Dropbox storage for PHP 5 applications"
},
"type": "library",
"extra": {
"branch-alias": {
"dev-master": "1.1-dev"
}
},
"autoload": {
"psr-4": {
"League\\Flysystem\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"MIT"
],
"authors": [
{
"name": "Frank de Jonge",
"email": "info@frenky.net"
}
],
"description": "Filesystem abstraction: Many filesystems, one API.",
"keywords": [
"Cloud Files",
"WebDAV",
"abstraction",
"aws",
"cloud",
"copy.com",
"dropbox",
"file systems",
"files",
"filesystem",
"filesystems",
"ftp",
"rackspace",
"remote",
"s3",
"sftp",
"storage"
],
"time": "2020-05-18T15:13:39+00:00"
},
{
"name": "mongodb/mongodb",
"version": "1.6.0",

3
storage/app/.gitignore vendored
View File

@ -0,0 +1,3 @@
!.gitignore
failovers.json
source_failover.lock

3
storage/logs/.gitignore vendored
View File

@ -1,2 +1,5 @@
*
!.gitignore
source-health-monitor.log
worker.error.log
worker.log