mirror of
https://github.com/jikan-me/jikan-rest.git
synced 2025-02-20 11:23:35 +08:00
refactor entire caching logic
This commit is contained in:
Irfan
parent
bdedc4c965
commit
e13bee3f27
@ -2,6 +2,7 @@ APP_ENV=production
|
||||
APP_DEBUG=false
|
||||
APP_KEY=
|
||||
APP_TIMEZONE=UTC
|
||||
APP_URL=http://localhost
|
||||
|
||||
DB_CONNECTION=mysql
|
||||
DB_HOST=127.0.0.1
|
||||
@ -11,7 +12,7 @@ DB_USERNAME=homestead
|
||||
DB_PASSWORD=secret
|
||||
|
||||
CACHE_DRIVER=redis
|
||||
QUEUE_DRIVER=sync
|
||||
QUEUE_DRIVER=redis
|
||||
|
||||
CACHE_DEFAULT_EXPIRE=86400
|
||||
CACHE_META_EXPIRE=300
|
||||
@ -22,7 +23,7 @@ CACHE_SEARCH_EXPIRE=432000
|
||||
THROTTLE=false
|
||||
THROTTLE_DECAY_MINUTES=1
|
||||
THROTTLE_MAX_PER_DECAY_MINUTES=30
|
||||
THROTTLE_MAX_PER_SECOND=2
|
||||
THROTTLE_MAX_PER_CONCURRENCY=2
|
||||
|
||||
SLAVE_INSTANCE=false
|
||||
SLAVE_KEY=
|
||||
|
||||
151
app/Http/Middleware/JikanResponseHandler.php
Normal file
151
app/Http/Middleware/JikanResponseHandler.php
Normal file
@ -0,0 +1,151 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* This middleware is the successor of JikanResponseLegacy; used for REST v3.3+
|
||||
*
|
||||
* It works by storing cache with no automated TTL handling by Redis
|
||||
*
|
||||
* If a request is past it's TTL, it queues an update instead of removing the cache followed by fetching a new one
|
||||
* Update queues are automated.
|
||||
*
|
||||
* Therefore,
|
||||
* - if MyAnimeList is down or rate-limits the response, stale cache is served
|
||||
* - if cache expires, the client doesn't have to wait longer for the server to fetch+parse the new response
|
||||
*/
|
||||
|
||||
|
||||
namespace App\Http\Middleware;
|
||||
|
||||
use App\Http\HttpHelper;
|
||||
use App\Jobs\UpdateCacheJob;
|
||||
use Closure;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
class JikanResponseHandler
|
||||
{
|
||||
private $requestUri;
|
||||
private $requestUriHash;
|
||||
private $requestType;
|
||||
private $requestCacheExpiry;
|
||||
private $requestCached = false;
|
||||
private $requestCacheTtl;
|
||||
|
||||
private $fingerprint;
|
||||
private $cacheExpiryFingerprint;
|
||||
|
||||
private $controllerName;
|
||||
private $controllerMethod;
|
||||
|
||||
public function handle(Request $request, Closure $next)
|
||||
{
|
||||
|
||||
if (empty($request->segments())) {return $next($request);}
|
||||
if (!isset($request->segments()[1])){return $next($request);}
|
||||
if (\in_array('meta', $request->segments())) {return $next($request);}
|
||||
if ($request->header('auth') === env('APP_KEY')) {return $next($request);}
|
||||
|
||||
|
||||
$this->requestUri = $request->getRequestUri();
|
||||
$this->requestUriHash = sha1(env('APP_URL') . $this->requestUri);
|
||||
$this->requestType = HttpHelper::requestType($request);
|
||||
|
||||
$this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
|
||||
$this->requestCacheExpiry = time() + $this->requestCacheTtl;
|
||||
|
||||
$this->fingerprint = "request:{$this->requestType}:{$this->requestUriHash}";
|
||||
$this->cacheExpiryFingerprint = "ttl:{$this->fingerprint}";
|
||||
|
||||
$this->requestCached = (bool) app('redis')->exists($this->fingerprint);
|
||||
|
||||
// Cache if it doesn't exist
|
||||
if (!$this->requestCached) {
|
||||
$response = $next($request);
|
||||
|
||||
if (HttpHelper::hasError($response)) {
|
||||
return $response;
|
||||
}
|
||||
|
||||
app('redis')->set($this->fingerprint, $response->original);
|
||||
app('redis')->set($this->cacheExpiryFingerprint, $this->requestCacheExpiry);
|
||||
}
|
||||
|
||||
// If cache is expired, queue for an update
|
||||
$this->requestCacheExpiry = (int) app('redis')->get($this->cacheExpiryFingerprint);
|
||||
|
||||
|
||||
if ($this->requestCacheExpiry < time()) {
|
||||
|
||||
$queueFingerprint = "queue_update:{$this->fingerprint}";
|
||||
|
||||
// Don't duplicate the queue for same request
|
||||
if (!app('redis')->exists($queueFingerprint)) {
|
||||
app('redis')->set($queueFingerprint, 1);
|
||||
dispatch(new UpdateCacheJob($request));
|
||||
} else {
|
||||
Log::info("Duplicate ({$queueFingerprint})");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// ETag
|
||||
if (
|
||||
$request->hasHeader('If-None-Match')
|
||||
&& app('redis')->exists($this->fingerprint)
|
||||
&& md5(app('redis')->get($this->fingerprint)) === $request->header('If-None-Match')
|
||||
) {
|
||||
return response('', 304);
|
||||
}
|
||||
|
||||
// Return cache
|
||||
$meta = $this->generateMeta($request);
|
||||
|
||||
$cache = app('redis')->get($this->fingerprint);
|
||||
$cacheMutable = json_decode(app('redis')->get($this->fingerprint), true);
|
||||
|
||||
|
||||
return response()
|
||||
->json(
|
||||
array_merge($meta, $cacheMutable)
|
||||
)
|
||||
->setEtag(
|
||||
md5($cache)
|
||||
)
|
||||
->withHeaders([
|
||||
'X-Request-Hash' => $this->fingerprint,
|
||||
'X-Request-Cached' => $this->requestCached,
|
||||
'X-Request-Cache-Expiry' => app('redis')->get($this->cacheExpiryFingerprint) - time()
|
||||
]);
|
||||
|
||||
}
|
||||
|
||||
private function generateMeta(Request $request) : array
|
||||
{
|
||||
$version = HttpHelper::requestAPIVersion($request);
|
||||
|
||||
$meta = [
|
||||
'request_hash' => $this->fingerprint,
|
||||
'request_cached' => $this->requestCached,
|
||||
'request_cache_expiry' => app('redis')->get($this->cacheExpiryFingerprint) - time()
|
||||
];
|
||||
|
||||
switch ($version) {
|
||||
case 2:
|
||||
$meta = array_merge([
|
||||
'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JULY 01st, 2019.',
|
||||
], $meta);
|
||||
break;
|
||||
case 4:
|
||||
// remove cache data from JSON response and send as headers
|
||||
unset($meta['request_cached'], $meta['request_cache_expiry']);
|
||||
$meta = array_merge([
|
||||
'DEVELOPMENT_NOTICE' => 'THIS VERSION IS IN TESTING. DO NOT USE FOR PRODUCTION.',
|
||||
'MIGRATION' => 'https://github.com/jikan-me/jikan-rest/blob/master/MIGRATION.MD',
|
||||
], $meta);
|
||||
break;
|
||||
}
|
||||
|
||||
|
||||
return $meta;
|
||||
}
|
||||
}
|
||||
@ -1,12 +1,21 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* This middleware was used up to REST v3.2; it was previously named `JikanResponse`
|
||||
*
|
||||
* It works by storing cache with TTL
|
||||
* Redis automatically removes any cache that's past it's TTL
|
||||
*
|
||||
* This middleware has been succeeded by JikanResponseHandler
|
||||
*/
|
||||
|
||||
namespace App\Http\Middleware;
|
||||
|
||||
use App\Http\HttpHelper;
|
||||
use Closure;
|
||||
use Illuminate\Http\Request;
|
||||
|
||||
class JikanResponse
|
||||
class JikanResponseLegacy
|
||||
{
|
||||
private $fingerprint;
|
||||
private $requestUri;
|
||||
@ -20,7 +29,7 @@ class JikanResponse
|
||||
if (empty($request->segments())) {return $next($request);}
|
||||
if (!isset($request->segments()[1])){return $next($request);}
|
||||
if (\in_array('meta', $request->segments())) {return $next($request);}
|
||||
|
||||
if ($request->hasHeader('auth') === env('APP_ADMIN_KEY')) {return $next($request);}
|
||||
|
||||
$this->requestUri = $request->getRequestUri();
|
||||
$this->requestType = HttpHelper::requestType($request);
|
||||
@ -84,10 +93,11 @@ class JikanResponse
|
||||
switch ($version) {
|
||||
case 2:
|
||||
$meta = array_merge([
|
||||
'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JUNE 20th, 2019.',
|
||||
'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JULY 01st, 2019.',
|
||||
], $meta);
|
||||
break;
|
||||
case 4:
|
||||
// remove cache data from JSON response and send as headers
|
||||
unset($meta['request_cached'], $meta['request_cache_expiry']);
|
||||
$meta = array_merge([
|
||||
'DEVELOPMENT_NOTICE' => 'THIS VERSION IS IN TESTING. DO NOT USE FOR PRODUCTION.',
|
||||
88
app/Jobs/UpdateCacheJob.php
Normal file
88
app/Jobs/UpdateCacheJob.php
Normal file
@ -0,0 +1,88 @@
|
||||
<?php
|
||||
|
||||
namespace App\Jobs;
|
||||
|
||||
use App\Http\HttpHelper;
|
||||
use GuzzleHttp\Client;
|
||||
use Illuminate\Http\Request;
|
||||
use Illuminate\Support\Facades\Log;
|
||||
|
||||
/**
|
||||
* Class UpdateCacheJob
|
||||
* @package App\Jobs
|
||||
*/
|
||||
class UpdateCacheJob extends Job
|
||||
{
|
||||
|
||||
public $tries = 1;
|
||||
|
||||
/**
|
||||
* @var string
|
||||
*/
|
||||
protected $requestUri;
|
||||
protected $requestUriHash;
|
||||
protected $requestType;
|
||||
protected $requestCacheTtl;
|
||||
protected $requestCacheExpiry;
|
||||
protected $fingerprint;
|
||||
protected $cacheExpiryFingerprint;
|
||||
protected $requestCached;
|
||||
|
||||
/**
|
||||
* Create a new job instance.
|
||||
*
|
||||
* @return void
|
||||
*/
|
||||
public function __construct(Request $request)
|
||||
{
|
||||
$this->requestUri = $request->getRequestUri();
|
||||
|
||||
$this->requestUriHash = sha1(env('APP_URL') . $this->requestUri);
|
||||
$this->requestType = HttpHelper::requestType($request);
|
||||
|
||||
$this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
|
||||
$this->requestCacheExpiry = time() + $this->requestCacheTtl;
|
||||
|
||||
$this->fingerprint = "request:{$this->requestType}:{$this->requestUriHash}";
|
||||
$this->cacheExpiryFingerprint = "ttl:{$this->fingerprint}";
|
||||
|
||||
$this->requestCached = (bool) app('redis')->exists($this->fingerprint);
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* @throws \GuzzleHttp\Exception\GuzzleException
|
||||
*/
|
||||
public function handle() : void
|
||||
{
|
||||
$queueFingerprint = "queue_update:{$this->fingerprint}";
|
||||
|
||||
$client = new Client();
|
||||
|
||||
$response = $client
|
||||
->request(
|
||||
'GET',
|
||||
env('APP_URL') . $this->requestUri,
|
||||
[
|
||||
'headers' => [
|
||||
'auth' => env('APP_ADMIN_KEY') // skip middleware
|
||||
]
|
||||
]
|
||||
);
|
||||
|
||||
$cache = json_decode($response->getBody()->getContents(), true);
|
||||
unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
|
||||
$cache = json_encode($cache);
|
||||
|
||||
|
||||
app('redis')->set($this->fingerprint, $cache);
|
||||
app('redis')->set($this->cacheExpiryFingerprint, $this->requestCacheExpiry);
|
||||
app('redis')->del($queueFingerprint);
|
||||
|
||||
}
|
||||
|
||||
public function failed(\Exception $e)
|
||||
{
|
||||
Log::error($e->getMessage());
|
||||
}
|
||||
}
|
||||
@ -66,11 +66,11 @@ $app->singleton(
|
||||
*/
|
||||
|
||||
$app->routeMiddleware([
|
||||
'slave-auth' => App\Http\Middleware\SlaveAuthentication::class,
|
||||
'blacklist' => App\Http\Middleware\Blacklist::class,
|
||||
'meta' => App\Http\Middleware\Meta::class,
|
||||
'jikan-response' => App\Http\Middleware\JikanResponse::class,
|
||||
'jikan-response' => App\Http\Middleware\JikanResponseHandler::class,
|
||||
'throttle' => App\Http\Middleware\Throttle::class,
|
||||
'slave-auth' => App\Http\Middleware\SlaveAuthentication::class,
|
||||
]);
|
||||
|
||||
/*
|
||||
@ -85,12 +85,14 @@ $app->routeMiddleware([
|
||||
*/
|
||||
|
||||
$app->configure('database');
|
||||
$app->configure('queue');
|
||||
|
||||
$app->register(Illuminate\Redis\RedisServiceProvider::class);
|
||||
|
||||
$guzzleClient = new \GuzzleHttp\Client();
|
||||
$app->instance('GuzzleClient', $guzzleClient);
|
||||
|
||||
$jikan = new \Jikan\MyAnimeList\MalClient($app->make('GuzzleClient'));
|
||||
$jikan = new \Jikan\MyAnimeList\MalClient(app('GuzzleClient'));
|
||||
$app->instance('JikanParser', $jikan);
|
||||
|
||||
|
||||
|
||||
85
config/queue.php
Normal file
85
config/queue.php
Normal file
@ -0,0 +1,85 @@
|
||||
<?php
|
||||
|
||||
return [
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Default Queue Driver
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| The Laravel queue API supports a variety of back-ends via an unified
|
||||
| API, giving you convenient access to each back-end using the same
|
||||
| syntax for each one. Here you may set the default queue driver.
|
||||
|
|
||||
| Supported: "null", "sync", "database", "beanstalkd", "sqs", "redis"
|
||||
|
|
||||
*/
|
||||
|
||||
'default' => env('QUEUE_DRIVER', 'redis'),
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Queue Connections
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| Here you may configure the connection information for each server that
|
||||
| is used by your application. A default configuration has been added
|
||||
| for each back-end shipped with Laravel. You are free to add more.
|
||||
|
|
||||
*/
|
||||
|
||||
'connections' => [
|
||||
|
||||
'sync' => [
|
||||
'driver' => 'sync',
|
||||
],
|
||||
|
||||
'database' => [
|
||||
'driver' => 'database',
|
||||
'table' => 'jobs',
|
||||
'queue' => 'default',
|
||||
'retry_after' => 60,
|
||||
],
|
||||
|
||||
'beanstalkd' => [
|
||||
'driver' => 'beanstalkd',
|
||||
'host' => 'localhost',
|
||||
'queue' => 'default',
|
||||
'retry_after' => 60,
|
||||
],
|
||||
|
||||
'sqs' => [
|
||||
'driver' => 'sqs',
|
||||
'key' => 'your-public-key',
|
||||
'secret' => 'your-secret-key',
|
||||
'queue' => 'your-queue-url',
|
||||
'region' => 'us-east-1',
|
||||
],
|
||||
|
||||
'redis' => [
|
||||
'driver' => 'redis',
|
||||
'connection' => env('QUEUE_REDIS_CONNECTION', 'default'),
|
||||
'queue' => 'default',
|
||||
'retry_after' => 60,
|
||||
// 'block_for' => 5,
|
||||
],
|
||||
|
||||
],
|
||||
|
||||
/*
|
||||
|--------------------------------------------------------------------------
|
||||
| Failed Queue Jobs
|
||||
|--------------------------------------------------------------------------
|
||||
|
|
||||
| These options configure the behavior of failed queue job logging so you
|
||||
| can control which database and table are used to store the jobs that
|
||||
| have failed. You may change them to any database / table you wish.
|
||||
|
|
||||
*/
|
||||
|
||||
'failed' => [
|
||||
'database' => 'mysql',
|
||||
'table' => env('QUEUE_FAILED_TABLE', 'failed_jobs'),
|
||||
],
|
||||
|
||||
];
|
||||
Loading…
x
Reference in New Issue
Block a user