From e13bee3f272d2b2210bb6bc2c507f729fe8a498c Mon Sep 17 00:00:00 2001
From: Irfan <irfan.dahir@protonmail.com>
Date: Sat, 11 May 2019 11:43:55 +0500
Subject: [PATCH] refactor entire caching logic

---
 .env.dist                                     |   5 +-
 app/Http/Middleware/JikanResponseHandler.php  | 151 ++++++++++++++++++
 ...anResponse.php => JikanResponseLegacy.php} |  16 +-
 app/Jobs/UpdateCacheJob.php                   |  88 ++++++++++
 bootstrap/app.php                             |   8 +-
 config/queue.php                              |  85 ++++++++++
 6 files changed, 345 insertions(+), 8 deletions(-)
 create mode 100644 app/Http/Middleware/JikanResponseHandler.php
 rename app/Http/Middleware/{JikanResponse.php => JikanResponseLegacy.php} (86%)
 create mode 100644 app/Jobs/UpdateCacheJob.php
 create mode 100644 config/queue.php

diff --git a/.env.dist b/.env.dist
index 9c4252f..93d4933 100644
--- a/.env.dist
+++ b/.env.dist
@@ -2,6 +2,7 @@ APP_ENV=production
 APP_DEBUG=false
 APP_KEY=
 APP_TIMEZONE=UTC
+APP_URL=http://localhost
 
 DB_CONNECTION=mysql
 DB_HOST=127.0.0.1
@@ -11,7 +12,7 @@ DB_USERNAME=homestead
 DB_PASSWORD=secret
 
 CACHE_DRIVER=redis
-QUEUE_DRIVER=sync
+QUEUE_DRIVER=redis
 
 CACHE_DEFAULT_EXPIRE=86400
 CACHE_META_EXPIRE=300
@@ -22,7 +23,7 @@ CACHE_SEARCH_EXPIRE=432000
 THROTTLE=false
 THROTTLE_DECAY_MINUTES=1
 THROTTLE_MAX_PER_DECAY_MINUTES=30
-THROTTLE_MAX_PER_SECOND=2
+THROTTLE_MAX_PER_CONCURRENCY=2
 
 SLAVE_INSTANCE=false
 SLAVE_KEY=
diff --git a/app/Http/Middleware/JikanResponseHandler.php b/app/Http/Middleware/JikanResponseHandler.php
new file mode 100644
index 0000000..1436d98
--- /dev/null
+++ b/app/Http/Middleware/JikanResponseHandler.php
@@ -0,0 +1,151 @@
+<?php
+
+/**
+ * This middleware is the successor of JikanResponseLegacy; used for REST v3.3+
+ *
+ * It works by storing cache with no automated TTL handling by Redis
+ *
+ * If a request is past it's TTL, it queues an update instead of removing the cache followed by fetching a new one
+ * Update queues are automated.
+ *
+ * Therefore,
+ * - if MyAnimeList is down or rate-limits the response, stale cache is served
+ * - if cache expires, the client doesn't have to wait longer for the server to fetch+parse the new response
+ */
+
+
+namespace App\Http\Middleware;
+
+use App\Http\HttpHelper;
+use App\Jobs\UpdateCacheJob;
+use Closure;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Log;
+
+class JikanResponseHandler
+{
+    private $requestUri;
+    private $requestUriHash;
+    private $requestType;
+    private $requestCacheExpiry;
+    private $requestCached = false;
+    private $requestCacheTtl;
+
+    private $fingerprint;
+    private $cacheExpiryFingerprint;
+
+    private $controllerName;
+    private $controllerMethod;
+
+    public function handle(Request $request, Closure $next)
+    {
+
+        if (empty($request->segments())) {return $next($request);}
+        if (!isset($request->segments()[1])){return $next($request);}
+        if (\in_array('meta', $request->segments())) {return $next($request);}
+        if ($request->header('auth') === env('APP_KEY')) {return $next($request);}
+
+
+        $this->requestUri = $request->getRequestUri();
+        $this->requestUriHash = sha1(env('APP_URL') . $this->requestUri);
+        $this->requestType = HttpHelper::requestType($request);
+
+        $this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
+        $this->requestCacheExpiry = time() + $this->requestCacheTtl;
+
+        $this->fingerprint = "request:{$this->requestType}:{$this->requestUriHash}";
+        $this->cacheExpiryFingerprint = "ttl:{$this->fingerprint}";
+
+        $this->requestCached = (bool) app('redis')->exists($this->fingerprint);
+
+        // Cache if it doesn't exist
+        if (!$this->requestCached) {
+            $response = $next($request);
+
+            if (HttpHelper::hasError($response)) {
+                return $response;
+            }
+
+            app('redis')->set($this->fingerprint, $response->original);
+            app('redis')->set($this->cacheExpiryFingerprint, $this->requestCacheExpiry);
+        }
+
+        // If cache is expired, queue for an update
+        $this->requestCacheExpiry = (int) app('redis')->get($this->cacheExpiryFingerprint);
+
+
+        if ($this->requestCacheExpiry < time()) {
+
+            $queueFingerprint = "queue_update:{$this->fingerprint}";
+
+            // Don't duplicate the queue for same request
+            if (!app('redis')->exists($queueFingerprint)) {
+                app('redis')->set($queueFingerprint, 1);
+                dispatch(new UpdateCacheJob($request));
+            } else {
+                Log::info("Duplicate ({$queueFingerprint})");
+            }
+        }
+
+
+        // ETag
+        if (
+            $request->hasHeader('If-None-Match')
+            && app('redis')->exists($this->fingerprint)
+            && md5(app('redis')->get($this->fingerprint)) === $request->header('If-None-Match')
+        ) {
+            return response('', 304);
+        }
+
+        // Return cache
+        $meta = $this->generateMeta($request);
+
+        $cache = app('redis')->get($this->fingerprint);
+        $cacheMutable = json_decode(app('redis')->get($this->fingerprint), true);
+
+
+        return response()
+            ->json(
+                array_merge($meta, $cacheMutable)
+            )
+            ->setEtag(
+                md5($cache)
+            )
+            ->withHeaders([
+                'X-Request-Hash' => $this->fingerprint,
+                'X-Request-Cached' => $this->requestCached,
+                'X-Request-Cache-Expiry' => app('redis')->get($this->cacheExpiryFingerprint) - time()
+            ]);
+
+    }
+
+    private function generateMeta(Request $request) : array
+    {
+        $version = HttpHelper::requestAPIVersion($request);
+
+        $meta = [
+            'request_hash' => $this->fingerprint,
+            'request_cached' => $this->requestCached,
+            'request_cache_expiry' => app('redis')->get($this->cacheExpiryFingerprint) - time()
+        ];
+
+        switch ($version) {
+            case 2:
+                $meta = array_merge([
+                    'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JULY 01st, 2019.',
+                ], $meta);
+                break;
+            case 4:
+                // remove cache data from JSON response and send as headers
+                unset($meta['request_cached'], $meta['request_cache_expiry']);
+                $meta = array_merge([
+                    'DEVELOPMENT_NOTICE' => 'THIS VERSION IS IN TESTING. DO NOT USE FOR PRODUCTION.',
+                    'MIGRATION' => 'https://github.com/jikan-me/jikan-rest/blob/master/MIGRATION.MD',
+                ], $meta);
+                break;
+        }
+
+
+        return $meta;
+    }
+}
diff --git a/app/Http/Middleware/JikanResponse.php b/app/Http/Middleware/JikanResponseLegacy.php
similarity index 86%
rename from app/Http/Middleware/JikanResponse.php
rename to app/Http/Middleware/JikanResponseLegacy.php
index fe6720a..d315379 100644
--- a/app/Http/Middleware/JikanResponse.php
+++ b/app/Http/Middleware/JikanResponseLegacy.php
@@ -1,12 +1,21 @@
 <?php
 
+/**
+ * This middleware was used up to REST v3.2; it was previously named `JikanResponse`
+ *
+ * It works by storing cache with TTL
+ * Redis automatically removes any cache that's past it's TTL
+ *
+ * This middleware has been succeeded by JikanResponseHandler
+ */
+
 namespace App\Http\Middleware;
 
 use App\Http\HttpHelper;
 use Closure;
 use Illuminate\Http\Request;
 
-class JikanResponse
+class JikanResponseLegacy
 {
     private $fingerprint;
     private $requestUri;
@@ -20,7 +29,7 @@ class JikanResponse
         if (empty($request->segments())) {return $next($request);}
         if (!isset($request->segments()[1])){return $next($request);}
         if (\in_array('meta', $request->segments())) {return $next($request);}
-
+        if ($request->hasHeader('auth') === env('APP_ADMIN_KEY')) {return $next($request);}
 
         $this->requestUri = $request->getRequestUri();
         $this->requestType = HttpHelper::requestType($request);
@@ -84,10 +93,11 @@ class JikanResponse
         switch ($version) {
             case 2:
                 $meta = array_merge([
-                    'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JUNE 20th, 2019.',
+                    'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JULY 01st, 2019.',
                 ], $meta);
                 break;
             case 4:
+                // remove cache data from JSON response and send as headers
                 unset($meta['request_cached'], $meta['request_cache_expiry']);
                 $meta = array_merge([
                     'DEVELOPMENT_NOTICE' => 'THIS VERSION IS IN TESTING. DO NOT USE FOR PRODUCTION.',
diff --git a/app/Jobs/UpdateCacheJob.php b/app/Jobs/UpdateCacheJob.php
new file mode 100644
index 0000000..8fd53fa
--- /dev/null
+++ b/app/Jobs/UpdateCacheJob.php
@@ -0,0 +1,88 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Http\HttpHelper;
+use GuzzleHttp\Client;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Log;
+
+/**
+ * Class UpdateCacheJob
+ * @package App\Jobs
+ */
+class UpdateCacheJob extends Job
+{
+
+    public $tries = 1;
+
+    /**
+     * @var string
+     */
+    protected $requestUri;
+    protected $requestUriHash;
+    protected $requestType;
+    protected $requestCacheTtl;
+    protected $requestCacheExpiry;
+    protected $fingerprint;
+    protected $cacheExpiryFingerprint;
+    protected $requestCached;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(Request $request)
+    {
+        $this->requestUri = $request->getRequestUri();
+
+        $this->requestUriHash = sha1(env('APP_URL') . $this->requestUri);
+        $this->requestType = HttpHelper::requestType($request);
+
+        $this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
+        $this->requestCacheExpiry = time() + $this->requestCacheTtl;
+
+        $this->fingerprint = "request:{$this->requestType}:{$this->requestUriHash}";
+        $this->cacheExpiryFingerprint = "ttl:{$this->fingerprint}";
+
+        $this->requestCached = (bool) app('redis')->exists($this->fingerprint);
+    }
+
+
+    /**
+     * @throws \GuzzleHttp\Exception\GuzzleException
+     */
+    public function handle() : void
+    {
+        $queueFingerprint = "queue_update:{$this->fingerprint}";
+
+        $client = new Client();
+
+        $response = $client
+            ->request(
+                'GET',
+                env('APP_URL') . $this->requestUri,
+                [
+                    'headers' => [
+                        'auth' => env('APP_ADMIN_KEY') // skip middleware
+                    ]
+                ]
+            );
+
+        $cache = json_decode($response->getBody()->getContents(), true);
+        unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
+        $cache = json_encode($cache);
+
+
+        app('redis')->set($this->fingerprint, $cache);
+        app('redis')->set($this->cacheExpiryFingerprint, $this->requestCacheExpiry);
+        app('redis')->del($queueFingerprint);
+
+    }
+
+    public function failed(\Exception $e)
+    {
+        Log::error($e->getMessage());
+    }
+}
diff --git a/bootstrap/app.php b/bootstrap/app.php
index 0a4ffa4..ab07c10 100755
--- a/bootstrap/app.php
+++ b/bootstrap/app.php
@@ -66,11 +66,11 @@ $app->singleton(
 */
 
 $app->routeMiddleware([
+    'slave-auth' => App\Http\Middleware\SlaveAuthentication::class,
     'blacklist' => App\Http\Middleware\Blacklist::class,
     'meta' => App\Http\Middleware\Meta::class,
-    'jikan-response' => App\Http\Middleware\JikanResponse::class,
+    'jikan-response' => App\Http\Middleware\JikanResponseHandler::class,
     'throttle' => App\Http\Middleware\Throttle::class,
-    'slave-auth' => App\Http\Middleware\SlaveAuthentication::class,
 ]);
 
 /*
@@ -85,12 +85,14 @@ $app->routeMiddleware([
 */
 
 $app->configure('database');
+$app->configure('queue');
+
 $app->register(Illuminate\Redis\RedisServiceProvider::class);
 
 $guzzleClient = new \GuzzleHttp\Client();
 $app->instance('GuzzleClient', $guzzleClient);
 
-$jikan = new \Jikan\MyAnimeList\MalClient($app->make('GuzzleClient'));
+$jikan = new \Jikan\MyAnimeList\MalClient(app('GuzzleClient'));
 $app->instance('JikanParser', $jikan);
 
 
diff --git a/config/queue.php b/config/queue.php
new file mode 100644
index 0000000..e904b27
--- /dev/null
+++ b/config/queue.php
@@ -0,0 +1,85 @@
+<?php
+
+return [
+
+    /*
+    |--------------------------------------------------------------------------
+    | Default Queue Driver
+    |--------------------------------------------------------------------------
+    |
+    | The Laravel queue API supports a variety of back-ends via an unified
+    | API, giving you convenient access to each back-end using the same
+    | syntax for each one. Here you may set the default queue driver.
+    |
+    | Supported: "null", "sync", "database", "beanstalkd", "sqs", "redis"
+    |
+    */
+
+    'default' => env('QUEUE_DRIVER', 'redis'),
+
+    /*
+    |--------------------------------------------------------------------------
+    | Queue Connections
+    |--------------------------------------------------------------------------
+    |
+    | Here you may configure the connection information for each server that
+    | is used by your application. A default configuration has been added
+    | for each back-end shipped with Laravel. You are free to add more.
+    |
+    */
+
+    'connections' => [
+
+        'sync' => [
+            'driver' => 'sync',
+        ],
+
+        'database' => [
+            'driver' => 'database',
+            'table' => 'jobs',
+            'queue' => 'default',
+            'retry_after' => 60,
+        ],
+
+        'beanstalkd' => [
+            'driver' => 'beanstalkd',
+            'host' => 'localhost',
+            'queue' => 'default',
+            'retry_after' => 60,
+        ],
+
+        'sqs' => [
+            'driver' => 'sqs',
+            'key' => 'your-public-key',
+            'secret' => 'your-secret-key',
+            'queue' => 'your-queue-url',
+            'region' => 'us-east-1',
+        ],
+
+        'redis' => [
+            'driver' => 'redis',
+            'connection' => env('QUEUE_REDIS_CONNECTION', 'default'),
+            'queue' => 'default',
+            'retry_after' => 60,
+//            'block_for' => 5,
+        ],
+
+    ],
+
+    /*
+    |--------------------------------------------------------------------------
+    | Failed Queue Jobs
+    |--------------------------------------------------------------------------
+    |
+    | These options configure the behavior of failed queue job logging so you
+    | can control which database and table are used to store the jobs that
+    | have failed. You may change them to any database / table you wish.
+    |
+    */
+
+    'failed' => [
+        'database' => 'mysql',
+        'table' => env('QUEUE_FAILED_TABLE', 'failed_jobs'),
+    ],
+
+];