add mongodb support for caching

app/DatabaseHandler.php

@@ -0,0 +1,26 @@
+<?php
+namespace App;
+
+use Illuminate\Support\Facades\DB;
+
+class DatabaseHandler
+{
+    public function resolve(string $table, $fingerprint, array $data)
+    {
+
+        if (DB::table($table)->where('fingerprint', $fingerprint)) {
+            return DB::table($table)->get();
+        }
+
+        DB::table('anime')->insert($data);
+    }
+
+    public static function getMappedTableName(string $controller)
+    {
+        return config('controller-to-table-mapping.'.$controller);
+    }
+
+    public function prepare(array $response)
+    {
+    }
+}

app/Exceptions/GithubReport.php

@@ -82,10 +82,12 @@ class GithubReport
         $report->jikanVersion = Versions::getVersion('jikan-me/jikan');
         $report->phpVersion = PHP_VERSION;
 
-        try {
-            $report->redisRunning = trim(app('redis')->ping()) === 'PONG' ? "Connected" : "Disconnected";
-        } catch (ConnectionException $e) {
-            $report->redisRunning = false;
+        if (env('CACHING') && env('CACHE_DRIVER') === 'redis') {
+            try {
+                $report->redisRunning = trim(app('redis')->ping()) === 'PONG' ? "Connected" : "Disconnected";
+            } catch (ConnectionException $e) {
+                $report->redisRunning = false;
+            }
         }
 
         $report->instanceType = 'UNKNOWN';

app/Exceptions/Handler.php

@@ -161,6 +161,10 @@ class Handler extends ExceptionHandler
      */
     private function set404Cache(Request $request, BadResponseException $e)
     {
+        if (!env('CACHING')) {
+            return;
+        }
+
         $fingerprint = "request:404:".sha1(env('APP_URL') . $request->getRequestUri());
 
         if (Cache::has($fingerprint)) {

app/Http/Controllers/V4/Controller.php

@@ -31,7 +31,7 @@ class Controller extends BaseController
      */
     public function __construct(MalClient $jikan)
     {
-        $this->serializer = SerializerFactory::createV3();
+        $this->serializer = SerializerFactory::createV4();
         $this->jikan = $jikan;
     }
 }

app/Http/Controllers/V4/MangaController.php

@@ -2,6 +2,7 @@
 
 namespace App\Http\Controllers\V4;
 
+use App\Http\HttpHelper;
 use Jikan\Request\Manga\MangaCharactersRequest;
 use Jikan\Request\Manga\MangaForumRequest;
 use Jikan\Request\Manga\MangaMoreInfoRequest;
@@ -23,9 +24,8 @@ class MangaController extends Controller
         $mangaSerialized = HttpHelper::serializeEmptyObjectsControllerLevel(
             json_decode($mangaSerialized, true)
         );
-        $mangaSerialized = json_encode($mangaSerialized);
 
-        return response($this->serializer->serialize($manga, 'json'));
+        return response($this->serializer->serialize($mangaSerialized, 'json'));
     }
 
     public function characters(int $id)
@@ -61,7 +61,7 @@ class MangaController extends Controller
     public function moreInfo(int $id)
     {
         $manga = ['moreinfo' => $this->jikan->getMangaMoreInfo(new MangaMoreInfoRequest($id))];
-        return response(json_encode($manga));
+        return response($this->serializer->serialize($manga, 'json'));
     }
 
     public function recommendations(int $id)

app/Http/Controllers/V4/UserController.php

@@ -16,8 +16,8 @@ class UserController extends Controller
 {
     public function profile(string $username)
     {
-        $person = $this->jikan->getUserProfile(new UserProfileRequest($username));
-        return response($this->serializer->serialize($person, 'json'));
+        $user = $this->jikan->getUserProfile(new UserProfileRequest($username));
+        return response($this->serializer->serialize($user, 'json'));
     }
 
     public function history(string $username, ?string $type = null)

app/Http/Middleware/CacheResolver.php

@@ -1,19 +1,5 @@
 <?php
 
-/**
- * This middleware is the successor of JikanResponseLegacy; used for REST v3.3+
- *
- * It works by storing cache with no automated TTL handling by Redis
- *
- * If a request is past it's TTL, it queues an update instead of removing the cache followed by fetching a new one
- * Update queues are automated.
- *
- * Therefore,
- * - if MyAnimeList is down or rate-limits the response, stale cache is served
- * - if cache expires, the client doesn't have to wait longer for the server to fetch+parse the new response
- */
-
-
 namespace App\Http\Middleware;
 
 use App\Http\HttpHelper;
@@ -22,7 +8,7 @@ use Closure;
 use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Cache;
 
-class JikanResponseHandler
+class CacheResolver
 {
     private $requestUri;
     private $requestUriHash;
@@ -52,6 +38,11 @@ class JikanResponseHandler
 
     public function handle(Request $request, Closure $next)
     {
+
+        if (!env('CACHING')) {
+            return $next($request);
+        }
+
         if ($request->header('auth') === env('APP_KEY')) {
             return $next($request);
         }

app/Http/Middleware/DatabaseResolver.php

@@ -0,0 +1,166 @@
+<?php
+
+namespace App\Http\Middleware;
+
+use App\DatabaseHandler;
+use App\Http\HttpHelper;
+use App\Jobs\UpdateCacheJob;
+use Closure;
+use Flipbox\LumenGenerator\LumenGeneratorServiceProvider;
+use Illuminate\Http\Request;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Date;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Schema;
+use Jenssegers\Mongodb\MongodbServiceProvider;
+use MongoDB\BSON\UTCDateTime;
+use MongoDB\Client;
+use MongoDB\Collection;
+
+class DatabaseResolver
+{
+    private $requestUri;
+    private $requestUriHash;
+    private $requestType;
+    private $requestCacheExpiry = 0;
+    private $requestCached = false;
+    private $requestCacheTtl;
+
+    private $fingerprint;
+    private $cacheExpiryFingerprint;
+
+    private $route;
+
+    private $queueable = true;
+
+    private const NON_QUEUEABLE = [
+        'UserController@profile',
+        'UserController@history',
+        'UserController@friends',
+        'UserController@animelist',
+        'UserController@mangalist',
+    ];
+
+    private const HIGH_PRIORITY_QUEUE = [
+        'ScheduleController@main'
+    ];
+
+    public function handle(Request $request, Closure $next)
+    {
+        if ($request->header('auth') === env('APP_KEY')) {
+            return $next($request);
+        }
+
+        if (empty($request->segments())) {
+            return $next($request);
+        }
+
+        if (!isset($request->segments()[1])) {
+            return $next($request);
+        }
+
+        if (\in_array('meta', $request->segments())) {
+            return $next($request);
+        }
+
+        $this->requestUriHash = HttpHelper::getRequestUriHash($request);
+        $this->requestType = HttpHelper::requestType($request);
+        $this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
+        $this->fingerprint = HttpHelper::resolveRequestFingerprint($request);
+        $this->cacheExpiryFingerprint = "ttl:{$this->fingerprint}";
+
+        $this->route = explode('\\', $request->route()[1]['uses']);
+        $this->route = end($this->route);
+
+        $db = new DatabaseHandler();
+        $table = $db::getMappedTableName($this->route);
+
+        $this->requestCached = DB::table($table)->where('request_hash', $this->fingerprint)->exists();
+
+        // Cache if it doesn't exist
+        if (!$this->requestCached) {
+            $response = $next($request);
+
+            if (HttpHelper::hasError($response)) {
+                return $response;
+            }
+
+            DB::table($table)->insert(array_merge(
+                [
+                    'expireAfterSeconds' => $this->requestCacheTtl,
+                    'request_hash' => $this->fingerprint
+                ],
+                json_decode($response->original, true)
+            ));
+        }
+
+
+        // Return response
+        $meta = $this->generateMeta($request);
+
+        $cache = DB::table($table)->where('request_hash', $this->fingerprint)->get();
+        $cacheMutable = json_decode($cache, true)[0];
+        $cacheMutable = $this->cacheMutation($cacheMutable);
+
+        $response = array_merge($meta, $cacheMutable);
+        unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id']);
+
+        // Build and return response
+        return response()
+            ->json(
+                $response
+            )
+            ->setEtag(
+                md5($cache)
+            )
+            ->withHeaders([
+                'X-Request-Hash' => $this->fingerprint,
+                'X-Request-Cached' => $this->requestCached,
+                'X-Request-Cache-Ttl' => (int) $this->requestCacheExpiry - time()
+            ])
+            ->setExpires((new \DateTime())->setTimestamp($this->requestCacheExpiry));
+    }
+
+    private function generateMeta(Request $request) : array
+    {
+        $version = HttpHelper::requestAPIVersion($request);
+
+        $meta = [
+            'request_hash' => $this->fingerprint,
+            'request_cached' => $this->requestCached,
+            'request_cache_expiry' => (int) $this->requestCacheExpiry - time()
+        ];
+
+        switch ($version) {
+            case 2:
+                $meta = array_merge([
+                    'DEPRECIATION_NOTICE' => 'THIS VERSION WILL BE DEPRECIATED ON JULY 01st, 2019.',
+                ], $meta);
+                break;
+            case 4:
+                // remove cache data from JSON response as it's sent as headers
+                unset($meta['request_cached'], $meta['request_cache_expiry']);
+                $meta = array_merge([
+                    'DEVELOPMENT_NOTICE' => 'THIS VERSION IS IN TESTING. DO NOT USE FOR PRODUCTION.',
+                    'MIGRATION' => 'https://github.com/jikan-me/jikan-rest/blob/master/MIGRATION.MD',
+                ], $meta);
+                break;
+        }
+
+        return $meta;
+    }
+
+    private function cacheMutation(array $data) : array
+    {
+        if (!($this->requestType === 'anime' || $this->requestType === 'manga')) {
+            return $data;
+        }
+
+        // Fix JSON response for empty related object
+        if (isset($data['related']) && \count($data['related']) === 0) {
+            $data['related'] = new \stdClass();
+        }
+
+        return $data;
+    }
+}

app/Http/Middleware/EtagMiddleware.php

@@ -17,6 +17,10 @@ class EtagMiddleware
      */
     public function handle($request, Closure $next)
     {
+        if (!env('CACHING')) {
+            return $next($request);
+        }
+
         if ($request->header('auth') === env('APP_KEY')) {
             return $next($request);
         }

app/Http/Middleware/JikanResponseLegacy.php

@@ -1,14 +1,5 @@
 <?php
 
-/**
- * This middleware was used up to REST v3.2; it was previously named `JikanResponse`
- *
- * It works by storing cache with TTL
- * Redis automatically removes any cache that's past it's TTL
- *
- * This middleware has been succeeded by JikanResponseHandler
- */
-
 namespace App\Http\Middleware;
 
 use App\Http\HttpHelper;

app/Http/Middleware/MicroCaching.php

@@ -17,10 +17,15 @@ class MicroCaching
      */
     public function handle($request, Closure $next)
     {
+
         if ($request->header('auth') === env('APP_KEY')) {
             return $next($request);
         }
 
+        if (!env('CACHING')) {
+            return $next($request);
+        }
+
         // Microcaching should not work alongside redis caching
         if (!env('MICROCACHING', false) || env('CACHE_DRIVER', 'file') === 'redis') {
             return $next($request);

app/Jobs/UpdateCacheJob.php

@@ -71,7 +71,7 @@ class UpdateCacheJob extends Job
             );
 
         $cache = json_decode($response->getBody()->getContents(), true);
-        unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
+        unset($cache['fingerprint'], $cache['request_cached'], $cache['request_cache_expiry']);
         $cache = json_encode($cache);
 
 

app/Providers/SerializerFactory.php

@@ -77,6 +77,40 @@ class SerializerFactory
         return $serializer;
     }
 
+    public static function createV4(): Serializer
+    {
+        $serializer = (new SerializerBuilder())
+            ->addMetadataDir(__DIR__.'/../../storage/app/metadata.v4')
+            ->configureHandlers(
+                function (HandlerRegistry $registry) {
+                    $registry->registerHandler(
+                        'serialization',
+                        MalUrl::class,
+                        'json',
+                        \Closure::fromCallable('self::convertMalUrl')
+                    );
+
+                    $registry->registerHandler(
+                        'serialization',
+                        DateRange::class,
+                        'json',
+                        \Closure::fromCallable('self::convertDateRange')
+                    );
+
+                    $registry->registerHandler(
+                        'serialization',
+                        \DateTimeImmutable::class,
+                        'json',
+                        \Closure::fromCallable('self::convertDateTimeImmutable')
+                    );
+                }
+            )
+            ->build();
+        $serializer->setSerializationContextFactory(new SerializationContextFactory());
+
+        return $serializer;
+    }
+
     private static function convertMalUrl($visitor, MalUrl $obj, array $type): array
     {
         return [

bootstrap/app.php

@@ -31,6 +31,8 @@ $app = new Laravel\Lumen\Application(
     realpath(__DIR__.'/../')
 );
 
+$app->register(Jenssegers\Mongodb\MongodbServiceProvider::class);
+
 $app->withFacades();
 $app->withEloquent();
 
@@ -71,10 +73,11 @@ $app->routeMiddleware([
     'blacklist' => App\Http\Middleware\Blacklist::class,
     'slave-auth' => App\Http\Middleware\SlaveAuthentication::class,
     'meta' => App\Http\Middleware\Meta::class,
-    'jikan-response' => App\Http\Middleware\JikanResponseHandler::class,
+    'cache-resolver' => App\Http\Middleware\CacheResolver::class,
     'throttle' => App\Http\Middleware\Throttle::class,
     'etag' => \App\Http\Middleware\EtagMiddleware::class,
-    'microcaching' => \App\Http\Middleware\MicroCaching::class
+    'microcaching' => \App\Http\Middleware\MicroCaching::class,
+    'database-resolver' => \App\Http\Middleware\DatabaseResolver::class
 ]);
 
 /*
@@ -90,9 +93,13 @@ $app->routeMiddleware([
 
 $app->configure('database');
 $app->configure('queue');
-$app->configure('cache');
+$app->configure('controller-to-table-mapping');
+
+if (env('CACHING')) {
+    $app->configure('cache');
+    $app->register(Illuminate\Redis\RedisServiceProvider::class);
+}
 
-$app->register(Illuminate\Redis\RedisServiceProvider::class);
 $app->register(Flipbox\LumenGenerator\LumenGeneratorServiceProvider::class);
 
 $guzzleClient = new \GuzzleHttp\Client();
@@ -119,13 +126,14 @@ $app->instance('JikanParser', $jikan);
 */
 
 $commonMiddleware = [
-    'blacklist',
-    'slave-auth',
-    'meta',
-    'etag',
-    'microcaching',
-    'jikan-response',
-    'throttle'
+//    'blacklist',
+//    'slave-auth',
+//    'meta',
+//    'etag',
+    'database-resolver',
+//    'microcaching',
+//    'cache-resolver',
+//    'throttle'
 ];
 
 $app->router->group(

composer.json

@@ -6,20 +6,22 @@
     "type": "project",
     "require": {
         "php": "^7.2.5",
-        "laravel/lumen-framework": "^7.0",
-        "vlucas/phpdotenv": "^4",
-        "danielmewes/php-rql": "dev-master",
-        "illuminate/redis": "^7",
-        "predis/predis": "^1.1",
-        "voku/anti-xss": "^4.0",
-        "divineomega/cachetphp": "^0.2.0",
-        "jms/serializer": "^1.13",
-        "symfony/yaml": "^4.1",
-        "fabpot/goutte": "3.2.3",
-        "jikan-me/jikan": "^3.0",
         "ext-json": "*",
+        "ext-mongodb": "*",
+        "danielmewes/php-rql": "dev-master",
+        "divineomega/cachetphp": "^0.2.0",
+        "fabpot/goutte": "3.2.3",
+        "flipbox/lumen-generator": "^6",
+        "illuminate/redis": "^7",
+        "jenssegers/mongodb": "^4.0",
+        "jikan-me/jikan": "^3.0",
+        "jms/serializer": "^1.13",
+        "laravel/lumen-framework": "^7.0",
         "ocramius/package-versions": "^1.4",
-        "flipbox/lumen-generator": "^6"
+        "predis/predis": "^1.1",
+        "symfony/yaml": "^4.1",
+        "vlucas/phpdotenv": "^4",
+        "voku/anti-xss": "^4.0"
     },
     "require-dev": {
         "fzaninotto/faker": "^1.9.1",

config/controller-to-table-mapping.php

@@ -0,0 +1,90 @@
+<?php
+
+return [
+
+    'AnimeController@main' => 'anime',
+    'AnimeController@characters_staff' => 'anime_characters_staff',
+    'AnimeController@episodes' => 'anime_episodes',
+    'AnimeController@episode' => 'anime_episode',
+    'AnimeController@news' => 'anime_news',
+    'AnimeController@forum' => 'anime_forum',
+    'AnimeController@videos' => 'anime_videos',
+    'AnimeController@pictures' => 'anime_pictures',
+    'AnimeController@stats' => 'anime_stats',
+    'AnimeController@moreInfo' => 'anime_moreinfo',
+    'AnimeController@recommendations' => 'anime_recommendations',
+    'AnimeController@userupdates' => 'anime_userupdates',
+    'AnimeController@reviews' => 'anime_reviews',
+
+    'MangaController@main' => 'manga',
+    'MangaController@characters' => 'manga_characters',
+    'MangaController@news' => 'manga_news',
+    'MangaController@forum' => 'manga_forum',
+    'MangaController@pictures' => 'manga_pictures',
+    'MangaController@stats' => 'manga_stats',
+    'MangaController@moreInfo' => 'manga_moreinfo',
+    'MangaController@recommendations' => 'manga_recommendations',
+    'MangaController@userupdates' => 'manga_userupdates',
+    'MangaController@reviews' => 'manga_reviews',
+
+    'CharacterController@main' => 'characters',
+    'CharacterController@pictures' => 'characters_pictures',
+
+    'PersonController@main' => 'people',
+    'PersonController@pictures' => 'people_pictures',
+
+    'SeasonController@archive' => 'season_archive',
+    'SeasonController@later' => 'season_later',
+    'SeasonController@main' => 'season',
+
+    'ScheduleController@main' => 'schedule',
+
+    'ProducerController@main' => 'producers',
+    'ProducerController@resource' => 'producers_anime',
+
+    'MagazineController@main' => 'magazines',
+    'MagazineController@resource' => 'magazines_manga',
+
+    'UserController@recentlyOnline' => 'users_recently_online',
+    'UserController@profile' => 'users',
+    'UserController@history' => 'users_history',
+    'UserController@friends' => 'users_friends',
+    'UserController@animelist' => 'users_animelist',
+    'UserController@mangalist' => 'users_mangalist',
+    'UserController@recommendations' => 'users_recommendations',
+    'UserController@reviews' => 'users_reviews',
+    'UserController@clubs' => 'users_clubs',
+
+    'GenreController@animeListing' => 'genres',
+    'GenreController@mangaListing' => 'genres',
+    'GenreController@anime' => 'genres_anime',
+    'GenreController@manga' => 'genres_manga',
+
+    'TopController@anime' => 'top_anime',
+    'TopController@manga' => 'top_manga',
+    'TopController@characters' => 'top_characters',
+    'TopController@people' => 'top_people',
+    'ReviewsController@bestVoted' => 'top_reviews',
+
+    'SearchController@anime' => 'search_anime',
+    'SearchController@manga' => 'search_manga',
+    'SearchController@character' => 'search_characters',
+    'SearchController@people' => 'search_people',
+    'SearchController@users' => 'search_users',
+    'SearchController@userById' => 'search_users_by_id',
+
+    'ClubController@main' => 'clubs',
+    'ClubController@members' => 'clubs_members',
+
+    'ReviewsController@anime' => 'reviews',
+    'ReviewsController@manga' => 'reviews',
+
+    'RecommendationsController@anime' => 'recommendations',
+    'RecommendationsController@manga' => 'recommendations',
+
+    'WatchController@recentEpisodes' => 'watch',
+    'WatchController@popularEpisodes' => 'watch',
+    'WatchController@recentPromos' => 'watch',
+    'WatchController@popularPromos' => 'watch',
+
+];

config/database.php

@@ -1,6 +1,26 @@
 <?php
 
 return [
+    'default' => env('DB_CONNECTION', 'mongodb'),
+
+    'connections' => [
+        'mongodb' => [
+            /*
+            'driver' => 'mongodb',
+            'host' => env('DB_HOST', 'localhost'),
+            'port' => env('DB_PORT', 27017),
+            'username' => env('DB_USERNAME', ''),
+            'password' => env('DB_PASSWORD', ''),
+            'options' => [
+                'db' => env('MONGODB_AUTHDATABASE', '')
+            ]
+            */
+            'driver' => 'mongodb',
+            'dsn'=> "mongodb+srv://".env('DB_USERNAME', 'jikan').":".env('DB_PASSWORD', '')."@".env('MONGODB_DSN', ''),
+            'database' => env('DB_DATABASE', ''),
+        ]
+    ],
+
     'redis' => [
         'client' => 'predis',
         'default' => [
@@ -9,5 +29,7 @@ return [
             'port' => env('REDIS_PORT', 6379),
             'database' => 0
         ]
-    ]
+    ],
+
+    'migrations' => 'migrations'
 ];

database/migrations/2020_05_21_051725_create_index.php

@@ -0,0 +1,44 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+class CreateIndex extends Migration
+{
+    /**
+     * Run the migrations.
+     *
+     * @return void
+     */
+    public function up()
+    {
+        $mappings = config('controller-to-table-mapping');
+        $mapped = [];
+
+        foreach ($mappings as $table) {
+            if (in_array($table, $mapped)) {
+                continue;
+            }
+
+            Schema::create($table, function (Blueprint $table) {
+                $table->index('request_hash', 'request_hash');
+                $table->timestamps();
+                $table->integer('expireAfterSeconds');
+            });
+
+            $mapped[] = $table;
+        }
+
+    }
+
+    /**
+     * Reverse the migrations.
+     *
+     * @return void
+     */
+    public function down()
+    {
+
+    }
+}

routes/web.v4.php

@@ -146,7 +146,7 @@ $router->group(
 
 $router->group(
     [
-        'prefix' => 'character/{id:[0-9]+}'
+        'prefix' => 'characters/{id:[0-9]+}'
     ],
     function () use ($router) {
         $router->get('/', [
@@ -161,7 +161,7 @@ $router->group(
 
 $router->group(
     [
-        'prefix' => 'person/{id:[0-9]+}'
+        'prefix' => 'people/{id:[0-9]+}'
     ],
     function () use ($router) {
         $router->get('/', [
@@ -334,7 +334,7 @@ $router->group(
             'uses' => 'SearchController@manga'
         ]);
 
-        $router->get('/character[/{page:[0-9]+}]', [
+        $router->get('/characters[/{page:[0-9]+}]', [
             'uses' => 'SearchController@character'
         ]);
 

storage/app/metadata.v4/Jikan.Model.Genre.AnimeGenre.yml

@@ -0,0 +1,5 @@
+Jikan\Model\Genre\AnimeGenre:
+  exclusion_policy: NONE
+  properties:
+    malUrl:
+      serialized_name: meta

storage/app/metadata.v4/Jikan.Model.Genre.MangaGenre.yml

@@ -0,0 +1,5 @@
+Jikan\Model\Genre\MangaGenre:
+  exclusion_policy: NONE
+  properties:
+    malUrl:
+      serialized_name: meta