malhuda/jikan-rest
1
0
Fork 0
mirror of https://github.com/jikan-me/jikan-rest.git synced 2025-02-20 11:23:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

bug fixes

Browse Source
This commit is contained in:
Irfan 2020-05-27 02:29:30 +05:00
parent 522e2f63ee
commit 7c60200295
10 changed files with 65 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.env.dist
View File

@ -24,7 +24,7 @@ MONGODB_DSN=
#
# If MyAnimeList is down or unavailable,
# Jikan will try to generate responses from internally
# populated database caches
# populated cache where available
#
# Note: DB_CACHING must be enabled
###

37
app/Http/Middleware/DatabaseResolver.php
View File

@ -53,7 +53,6 @@ class DatabaseResolver
public function handle(Request $request, Closure $next)
{
if ($request->header('auth') === env('APP_KEY')) {
return $next($request);
}
@ -84,13 +83,19 @@ class DatabaseResolver
$this->requestCached = DB::table($this->table)->where('request_hash', $this->fingerprint)->exists();
// Is the request queueable?
if (\in_array($this->route, self::NON_QUEUEABLE) || env('CACHE_METHOD', 'legacy') === 'legacy') {
if (\in_array($this->route, self::NON_QUEUEABLE) || env('DB_METHOD', 'legacy') === 'legacy') {
$this->queueable = false;
}
// If cache does not exist
if (!$this->requestCached) {
return $this->fetchFresh($request, $next);
$response = $next($request);
if (HttpHelper::hasError($response)) {
return $response;
}
$this->insertCache($response);
}
// Fetch Cache & Generate Meta
@ -103,12 +108,17 @@ class DatabaseResolver
// If cache is expired, handle it depending on whether it's queueable
$expiresAt = $cacheMutable['expiresAt']['$date']['$numberLong']/1000;
if ($this->requestCached && $expiresAt > time() && !$this->queueable) {
return $this->fetchFresh($request, $next);
if ($this->requestCached && $expiresAt < time() && !$this->queueable) {
$response = $next($request);
if (HttpHelper::hasError($response)) {
return $response;
}
$this->insertCache($response);
}
if ( $this->queueable && $expiresAt > time()) {
$queueFingerprint = "queue_update:{$this->fingerprint}";
if ( $this->queueable && $expiresAt < time()) {
$queueHighPriority = \in_array($this->route, self::HIGH_PRIORITY_QUEUE);
// Don't duplicate the job in the queue for same request
@ -116,15 +126,14 @@ class DatabaseResolver
if (!$job->exists()) {
dispatch(
(new UpdateDatabaseJob($request))
(new UpdateDatabaseJob($request, $this->table))
->onQueue($queueHighPriority ? 'high' : 'low')
);
}
}
$response = array_merge($meta, $cacheMutable);
unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id']);
unset($response['createdAt'], $response['expireAfterSeconds'], $response['_id'], $response['expiresAt']);
// Build and return response
return response()
@ -185,14 +194,8 @@ class DatabaseResolver
return $data;
}
public function fetchFresh($request, $next)
public function insertCache($response)
{
$response = $next($request);
if (HttpHelper::hasError($response)) {
return $response;
}
DB::table($this->table)->insert(array_merge(
[
'expiresAt' => new UTCDateTime((time()+$this->requestCacheTtl)*1000),

1
app/Jobs/UpdateCacheJob.php
View File

@ -74,7 +74,6 @@ class UpdateCacheJob extends Job
unset($cache['fingerprint'], $cache['request_cached'], $cache['request_cache_expiry']);
$cache = json_encode($cache);
Cache::forever($this->fingerprint, $cache);
Cache::forever($this->cacheExpiryFingerprint, time() + $this->requestCacheTtl);
app('redis')->del($queueFingerprint);

38
app/Jobs/UpdateDatabaseJob.php
View File

@ -8,6 +8,7 @@ use Illuminate\Http\Request;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Log;
use Illuminate\Support\Facades\Cache;
use MongoDB\BSON\UTCDateTime;
/**
@ -22,51 +23,56 @@ class UpdateDatabaseJob extends Job
public $retryAfter = 60;
/**
* @var string
*/
protected $request;
protected $requestUri;
protected $requestType;
protected $requestCacheTtl;
protected $fingerprint;
protected $cacheExpiryFingerprint;
protected $requestCached;
protected $table;
/**
* Create a new job instance.
*
* @return void
* @param Request $request
* @param $table
*/
public function __construct(Request $request)
public function __construct(Request $request, $table)
{
$this->table = $table;
$this->fingerprint = HttpHelper::resolveRequestFingerprint($request);
$this->requestCached = DB::table(env('QUEUE_TABLE', 'jobs'))->where('request_hash', $this->fingerprint);
$this->requestType = HttpHelper::requestType($request);
$this->requestCacheTtl = HttpHelper::requestCacheExpiry($this->requestType);
}
/**
* @throws \GuzzleHttp\Exception\GuzzleException
*/
public function handle() : void
{
$client = new Client();
$response = $client
$response = app('GuzzleClient')
->request(
'GET',
env('APP_URL') . $this->requestUri,
[
'headers' => [
'auth' => env('APP_KEY') // skip middleware
'auth' => env('APP_KEY') // skips middleware
]
]
);
$cache = json_decode($response->getBody()->getContents(), true);
unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry']);
unset($cache['request_hash'], $cache['request_cached'], $cache['request_cache_expiry'], $cache['DEVELOPMENT_NOTICE'], $cache['MIGRATION']);
$cache = json_encode($cache);
DB::table($this->table)
->where('request_hash', $this->fingerprint)
->update(array_merge(
[
'expiresAt' => new UTCDateTime((time()+$this->requestCacheTtl)*1000),
'request_hash' => $this->fingerprint
],
json_decode($cache, true)
));
sleep((int) env('QUEUE_DELAY_PER_JOB', 5));
}

2
app/Listeners/SourceHealthListener.php
View File

@ -25,7 +25,7 @@ class SourceHealthListener
public function __construct()
{
$this->logger = new Logger('source-health-monitor');
$this->logger->pushHandler(new StreamHandler(storage_path().'/logs/source-health-monitor.log'), Logger::DEBUG);
$this->logger->pushHandler(new StreamHandler(storage_path().'/logs/source-health-monitor.log'), env('APP_DEBUG') ? Logger::DEBUG : Logger::WARNING);
if (SourceHealthServiceProvider::isFailoverEnabled()) {
$lastFailoverLockTimestamp = $this->getLastFailoverLockTimestamp();

10
composer.lock generated
View File

@ -2210,16 +2210,16 @@
},
{
"name": "jikan-me/jikan",
"version": "v3.0.0-alpha.4",
"version": "v3.0.0-alpha.5",
"source": {
"type": "git",
"url": "https://github.com/jikan-me/jikan.git",
"reference": "c5f4052c40454bf3a7c5a092fed6ed24650426cb"
"reference": "e1d11d663669a6e6e5d7ac536138834846764596"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/jikan-me/jikan/zipball/c5f4052c40454bf3a7c5a092fed6ed24650426cb",
"reference": "c5f4052c40454bf3a7c5a092fed6ed24650426cb",
"url": "https://api.github.com/repos/jikan-me/jikan/zipball/e1d11d663669a6e6e5d7ac536138834846764596",
"reference": "e1d11d663669a6e6e5d7ac536138834846764596",
"shasum": ""
},
"require": {
@ -2256,7 +2256,7 @@
}
],
"description": "Jikan is an unofficial MyAnimeList API",
"time": "2020-05-20T07:44:20+00:00"
"time": "2020-05-26T18:52:10+00:00"
},
{
"name": "jms/metadata",

3
config/queue.php
View File

@ -15,7 +15,7 @@ return [
|
*/
'default' => env('QUEUE_CONNECTION', 'redis'),
'default' => env('QUEUE_CONNECTION', 'mongodb'),
/*
|--------------------------------------------------------------------------
@ -37,6 +37,7 @@ return [
'database' => [
'driver' => 'mongodb',
'connection' => 'mongodb',
'dsn'=> "mongodb+srv://".env('DB_USERNAME', 'jikan').":".env('DB_PASSWORD', '')."@".env('MONGODB_DSN', ''),
'table' => env('QUEUE_TABLE', 'jobs'),
'queue' => 'low',
// 'retry_after' => 60,

2
database/migrations/2020_05_21_051725_create_index.php
View File

@ -22,7 +22,7 @@ class CreateIndex extends Migration
}
Schema::create($table, function (Blueprint $table) {
$table->index(['request_hash' => 1], 'request_hash');
$table->unique(['request_hash' => 1], 'request_hash');
});
$mapped[] = $table;

16
database/migrations/2020_05_21_214747_create_queue_index.php
View File

@ -13,8 +13,18 @@ class CreateQueueIndex extends Migration
*/
public function up()
{
Schema::create(env('QUEUE_TABLE', 'jobs_failed'), function (Blueprint $table) {
$table->index(['request_hash' => 1], 'request_hash');
Schema::create(env('QUEUE_TABLE', 'jobs'), function (Blueprint $table) {
$table->index(['queue', 'reserved_at']);
$table->bigIncrements('id');
$table->string('queue');
$table->longText('payload');
$table->tinyInteger('attempts')->unsigned();
$table->unsignedInteger('reserved_at')->nullable();
$table->unsignedInteger('available_at');
$table->unsignedInteger('created_at');
});
Schema::create(env('QUEUE_TABLE', 'jobs'), function (Blueprint $table) {
$table->index(['queue', 'reserved_at']);
$table->bigIncrements('id');
$table->string('queue');
@ -33,6 +43,6 @@ class CreateQueueIndex extends Migration
*/
public function down()
{
Schema::dropIfExists(env('QUEUE_TABLE', 'jobs_failed'));
Schema::dropIfExists(env('QUEUE_TABLE', 'jobs'));
}
}

1
database/migrations/2020_05_21_215040_create_queue_failed_index.php
View File

@ -14,7 +14,6 @@ class CreateQueueFailedIndex extends Migration
public function up()
{
Schema::create(env('QUEUE_FAILED_TABLE', 'jobs_failed'), function (Blueprint $table) {
$table->index(['request_hash' => 1], 'request_hash');
$table->increments('id');
$table->text('connection');
$table->text('queue');