malhuda/jikan-rest
1
0
Fork 0
mirror of https://github.com/jikan-me/jikan-rest.git synced 2025-02-20 11:23:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

improve indexer:manga

Browse Source
This commit is contained in:
Irfan 2021-07-18 23:56:58 +05:00
parent 1dbbbd649b
commit dbdca67f1e
3 changed files with 135 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

148
app/Console/Commands/Indexer/MangaIndexer.php
View File

@ -2,10 +2,16 @@
namespace App\Console\Commands\Indexer;
use App\Exceptions\Console\CommandAlreadyRunningException;
use App\Exceptions\Console\FileNotFoundException;
use Illuminate\Console\Command;
use Illuminate\Support\Facades\Storage;
/**
* Class MangaIndexer
* @package App\Console\Commands\Indexer
*/
class MangaIndexer extends Command
{
/**
@ -13,7 +19,12 @@ class MangaIndexer extends Command
*`
* @var string
*/
protected $signature = 'indexer:manga';
protected $signature = 'indexer:manga
{--failed : Run only entries that failed to index last time}
{--resume : Resume from the last position}
{--reverse : Start from the end of the array}
{--index=0 : Start from a specific index}
{--delay=3 : Set a delay between requests}';
/**
* The console command description.
@ -22,6 +33,11 @@ class MangaIndexer extends Command
*/
protected $description = 'Index all manga';
/**
* @var array
*/
private array $ids;
/**
* Create a new command instance.
*
@ -36,48 +52,122 @@ class MangaIndexer extends Command
* Execute the console command.
*
* @return mixed
* @throws FileNotFoundException
* @throws CommandAlreadyRunningException
*/
public function handle()
{
echo "Note: MangaIndexer uses seanbreckenridge/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n";
$failed = $this->option('failed') ?? false;
$resume = $this->option('resume') ?? false;
$reverse = $this->option('reverse') ?? false;
$delay = $this->option('delay') ?? 3;
$index = $this->option('index') ?? 0;
/**
*
*/
// https://github.com/seanbreckenridge/mal-id-cache
echo "Fetching MAL ID Cache https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json...\n";
$index = (int)$index;
$delay = (int)$delay;
$this->info("Info: MangaIndexer uses seanbreckenridge/mal-id-cache fetch available MAL IDs and updates/indexes them\n\n");
if ($failed && Storage::exists('indexer/indexer_manga.save')) {
$this->ids = $this->loadFailedMalIds();
}
if (!$failed) {
$this->ids = $this->fetchMalIds();
}
// start from the end
if ($reverse) {
$this->ids = array_reverse($this->ids);
}
// Resume
if ($resume && Storage::exists('indexer/indexer_manga.save')) {
$index = (int)Storage::get('indexer/indexer_manga.save');
$this->info("Resuming from index: {$index}");
}
// check if index even exists
if ($index > 0 && !isset($this->ids[$index])) {
$index = 0;
$this->warn('Invalid index; set back to 0');
}
// initialize and index
Storage::put('indexer/indexer_manga.save', 0);
echo "Loading MAL IDs\n";
$count = count($this->ids);
$failedIds = [];
$success = [];
echo "{$count} entries available\n";
for ($i = $index; $i <= ($count - 1); $i++) {
$id = $this->ids[$i];
$url = env('APP_URL') . "/v4/manga/{$id}";
echo "Indexing/Updating " . ($i + 1) . "/{$count} {$url} [MAL ID: {$id}] \n";
try {
$response = json_decode(file_get_contents($url), true);
if (isset($response['error']) && $response['status'] != 404) {
echo "[SKIPPED] Failed to fetch {$url} - {$response['error']}\n";
$failedIds[] = $id;
Storage::put('indexer/indexer_manga.failed', json_encode($failedIds));
}
sleep($delay);
} catch (\Exception $e) {
echo "[SKIPPED] Failed to fetch {$url}\n";
$failedIds[] = $id;
Storage::put('indexer/indexer_manga.failed', json_encode($failedIds));
}
$success[] = $id;
Storage::put('indexer/indexer_manga.save', $i);
}
Storage::delete('indexer/indexer_manga.save');
echo "---------\nIndexing complete\n";
echo count($success) . " entries indexed or updated\n";
echo count($failedIds) . " entries failed to index or update. Re-run with --failed to requeue failed entries only\n";
}
/**
* @return array
* @url https://github.com/seanbreckenridge/mal-id-cache
*/
private function fetchMalIds() : array
{
$this->info("Fetching MAL ID Cache https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json...\n");
$ids = json_decode(
file_get_contents('https://raw.githubusercontent.com/seanbreckenridge/mal-id-cache/master/cache/manga_cache.json'),
true
);
$ids = $ids['sfw'] + $ids['nsfw']; // merge
Storage::put('manga_mal_id.json', json_encode($ids));
echo "Loading MAL IDs\n";
$ids = json_decode(Storage::get('manga_mal_id.json'));
$count = count($ids);
$this->ids = $ids['sfw'] + $ids['nsfw']; // merge
Storage::put('indexer/manga_mal_id.json', json_encode($this->ids));
echo "{$count} entries available\n";
foreach ($ids as $i => $id) {
$url = env('APP_URL') . "/v4/manga/{$id}";
return json_decode(Storage::get('indexer/manga_mal_id.json'));
}
echo "Indexing/Updating ".($i+1)."/{$count} {$url} [MAL ID: {$id}] \n";
try {
$response = json_decode(file_get_contents($url), true);
if (isset($response['error'])) {
echo "[SKIPPED] Failed to fetch {$url} - {$response['error']}\n";
}
sleep(3);
} catch (\Exception $e) {
echo "[SKIPPED] Failed to fetch {$url}\n";
}
/**
* @return array
* @throws FileNotFoundException
*/
private function loadFailedMalIds() : array
{
if (!Storage::exists('indexer/indexer_manga.failed')) {
throw new FileNotFoundException('"indexer/indexer_manga.failed" does not exist');
}
echo str_pad("Indexing complete", 100).PHP_EOL;
return json_decode(Storage::get('indexer/indexer_manga.failed'));
}
}

8
app/Exceptions/Console/CommandAlreadyRunningException.php Normal file
View File

@ -0,0 +1,8 @@
<?php
namespace App\Exceptions\Console;
class CommandAlreadyRunningException extends \Exception
{
}

8
app/Exceptions/Console/FileNotFoundException.php Normal file
View File

@ -0,0 +1,8 @@
<?php
namespace App\Exceptions\Console;
class FileNotFoundException extends \Exception
{
}