malhuda/jikan-rest
1
0
Fork 0
mirror of https://github.com/jikan-me/jikan-rest.git synced 2025-02-20 11:23:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

refactored the code for better readability

Browse Source
This commit is contained in:
pushrbx 2024-11-05 09:25:52 +00:00
parent 5148950125
commit 6c2d2d3e9d
No known key found for this signature in database
GPG Key ID: A16A474BBC2C91D9
1 changed files with 142 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

141
app/Console/Commands/Indexer/IncrementalIndexer.php
View File

@ -30,60 +30,28 @@ class IncrementalIndexer extends Command
];
}
public function handle(): int
private function getExistingIds(string $mediaType): array
{
$validator = Validator::make(
[
'mediaType' => $this->argument('mediaType'),
'delay' => $this->option('delay'),
'resume' => $this->option('resume') ?? false,
'failed' => $this->option('failed') ?? false
],
[
'mediaType' => 'required|in:anime,manga,character,people',
'delay' => 'integer|min:1',
'resume' => 'bool|prohibited_with:failed',
'failed' => 'bool|prohibited_with:resume'
]
);
if ($validator->fails()) {
$this->error($validator->errors()->toJson());
return 1;
}
$this->trap(SIGTERM, fn () => $this->cancelled = true);
$resume = $this->option('resume') ?? false;
$onlyFailed = $this->option('failed') ?? false;
$existingIdsHash = "";
$existingIdsRaw = "";
/**
* @var $mediaTypes array
*/
$mediaTypes = $this->argument("mediaType");
foreach ($mediaTypes as $mediaType)
{
$idsToFetch = [];
$failedIds = [];
$success = [];
if ($onlyFailed && Storage::exists("indexer/incremental/{$mediaType}_failed.json"))
{
$idsToFetch["sfw"] = json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json"));
}
else
{
if (Storage::exists("indexer/incremental/$mediaType.json"))
{
$existingIdsRaw = Storage::get("indexer/incremental/$mediaType.json");
$existingIdsHash = sha1($existingIdsRaw);
}
return [$existingIdsHash, $existingIdsRaw];
}
private function getIdsToFetch(string $mediaType): array
{
$idsToFetch = [];
[$existingIdsHash, $existingIdsRaw] = $this->getExistingIds($mediaType);
if ($this->cancelled)
{
return 127;
return [];
}
$newIdsRaw = file_get_contents("https://raw.githubusercontent.com/purarue/mal-id-cache/master/cache/${mediaType}_cache.json");
@ -92,7 +60,7 @@ class IncrementalIndexer extends Command
/** @noinspection PhpConditionAlreadyCheckedInspection */
if ($this->cancelled)
{
return 127;
return [];
}
if ($newIdsHash !== $existingIdsHash)
@ -114,19 +82,29 @@ class IncrementalIndexer extends Command
Storage::put("indexer/incremental/$mediaType.json.tmp", $newIdsRaw);
}
return $idsToFetch;
}
$idCount = count($idsToFetch);
if ($idCount > 0)
private function getFailedIdsToFetch(string $mediaType): array
{
return json_decode(Storage::get("indexer/incremental/{$mediaType}_failed.json"));
}
private function fetchIds(string $mediaType, array $idsToFetch, bool $resume): void
{
$index = 0;
$success = [];
$failedIds = [];
$idCount = count($idsToFetch);
if ($resume && Storage::exists("indexer/incremental/{$mediaType}_resume.save"))
{
$index = (int)Storage::get("indexer/incremental/{$mediaType}_resume.save");
$this->info("Resuming from index: $index");
}
if ($index > 0 && !isset($this->ids[$index])) {
if ($index > 0 && !isset($this->ids[$index]))
{
$index = 0;
$this->warn('Invalid index; set back to 0');
}
@ -135,11 +113,12 @@ class IncrementalIndexer extends Command
$this->info("$idCount $mediaType entries available");
$ids = array_merge($idsToFetch['sfw'], $idsToFetch['nsfw']);
for ($i = $index; $i <= ($idCount - 1); $i++)
{
if ($this->cancelled)
{
return 127;
return;
}
$id = $ids[$index];
@ -150,10 +129,12 @@ class IncrementalIndexer extends Command
try
{
$response = json_decode(file_get_contents($url), true);
if (isset($response['error']) && $response['status'] != 404)
if (!isset($response['error']) || $response['status'] == 404)
{
$this->error("[SKIPPED] Failed to fetch $url - {$response['error']}");
continue;
}
$this->error("[SKIPPED] Failed to fetch $url - {$response['error']}");
}
catch (\Exception)
{
@ -167,15 +148,77 @@ class IncrementalIndexer extends Command
}
Storage::delete("indexer/incremental/{$mediaType}_resume.save");
$this->info("--- Indexing of $mediaType is complete.");
$this->info(count($success) . ' entries indexed or updated.');
if (count($failedIds) > 0)
{
$this->info(count($failedIds) . ' entries failed to index or update. Re-run with --failed to requeue failed entries only.');
}
// finalize the latest state
Storage::move("indexer/incremental/$mediaType.json.tmp", "indexer/incremental/$mediaType.json");
}
public function handle(): int
{
// validate inputs
$validator = Validator::make(
[
'mediaType' => $this->argument('mediaType'),
'delay' => $this->option('delay'),
'resume' => $this->option('resume') ?? false,
'failed' => $this->option('failed') ?? false
],
[
'mediaType' => 'required|in:anime,manga,character,people',
'delay' => 'integer|min:1',
'resume' => 'bool|prohibited_with:failed',
'failed' => 'bool|prohibited_with:resume'
]
);
if ($validator->fails()) {
$this->error($validator->errors()->toJson());
return 1;
}
// we want to handle signals from the OS
$this->trap(SIGTERM, fn () => $this->cancelled = true);
$resume = $this->option('resume') ?? false;
$onlyFailed = $this->option('failed') ?? false;
/**
* @var $mediaTypes array
*/
$mediaTypes = $this->argument("mediaType");
foreach ($mediaTypes as $mediaType)
{
$idsToFetch = [];
// if "--failed" option is specified just run the failed ones
if ($onlyFailed && Storage::exists("indexer/incremental/{$mediaType}_failed.json"))
{
$idsToFetch["sfw"] = $this->getFailedIdsToFetch($mediaType);
}
else
{
$idsToFetch = $this->getIdsToFetch($mediaType);
}
$idCount = count($idsToFetch);
if ($idCount == 0)
{
if ($this->cancelled)
{
return 127;
}
continue;
}
$this->fetchIds($mediaType, $idsToFetch, $resume);
}
return 0;