From 1d3aa7b79458776286116fe35dbb31a3191c3db7 Mon Sep 17 00:00:00 2001 From: pushrbx Date: Sat, 15 Jul 2023 14:44:42 +0100 Subject: [PATCH] various hot fixes and improvements - improved searching with low letter count - removed the ability to order anime by type and rating - fixed schedules endpoint's filter parameter yet again - improved configuration - fixed filtering anime by producers - fixed filtering manga by magazines - fixed search analytics in case of short search terms --- app/Anime.php | 28 +++++++++++-------- app/Dto/QueryAnimeSchedulesCommand.php | 5 ++-- app/Enums/AnimeOrderByEnum.php | 4 +-- app/Features/QueryAnimeSchedulesHandler.php | 2 +- app/Manga.php | 18 ++++++------ app/Services/DefaultQueryBuilderService.php | 3 ++ .../DefaultSearchAnalyticsService.php | 16 +++++++++-- app/Services/TypeSenseScoutSearchService.php | 17 +++++++++-- app/Support/JikanConfig.php | 22 +++++++-------- config/jikan.php | 10 +++---- routes/web.v4.php | 2 +- 11 files changed, 75 insertions(+), 52 deletions(-) diff --git a/app/Anime.php b/app/Anime.php index 33a73cf..61df1e1 100644 --- a/app/Anime.php +++ b/app/Anime.php @@ -182,10 +182,14 @@ class Anime extends JikanApiSearchableModel } $producer = (int)$value; - return $query - ->orWhere('producers.mal_id', $producer) - ->orWhere('licensors.mal_id', $producer) - ->orWhere('studios.mal_id', $producer); + /** @noinspection PhpParamsInspection */ + return $query->whereRaw([ + '$or' => [ + ['producers.mal_id' => $producer], + ['licensors.mal_id' => $producer], + ['studios.mal_id' => $producer] + ] + ]); } /** @noinspection PhpUnused */ @@ -195,16 +199,16 @@ class Anime extends JikanApiSearchableModel return $query; } - $producers = explode(',', $value); + $producers = collect(explode(',', $value))->filter()->toArray(); + $orFilters = []; foreach ($producers as $producer) { - if (empty($producer)) { - continue; - } - - $query = $this->filterByProducer($query, $value); + $producer = (int)$producer; + $orFilters[] = ['producers.mal_id' => $producer]; + $orFilters[] = ['licensors.mal_id' => $producer]; + $orFilters[] = ['studios.mal_id' => $producer]; } - - return $query; + /** @noinspection PhpParamsInspection */ + return $query->whereRaw(['$or' => $orFilters]); } /** @noinspection PhpUnused */ diff --git a/app/Dto/QueryAnimeSchedulesCommand.php b/app/Dto/QueryAnimeSchedulesCommand.php index 6fb1d9d..895580e 100644 --- a/app/Dto/QueryAnimeSchedulesCommand.php +++ b/app/Dto/QueryAnimeSchedulesCommand.php @@ -28,8 +28,7 @@ final class QueryAnimeSchedulesCommand extends Data implements DataRequest #[ WithCast(EnumCast::class, AnimeScheduleFilterEnum::class), - EnumValidation(AnimeScheduleFilterEnum::class), - MapOutputName("filter") + EnumValidation(AnimeScheduleFilterEnum::class) ] - public ?AnimeScheduleFilterEnum $dayFilter; + public ?AnimeScheduleFilterEnum $filter; } diff --git a/app/Enums/AnimeOrderByEnum.php b/app/Enums/AnimeOrderByEnum.php index a8c0ee2..588aa9e 100644 --- a/app/Enums/AnimeOrderByEnum.php +++ b/app/Enums/AnimeOrderByEnum.php @@ -7,14 +7,12 @@ use Spatie\Enum\Laravel\Enum; /** * @method static self mal_id() * @method static self title() - * @method static self type() * @method static self rating() * @method static self start_date() * @method static self end_date() * @method static self episodes() * @method static self score() * @method static self scored_by() - * @method static self rank() * @method static self popularity() * @method static self members() * @method static self favorites() @@ -23,7 +21,7 @@ use Spatie\Enum\Laravel\Enum; * schema="anime_search_query_orderby", * description="Available Anime order_by properties", * type="string", - * enum={"mal_id", "title", "type", "rating", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" } + * enum={"mal_id", "title", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" } * ) */ final class AnimeOrderByEnum extends Enum diff --git a/app/Features/QueryAnimeSchedulesHandler.php b/app/Features/QueryAnimeSchedulesHandler.php index 3363d02..573d94c 100644 --- a/app/Features/QueryAnimeSchedulesHandler.php +++ b/app/Features/QueryAnimeSchedulesHandler.php @@ -25,7 +25,7 @@ final class QueryAnimeSchedulesHandler implements RequestHandler { $requestParams = collect($request->all()); $limit = $requestParams->get("limit"); - $results = $this->repository->getCurrentlyAiring($request->dayFilter); + $results = $this->repository->getCurrentlyAiring($request->filter); // apply sfw, kids and unapproved filters /** @noinspection PhpUndefinedMethodInspection */ $results = $results->filter($requestParams); diff --git a/app/Manga.php b/app/Manga.php index 8f5af41..08e2409 100644 --- a/app/Manga.php +++ b/app/Manga.php @@ -101,7 +101,7 @@ class Manga extends JikanApiSearchableModel $magazine = (int)$value; return $query - ->orWhere('serializations.mal_id', $magazine); + ->where('serializations.mal_id', $magazine); } /** @noinspection PhpUnused */ @@ -111,16 +111,14 @@ class Manga extends JikanApiSearchableModel return $query; } - $magazines = explode(',', $value); - foreach ($magazines as $magazine) { - if (empty($magazine)) { - continue; - } + $magazines = collect(explode(',', $value))->filter()->map(fn($x) => (int)$x)->toArray(); - $query = $this->filterByMagazine($query, $value); - } - - return $query; + /** @noinspection PhpParamsInspection */ + return $query->whereRaw([ + "serializations.mal_id" => [ + '$in' => $magazines + ] + ]); } /** @noinspection PhpUnused */ diff --git a/app/Services/DefaultQueryBuilderService.php b/app/Services/DefaultQueryBuilderService.php index 8018c84..8194abc 100644 --- a/app/Services/DefaultQueryBuilderService.php +++ b/app/Services/DefaultQueryBuilderService.php @@ -24,6 +24,9 @@ final class DefaultQueryBuilderService implements QueryBuilderService ->search($requestParameters->get("q"), $searchEngineOptions["order_by"], $searchEngineOptions["sort_direction_descending"]); } else { $builder = $this->searchService->setFilterParameters($requestParameters)->query(); + if (!$requestParameters->has("order_by")) { + $builder = $builder->orderBy("mal_id"); + } } return $builder; diff --git a/app/Services/DefaultSearchAnalyticsService.php b/app/Services/DefaultSearchAnalyticsService.php index 8e18d09..9b9200c 100644 --- a/app/Services/DefaultSearchAnalyticsService.php +++ b/app/Services/DefaultSearchAnalyticsService.php @@ -4,11 +4,12 @@ namespace App\Services; use App\SearchMetric; use App\Contracts\SearchAnalyticsService; use Illuminate\Support\Collection; +use Typesense\Documents; /** * The default search analytics service implementation, which saves the stats to the database and indexes it in Typesense. - * + * * By indexing search terms in Typesense we can use it to provide search suggestions of popular searches. * @package App\Services */ @@ -17,13 +18,22 @@ final class DefaultSearchAnalyticsService implements SearchAnalyticsService public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void { /** - * @var \Laravel\Scout\Builder $existingMetrics + * @var Collection $existingMetrics */ - $existingMetrics = SearchMetric::search($searchTerm); + $existingMetrics = SearchMetric::search($searchTerm, function (Documents $documents, string $query, array $options) { + if (strlen($query) <= 3) { + $options['prioritize_token_position'] = 'true'; + } + + return $documents->search($options); + })->take(1)->get(); $hitList = $hits->pluck("id")->values()->map(fn($x) => (int)$x)->all(); if ($existingMetrics->count() > 0) { + /** + * @var SearchMetric $metric + */ $metric = $existingMetrics->first(); $metric->hits = $hitList; $metric->hits_count = $hitsCount; diff --git a/app/Services/TypeSenseScoutSearchService.php b/app/Services/TypeSenseScoutSearchService.php index f7bfca5..6265c81 100644 --- a/app/Services/TypeSenseScoutSearchService.php +++ b/app/Services/TypeSenseScoutSearchService.php @@ -61,7 +61,6 @@ class TypeSenseScoutSearchService implements ScoutSearchService $options['per_page'] = min($this->maxItemsPerPage, 250); } - $options = $this->skipTypoCheckingForShortQueries($query, $options); $modelInstance = $this->repository->createEntity(); if ($modelInstance instanceof JikanApiSearchableModel) { @@ -69,6 +68,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService $options = $this->setSortOrder($options, $modelInstance); $options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending); } + $options = $this->adaptToShortQueries($query, $options); $results = $documents->search($options); $this->recordSearchTelemetry($query, $results); @@ -77,7 +77,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService }; } - private function skipTypoCheckingForShortQueries(string $query, array $options): array + private function adaptToShortQueries(string $query, array $options): array { if (strlen($query) <= 3) { $options['num_typos'] = 0; @@ -85,7 +85,18 @@ class TypeSenseScoutSearchService implements ScoutSearchService $options['drop_tokens_threshold'] = 0; $options['exhaustive_search'] = 'false'; $options['infix'] = 'off'; - $options['prefix'] = 'false'; + $options['prioritize_token_position'] = 'true'; + + if (Str::startsWith($options["sort_by"], "_text_match")) { + $options["sort_by"] = "_text_match:desc,title:asc"; + } else { + // move text_match to the beginning if there is an orderby parameter set. + $options["sort_by"] = Str::replace("(buckets:". $this->jikanConfig->textMatchBuckets().")", "", $options["sort_by"]); + $parts = collect(explode(",", $options["sort_by"])); + $last = $parts->pop(); + $parts = $parts->prepend($last); + $options["sort_by"] = $parts->implode(","); + } } return $options; diff --git a/app/Support/JikanConfig.php b/app/Support/JikanConfig.php index 17b6cf9..228ae38 100644 --- a/app/Support/JikanConfig.php +++ b/app/Support/JikanConfig.php @@ -3,6 +3,7 @@ namespace App\Support; use Illuminate\Support\Collection; +use Illuminate\Support\Arr; /** * Jikan behavior config @@ -32,16 +33,15 @@ final class JikanConfig public function __construct(array $config) { - $config = collect($config); - $this->perEndpointCacheTtl = $config->get("per_endpoint_cache_ttl", []); - $this->defaultCacheExpire = $config->get("default_cache_expire", 0); - $this->microCachingEnabled = in_array($config->get("micro_caching_enabled", false), [true, 1, "1", "true"]); - $this->textMatchBuckets = $config->get("typesense_options.text_match_buckets", 85); - $this->exhaustiveSearch = (string) $config->get("typesense_options.exhaustive_search", "false"); - $this->config = $config; - $this->typoTokensThreshold = $config->get("typesense_options.typo_tokens_threshold", $this->maxResultsPerPage()); - $this->dropTokensThreshold = $config->get("typesense_options.drop_tokens_threshold", $this->maxResultsPerPage()); - $this->searchCutOffMs = $config->get("typesense_options.search_cutoff_ms", 450); + $this->perEndpointCacheTtl = Arr::get($config, "per_endpoint_cache_ttl", []); + $this->defaultCacheExpire = Arr::get($config, "default_cache_expire", 0); + $this->microCachingEnabled = in_array(Arr::get($config, "micro_caching_enabled", false), [true, 1, "1", "true"]); + $this->textMatchBuckets = Arr::get($config,"typesense_options.text_match_buckets", 1); + $this->exhaustiveSearch = (string) Arr::get($config, "typesense_options.exhaustive_search", "false"); + $this->config = collect($config); + $this->typoTokensThreshold = Arr::get($config, "typesense_options.typo_tokens_threshold") ?? $this->maxResultsPerPage(); + $this->dropTokensThreshold = Arr::get($config, "typesense_options.drop_tokens_threshold") ?? $this->maxResultsPerPage(); + $this->searchCutOffMs = Arr::get($config, "typesense_options.search_cutoff_ms", 450); } public function cacheTtlForEndpoint(string $endpoint): ?int @@ -61,7 +61,7 @@ final class JikanConfig public function maxResultsPerPage(?int $defaultValue = null): int { - return $this->config->get("max_results_per_page", $defaultValue ?? 25); + return (int) $this->config->get("max_results_per_page", $defaultValue ?? 25); } public function textMatchBuckets(): int diff --git a/config/jikan.php b/config/jikan.php index 896b726..662e022 100644 --- a/config/jikan.php +++ b/config/jikan.php @@ -1,14 +1,14 @@ (int) env('MAX_RESULTS_PER_PAGE', 25), + 'max_results_per_page' => env('MAX_RESULTS_PER_PAGE', 25), 'micro_caching_enabled' => env('MICROCACHING', false), 'default_cache_expire' => env('CACHE_DEFAULT_EXPIRE', 86400), 'typesense_options' => [ - 'text_match_buckets' => env('TYPESENSE_TEXT_MATCH_BUCKETS', 85), - 'typo_tokens_threshold' => (int) env('TYPESENSE_TYPO_TOKENS_THRESHOLD'), - 'drop_tokens_threshold' => (int) env('TYPESENSE_DROP_TOKENS_THRESHOLD'), - 'search_cutoff_ms' => (int) env('TYPESENSE_SEARCH_CUTOFF_MS', 450), + 'text_match_buckets' => env('TYPESENSE_TEXT_MATCH_BUCKETS', 1), + 'typo_tokens_threshold' => env('TYPESENSE_TYPO_TOKENS_THRESHOLD'), + 'drop_tokens_threshold' => env('TYPESENSE_DROP_TOKENS_THRESHOLD'), + 'search_cutoff_ms' => env('TYPESENSE_SEARCH_CUTOFF_MS', 450), 'exhaustive_search' => env('TYPESENSE_ENABLE_EXHAUSTIVE_SEARCH', 'false') ], 'per_endpoint_cache_ttl' => [ diff --git a/routes/web.v4.php b/routes/web.v4.php index 84b2137..0d06474 100644 --- a/routes/web.v4.php +++ b/routes/web.v4.php @@ -269,7 +269,7 @@ $router->group( } ); -$router->get('schedules[/{dayFilter:[A-Za-z]+}]', [ +$router->get('schedules[/{filter:[A-Za-z]+}]', [ 'uses' => 'ScheduleController@main' ]);