various hot fixes and improvements

- improved searching with low letter count
- removed the ability to order anime by type and rating
- fixed schedules endpoint's filter parameter yet again
- improved configuration
- fixed filtering anime by producers
- fixed filtering manga by magazines
- fixed search analytics in case of short search terms
This commit is contained in:
pushrbx 2023-07-15 14:44:42 +01:00
parent 0ac49539ba
commit 1d3aa7b794
11 changed files with 75 additions and 52 deletions

View File

@ -182,10 +182,14 @@ class Anime extends JikanApiSearchableModel
} }
$producer = (int)$value; $producer = (int)$value;
return $query /** @noinspection PhpParamsInspection */
->orWhere('producers.mal_id', $producer) return $query->whereRaw([
->orWhere('licensors.mal_id', $producer) '$or' => [
->orWhere('studios.mal_id', $producer); ['producers.mal_id' => $producer],
['licensors.mal_id' => $producer],
['studios.mal_id' => $producer]
]
]);
} }
/** @noinspection PhpUnused */ /** @noinspection PhpUnused */
@ -195,16 +199,16 @@ class Anime extends JikanApiSearchableModel
return $query; return $query;
} }
$producers = explode(',', $value); $producers = collect(explode(',', $value))->filter()->toArray();
$orFilters = [];
foreach ($producers as $producer) { foreach ($producers as $producer) {
if (empty($producer)) { $producer = (int)$producer;
continue; $orFilters[] = ['producers.mal_id' => $producer];
} $orFilters[] = ['licensors.mal_id' => $producer];
$orFilters[] = ['studios.mal_id' => $producer];
$query = $this->filterByProducer($query, $value);
} }
/** @noinspection PhpParamsInspection */
return $query; return $query->whereRaw(['$or' => $orFilters]);
} }
/** @noinspection PhpUnused */ /** @noinspection PhpUnused */

View File

@ -28,8 +28,7 @@ final class QueryAnimeSchedulesCommand extends Data implements DataRequest
#[ #[
WithCast(EnumCast::class, AnimeScheduleFilterEnum::class), WithCast(EnumCast::class, AnimeScheduleFilterEnum::class),
EnumValidation(AnimeScheduleFilterEnum::class), EnumValidation(AnimeScheduleFilterEnum::class)
MapOutputName("filter")
] ]
public ?AnimeScheduleFilterEnum $dayFilter; public ?AnimeScheduleFilterEnum $filter;
} }

View File

@ -7,14 +7,12 @@ use Spatie\Enum\Laravel\Enum;
/** /**
* @method static self mal_id() * @method static self mal_id()
* @method static self title() * @method static self title()
* @method static self type()
* @method static self rating() * @method static self rating()
* @method static self start_date() * @method static self start_date()
* @method static self end_date() * @method static self end_date()
* @method static self episodes() * @method static self episodes()
* @method static self score() * @method static self score()
* @method static self scored_by() * @method static self scored_by()
* @method static self rank()
* @method static self popularity() * @method static self popularity()
* @method static self members() * @method static self members()
* @method static self favorites() * @method static self favorites()
@ -23,7 +21,7 @@ use Spatie\Enum\Laravel\Enum;
* schema="anime_search_query_orderby", * schema="anime_search_query_orderby",
* description="Available Anime order_by properties", * description="Available Anime order_by properties",
* type="string", * type="string",
* enum={"mal_id", "title", "type", "rating", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" } * enum={"mal_id", "title", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" }
* ) * )
*/ */
final class AnimeOrderByEnum extends Enum final class AnimeOrderByEnum extends Enum

View File

@ -25,7 +25,7 @@ final class QueryAnimeSchedulesHandler implements RequestHandler
{ {
$requestParams = collect($request->all()); $requestParams = collect($request->all());
$limit = $requestParams->get("limit"); $limit = $requestParams->get("limit");
$results = $this->repository->getCurrentlyAiring($request->dayFilter); $results = $this->repository->getCurrentlyAiring($request->filter);
// apply sfw, kids and unapproved filters // apply sfw, kids and unapproved filters
/** @noinspection PhpUndefinedMethodInspection */ /** @noinspection PhpUndefinedMethodInspection */
$results = $results->filter($requestParams); $results = $results->filter($requestParams);

View File

@ -101,7 +101,7 @@ class Manga extends JikanApiSearchableModel
$magazine = (int)$value; $magazine = (int)$value;
return $query return $query
->orWhere('serializations.mal_id', $magazine); ->where('serializations.mal_id', $magazine);
} }
/** @noinspection PhpUnused */ /** @noinspection PhpUnused */
@ -111,16 +111,14 @@ class Manga extends JikanApiSearchableModel
return $query; return $query;
} }
$magazines = explode(',', $value); $magazines = collect(explode(',', $value))->filter()->map(fn($x) => (int)$x)->toArray();
foreach ($magazines as $magazine) {
if (empty($magazine)) {
continue;
}
$query = $this->filterByMagazine($query, $value); /** @noinspection PhpParamsInspection */
} return $query->whereRaw([
"serializations.mal_id" => [
return $query; '$in' => $magazines
]
]);
} }
/** @noinspection PhpUnused */ /** @noinspection PhpUnused */

View File

@ -24,6 +24,9 @@ final class DefaultQueryBuilderService implements QueryBuilderService
->search($requestParameters->get("q"), $searchEngineOptions["order_by"], $searchEngineOptions["sort_direction_descending"]); ->search($requestParameters->get("q"), $searchEngineOptions["order_by"], $searchEngineOptions["sort_direction_descending"]);
} else { } else {
$builder = $this->searchService->setFilterParameters($requestParameters)->query(); $builder = $this->searchService->setFilterParameters($requestParameters)->query();
if (!$requestParameters->has("order_by")) {
$builder = $builder->orderBy("mal_id");
}
} }
return $builder; return $builder;

View File

@ -4,11 +4,12 @@ namespace App\Services;
use App\SearchMetric; use App\SearchMetric;
use App\Contracts\SearchAnalyticsService; use App\Contracts\SearchAnalyticsService;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Typesense\Documents;
/** /**
* The default search analytics service implementation, which saves the stats to the database and indexes it in Typesense. * The default search analytics service implementation, which saves the stats to the database and indexes it in Typesense.
* *
* By indexing search terms in Typesense we can use it to provide search suggestions of popular searches. * By indexing search terms in Typesense we can use it to provide search suggestions of popular searches.
* @package App\Services * @package App\Services
*/ */
@ -17,13 +18,22 @@ final class DefaultSearchAnalyticsService implements SearchAnalyticsService
public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void
{ {
/** /**
* @var \Laravel\Scout\Builder $existingMetrics * @var Collection $existingMetrics
*/ */
$existingMetrics = SearchMetric::search($searchTerm); $existingMetrics = SearchMetric::search($searchTerm, function (Documents $documents, string $query, array $options) {
if (strlen($query) <= 3) {
$options['prioritize_token_position'] = 'true';
}
return $documents->search($options);
})->take(1)->get();
$hitList = $hits->pluck("id")->values()->map(fn($x) => (int)$x)->all(); $hitList = $hits->pluck("id")->values()->map(fn($x) => (int)$x)->all();
if ($existingMetrics->count() > 0) { if ($existingMetrics->count() > 0) {
/**
* @var SearchMetric $metric
*/
$metric = $existingMetrics->first(); $metric = $existingMetrics->first();
$metric->hits = $hitList; $metric->hits = $hitList;
$metric->hits_count = $hitsCount; $metric->hits_count = $hitsCount;

View File

@ -61,7 +61,6 @@ class TypeSenseScoutSearchService implements ScoutSearchService
$options['per_page'] = min($this->maxItemsPerPage, 250); $options['per_page'] = min($this->maxItemsPerPage, 250);
} }
$options = $this->skipTypoCheckingForShortQueries($query, $options);
$modelInstance = $this->repository->createEntity(); $modelInstance = $this->repository->createEntity();
if ($modelInstance instanceof JikanApiSearchableModel) { if ($modelInstance instanceof JikanApiSearchableModel) {
@ -69,6 +68,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService
$options = $this->setSortOrder($options, $modelInstance); $options = $this->setSortOrder($options, $modelInstance);
$options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending); $options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending);
} }
$options = $this->adaptToShortQueries($query, $options);
$results = $documents->search($options); $results = $documents->search($options);
$this->recordSearchTelemetry($query, $results); $this->recordSearchTelemetry($query, $results);
@ -77,7 +77,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService
}; };
} }
private function skipTypoCheckingForShortQueries(string $query, array $options): array private function adaptToShortQueries(string $query, array $options): array
{ {
if (strlen($query) <= 3) { if (strlen($query) <= 3) {
$options['num_typos'] = 0; $options['num_typos'] = 0;
@ -85,7 +85,18 @@ class TypeSenseScoutSearchService implements ScoutSearchService
$options['drop_tokens_threshold'] = 0; $options['drop_tokens_threshold'] = 0;
$options['exhaustive_search'] = 'false'; $options['exhaustive_search'] = 'false';
$options['infix'] = 'off'; $options['infix'] = 'off';
$options['prefix'] = 'false'; $options['prioritize_token_position'] = 'true';
if (Str::startsWith($options["sort_by"], "_text_match")) {
$options["sort_by"] = "_text_match:desc,title:asc";
} else {
// move text_match to the beginning if there is an orderby parameter set.
$options["sort_by"] = Str::replace("(buckets:". $this->jikanConfig->textMatchBuckets().")", "", $options["sort_by"]);
$parts = collect(explode(",", $options["sort_by"]));
$last = $parts->pop();
$parts = $parts->prepend($last);
$options["sort_by"] = $parts->implode(",");
}
} }
return $options; return $options;

View File

@ -3,6 +3,7 @@
namespace App\Support; namespace App\Support;
use Illuminate\Support\Collection; use Illuminate\Support\Collection;
use Illuminate\Support\Arr;
/** /**
* Jikan behavior config * Jikan behavior config
@ -32,16 +33,15 @@ final class JikanConfig
public function __construct(array $config) public function __construct(array $config)
{ {
$config = collect($config); $this->perEndpointCacheTtl = Arr::get($config, "per_endpoint_cache_ttl", []);
$this->perEndpointCacheTtl = $config->get("per_endpoint_cache_ttl", []); $this->defaultCacheExpire = Arr::get($config, "default_cache_expire", 0);
$this->defaultCacheExpire = $config->get("default_cache_expire", 0); $this->microCachingEnabled = in_array(Arr::get($config, "micro_caching_enabled", false), [true, 1, "1", "true"]);
$this->microCachingEnabled = in_array($config->get("micro_caching_enabled", false), [true, 1, "1", "true"]); $this->textMatchBuckets = Arr::get($config,"typesense_options.text_match_buckets", 1);
$this->textMatchBuckets = $config->get("typesense_options.text_match_buckets", 85); $this->exhaustiveSearch = (string) Arr::get($config, "typesense_options.exhaustive_search", "false");
$this->exhaustiveSearch = (string) $config->get("typesense_options.exhaustive_search", "false"); $this->config = collect($config);
$this->config = $config; $this->typoTokensThreshold = Arr::get($config, "typesense_options.typo_tokens_threshold") ?? $this->maxResultsPerPage();
$this->typoTokensThreshold = $config->get("typesense_options.typo_tokens_threshold", $this->maxResultsPerPage()); $this->dropTokensThreshold = Arr::get($config, "typesense_options.drop_tokens_threshold") ?? $this->maxResultsPerPage();
$this->dropTokensThreshold = $config->get("typesense_options.drop_tokens_threshold", $this->maxResultsPerPage()); $this->searchCutOffMs = Arr::get($config, "typesense_options.search_cutoff_ms", 450);
$this->searchCutOffMs = $config->get("typesense_options.search_cutoff_ms", 450);
} }
public function cacheTtlForEndpoint(string $endpoint): ?int public function cacheTtlForEndpoint(string $endpoint): ?int
@ -61,7 +61,7 @@ final class JikanConfig
public function maxResultsPerPage(?int $defaultValue = null): int public function maxResultsPerPage(?int $defaultValue = null): int
{ {
return $this->config->get("max_results_per_page", $defaultValue ?? 25); return (int) $this->config->get("max_results_per_page", $defaultValue ?? 25);
} }
public function textMatchBuckets(): int public function textMatchBuckets(): int

View File

@ -1,14 +1,14 @@
<?php <?php
return [ return [
'max_results_per_page' => (int) env('MAX_RESULTS_PER_PAGE', 25), 'max_results_per_page' => env('MAX_RESULTS_PER_PAGE', 25),
'micro_caching_enabled' => env('MICROCACHING', false), 'micro_caching_enabled' => env('MICROCACHING', false),
'default_cache_expire' => env('CACHE_DEFAULT_EXPIRE', 86400), 'default_cache_expire' => env('CACHE_DEFAULT_EXPIRE', 86400),
'typesense_options' => [ 'typesense_options' => [
'text_match_buckets' => env('TYPESENSE_TEXT_MATCH_BUCKETS', 85), 'text_match_buckets' => env('TYPESENSE_TEXT_MATCH_BUCKETS', 1),
'typo_tokens_threshold' => (int) env('TYPESENSE_TYPO_TOKENS_THRESHOLD'), 'typo_tokens_threshold' => env('TYPESENSE_TYPO_TOKENS_THRESHOLD'),
'drop_tokens_threshold' => (int) env('TYPESENSE_DROP_TOKENS_THRESHOLD'), 'drop_tokens_threshold' => env('TYPESENSE_DROP_TOKENS_THRESHOLD'),
'search_cutoff_ms' => (int) env('TYPESENSE_SEARCH_CUTOFF_MS', 450), 'search_cutoff_ms' => env('TYPESENSE_SEARCH_CUTOFF_MS', 450),
'exhaustive_search' => env('TYPESENSE_ENABLE_EXHAUSTIVE_SEARCH', 'false') 'exhaustive_search' => env('TYPESENSE_ENABLE_EXHAUSTIVE_SEARCH', 'false')
], ],
'per_endpoint_cache_ttl' => [ 'per_endpoint_cache_ttl' => [

View File

@ -269,7 +269,7 @@ $router->group(
} }
); );
$router->get('schedules[/{dayFilter:[A-Za-z]+}]', [ $router->get('schedules[/{filter:[A-Za-z]+}]', [
'uses' => 'ScheduleController@main' 'uses' => 'ScheduleController@main'
]); ]);