diff --git a/app/Contracts/SearchAnalyticsService.php b/app/Contracts/SearchAnalyticsService.php new file mode 100644 index 0000000..bedfb7f --- /dev/null +++ b/app/Contracts/SearchAnalyticsService.php @@ -0,0 +1,10 @@ +registerMacros(); + // the registration of request handlers should happen after the load of all service providers. + $this->registerRequestHandlers(); } /** @@ -81,6 +83,7 @@ class AppServiceProvider extends ServiceProvider public function register(): void { $this->app->singleton(JikanConfig::class, fn() => new JikanConfig(config("jikan"))); + $this->app->singleton(\App\Contracts\SearchAnalyticsService::class, \App\Services\DefaultSearchAnalyticsService::class); $this->app->alias(JikanConfig::class, "jikan-config"); // cache options class is used to share the request scope level cache settings $this->app->singleton(CacheOptions::class); @@ -88,10 +91,9 @@ class AppServiceProvider extends ServiceProvider $this->app->singleton(PrivateFieldMapperService::class, DefaultPrivateFieldMapperService::class); $this->app->bind(QueryBuilderPaginatorService::class, DefaultBuilderPaginatorService::class); if (static::getSearchIndexDriver($this->app) === "typesense") { - $this->app->singleton(App\Services\TypesenseCollectionDescriptor::class); + $this->app->singleton(\App\Services\TypesenseCollectionDescriptor::class); } $this->registerModelRepositories(); - $this->registerRequestHandlers(); } private function getSearchService(Repository $repository): SearchService @@ -107,7 +109,6 @@ class AppServiceProvider extends ServiceProvider $scoutSearchService = $this->app->make($serviceClass, [ "repository" => $repository, - "config" => $this->app->make("jikan-config") ]); $result = new SearchEngineSearchService($scoutSearchService, $repository); } @@ -177,10 +178,9 @@ class AppServiceProvider extends ServiceProvider $requestHandlers = []; foreach ($searchRequestHandlersDescriptors as $handlerClass => $repositoryInstance) { $requestHandlers[] = $app->make($handlerClass, [ - "queryBuilderService" => new DefaultQueryBuilderService( - $this->getSearchService($repositoryInstance), - $app->make(QueryBuilderPaginatorService::class) - ) + "queryBuilderService" => $app->make(DefaultQueryBuilderService::class, [ + "searchService" => $this->getSearchService($repositoryInstance) + ]), ]); } diff --git a/app/SearchMetric.php b/app/SearchMetric.php new file mode 100644 index 0000000..a34c44b --- /dev/null +++ b/app/SearchMetric.php @@ -0,0 +1,121 @@ + $this->_id, + "search_term" => $this->search_term, + "request_count" => $this->request_count, + "hits" => $this->hits, + "hits_count" => $this->hits_count, + "index_name" => $this->index_name + ]; + } + + public function getCollectionSchema(): array + { + return [ + "name" => $this->searchableAs(), + "fields" => [ + [ + "name" => "id", + "type" => "string", + "optional" => false + ], + [ + "name" => "search_term", + "type" => "string", + "optional" => false, + "sort" => false, + "infix" => true + ], + [ + "name" => "request_count", + "type" => "int64", + "sort" => true, + "optional" => false, + ], + [ + "name" => "hits", + "type" => "int64[]", + "sort" => false, + "optional" => false, + ], + [ + "name" => "hits_count", + "type" => "int64", + "sort" => true, + "optional" => false, + ], + [ + "name" => "index_name", + "type" => "string", + "sort" => false, + "optional" => false, + "facet" => true + ] + ] + ]; + } + + public function getTypeSenseQueryByWeights(): string|null + { + return "1"; + } + + public function getScoutKey(): mixed + { + return $this->_id; + } + + public function getScoutKeyName(): mixed + { + return '_id'; + } + + public function getKeyType(): string + { + return 'string'; + } + + public function typesenseQueryBy(): array + { + return ["search_term"]; + } + + public function queryScoutModelsByIds(Builder $builder, array $ids): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder + { + $query = static::usesSoftDelete() + ? $this->withTrashed() : $this->newQuery(); + + if ($builder->queryCallback) { + call_user_func($builder->queryCallback, $query); + } + + $whereIn = in_array($this->getKeyType(), ['int', 'integer']) ? + 'whereIntegerInRaw' : + 'whereIn'; + + return $query->{$whereIn}( + $this->getScoutKeyName(), array_map(fn ($x) => new ObjectId($x), $ids) + ); + } +} diff --git a/app/Services/DefaultSearchAnalyticsService.php b/app/Services/DefaultSearchAnalyticsService.php new file mode 100644 index 0000000..8e18d09 --- /dev/null +++ b/app/Services/DefaultSearchAnalyticsService.php @@ -0,0 +1,43 @@ +pluck("id")->values()->map(fn($x) => (int)$x)->all(); + + if ($existingMetrics->count() > 0) { + $metric = $existingMetrics->first(); + $metric->hits = $hitList; + $metric->hits_count = $hitsCount; + $metric->request_count = $metric->request_count + 1; + $metric->index_name = $indexName; + $metric->save(); + } else { + SearchMetric::create([ + "search_term" => $searchTerm, + "request_count" => 1, + "hits" => $hitList, + "hits_count" => $hitsCount, + "index_name" => $indexName + ]); + } + } +} diff --git a/app/Services/TypeSenseScoutSearchService.php b/app/Services/TypeSenseScoutSearchService.php index 4965e9f..7b00ab9 100644 --- a/app/Services/TypeSenseScoutSearchService.php +++ b/app/Services/TypeSenseScoutSearchService.php @@ -9,12 +9,17 @@ use Illuminate\Support\Str; use Illuminate\Support\Arr; use Illuminate\Support\Facades\App; use Typesense\Documents; +use App\Contracts\SearchAnalyticsService; +use App\Services\TypesenseCollectionDescriptor; class TypeSenseScoutSearchService implements ScoutSearchService { private int $maxItemsPerPage; - public function __construct(private readonly Repository $repository, JikanConfig $config) + public function __construct(private readonly Repository $repository, + JikanConfig $config, + private readonly TypesenseCollectionDescriptor $collectionDescriptor, + private readonly SearchAnalyticsService $searchAnalytics) { $this->maxItemsPerPage = (int) $config->maxResultsPerPage(); if ($this->maxItemsPerPage > 250) { @@ -32,7 +37,12 @@ class TypeSenseScoutSearchService implements ScoutSearchService public function search(string $q, ?string $orderByField = null, bool $sortDirectionDescending = false): \Laravel\Scout\Builder { - return $this->repository->search($q, function (Documents $documents, string $query, array $options) use ($orderByField, $sortDirectionDescending) { + return $this->repository->search($q, $this->middleware($orderByField, $sortDirectionDescending)); + } + + private function middleware(?string $orderByField = null, bool $sortDirectionDescending = false): \Closure + { + return function (Documents $documents, string $query, array $options) use ($orderByField, $sortDirectionDescending) { // let's enable exhaustive search // which will make Typesense consider all variations of prefixes and typo corrections of the words // in the query exhaustively, without stopping early when enough results are found. @@ -53,75 +63,105 @@ class TypeSenseScoutSearchService implements ScoutSearchService $options['per_page'] = min($this->maxItemsPerPage, 250); } - // skip typo checking for short queries - if (strlen($query) <= 3) { - $options['num_typos'] = 0; - $options['typo_tokens_threshold'] = 0; - $options['drop_tokens_threshold'] = 0; - $options['exhaustive_search'] = 'false'; - $options['infix'] = 'off'; - $options['prefix'] = 'false'; - } - + $options = $this->skipTypoCheckingForShortQueries($query, $options); $modelInstance = $this->repository->createEntity(); - // get the weights of the query_by fields, if they are provided by the model. + if ($modelInstance instanceof JikanApiSearchableModel) { - $queryByWeights = $modelInstance->getTypeSenseQueryByWeights(); - if (!is_null($queryByWeights)) { - $options['query_by_weights'] = $queryByWeights; - } - - // if the model specifies search index sort order, use it - // this is the default sort order for the model - $sortByFields = $modelInstance->getSearchIndexSortBy(); - if (!is_null($sortByFields)) { - $sortBy = ""; - foreach ($sortByFields as $f) { - $sortBy .= $f['field'] . ':' . $f['direction']; - $sortBy .= ','; - } - $sortBy = rtrim($sortBy, ','); - $options['sort_by'] = $sortBy; - } - - // todo: try to avoid service lookup, resolve things via constructor instead. - // this is currently a workaround as the search service resolution in the service provider is complex, - // and it gives errors when you try to resolve the Typesense class from the LaraveTypesense driver package. - // here we'd like to get all the searchable attributes of the model, so we can override the sort order. - // we use these attribute names to validate the incoming field name against them, otherwise ignoring them. - $collectionDescriptor = App::make(TypesenseCollectionDescriptor::class); - $modelAttrNames = $collectionDescriptor->getSearchableAttributes($modelInstance); - - // fixme: this shouldn't be here, but it's a quick fix for the time being - if ($orderByField === "aired.from") { - $orderByField = "start_date"; - } - - if ($orderByField === "aired.to") { - $orderByField = "end_date"; - } - - if ($orderByField === "published.from") { - $orderByField = "start_date"; - } - - if ($orderByField === "published.to") { - $orderByField = "end_date"; - } - // fixme end - - // override ordering field - if (!is_null($orderByField) && Arr::has($modelAttrNames, $orderByField)) { - $options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->maxItemsPerPage."):desc"; - } - - // override overall sorting direction - if (is_null($orderByField) && $sortDirectionDescending && array_key_exists("sort_by", $options) && Str::contains($options["sort_by"], "asc")) { - $options["sort_by"] = Str::replace("asc", "desc", $options["sort_by"]); - } + $options = $this->setQueryByWeights($options, $modelInstance); + $options = $this->setSortOrder($options, $modelInstance); + $options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending); } - return $documents->search($options); - }); + $results = $documents->search($options); + $this->recordSearchTelemetry($query, $results); + + return $results; + }; + } + + private function skipTypoCheckingForShortQueries(string $query, array $options): array + { + if (strlen($query) <= 3) { + $options['num_typos'] = 0; + $options['typo_tokens_threshold'] = 0; + $options['drop_tokens_threshold'] = 0; + $options['exhaustive_search'] = 'false'; + $options['infix'] = 'off'; + $options['prefix'] = 'false'; + } + + return $options; + } + + private function setQueryByWeights(array $options, JikanApiSearchableModel $modelInstance): array + { + $queryByWeights = $modelInstance->getTypeSenseQueryByWeights(); + if (!is_null($queryByWeights)) { + $options['query_by_weights'] = $queryByWeights; + } + + return $options; + } + + private function setSortOrder(array $options, JikanApiSearchableModel $modelInstance): array + { + $sortByFields = $modelInstance->getSearchIndexSortBy(); + if (!is_null($sortByFields)) { + $sortBy = ""; + foreach ($sortByFields as $f) { + $sortBy .= $f['field'] . ':' . $f['direction']; + $sortBy .= ','; + } + $sortBy = rtrim($sortBy, ','); + $options['sort_by'] = $sortBy; + } + + return $options; + } + + private function overrideSortingOrder(array $options, JikanApiSearchableModel $modelInstance, ?string $orderByField, bool $sortDirectionDescending): array + { + $modelAttrNames = $this->collectionDescriptor->getSearchableAttributes($modelInstance); + + // fixme: this shouldn't be here, but it's a quick fix for the time being + if ($orderByField === "aired.from") { + $orderByField = "start_date"; + } + + if ($orderByField === "aired.to") { + $orderByField = "end_date"; + } + + if ($orderByField === "published.from") { + $orderByField = "start_date"; + } + + if ($orderByField === "published.to") { + $orderByField = "end_date"; + } + // fixme end + + // override ordering field + if (!is_null($orderByField) && Arr::has($modelAttrNames, $orderByField)) { + $options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->maxItemsPerPage."):desc"; + } + + // override overall sorting direction + if (is_null($orderByField) && $sortDirectionDescending && array_key_exists("sort_by", $options) && Str::contains($options["sort_by"], "asc")) { + $options["sort_by"] = Str::replace("asc", "desc", $options["sort_by"]); + } + + return $options; + } + + private function recordSearchTelemetry(string $query, array $typesenseApiResponse): void + { + $hits = collect($typesenseApiResponse["hits"]); + $this->searchAnalytics->logSearch( + $query, + $typesenseApiResponse["found"], + $hits->pluck('document')->values(), + $typesenseApiResponse["request_params"]["collection_name"] + ); } }