malhuda/jikan-rest
1
0
Fork 0
mirror of https://github.com/jikan-me/jikan-rest.git synced 2025-02-20 11:23:35 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity

added search analytics

Browse Source
This commit is contained in:
pushrbx 2023-06-28 16:30:24 +01:00
parent eed1a33813
commit 714d4cb490
5 changed files with 289 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
app/Contracts/SearchAnalyticsService.php Normal file
View File

@ -0,0 +1,10 @@
<?php
namespace App\Contracts;
use Illuminate\Support\Collection;
interface SearchAnalyticsService
{
public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void;
}

14
app/Providers/AppServiceProvider.php
View File

@ -69,6 +69,8 @@ class AppServiceProvider extends ServiceProvider
public function boot(): void
{
$this->registerMacros();
// the registration of request handlers should happen after the load of all service providers.
$this->registerRequestHandlers();
}
/**
@ -81,6 +83,7 @@ class AppServiceProvider extends ServiceProvider
public function register(): void
{
$this->app->singleton(JikanConfig::class, fn() => new JikanConfig(config("jikan")));
$this->app->singleton(\App\Contracts\SearchAnalyticsService::class, \App\Services\DefaultSearchAnalyticsService::class);
$this->app->alias(JikanConfig::class, "jikan-config");
// cache options class is used to share the request scope level cache settings
$this->app->singleton(CacheOptions::class);
@ -88,10 +91,9 @@ class AppServiceProvider extends ServiceProvider
$this->app->singleton(PrivateFieldMapperService::class, DefaultPrivateFieldMapperService::class);
$this->app->bind(QueryBuilderPaginatorService::class, DefaultBuilderPaginatorService::class);
if (static::getSearchIndexDriver($this->app) === "typesense") {
$this->app->singleton(App\Services\TypesenseCollectionDescriptor::class);
$this->app->singleton(\App\Services\TypesenseCollectionDescriptor::class);
}
$this->registerModelRepositories();
$this->registerRequestHandlers();
}
private function getSearchService(Repository $repository): SearchService
@ -107,7 +109,6 @@ class AppServiceProvider extends ServiceProvider
$scoutSearchService = $this->app->make($serviceClass, [
"repository" => $repository,
"config" => $this->app->make("jikan-config")
]);
$result = new SearchEngineSearchService($scoutSearchService, $repository);
}
@ -177,10 +178,9 @@ class AppServiceProvider extends ServiceProvider
$requestHandlers = [];
foreach ($searchRequestHandlersDescriptors as $handlerClass => $repositoryInstance) {
$requestHandlers[] = $app->make($handlerClass, [
"queryBuilderService" => new DefaultQueryBuilderService(
$this->getSearchService($repositoryInstance),
$app->make(QueryBuilderPaginatorService::class)
)
"queryBuilderService" => $app->make(DefaultQueryBuilderService::class, [
"searchService" => $this->getSearchService($repositoryInstance)
]),
]);
}

121
app/SearchMetric.php Normal file
View File

@ -0,0 +1,121 @@
<?php
namespace App;
use MongoDB\BSON\ObjectId;
use Laravel\Scout\Builder;
class SearchMetric extends JikanApiSearchableModel
{
protected $table = 'search_metrics';
protected $appends = ['hits'];
protected $fillable = ["search_term", "request_count", "hits", "hits_count", "index_name"];
protected $hidden = ["_id"];
public function searchableAs(): string
{
return "jikan_search_metrics";
}
public function toSearchableArray()
{
return [
"id" => $this->_id,
"search_term" => $this->search_term,
"request_count" => $this->request_count,
"hits" => $this->hits,
"hits_count" => $this->hits_count,
"index_name" => $this->index_name
];
}
public function getCollectionSchema(): array
{
return [
"name" => $this->searchableAs(),
"fields" => [
[
"name" => "id",
"type" => "string",
"optional" => false
],
[
"name" => "search_term",
"type" => "string",
"optional" => false,
"sort" => false,
"infix" => true
],
[
"name" => "request_count",
"type" => "int64",
"sort" => true,
"optional" => false,
],
[
"name" => "hits",
"type" => "int64[]",
"sort" => false,
"optional" => false,
],
[
"name" => "hits_count",
"type" => "int64",
"sort" => true,
"optional" => false,
],
[
"name" => "index_name",
"type" => "string",
"sort" => false,
"optional" => false,
"facet" => true
]
]
];
}
public function getTypeSenseQueryByWeights(): string|null
{
return "1";
}
public function getScoutKey(): mixed
{
return $this->_id;
}
public function getScoutKeyName(): mixed
{
return '_id';
}
public function getKeyType(): string
{
return 'string';
}
public function typesenseQueryBy(): array
{
return ["search_term"];
}
public function queryScoutModelsByIds(Builder $builder, array $ids): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$query = static::usesSoftDelete()
? $this->withTrashed() : $this->newQuery();
if ($builder->queryCallback) {
call_user_func($builder->queryCallback, $query);
}
$whereIn = in_array($this->getKeyType(), ['int', 'integer']) ?
'whereIntegerInRaw' :
'whereIn';
return $query->{$whereIn}(
$this->getScoutKeyName(), array_map(fn ($x) => new ObjectId($x), $ids)
);
}
}

43
app/Services/DefaultSearchAnalyticsService.php Normal file
View File

@ -0,0 +1,43 @@
<?php
namespace App\Services;
use App\SearchMetric;
use App\Contracts\SearchAnalyticsService;
use Illuminate\Support\Collection;
/**
* The default search analytics service implementation, which saves the stats to the database and indexes it in Typesense.
*
* By indexing search terms in Typesense we can use it to provide search suggestions of popular searches.
* @package App\Services
*/
final class DefaultSearchAnalyticsService implements SearchAnalyticsService
{
public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void
{
/**
* @var \Laravel\Scout\Builder $existingMetrics
*/
$existingMetrics = SearchMetric::search($searchTerm);
$hitList = $hits->pluck("id")->values()->map(fn($x) => (int)$x)->all();
if ($existingMetrics->count() > 0) {
$metric = $existingMetrics->first();
$metric->hits = $hitList;
$metric->hits_count = $hitsCount;
$metric->request_count = $metric->request_count + 1;
$metric->index_name = $indexName;
$metric->save();
} else {
SearchMetric::create([
"search_term" => $searchTerm,
"request_count" => 1,
"hits" => $hitList,
"hits_count" => $hitsCount,
"index_name" => $indexName
]);
}
}
}

176
app/Services/TypeSenseScoutSearchService.php
View File

@ -9,12 +9,17 @@ use Illuminate\Support\Str;
use Illuminate\Support\Arr;
use Illuminate\Support\Facades\App;
use Typesense\Documents;
use App\Contracts\SearchAnalyticsService;
use App\Services\TypesenseCollectionDescriptor;
class TypeSenseScoutSearchService implements ScoutSearchService
{
private int $maxItemsPerPage;
public function __construct(private readonly Repository $repository, JikanConfig $config)
public function __construct(private readonly Repository $repository,
JikanConfig $config,
private readonly TypesenseCollectionDescriptor $collectionDescriptor,
private readonly SearchAnalyticsService $searchAnalytics)
{
$this->maxItemsPerPage = (int) $config->maxResultsPerPage();
if ($this->maxItemsPerPage > 250) {
@ -32,7 +37,12 @@ class TypeSenseScoutSearchService implements ScoutSearchService
public function search(string $q, ?string $orderByField = null,
bool $sortDirectionDescending = false): \Laravel\Scout\Builder
{
return $this->repository->search($q, function (Documents $documents, string $query, array $options) use ($orderByField, $sortDirectionDescending) {
return $this->repository->search($q, $this->middleware($orderByField, $sortDirectionDescending));
}
private function middleware(?string $orderByField = null, bool $sortDirectionDescending = false): \Closure
{
return function (Documents $documents, string $query, array $options) use ($orderByField, $sortDirectionDescending) {
// let's enable exhaustive search
// which will make Typesense consider all variations of prefixes and typo corrections of the words
// in the query exhaustively, without stopping early when enough results are found.
@ -53,75 +63,105 @@ class TypeSenseScoutSearchService implements ScoutSearchService
$options['per_page'] = min($this->maxItemsPerPage, 250);
}
// skip typo checking for short queries
if (strlen($query) <= 3) {
$options['num_typos'] = 0;
$options['typo_tokens_threshold'] = 0;
$options['drop_tokens_threshold'] = 0;
$options['exhaustive_search'] = 'false';
$options['infix'] = 'off';
$options['prefix'] = 'false';
}
$options = $this->skipTypoCheckingForShortQueries($query, $options);
$modelInstance = $this->repository->createEntity();
// get the weights of the query_by fields, if they are provided by the model.
if ($modelInstance instanceof JikanApiSearchableModel) {
$queryByWeights = $modelInstance->getTypeSenseQueryByWeights();
if (!is_null($queryByWeights)) {
$options['query_by_weights'] = $queryByWeights;
}
// if the model specifies search index sort order, use it
// this is the default sort order for the model
$sortByFields = $modelInstance->getSearchIndexSortBy();
if (!is_null($sortByFields)) {
$sortBy = "";
foreach ($sortByFields as $f) {
$sortBy .= $f['field'] . ':' . $f['direction'];
$sortBy .= ',';
}
$sortBy = rtrim($sortBy, ',');
$options['sort_by'] = $sortBy;
}
// todo: try to avoid service lookup, resolve things via constructor instead.
// this is currently a workaround as the search service resolution in the service provider is complex,
// and it gives errors when you try to resolve the Typesense class from the LaraveTypesense driver package.
// here we'd like to get all the searchable attributes of the model, so we can override the sort order.
// we use these attribute names to validate the incoming field name against them, otherwise ignoring them.
$collectionDescriptor = App::make(TypesenseCollectionDescriptor::class);
$modelAttrNames = $collectionDescriptor->getSearchableAttributes($modelInstance);
// fixme: this shouldn't be here, but it's a quick fix for the time being
if ($orderByField === "aired.from") {
$orderByField = "start_date";
}
if ($orderByField === "aired.to") {
$orderByField = "end_date";
}
if ($orderByField === "published.from") {
$orderByField = "start_date";
}
if ($orderByField === "published.to") {
$orderByField = "end_date";
}
// fixme end
// override ordering field
if (!is_null($orderByField) && Arr::has($modelAttrNames, $orderByField)) {
$options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->maxItemsPerPage."):desc";
}
// override overall sorting direction
if (is_null($orderByField) && $sortDirectionDescending && array_key_exists("sort_by", $options) && Str::contains($options["sort_by"], "asc")) {
$options["sort_by"] = Str::replace("asc", "desc", $options["sort_by"]);
}
$options = $this->setQueryByWeights($options, $modelInstance);
$options = $this->setSortOrder($options, $modelInstance);
$options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending);
}
return $documents->search($options);
});
$results = $documents->search($options);
$this->recordSearchTelemetry($query, $results);
return $results;
};
}
private function skipTypoCheckingForShortQueries(string $query, array $options): array
{
if (strlen($query) <= 3) {
$options['num_typos'] = 0;
$options['typo_tokens_threshold'] = 0;
$options['drop_tokens_threshold'] = 0;
$options['exhaustive_search'] = 'false';
$options['infix'] = 'off';
$options['prefix'] = 'false';
}
return $options;
}
private function setQueryByWeights(array $options, JikanApiSearchableModel $modelInstance): array
{
$queryByWeights = $modelInstance->getTypeSenseQueryByWeights();
if (!is_null($queryByWeights)) {
$options['query_by_weights'] = $queryByWeights;
}
return $options;
}
private function setSortOrder(array $options, JikanApiSearchableModel $modelInstance): array
{
$sortByFields = $modelInstance->getSearchIndexSortBy();
if (!is_null($sortByFields)) {
$sortBy = "";
foreach ($sortByFields as $f) {
$sortBy .= $f['field'] . ':' . $f['direction'];
$sortBy .= ',';
}
$sortBy = rtrim($sortBy, ',');
$options['sort_by'] = $sortBy;
}
return $options;
}
private function overrideSortingOrder(array $options, JikanApiSearchableModel $modelInstance, ?string $orderByField, bool $sortDirectionDescending): array
{
$modelAttrNames = $this->collectionDescriptor->getSearchableAttributes($modelInstance);
// fixme: this shouldn't be here, but it's a quick fix for the time being
if ($orderByField === "aired.from") {
$orderByField = "start_date";
}
if ($orderByField === "aired.to") {
$orderByField = "end_date";
}
if ($orderByField === "published.from") {
$orderByField = "start_date";
}
if ($orderByField === "published.to") {
$orderByField = "end_date";
}
// fixme end
// override ordering field
if (!is_null($orderByField) && Arr::has($modelAttrNames, $orderByField)) {
$options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->maxItemsPerPage."):desc";
}
// override overall sorting direction
if (is_null($orderByField) && $sortDirectionDescending && array_key_exists("sort_by", $options) && Str::contains($options["sort_by"], "asc")) {
$options["sort_by"] = Str::replace("asc", "desc", $options["sort_by"]);
}
return $options;
}
private function recordSearchTelemetry(string $query, array $typesenseApiResponse): void
{
$hits = collect($typesenseApiResponse["hits"]);
$this->searchAnalytics->logSearch(
$query,
$typesenseApiResponse["found"],
$hits->pluck('document')->values(),
$typesenseApiResponse["request_params"]["collection_name"]
);
}
}