initial attempt to use typesense as search engine

- currently only anime entries.
This commit is contained in:
pushrbx 2022-05-30 18:37:28 +01:00
parent 021e6f7cea
commit 38113a4d1f
7 changed files with 670 additions and 6 deletions

View File

@ -9,9 +9,13 @@ use Jikan\Helper\Parser;
use Jikan\Jikan;
use Jikan\Model\Common\YoutubeMeta;
use Jikan\Request\Anime\AnimeRequest;
use Laravel\Scout\Builder;
use Typesense\LaravelTypesense\Interfaces\TypesenseDocument;
use Laravel\Scout\Searchable;
class Anime extends Model
class Anime extends Model implements TypesenseDocument
{
use JikanSearchable;
/**
* The attributes that are mass assignable.
@ -131,4 +135,117 @@ class Anime extends Model
)
);
}
/**
* Get the name of the index associated with the model.
*
* @return string
*/
public function searchableAs(): string
{
return 'anime_index';
}
/**
* Get the value used to index the model.
*
* @return mixed
*/
public function getScoutKey(): mixed
{
return $this->mal_id;
}
/**
* Get the key name used to index the model.
*
* @return mixed
*/
public function getScoutKeyName(): mixed
{
return 'mal_id';
}
/**
* Get the indexable data array for the model.
*
* @return array
*/
public function toSearchableArray(): array
{
$serializer = app('SerializerV4');
$result = [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'start_date' => $this->aired['from'] ? Parser::parseDate($this->aired['from'])->getTimestamp() : 0,
'end_date' => $this->aired['to'] ? Parser::parseDate($this->aired['to'])->getTimestamp() : 0,
'url' => $this->url,
'images' => $this->images,
'trailer' => $this->trailer,
'title' => $this->title,
'title_english' => $this->title_english,
'title_japanese' => $this->title_japanese,
'title_synonyms' => $this->title_synonyms,
'type' => $this->type,
'source' => $this->source,
'episodes' => $this->episodes,
'status' => $this->status,
'airing' => $this->airing,
'duration' => $this->duration,
'rating' => $this->rating,
'score' => $this->score,
'scored_by' => $this->scored_by,
'rank' => $this->rank,
'popularity' => $this->popularity,
'members' => $this->members,
'favorites' => $this->favorites,
'synopsis' => $this->synopsis,
'background' => $this->background,
'season' => $this->season,
'year' => $this->year,
'broadcast' => $this->broadcast,
'producers' => $serializer->serialize($this->producers, 'json'),
'licensors' => $serializer->serialize($this->licensors, 'json'),
'studios' => $serializer->serialize($this->studios, 'json'),
'genres' => $serializer->serialize($this->genres, 'json'),
'explicit_genres' => $serializer->serialize($this->explicit_genres, 'json'),
'themes' => $serializer->serialize($this->themes, 'json'),
'demographics' => $serializer->serialize($this->demographics, 'json'),
];
return $result;
}
/**
* The fields to be queried against. See https://typesense.org/docs/0.21.0/api/documents.html#search.
*
* @return array
*/
public function typesenseQueryBy(): array
{
return [
'title',
'title_english',
'title_japanese',
'title_synonyms'
];
}
/**
* The Typesense schema to be created.
*
* @return array
*/
public function getCollectionSchema(): array
{
return [
'name' => $this->searchableAs(),
'fields' => [
[
'name' => '.*',
'type' => 'auto',
]
]
];
}
}

View File

@ -9,6 +9,7 @@ use App\Http\HttpHelper;
use App\Http\HttpResponse;
use App\Http\Middleware\Throttle;
use App\Http\QueryBuilder\SearchQueryBuilderAnime;
use App\Http\QueryBuilder\ScoutSearchQueryBuilderAnime;
use App\Http\QueryBuilder\SearchQueryBuilderCharacter;
use App\Http\QueryBuilder\SearchQueryBuilderClub;
use App\Http\QueryBuilder\SearchQueryBuilderManga;
@ -210,16 +211,13 @@ class SearchController extends Controller
}
}
$results = SearchQueryBuilderAnime::query(
$request,
Anime::query()
$results = ScoutSearchQueryBuilderAnime::query(
$request
);
$results = $results
->paginate(
$limit,
['*'],
null,
$page
);

View File

@ -0,0 +1,356 @@
<?php
namespace App\Http\QueryBuilder;
use App\Anime;
use App\Http\HttpHelper;
use Illuminate\Http\Request;
use Jenssegers\Mongodb\Eloquent\Builder;
/**
* Class SearchQueryBuilderAnime
* @package App\Http\QueryBuilder
*/
class ScoutSearchQueryBuilderAnime
{
/**
* @OA\Schema(
* schema="anime_search_query_type",
* description="Available Anime types",
* type="string",
* enum={"tv","movie","ova","special","ona","music"}
* )
*/
const MAP_TYPES = [
'tv' => 'TV',
'movie' => 'Movie',
'ova' => 'OVA',
'special' => 'Special',
'ona' => 'ONA',
'music' => 'Music'
];
/**
* @OA\Schema(
* schema="anime_search_query_status",
* description="Available Anime statuses",
* type="string",
* enum={"airing","complete","upcoming"}
* )
*/
const MAP_STATUS = [
'airing' => 'Currently Airing',
'complete' => 'Finished Airing',
'upcoming' => 'Not yet aired',
];
/**
* @OA\Schema(
* schema="anime_search_query_rating",
* description="Available Anime audience ratings<br><br><b>Ratings</b><br><ul><li>G - All Ages</li><li>PG - Children</li><li>PG-13 - Teens 13 or older</li><li>R - 17+ (violence & profanity)</li><li>R+ - Mild Nudity</li><li>Rx - Hentai</li></ul>",
* type="string",
* enum={"g","pg","pg13","r17","r","rx"}
* )
*/
const MAP_RATING = [
'g' => 'G - All Ages',
'pg' => 'PG - Children',
'pg13' => 'PG-13 - Teens 13 or older',
'r17' => 'R - 17+ (violence & profanity)',
'r' => 'R+ - Mild Nudity',
'rx' => 'Rx - Hentai'
];
/**
* @OA\Schema(
* schema="anime_search_query_orderby",
* description="Available Anime order_by properties",
* type="string",
* enum={"mal_id", "title", "type", "rating", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" }
* )
*/
const ORDER_BY = [
'mal_id' => 'mal_id',
'title' => 'title',
'type' => 'type',
'rating' => 'rating',
'start_date' => 'aired.from',
'end_date' => 'aired.to',
'episodes' => 'episodes',
'score' => 'score',
'scored_by' => 'scored_by',
'rank' => 'rank',
'popularity' => 'popularity',
'members' => 'members',
'favorites' => 'favorites'
];
/**
* @param Request $request
* @param Builder $results
* @return Builder
*/
public static function query(Request $request) : \Laravel\Scout\Builder
{
$requestType = HttpHelper::requestType($request);
$query = $request->get('q');
$type = self::mapType($request->get('type'));
$score = $request->get('score') ?? 0;
$status = self::mapStatus($request->get('status'));
$rating = self::mapRating($request->get('rating'));
$sfw = $request->get('sfw');
$genres = $request->get('genres');
$genresExclude = $request->get('genres_exclude');
$orderBy = self::mapOrderBy($request->get('order_by'));
$sort = self::mapSort($request->get('sort'));
$letter = $request->get('letter');
$producer = $request->get('producers');
$minScore = $request->get('min_score');
$maxScore = $request->get('max_score');
$startDate = $request->get('start_date');
$endDate = $request->get('end_date');
$results = Anime::search($query);
if (empty($query) && is_null($orderBy)) {
$results = $results
->orderBy('mal_id');
}
if (!is_null($startDate)) {
$startDate = explode('-', $startDate);
$startDate = (new \DateTime())
->setDate(
$startDate[0] ?? date('Y'),
$startDate[1] ?? 1,
$startDate[2] ?? 1
)
->format(\DateTimeInterface::ISO8601);
$results = $results
->where('aired.from', '>=', $startDate);
}
if (!is_null($endDate)) {
$endDate = explode('-', $endDate);
$endDate = (new \DateTime())
->setDate(
$endDate[0] ?? date('Y'),
$endDate[1] ?? 1,
$endDate[2] ?? 1
)
->format(\DateTimeInterface::ISO8601);
$results = $results
->where('aired.to', '<=', $endDate);
}
if (!is_null($type)) {
$results = $results
->where('type', $type);
}
if ($score !== 0) {
$score = (float) $score;
$results = $results
->where('score', '>=', $score);
}
if ($minScore !== null) {
$minScore = (float) $minScore;
$results = $results
->where('score', '>=', $minScore);
}
if ($maxScore !== null) {
$maxScore = (float) $maxScore;
$results = $results
->where('score', '<=', $maxScore);
}
if (!is_null($status)) {
$results = $results
->where('status', $status);
}
if (!is_null($rating)) {
$results = $results
->where('rating', $rating);
}
if (!is_null($producer)) {
$producer = (int) $producer;
$results = $results
->where('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer);
}
if (!is_null($genres)) {
$genres = explode(',', $genres);
foreach ($genres as $genre) {
if (empty($genre)) {
continue;
}
$genre = (int) $genre;
$results = $results
->where(function($query) use ($genre) {
$query
->where('genres.mal_id', $genre)
->orWhere('demographics.mal_id', $genre)
->orWhere('themes.mal_id', $genre)
->orWhere('explicit_genres.mal_id', $genre);
});
}
}
if (!is_null($genresExclude)) {
$genresExclude = explode(',', $genresExclude);
foreach ($genresExclude as $genreExclude) {
if (empty($genreExclude)) {
continue;
}
$genreExclude = (int) $genreExclude;
$results = $results
->where(function($query) use ($genreExclude) {
$query
->where('genres.mal_id', '!=', $genreExclude)
->where('demographics.mal_id', '!=', $genreExclude)
->where('themes.mal_id', '!=', $genreExclude)
->where('explicit_genres.mal_id', '!=', $genreExclude);
});
;
}
}
if (!is_null($sfw)) {
$results = $results
->where('rating', '!=', self::MAP_RATING['rx']);
}
if (!is_null($orderBy)) {
$results = $results
->orderBy($orderBy, $sort ?? 'asc');
}
return $results;
}
/**
* @param Request $request
* @param Builder $results
* @return array
*/
public static function paginate(Request $request, Builder $results)
{
$page = $request->get('page') ?? 1;
$limit = $request->get('limit') ?? env('MAX_RESULTS_PER_PAGE', 25);
$limit = (int) $limit;
if ($limit <= 0) {
$limit = 1;
}
if ($limit > env('MAX_RESULTS_PER_PAGE', 25)) {
$limit = env('MAX_RESULTS_PER_PAGE', 25);
}
if ($page <= 0) {
$page = 1;
}
$paginated = $results
->paginate(
$limit,
null,
null,
$page
);
$items = $paginated->items();
foreach ($items as &$item) {
unset($item['_id']);
}
return [
'per_page' => $paginated->perPage(),
'total' => $paginated->total(),
'current_page' => $paginated->currentPage(),
'last_page' => $paginated->lastPage(),
'data' => $items
];
}
/**
* @param string|null $type
* @return string|null
*/
public static function mapType(?string $type = null) : ?string
{
$type = strtolower($type);
return self::MAP_TYPES[$type] ?? null;
}
/**
* @param string|null $status
* @return string|null
*/
public static function mapStatus(?string $status = null) : ?string
{
$status = strtolower($status);
return self::MAP_STATUS[$status] ?? null;
}
/**
* @param string|null $rating
* @return string|null
*/
public static function mapRating(?string $rating = null) : ?string
{
$rating = strtolower($rating);
return self::MAP_RATING[$rating] ?? null;
}
/**
* @param string|null $sort
* @return string|null
*/
public static function mapSort(?string $sort = null) : ?string
{
$sort = strtolower($sort);
return $sort === 'desc' ? 'desc' : 'asc';
}
/**
* @param string|null $orderBy
* @return string|null
*/
public static function mapOrderBy(?string $orderBy) : ?string
{
$orderBy = strtolower($orderBy);
return self::ORDER_BY[$orderBy] ?? null;
}
}

28
app/JikanSearchable.php Normal file
View File

@ -0,0 +1,28 @@
<?php
namespace App;
use Laravel\Scout\Builder;
use Laravel\Scout\Searchable;
trait JikanSearchable
{
use Searchable;
public function queryScoutModelsByIds(Builder $builder, array $ids)
{
$query = static::usesSoftDelete()
? $this->withTrashed() : $this->newQuery();
if ($builder->queryCallback) {
call_user_func($builder->queryCallback, $query);
}
$whereIn = in_array($this->getKeyType(), ['int', 'integer']) ?
'whereIntegerInRaw' :
'whereIn';
return $query->{$whereIn}(
$this->getScoutKeyName(), array_map(function($v) { return (int)$v; }, $ids)
);
}
}

View File

@ -35,9 +35,14 @@ $app->register(Jenssegers\Mongodb\MongodbServiceProvider::class);
$app->withFacades();
$app->instance('path.config', app()->basePath() . DIRECTORY_SEPARATOR . 'config');
$app->instance('path.storage', app()->basePath() . DIRECTORY_SEPARATOR . 'storage');
$app->withEloquent();
$app->configure('swagger-lume');
$app->configure('scout');
/*
|--------------------------------------------------------------------------
@ -142,6 +147,8 @@ $jikan = new \Jikan\MyAnimeList\MalClient(app('HttpClient'));
$app->instance('JikanParser', $jikan);
$app->instance('SerializerV4', SerializerFactory::createV4());
$app->register(Laravel\Scout\ScoutServiceProvider::class);
$app->register(Typesense\LaravelTypesense\TypesenseServiceProvider::class);
/*

View File

@ -8,6 +8,7 @@
"php": "^8.0",
"ext-json": "*",
"ext-mongodb": "*",
"amphp/http-client": "^4.6",
"danielmewes/php-rql": "dev-master",
"darkaonline/swagger-lume": "^9.0",
"fabpot/goutte": "^4.0",
@ -20,9 +21,11 @@
"laravel/lumen-framework": "^9.0",
"league/flysystem": "^3.0",
"ocramius/package-versions": "^2.5",
"php-http/guzzle6-adapter": "^2.0",
"predis/predis": "^1.1",
"sentry/sentry-laravel": "^2.8",
"symfony/yaml": "^4.1",
"typesense/laravel-scout-typesense-driver": "^5.0",
"vlucas/phpdotenv": "^5",
"zircote/swagger-php": "3.*"
},

155
config/scout.php Normal file
View File

@ -0,0 +1,155 @@
<?php
return [
/*
|--------------------------------------------------------------------------
| Default Search Engine
|--------------------------------------------------------------------------
|
| This option controls the default search connection that gets used while
| using Laravel Scout. This connection is used when syncing all models
| to the search service. You should adjust this based on your needs.
|
| Supported: "algolia", "meilisearch", "database", "collection", "null"
|
*/
'driver' => env('SCOUT_DRIVER', 'typesense'),
/*
|--------------------------------------------------------------------------
| Index Prefix
|--------------------------------------------------------------------------
|
| Here you may specify a prefix that will be applied to all search index
| names used by Scout. This prefix may be useful if you have multiple
| "tenants" or applications sharing the same search infrastructure.
|
*/
'prefix' => env('SCOUT_PREFIX', ''),
/*
|--------------------------------------------------------------------------
| Queue Data Syncing
|--------------------------------------------------------------------------
|
| This option allows you to control if the operations that sync your data
| with your search engines are queued. When this is set to "true" then
| all automatic data syncing will get queued for better performance.
|
*/
'queue' => env('SCOUT_QUEUE', false),
/*
|--------------------------------------------------------------------------
| Database Transactions
|--------------------------------------------------------------------------
|
| This configuration option determines if your data will only be synced
| with your search indexes after every open database transaction has
| been committed, thus preventing any discarded data from syncing.
|
*/
'after_commit' => false,
/*
|--------------------------------------------------------------------------
| Chunk Sizes
|--------------------------------------------------------------------------
|
| These options allow you to control the maximum chunk size when you are
| mass importing data into the search engine. This allows you to fine
| tune each of these chunk sizes based on the power of the servers.
|
*/
'chunk' => [
'searchable' => 500,
'unsearchable' => 500,
],
/*
|--------------------------------------------------------------------------
| Soft Deletes
|--------------------------------------------------------------------------
|
| This option allows to control whether to keep soft deleted records in
| the search indexes. Maintaining soft deleted records can be useful
| if your application still needs to search for the records later.
|
*/
'soft_delete' => false,
/*
|--------------------------------------------------------------------------
| Identify User
|--------------------------------------------------------------------------
|
| This option allows you to control whether to notify the search engine
| of the user performing the search. This is sometimes useful if the
| engine supports any analytics based on this application's users.
|
| Supported engines: "algolia"
|
*/
'identify' => env('SCOUT_IDENTIFY', false),
/*
|--------------------------------------------------------------------------
| Algolia Configuration
|--------------------------------------------------------------------------
|
| Here you may configure your Algolia settings. Algolia is a cloud hosted
| search engine which works great with Scout out of the box. Just plug
| in your application ID and admin API key to get started searching.
|
*/
'algolia' => [
'id' => env('ALGOLIA_APP_ID', ''),
'secret' => env('ALGOLIA_SECRET', ''),
],
/*
|--------------------------------------------------------------------------
| TypeSense Configuration
|--------------------------------------------------------------------------
|
| Here you may configure your TypeSense settings. TypeSense is an open
| source search engine. Below, you can state
| the host and key information for your own TypeSense installation.
|
| See: https://github.com/typesense/laravel-scout-typesense-driver
| https://typesense.org/docs/
|
*/
'typesense' => [
'api_key' => env('TYPESENSE_API_KEY','abcd'),
'nodes' => [
[
'host' => env('TYPESENSE_HOST', 'localhost'),
'port' => env('TYPESENSE_PORT', '8108'),
'path' => '',
'protocol' => env('TYPESENSE_PROTOCOL', 'http'),
],
],
'nearest_node' => [
'host' => env('TYPESENSE_HOST', 'localhost'),
'port' => env('TYPESENSE_PORT', '8108'),
'path' => '',
'protocol' => env('TYPESENSE_PROTOCOL', 'http'),
],
'connection_timeout_seconds' => 2,
'healthcheck_interval_seconds' => 30,
'num_retries' => 3,
'retry_interval_seconds' => 1,
],
];