From 4f6d832b7bca90e1a52df778a6fc7e2e4310fb0f Mon Sep 17 00:00:00 2001 From: Irfan Date: Tue, 22 Feb 2022 04:43:56 +0500 Subject: [PATCH] add laravel scout with meilisearch --- app/Anime.php | 80 +++- .../Controllers/V4DB/SearchController.php | 21 +- .../ScoutSearchQueryBuilderAnime.php | 363 ++++++++++++++++++ .../QueryBuilder/SearchQueryBuilderAnime.php | 43 ++- bootstrap/app.php | 9 +- composer.json | 3 + composer.lock | 141 ++++++- config/scout.php | 137 +++++++ 8 files changed, 777 insertions(+), 20 deletions(-) create mode 100644 app/Http/QueryBuilder/ScoutSearchQueryBuilderAnime.php create mode 100644 config/scout.php diff --git a/app/Anime.php b/app/Anime.php index 754fdfc..8c0f307 100644 --- a/app/Anime.php +++ b/app/Anime.php @@ -9,10 +9,13 @@ use Jikan\Helper\Parser; use Jikan\Jikan; use Jikan\Model\Common\YoutubeMeta; use Jikan\Request\Anime\AnimeRequest; +use Laravel\Scout\Searchable; class Anime extends Model { + use Searchable; + /** * The attributes that are mass assignable. * @@ -119,7 +122,7 @@ class Anime extends Model ]; } - public static function scrape(int $id) + public static function scrape(int $id): array { $data = app('JikanParser')->getAnime(new AnimeRequest($id)); @@ -131,4 +134,79 @@ class Anime extends Model ) ); } + + /** + * Get the name of the index associated with the model. + * + * @return string + */ + public function searchableAs(): string + { + return 'anime_index'; + } + + /** + * Get the value used to index the model. + * + * @return mixed + */ + public function getScoutKey(): mixed + { + return $this->mal_id; + } + + /** + * Get the key name used to index the model. + * + * @return mixed + */ + public function getScoutKeyName(): mixed + { + return 'mal_id'; + } + + /** + * Get the indexable data array for the model. + * + * @return array + */ + public function toSearchableArray() + { + return [ + 'mal_id' => $this->mal_id, + 'url' => $this->url, + 'images' => $this->images, + 'trailer' => $this->trailer, + 'title' => $this->title, + 'title_english' => $this->title_english, + 'title_japanese' => $this->title_japanese, + 'title_synonyms' => $this->title_synonyms, + 'type' => $this->type, + 'source' => $this->source, + 'episodes' => $this->episodes, + 'status' => $this->status, + 'airing' => $this->airing, + 'aired' => $this->aired, + 'duration' => $this->duration, + 'rating' => $this->rating, + 'score' => $this->score, + 'scored_by' => $this->scored_by, + 'rank' => $this->rank, + 'popularity' => $this->popularity, + 'members' => $this->members, + 'favorites' => $this->favorites, + 'synopsis' => $this->synopsis, + 'background' => $this->background, + 'season' => $this->season, + 'year' => $this->year, + 'broadcast' => $this->broadcast, + 'producers' => $this->producers, + 'licensors' => $this->licensors, + 'studios' => $this->studios, + 'genres' => $this->genres, + 'explicit_genres' => $this->explicit_genres, + 'themes' => $this->themes, + 'demographics' => $this->demographics, + ]; + } } \ No newline at end of file diff --git a/app/Http/Controllers/V4DB/SearchController.php b/app/Http/Controllers/V4DB/SearchController.php index aedc979..2f93f23 100644 --- a/app/Http/Controllers/V4DB/SearchController.php +++ b/app/Http/Controllers/V4DB/SearchController.php @@ -8,6 +8,7 @@ use App\Club; use App\Http\HttpHelper; use App\Http\HttpResponse; use App\Http\Middleware\Throttle; +use App\Http\QueryBuilder\ScoutSearchQueryBuilderAnime; use App\Http\QueryBuilder\SearchQueryBuilderAnime; use App\Http\QueryBuilder\SearchQueryBuilderCharacter; use App\Http\QueryBuilder\SearchQueryBuilderClub; @@ -196,18 +197,20 @@ class SearchController extends Controller } } - $results = SearchQueryBuilderAnime::query( - $request, - Anime::query() + $results = ScoutSearchQueryBuilderAnime::query( + $request ); $results = $results - ->paginate( - $limit, - ['*'], - null, - $page - ); + ->paginate($page); + +// $results = $results +// ->paginate( +// $limit, +// ['*'], +// null, +// $page +// ); return new AnimeCollection( $results diff --git a/app/Http/QueryBuilder/ScoutSearchQueryBuilderAnime.php b/app/Http/QueryBuilder/ScoutSearchQueryBuilderAnime.php new file mode 100644 index 0000000..065764b --- /dev/null +++ b/app/Http/QueryBuilder/ScoutSearchQueryBuilderAnime.php @@ -0,0 +1,363 @@ + 'TV', + 'movie' => 'Movie', + 'ova' => 'OVA', + 'special' => 'Special', + 'ona' => 'ONA', + 'music' => 'Music' + ]; + + /** + * @OA\Schema( + * schema="anime_search_query_status", + * description="Available Anime statuses", + * type="string", + * enum={"airing","complete","upcoming"} + * ) + */ + const MAP_STATUS = [ + 'airing' => 'Currently Airing', + 'complete' => 'Finished Airing', + 'upcoming' => 'Not yet aired', + ]; + + /** + * @OA\Schema( + * schema="anime_search_query_rating", + * description="Available Anime audience ratings

Ratings
", + * type="string", + * enum={"g","pg","pg13","r17","r","rx"} + * ) + */ + const MAP_RATING = [ + 'g' => 'G - All Ages', + 'pg' => 'PG - Children', + 'pg13' => 'PG-13 - Teens 13 or older', + 'r17' => 'R - 17+ (violence & profanity)', + 'r' => 'R+ - Mild Nudity', + 'rx' => 'Rx - Hentai' + ]; + + /** + * @OA\Schema( + * schema="anime_search_query_orderby", + * description="Available Anime order_by properties", + * type="string", + * enum={"mal_id", "title", "type", "rating", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" } + * ) + */ + const ORDER_BY = [ + 'mal_id' => 'mal_id', + 'title' => 'title', + 'type' => 'type', + 'rating' => 'rating', + 'start_date' => 'aired.from', + 'end_date' => 'aired.to', + 'episodes' => 'episodes', + 'score' => 'score', + 'scored_by' => 'scored_by', + 'rank' => 'rank', + 'popularity' => 'popularity', + 'members' => 'members', + 'favorites' => 'favorites' + ]; + + /** + * @param Request $request + * @param Builder $results + * @return Builder + */ + public static function query(Request $request) : \Laravel\Scout\Builder + { + $requestType = HttpHelper::requestType($request); + $query = $request->get('q'); + $type = self::mapType($request->get('type')); + $score = $request->get('score') ?? 0; + $status = self::mapStatus($request->get('status')); + $rating = self::mapRating($request->get('rating')); + $sfw = $request->get('sfw'); + $genres = $request->get('genres'); + $genresExclude = $request->get('genres_exclude'); + $orderBy = self::mapOrderBy($request->get('order_by')); + $sort = self::mapSort($request->get('sort')); + $letter = $request->get('letter'); + $producer = $request->get('producers'); + $minScore = $request->get('min_score'); + $maxScore = $request->get('max_score'); + $startDate = $request->get('start_date'); + $endDate = $request->get('end_date'); + + if (!empty($query) && is_null($letter)) { + $results = Anime::search($query); + } + + if (!is_null($letter)) { + $results = $results + ->where('title', 'like', "{$letter}%"); + } + + if (empty($query) && is_null($orderBy)) { + $results = $results + ->orderBy('mal_id'); + } + + if (!is_null($startDate)) { + + $startDate = explode('-', $startDate); + + $startDate = (new \DateTime()) + ->setDate( + $startDate[0] ?? date('Y'), + $startDate[1] ?? 1, + $startDate[2] ?? 1 + ) + ->format(\DateTimeInterface::ISO8601); + + $results = $results + ->where('aired.from', '>=', $startDate); + } + + if (!is_null($endDate)) { + + $endDate = explode('-', $endDate); + + $endDate = (new \DateTime()) + ->setDate( + $endDate[0] ?? date('Y'), + $endDate[1] ?? 1, + $endDate[2] ?? 1 + ) + ->format(\DateTimeInterface::ISO8601); + + $results = $results + ->where('aired.to', '<=', $endDate); + } + + if (!is_null($type)) { + $results = $results + ->where('type', $type); + } + + if ($score !== 0) { + $score = (float) $score; + + $results = $results + ->where('score', '>=', $score); + } + + if ($minScore !== null) { + $minScore = (float) $minScore; + + $results = $results + ->where('score', '>=', $minScore); + } + + if ($maxScore !== null) { + $maxScore = (float) $maxScore; + + $results = $results + ->where('score', '<=', $maxScore); + } + + if (!is_null($status)) { + $results = $results + ->where('status', $status); + } + + if (!is_null($rating)) { + $results = $results + ->where('rating', $rating); + } + + if (!is_null($producer)) { + + $producer = (int) $producer; + + $results = $results + ->where('producers.mal_id', $producer) + ->orWhere('licensors.mal_id', $producer) + ->orWhere('studios.mal_id', $producer); + } + + if (!is_null($genres)) { + $genres = explode(',', $genres); + + foreach ($genres as $genre) { + if (empty($genre)) { + continue; + } + + $genre = (int) $genre; + + $results = $results + ->where(function($query) use ($genre) { + $query + ->where('genres.mal_id', $genre) + ->orWhere('demographics.mal_id', $genre) + ->orWhere('themes.mal_id', $genre) + ->orWhere('explicit_genres.mal_id', $genre); + }); + } + } + + if (!is_null($genresExclude)) { + $genresExclude = explode(',', $genresExclude); + + foreach ($genresExclude as $genreExclude) { + if (empty($genreExclude)) { + continue; + } + + $genreExclude = (int) $genreExclude; + + $results = $results + ->where(function($query) use ($genreExclude) { + $query + ->where('genres.mal_id', '!=', $genreExclude) + ->where('demographics.mal_id', '!=', $genreExclude) + ->where('themes.mal_id', '!=', $genreExclude) + ->where('explicit_genres.mal_id', '!=', $genreExclude); + }); + ; + } + } + + if (!is_null($sfw)) { + $results = $results + ->where('rating', '!=', self::MAP_RATING['rx']); + } + + if (!is_null($orderBy)) { + $results = $results + ->orderBy($orderBy, $sort ?? 'asc'); + } + + return $results; + } + + /** + * @param Request $request + * @param Builder $results + * @return array + */ + public static function paginate(Request $request, Builder $results) + { + $page = $request->get('page') ?? 1; + $limit = $request->get('limit') ?? env('MAX_RESULTS_PER_PAGE', 25); + + $limit = (int) $limit; + + if ($limit <= 0) { + $limit = 1; + } + + if ($limit > env('MAX_RESULTS_PER_PAGE', 25)) { + $limit = env('MAX_RESULTS_PER_PAGE', 25); + } + + if ($page <= 0) { + $page = 1; + } + + $paginated = $results + ->paginate( + $limit, + null, + null, + $page + ); + + $items = $paginated->items(); + foreach ($items as &$item) { + unset($item['_id']); + } + + return [ + 'per_page' => $paginated->perPage(), + 'total' => $paginated->total(), + 'current_page' => $paginated->currentPage(), + 'last_page' => $paginated->lastPage(), + 'data' => $items + ]; + } + + /** + * @param string|null $type + * @return string|null + */ + public static function mapType(?string $type = null) : ?string + { + $type = strtolower($type); + + return self::MAP_TYPES[$type] ?? null; + } + + /** + * @param string|null $status + * @return string|null + */ + public static function mapStatus(?string $status = null) : ?string + { + $status = strtolower($status); + + return self::MAP_STATUS[$status] ?? null; + } + + /** + * @param string|null $rating + * @return string|null + */ + public static function mapRating(?string $rating = null) : ?string + { + $rating = strtolower($rating); + + return self::MAP_RATING[$rating] ?? null; + } + + /** + * @param string|null $sort + * @return string|null + */ + public static function mapSort(?string $sort = null) : ?string + { + $sort = strtolower($sort); + + return $sort === 'desc' ? 'desc' : 'asc'; + } + + /** + * @param string|null $orderBy + * @return string|null + */ + public static function mapOrderBy(?string $orderBy) : ?string + { + $orderBy = strtolower($orderBy); + + return self::ORDER_BY[$orderBy] ?? null; + } +} \ No newline at end of file diff --git a/app/Http/QueryBuilder/SearchQueryBuilderAnime.php b/app/Http/QueryBuilder/SearchQueryBuilderAnime.php index e8d8c07..0a656bc 100644 --- a/app/Http/QueryBuilder/SearchQueryBuilderAnime.php +++ b/app/Http/QueryBuilder/SearchQueryBuilderAnime.php @@ -113,19 +113,46 @@ class SearchQueryBuilderAnime implements SearchQueryBuilderInterface if (!empty($query) && is_null($letter)) { - $results = $results - ->where('title', 'like', "%{$query}%") - ->orWhere('title_english', 'like', "%{$query}%") - ->orWhere('title_japanese', 'like', "%{$query}%") - ->orWhere('title_synonyms', 'like', "%{$query}%"); +// $results = $results +// ->where('title', 'like', "%{$query}%") +// ->orWhere('title_english', 'like', "%{$query}%") +// ->orWhere('title_japanese', 'like', "%{$query}%") +// ->orWhere('title_synonyms', 'like', "%{$query}%"); // needs elastic search // $results = $results -// ->whereRaw([ -// '$text' => [ -// '$search' => $query +// ->aggregate([ +// [ +// '$match' => [ +// '$text' => [ '$search' => $query ] +// ] +// ], +// [ +// '$sort' => [ +// 'score' => [ +// '$meta' => 'textScore' +// ] +// ] // ] // ]); + + + $results = $results + ->whereRaw([ + '$text' => [ + '$search' => $query + ], + ],[ + 'score' => [ + '$meta' => 'textScore' + ] + ]) + ->orderBy('score', ['$meta' => 'textScore']); +// ->orWhere('title', 'like', "%{$query}%") +// ->orWhere('title_english', 'like', "%{$query}%") +// ->orWhere('title_japanese', 'like', "%{$query}%") +// ->orWhere('title_synonyms', 'like', "%{$query}%"); + ; } if (!is_null($letter)) { diff --git a/bootstrap/app.php b/bootstrap/app.php index 7d5d8fc..683fb53 100755 --- a/bootstrap/app.php +++ b/bootstrap/app.php @@ -33,11 +33,15 @@ $app = new Laravel\Lumen\Application( $app->register(Jenssegers\Mongodb\MongodbServiceProvider::class); - $app->withFacades(); + +$app->instance('path.config', app()->basePath() . DIRECTORY_SEPARATOR . 'config'); +$app->instance('path.storage', app()->basePath() . DIRECTORY_SEPARATOR . 'storage'); + $app->withEloquent(); $app->configure('swagger-lume'); +$app->configure('scout'); /* |-------------------------------------------------------------------------- @@ -138,6 +142,9 @@ $app->instance('JikanParser', $jikan); $app->instance('SerializerV4', SerializerFactory::createV4()); +$app->register(Laravel\Scout\ScoutServiceProvider::class); + + /* |-------------------------------------------------------------------------- diff --git a/composer.json b/composer.json index 6545f27..8772fb5 100755 --- a/composer.json +++ b/composer.json @@ -13,13 +13,16 @@ "divineomega/cachetphp": "^0.2.0", "fabpot/goutte": "^4.0", "flipbox/lumen-generator": "^8", + "http-interop/http-factory-guzzle": "^1.2", "illuminate/redis": "^8", "jenssegers/mongodb": "^3.8", "jikan-me/jikan": "3.0.0.x-dev", "jms/serializer": "^3.0", "laravel/legacy-factories": "^1.1", "laravel/lumen-framework": "^8.0", + "laravel/scout": "^9.4", "league/flysystem": "^1.0", + "meilisearch/meilisearch-php": "^0.22.0", "ocramius/package-versions": "^2.5", "predis/predis": "^1.1", "sentry/sentry-laravel": "^2.8", diff --git a/composer.lock b/composer.lock index 946de40..8f10f87 100755 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "9d047984b287a81d988936ead7f47eb5", + "content-hash": "e8b30b1100a74d9128d54944da8f559f", "packages": [ { "name": "brick/math", @@ -3299,6 +3299,78 @@ }, "time": "2021-12-22T10:11:35+00:00" }, + { + "name": "laravel/scout", + "version": "v9.4.4", + "source": { + "type": "git", + "url": "https://github.com/laravel/scout.git", + "reference": "06c6da8eb76b98229d8e6bee13ca23904956667e" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/laravel/scout/zipball/06c6da8eb76b98229d8e6bee13ca23904956667e", + "reference": "06c6da8eb76b98229d8e6bee13ca23904956667e", + "shasum": "" + }, + "require": { + "illuminate/bus": "^8.0|^9.0", + "illuminate/contracts": "^8.0|^9.0", + "illuminate/database": "^8.0|^9.0", + "illuminate/http": "^8.0|^9.0", + "illuminate/pagination": "^8.0|^9.0", + "illuminate/queue": "^8.0|^9.0", + "illuminate/support": "^8.0|^9.0", + "php": "^7.3|^8.0" + }, + "require-dev": { + "meilisearch/meilisearch-php": "^0.19", + "mockery/mockery": "^1.0", + "orchestra/testbench": "^6.17|^7.0", + "phpunit/phpunit": "^9.3" + }, + "suggest": { + "algolia/algoliasearch-client-php": "Required to use the Algolia engine (^2.2).", + "meilisearch/meilisearch-php": "Required to use the MeiliSearch engine (^0.17)." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "9.x-dev" + }, + "laravel": { + "providers": [ + "Laravel\\Scout\\ScoutServiceProvider" + ] + } + }, + "autoload": { + "psr-4": { + "Laravel\\Scout\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Taylor Otwell", + "email": "taylor@laravel.com" + } + ], + "description": "Laravel Scout provides a driver based solution to searching your Eloquent models.", + "keywords": [ + "algolia", + "laravel", + "search" + ], + "support": { + "issues": "https://github.com/laravel/scout/issues", + "source": "https://github.com/laravel/scout" + }, + "time": "2022-02-15T18:13:05+00:00" + }, { "name": "laravel/serializable-closure", "version": "v1.0.5", @@ -3508,6 +3580,73 @@ ], "time": "2021-11-21T11:48:40+00:00" }, + { + "name": "meilisearch/meilisearch-php", + "version": "v0.22.0", + "source": { + "type": "git", + "url": "https://github.com/meilisearch/meilisearch-php.git", + "reference": "0229ce11be0ac2ede91577bbcd6bdfe17af63e05" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/meilisearch/meilisearch-php/zipball/0229ce11be0ac2ede91577bbcd6bdfe17af63e05", + "reference": "0229ce11be0ac2ede91577bbcd6bdfe17af63e05", + "shasum": "" + }, + "require": { + "ext-json": "*", + "php": "^7.4 || ^8.0", + "php-http/client-common": "^2.0", + "php-http/discovery": "^1.7", + "php-http/httplug": "^2.1" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "guzzlehttp/guzzle": "^7.1", + "http-interop/http-factory-guzzle": "^1.0", + "phpstan/extension-installer": "^1.1", + "phpstan/phpstan": "^1.4", + "phpstan/phpstan-deprecation-rules": "^1.0", + "phpstan/phpstan-phpunit": "^1.0", + "phpstan/phpstan-strict-rules": "^1.1", + "phpunit/phpunit": "^9.5" + }, + "suggest": { + "guzzlehttp/guzzle": "Use Guzzle ^7 as HTTP client", + "http-interop/http-factory-guzzle": "Factory for guzzlehttp/guzzle" + }, + "type": "library", + "autoload": { + "psr-4": { + "MeiliSearch\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Clementine Urquizar", + "email": "clementine@meilisearch.com" + } + ], + "description": "PHP wrapper for the Meilisearch API", + "keywords": [ + "api", + "client", + "instant", + "meilisearch", + "php", + "search" + ], + "support": { + "issues": "https://github.com/meilisearch/meilisearch-php/issues", + "source": "https://github.com/meilisearch/meilisearch-php/tree/v0.22.0" + }, + "time": "2022-02-14T16:00:33+00:00" + }, { "name": "mongodb/mongodb", "version": "1.10.1", diff --git a/config/scout.php b/config/scout.php new file mode 100644 index 0000000..8d419d4 --- /dev/null +++ b/config/scout.php @@ -0,0 +1,137 @@ + env('SCOUT_DRIVER', 'meilisearch'), + + /* + |-------------------------------------------------------------------------- + | Index Prefix + |-------------------------------------------------------------------------- + | + | Here you may specify a prefix that will be applied to all search index + | names used by Scout. This prefix may be useful if you have multiple + | "tenants" or applications sharing the same search infrastructure. + | + */ + + 'prefix' => env('SCOUT_PREFIX', ''), + + /* + |-------------------------------------------------------------------------- + | Queue Data Syncing + |-------------------------------------------------------------------------- + | + | This option allows you to control if the operations that sync your data + | with your search engines are queued. When this is set to "true" then + | all automatic data syncing will get queued for better performance. + | + */ + + 'queue' => env('SCOUT_QUEUE', false), + + /* + |-------------------------------------------------------------------------- + | Database Transactions + |-------------------------------------------------------------------------- + | + | This configuration option determines if your data will only be synced + | with your search indexes after every open database transaction has + | been committed, thus preventing any discarded data from syncing. + | + */ + + 'after_commit' => false, + + /* + |-------------------------------------------------------------------------- + | Chunk Sizes + |-------------------------------------------------------------------------- + | + | These options allow you to control the maximum chunk size when you are + | mass importing data into the search engine. This allows you to fine + | tune each of these chunk sizes based on the power of the servers. + | + */ + + 'chunk' => [ + 'searchable' => 500, + 'unsearchable' => 500, + ], + + /* + |-------------------------------------------------------------------------- + | Soft Deletes + |-------------------------------------------------------------------------- + | + | This option allows to control whether to keep soft deleted records in + | the search indexes. Maintaining soft deleted records can be useful + | if your application still needs to search for the records later. + | + */ + + 'soft_delete' => false, + + /* + |-------------------------------------------------------------------------- + | Identify User + |-------------------------------------------------------------------------- + | + | This option allows you to control whether to notify the search engine + | of the user performing the search. This is sometimes useful if the + | engine supports any analytics based on this application's users. + | + | Supported engines: "algolia" + | + */ + + 'identify' => env('SCOUT_IDENTIFY', false), + + /* + |-------------------------------------------------------------------------- + | Algolia Configuration + |-------------------------------------------------------------------------- + | + | Here you may configure your Algolia settings. Algolia is a cloud hosted + | search engine which works great with Scout out of the box. Just plug + | in your application ID and admin API key to get started searching. + | + */ + + 'algolia' => [ + 'id' => env('ALGOLIA_APP_ID', ''), + 'secret' => env('ALGOLIA_SECRET', ''), + ], + + /* + |-------------------------------------------------------------------------- + | MeiliSearch Configuration + |-------------------------------------------------------------------------- + | + | Here you may configure your MeiliSearch settings. MeiliSearch is an open + | source search engine with minimal configuration. Below, you can state + | the host and key information for your own MeiliSearch installation. + | + | See: https://docs.meilisearch.com/guides/advanced_guides/configuration.html + | + */ + + 'meilisearch' => [ + 'host' => env('MEILISEARCH_HOST', 'http://localhost:7700'), + 'key' => env('MEILISEARCH_KEY', null), + ], + +];