(wip) added a new system for resolving query string parameters to ORM commands.

This commit is contained in:
pushrbx 2022-06-08 17:05:12 +01:00
parent b0e3b05a7c
commit faa0031409
30 changed files with 884 additions and 313 deletions

View File

@ -3,20 +3,14 @@
namespace App;
use App\Http\HttpHelper;
use Jenssegers\Mongodb\Eloquent\Model;
use Jikan\Helper\Media;
use Jikan\Helper\Parser;
use Jikan\Jikan;
use Jikan\Model\Common\YoutubeMeta;
use Jikan\Request\Anime\AnimeRequest;
use Laravel\Scout\Builder;
use Typesense\LaravelTypesense\Interfaces\TypesenseDocument;
use Laravel\Scout\Searchable;
class Anime extends Model implements TypesenseDocument
class Anime extends JikanApiSearchableModel
{
use JikanSearchable;
// note that here we skip "score", "min_score", "max_score", "rating" and others because they need special logic
// to set the correct filtering on the ORM.
protected array $filters = ["order_by", "status", "type"];
/**
* The attributes that are mass assignable.
*
@ -136,16 +130,6 @@ class Anime extends Model implements TypesenseDocument
);
}
/**
* Get the name of the index associated with the model.
*
* @return string
*/
public function searchableAs(): string
{
return 'anime_index';
}
/**
* Get the value used to index the model.
*
@ -167,19 +151,17 @@ class Anime extends Model implements TypesenseDocument
}
/**
* Get the indexable data array for the model.
* Converts the model to an index-able data array.
*
* @return array
*/
public function toSearchableArray(): array
{
$serializer = app('SerializerV4');
$result = [
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'start_date' => $this->aired['from'] ? Parser::parseDate($this->aired['from'])->getTimestamp() : 0,
'end_date' => $this->aired['to'] ? Parser::parseDate($this->aired['to'])->getTimestamp() : 0,
'images' => $this->images,
'start_date' => $this->convertToTimestamp($this->aired['from']),
'end_date' => $this->convertToTimestamp($this->aired['to']),
'title' => $this->title,
'title_english' => $this->title_english,
'title_japanese' => $this->title_japanese,
@ -196,19 +178,14 @@ class Anime extends Model implements TypesenseDocument
'members' => $this->members,
'favorites' => $this->favorites,
'synopsis' => $this->synopsis,
'background' => $this->background,
'season' => $this->season,
'year' => $this->year,
'producers' => $this->getMalIdsOfField($this->producers),
'studios' => $this->getMalIdsOfField($this->studios),
'licensors' => $this->getMalIdsOfField($this->licensors),
'genres' => $this->getMalIdsOfField($this->genres),
'explicit_genres' => $this->getMalIdsOfField($this->explicit_genres)
];
// todo: test this with artisan::tinker
return array_merge($result,
// this will add nested fields in the format of "producers.0.mal_id=123" and etc
$this->toTypeSenseCompatibleNestedField("producers"),
$this->toTypeSenseCompatibleNestedField("studios"),
$this->toTypeSenseCompatibleNestedField("genres"),
$this->toTypeSenseCompatibleNestedField("explicit_genres")
);
}
/**
@ -225,22 +202,4 @@ class Anime extends Model implements TypesenseDocument
'title_synonyms'
];
}
/**
* The Typesense schema to be created.
*
* @return array
*/
public function getCollectionSchema(): array
{
return [
'name' => $this->searchableAs(),
'fields' => [
[
'name' => '.*',
'type' => 'auto',
]
]
];
}
}
}

View File

@ -2,17 +2,12 @@
namespace App;
use App\Http\HttpHelper;
use Jenssegers\Mongodb\Eloquent\Model;
use Jikan\Helper\Media;
use Jikan\Helper\Parser;
use Jikan\Jikan;
use Jikan\Model\Common\YoutubeMeta;
use Jikan\Request\Anime\AnimeRequest;
use Jikan\Request\Character\CharacterRequest;
class Character extends Model
class Character extends JikanApiSearchableModel
{
protected array $filters = ["order_by"];
/**
* The attributes that are mass assignable.
@ -60,4 +55,23 @@ class Character extends Model
true
);
}
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'name' => $this->name,
'name_kanji' => $this->name_kanji,
'member_favorites' => $this->member_favorites
];
}
public function typesenseQueryBy(): array
{
return [
'name',
'name_kanji'
];
}
}

View File

@ -2,13 +2,11 @@
namespace App;
use App\Http\HttpHelper;
use Jenssegers\Mongodb\Eloquent\Model;
use Jikan\Request\Anime\AnimeRequest;
use Jikan\Request\Club\ClubRequest;
class Club extends Model
class Club extends JikanApiSearchableModel
{
protected array $filters = ["order_by"];
/**
* The attributes that are mass assignable.
@ -52,4 +50,21 @@ class Club extends Model
true
);
}
}
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'title' => $this->title,
'category' => $this->category,
'created' => $this->convertToTimestamp($this->created),
'type' => $this->type
];
}
public function typesenseQueryBy(): array
{
return ['title'];
}
}

View File

@ -0,0 +1,31 @@
<?php
namespace App\Filters;
use Illuminate\Database\Eloquent\Builder;
abstract class BaseClause
{
protected $query;
protected $filter;
protected $values;
public function __construct($values, $filter)
{
$this->values = $values;
$this->filter = $filter;
}
public function handle($query, $nextFilter)
{
$query = $nextFilter($query);
if(static::validate($this->values) === false) {
return $query;
}
return static::apply($query);
}
abstract protected function apply($query): Builder;
abstract protected function validate($value): bool;
}

View File

@ -0,0 +1,56 @@
<?php
namespace App\Filters;
use Illuminate\Database\Eloquent\Builder;
use Illuminate\Pipeline\Pipeline;
trait FilterQueryString
{
use FilterResolver;
private array $availableFilters = [
'default' => WhereClause::class,
'order_by' => OrderbyClause::class
];
/** @noinspection PhpUnused */
public function scopeFilter(Builder $query, array $queryParameters, ...$filters)
{
$filters = collect($this->getFilters($queryParameters, $filters))->map(function ($values, $filter) {
return $this->resolve($filter, $values);
})->toArray();
return app(Pipeline::class)
->send($query)
->through($filters)
->thenReturn();
}
private function _normalizeOrderBy(array $filters): array
{
// fixme: this can be done more elegantly, for now this is here as a quick hack.
if (array_key_exists("sort", $filters) && array_key_exists("order_by", $filters)) {
// we put the order by field and the sort direction in one array element.
// the OrderByClause class will explode the string by the comma and set the correct field.
$filters["order_by"] = $filters["order_by"] . "," . $filters["sort"];
unset($filters["sort"]);
}
return $filters;
}
private function getFilters(array $queryParameters, array $filters): array
{
$filter = function ($key) use($filters) {
$filters = $filters ?: $this->filters ?: [];
// if model class sets the "unguardFilters" variable to true, then we skip the filter validation
return !($this->unguardFilters != true) || in_array($key, $filters);
};
$result = array_filter($queryParameters, $filter, ARRAY_FILTER_USE_KEY) ?? [];
return $this->_normalizeOrderBy($result);
}
}

View File

@ -0,0 +1,38 @@
<?php
namespace App\Filters;
trait FilterResolver
{
private function resolve($filterName, $values)
{
if($this->isCustomFilter($filterName)) {
return $this->resolveCustomFilter($filterName, $values);
}
$availableFilter = $this->availableFilters[$filterName] ?? $this->availableFilters['default'];
return app($availableFilter, ['filter' => $filterName, 'values' => $values]);
}
private function resolveCustomFilter($filterName, $values)
{
return $this->getClosure($this->makeCallable($filterName), $values);
}
private function makeCallable($filter)
{
return static::class.'@'.$filter;
}
private function isCustomFilter($filterName)
{
return method_exists($this, $filterName);
}
private function getClosure($callable, $values)
{
return function ($query, $nextFilter) use ($callable, $values) {
return app()->call($callable, ['query' => $nextFilter($query), 'value' => $values]);
};
}
}

View File

@ -0,0 +1,41 @@
<?php
namespace App\Filters;
use Illuminate\Database\Eloquent\Builder;
class OrderbyClause extends BaseClause
{
protected function apply($query): Builder
{
foreach ($this->normalizeValues() as $field => $order) {
$query->orderBy($field, $order);
}
return $query;
}
protected function validate($value): bool
{
return !in_array(null, (array)$value);
}
private function normalizeValues(): array
{
$normalized = [];
foreach ((array)$this->values as $value) {
$exploded = explode(',', $value);
if (!empty($exploded[1]) and in_array($exploded[1], ['asc', 'desc'])) {
$normalized[$exploded[0]] = $exploded[1];
continue;
}
$normalized[$exploded[0]] = 'asc';
}
return $normalized;
}
}

View File

@ -0,0 +1,39 @@
<?php
namespace App\Filters;
use Illuminate\Database\Eloquent\Builder;
class WhereClause extends BaseClause
{
protected function apply($query): Builder
{
$method = is_array($this->values) ? 'orWhere' : 'andWhere';
return $this->{$method}($query, $this->filter, $this->values);
}
protected function validate($value): bool
{
return !is_null($value);
}
private function orWhere($query, $filter, $values): Builder
{
$query->where(function($query) use($values, $filter) {
foreach((array)$values as $value) {
$query->orWhere($filter, $value);
}
});
return $query;
}
private function andWhere($query, $filter, $values): Builder
{
foreach((array)$values as $value) {
$query->where($filter, $value);
}
return $query;
}
}

View File

@ -9,7 +9,7 @@ use Jikan\Request\Genre\AnimeGenresRequest;
* Class Magazine
* @package App
*/
class GenreAnime extends Model
class GenreAnime extends JikanApiSearchableModel
{
/**
@ -51,4 +51,21 @@ class GenreAnime extends Model
true
);
}
}
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'name' => $this->name,
'count' => $this->count
];
}
public function typesenseQueryBy(): array
{
return [
'name'
];
}
}

View File

@ -9,7 +9,7 @@ use Jikan\Request\Genre\AnimeGenresRequest;
* Class Magazine
* @package App
*/
class GenreManga extends Model
class GenreManga extends JikanApiSearchableModel
{
/**
@ -50,4 +50,21 @@ class GenreManga extends Model
true
);
}
}
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'name' => $this->name,
'count' => $this->count
];
}
public function typesenseQueryBy(): array
{
return [
'name'
];
}
}

View File

@ -4,6 +4,7 @@ namespace App\Http\Controllers\V4DB;
use App\Character;
use App\Club;
use App\Http\Controllers\V4DB\Traits\JikanApiQueryBuilder;
use App\Http\QueryBuilder\SearchQueryBuilderCharacter;
use App\Http\QueryBuilder\SearchQueryBuilderClub;
use App\Http\QueryBuilder\SearchQueryBuilderManga;
@ -27,6 +28,7 @@ use Jikan\Request\User\UsernameByIdRequest;
class SearchController extends Controller
{
use JikanApiQueryBuilder;
private $request;
const MAX_RESULTS_PER_PAGE = 25;
private SearchQueryBuilderProvider $searchQueryBuilderProvider;
@ -37,18 +39,6 @@ class SearchController extends Controller
$this->searchQueryBuilderProvider = $searchQueryBuilderProvider;
}
private function getQueryBuilder(string $name, Request $request): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder
{
$queryBuilder = $this->searchQueryBuilderProvider->getQueryBuilder($name);
return $queryBuilder->query($request);
}
private function getPaginator(string $name, Request $request, \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $results): \Illuminate\Contracts\Pagination\LengthAwarePaginator
{
$queryBuilder = $this->searchQueryBuilderProvider->getQueryBuilder($name);
return $queryBuilder->paginateBuilder($request, $results);
}
/**
* @OA\Parameter(
* name="page",
@ -198,28 +188,7 @@ class SearchController extends Controller
*/
public function anime(Request $request)
{
$this->request = $request;
$page = $this->request->get('page') ?? 1;
$limit = $this->request->get('limit') ?? self::MAX_RESULTS_PER_PAGE;
if (!empty($limit)) {
$limit = (int) $limit;
if ($limit <= 0) {
$limit = 1;
}
if ($limit > self::MAX_RESULTS_PER_PAGE) {
$limit = self::MAX_RESULTS_PER_PAGE;
}
}
$results = $this->getQueryBuilder("anime", $request);
$paginator = $this->getPaginator("anime", $request, $results);
return new AnimeCollection(
$paginator
);
return $this->preparePaginatedResponse(AnimeCollection::class, "anime", $request);
}
/**
@ -345,28 +314,7 @@ class SearchController extends Controller
*/
public function manga(Request $request)
{
$this->request = $request;
$page = $this->request->get('page') ?? 1;
$limit = $this->request->get('limit') ?? self::MAX_RESULTS_PER_PAGE;
if (!empty($limit)) {
$limit = (int) $limit;
if ($limit <= 0) {
$limit = 1;
}
if ($limit > self::MAX_RESULTS_PER_PAGE) {
$limit = self::MAX_RESULTS_PER_PAGE;
}
}
$results = $this->getQueryBuilder("manga", $request);
$paginator = $this->getPaginator("manga", $request, $results);
return new MangaCollection(
$paginator
);
return $this->preparePaginatedResponse(MangaCollection::class, "manga", $request);
}
/**
@ -772,37 +720,6 @@ class SearchController extends Controller
*/
public function clubs(Request $request)
{
$this->request = $request;
$page = $this->request->get('page') ?? 1;
$limit = $this->request->get('limit') ?? self::MAX_RESULTS_PER_PAGE;
if (!empty($limit)) {
$limit = (int) $limit;
if ($limit <= 0) {
$limit = 1;
}
if ($limit > self::MAX_RESULTS_PER_PAGE) {
$limit = self::MAX_RESULTS_PER_PAGE;
}
}
$results = SearchQueryBuilderClub::query(
$request,
Club::query()
);
$results = $results
->paginate(
$limit,
['*'],
null,
$page
);
return new ClubCollection(
$results
);
return $this->preparePaginatedResponse(ClubCollection::class, "club", $request);
}
}

View File

@ -0,0 +1,36 @@
<?php
namespace App\Http\Controllers\V4DB\Traits;
use App\Providers\SearchQueryBuilderProvider;
use Illuminate\Http\Request;
trait JikanApiQueryBuilder
{
private SearchQueryBuilderProvider $searchQueryBuilderProvider;
private function getQueryBuilder(string $name, Request $request): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder
{
$queryBuilder = $this->searchQueryBuilderProvider->getQueryBuilder($name);
return $queryBuilder->query($request);
}
private function getPaginator(string $name, Request $request, \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $results): \Illuminate\Contracts\Pagination\LengthAwarePaginator
{
$queryBuilder = $this->searchQueryBuilderProvider->getQueryBuilder($name);
return $queryBuilder->paginateBuilder($request, $results);
}
/**
* @template T
* @param T $resourceCollectionClass
* @param \Illuminate\Http\Request $request
* @return T
*/
private function preparePaginatedResponse(string|object $resourceCollectionClass, string $resourceTypeName, Request $request)
{
$results = $this->getQueryBuilder($resourceTypeName, $request);
$paginator = $this->getPaginator($resourceTypeName, $request, $results);
return new $resourceCollectionClass($paginator);
}
}

View File

@ -73,33 +73,46 @@ class AnimeSearchQueryBuilder extends MediaSearchQueryBuilder
'type' => 'type',
];
protected function buildQuery(array $requestParameters, \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
protected function buildQuery(array $requestParameters, \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$builder = parent::buildQuery($requestParameters, $results);
extract($requestParameters);
if (!is_null($rating)) {
$builder = $builder->where('rating', $rating);
$builder = $builder->where('rating', $this->mapRating($rating));
}
if (!is_null($producer)) {
$producer = (int)$producer;
if (!is_null($producer)) $producers = $producer;
$builder = $builder
->where('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer);
if (!is_null($producers)) {
$producers = explode(',', $producers);
foreach ($producers as $producer) {
if (empty($producer)) {
continue;
}
$producer = (int)$producer;
// todo: this might be wrong, because the filtering like this should only happen in mongodb maybe?
// https://laravel.com/docs/9.x/scout#customizing-the-eloquent-results-query
$results = $results
->query(fn($query) => $query
->orWhere('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer));
}
}
return $builder;
}
protected function filterByStartDate(\Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $startDate): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder
protected function filterByStartDate(\Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $startDate): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder
{
return $builder->where('aired.from', '>=', $startDate);
}
protected function filterByEndDate(\Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $endDate): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder
protected function filterByEndDate(\Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $endDate): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder
{
return $builder->where('aired.to', '<=', $endDate);
}
@ -134,4 +147,11 @@ class AnimeSearchQueryBuilder extends MediaSearchQueryBuilder
{
return "anime";
}
protected function mapRating(?string $rating = null): ?string
{
$rating = strtolower($rating);
return self::MAP_RATING[$rating] ?? null;
}
}

View File

@ -0,0 +1,111 @@
<?php
namespace App\Http\QueryBuilder;
use App\Club;
use App\Http\QueryBuilder\Traits\TypeResolver;
class ClubSearchQueryBuilder extends SearchQueryBuilder
{
use TypeResolver;
protected string $displayNameFieldName = "title";
protected array $parameterNames = ["category", "type"];
/**
* @OA\Schema(
* schema="club_search_query_type",
* description="Club Search Query Type",
* type="string",
* enum={"public","private","secret"}
* )
*/
const MAP_TYPES = [
'public' => 'public',
'private' => 'private',
'secret' => 'secret'
];
/**
* @OA\Schema(
* schema="club_search_query_category",
* description="Club Search Query Category",
* type="string",
* enum={
* "anime","manga","actors_and_artists","characters",
* "cities_and_neighborhoods","companies","conventions","games",
* "japan","music","other","schools"
* }
* )
*/
const MAP_CATEGORY = [
'anime' => 'Anime',
'manga' => 'Manga',
'actors_and_artists' => 'Actors & Artists',
'characters' => 'Characters',
'cities_and_neighborhoods' => 'Cities & Neighborhoods',
'companies' => 'Companies',
'conventions' => 'Conventions',
'games' => 'Games',
'japan' => 'Japan',
'music' => 'Music',
'other' => 'Other',
'schools' => 'Schools'
];
/**
* @OA\Schema(
* schema="club_search_query_orderby",
* description="Club Search Query OrderBy",
* type="string",
* enum={"mal_id","title","members_count","pictures_count","created"}
* )
*/
const ORDER_BY = [
'mal_id', 'title', 'members_count', 'pictures_count', 'created'
];
protected function getModelClass(): object|string
{
return Club::class;
}
protected function sanitizeParameters($parameters): array
{
$parameters = parent::sanitizeParameters($parameters);
$parameters["category"] = $this->mapCategory($parameters["category"]);
$parameters["type"] = $this->mapType($parameters["type"]);
return $parameters;
}
protected function buildQuery(array $requestParameters, \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $results;
}
protected function getOrderByFieldMap(): array
{
return self::ORDER_BY;
}
function getIdentifier(): string
{
return "club";
}
protected function getTypeMap(): array
{
return self::MAP_TYPES;
}
/**
* @param string|null $category
* @return string|null
*/
private function mapCategory(?string $category = null) : ?string
{
$category = strtolower($category);
return self::MAP_CATEGORY[$category] ?? null;
}
}

View File

@ -58,10 +58,11 @@ class MangaSearchQueryBuilder extends MediaSearchQueryBuilder
'end_date' => 'published.to',
];
protected function buildQuery(array $requestParameters, \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
protected function buildQuery(array $requestParameters, \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$builder = parent::buildQuery($requestParameters, $results);
extract($requestParameters);
$magazine = $requestParameters['magazine'];
$magazines = $requestParameters['magazines'];
if (!is_null($magazine)) $magazines = $magazine;
@ -83,12 +84,12 @@ class MangaSearchQueryBuilder extends MediaSearchQueryBuilder
return $builder;
}
protected function filterByStartDate(\Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $startDate): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder
protected function filterByStartDate(\Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $startDate): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder
{
return $builder->where('published.from', '>=', $startDate);
}
protected function filterByEndDate(\Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $endDate): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder
protected function filterByEndDate(\Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $endDate): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder
{
return $builder->where('published.to', '<=', $endDate);
}

View File

@ -2,11 +2,13 @@
namespace App\Http\QueryBuilder;
use App\Http\QueryBuilder\Traits\StatusResolver;
use App\Http\QueryBuilder\Traits\TypeResolver;
use App\IsoDateFormatter;
abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
{
use IsoDateFormatter;
use IsoDateFormatter, StatusResolver, TypeResolver;
private array $mediaParameterNames = ["score", "sfw", "genres", "genres_exclude", "min_score", "max_score",
"start_date", "end_date", "status"];
@ -22,28 +24,6 @@ abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
'favorites' => 'favorites'
];
/**
* @param string|null $status
* @return string|null
*/
public function mapStatus(?string $status = null): ?string
{
$status = strtolower($status);
return $this->getStatusMap()[$status] ?? null;
}
/**
* @param string|null $type
* @return string|null
*/
public function mapType(?string $type = null): ?string
{
$type = strtolower($type);
return $this->getTypeMap()[$type] ?? null;
}
protected function getParameterNames(): array
{
$parameterNames = parent::getParameterNames();
@ -64,7 +44,7 @@ abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
return $parameters;
}
private function filterByGenre(\Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $builder, int $genre, $exclude = false): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
private function filterByGenre(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $builder, int $genre, $exclude = false): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $builder->where(function ($query) use ($genre, $exclude) {
$operator = $exclude ? '!=' : null;
@ -76,7 +56,7 @@ abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
});
}
private function filterByGenres(\Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $builder, string $genres, $exclude = false): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
private function filterByGenres(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $builder, string $genres, $exclude = false): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$genres = explode(',', $genres);
foreach ($genres as $genre) {
@ -92,7 +72,7 @@ abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
return $builder;
}
protected function buildQuery(array $requestParameters, \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
protected function buildQuery(array $requestParameters, \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $results): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$builder = $results;
extract($requestParameters);
@ -105,11 +85,6 @@ abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
$builder = $this->filterByEndDate($builder, $this->formatIsoDateTime($end_date));
}
if (!is_null($type)) {
$builder = $builder
->where('type', $type);
}
if ($score !== 0) {
$score = (float)$score;
@ -152,13 +127,9 @@ abstract class MediaSearchQueryBuilder extends SearchQueryBuilder
return self::ORDER_BY;
}
protected abstract function filterByStartDate(\Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $startDate): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder;
protected abstract function filterByStartDate(\Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $startDate): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder;
protected abstract function filterByEndDate(\Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $endDate): \Jenssegers\Mongodb\Eloquent\Builder|\Laravel\Scout\Builder;
protected abstract function getStatusMap(): array;
protected abstract function getTypeMap(): array;
protected abstract function filterByEndDate(\Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder $builder, string $endDate): \Illuminate\Database\Eloquent\Builder|\Laravel\Scout\Builder;
protected abstract function getAdultRating(): string;
}

View File

@ -6,6 +6,7 @@ use Illuminate\Http\Request;
use JetBrains\PhpStorm\ArrayShape;
use Laravel\Scout\Searchable;
use Jenssegers\Mongodb\Eloquent\Model;
use Typesense\Documents;
abstract class SearchQueryBuilder implements SearchQueryBuilderService
{
@ -14,6 +15,8 @@ abstract class SearchQueryBuilder implements SearchQueryBuilderService
protected string $displayNameFieldName = "name";
protected bool $searchIndexesEnabled;
private $modelClassTraitsCache = null;
public function __construct(bool $searchIndexesEnabled)
{
$this->searchIndexesEnabled = $searchIndexesEnabled;
@ -47,18 +50,29 @@ abstract class SearchQueryBuilder implements SearchQueryBuilderService
/**
* @throws \Exception
* @throws \Http\Client\Exception
*/
private function getQueryBuilder($requestParameters): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
private function getQueryBuilder($requestParameters): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$modelClass = $this->getModelClass();
$traits = class_uses_recursive($modelClass);
if (!in_array(Model::class, class_parents($modelClass))) {
throw new \Exception("Programming error: The getModelClass method should return a class which
inherits from \Jenssegers\Mongodb\Eloquent\Model.");
}
if ($this->isSearchIndexUsed()) {
if ($this->isSearchIndexUsed() && !empty($requestParameters['q'])) {
if (env('SCOUT_DRIVER') == 'typesense')
{
// if search index is typesense, let's enable exhaustive search
// which will make Typesense consider all variations of prefixes and typo corrections of the words
// in the query exhaustively, without stopping early when enough results are found.
return $modelClass::search($requestParameters["q"], function(Documents $documents, string $query, array $options) {
$options['exhaustive_search'] = true;
return $documents->search($options);
});
}
return $modelClass::search($requestParameters["q"]);
}
@ -68,7 +82,10 @@ abstract class SearchQueryBuilder implements SearchQueryBuilderService
public function isSearchIndexUsed(): bool
{
$modelClass = $this->getModelClass();
$traits = class_uses_recursive($modelClass);
if (is_null($this->modelClassTraitsCache)) {
$this->modelClassTraitsCache = class_uses_recursive($modelClass);
}
$traits = $this->modelClassTraitsCache;
return in_array(Searchable::class, $traits) && $this->searchIndexesEnabled;
}
@ -82,54 +99,63 @@ abstract class SearchQueryBuilder implements SearchQueryBuilderService
protected abstract function getModelClass(): object|string;
protected abstract function buildQuery(array $requestParameters, \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $results): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder;
protected abstract function buildQuery(array $requestParameters, \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $results): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder;
protected abstract function getOrderByFieldMap(): array;
/**
* @throws \Exception
* @throws \Http\Client\Exception
*/
public function query(Request $request): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder
public function query(Request $request): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$requestParameters = $this->getSanitizedParametersFromRequest($request);
extract($requestParameters);
$results = $this->getQueryBuilder($requestParameters);
if (!is_null($letter)) {
$results = $results
->where($this->displayNameFieldName, 'like', "{$letter}%");
}
// if search index is enabled, this way we only do the full-text search on the index, and filter further in mongodb.
// the $results variable can be a Builder from the Mongodb Eloquent or from Scout. Both of them have a query
// method which will result in a Mongodb Eloquent Builder.
return $results->query(function(\Illuminate\Database\Eloquent\Builder $query) use($requestParameters) {
$letter = $requestParameters['letter'];
$q = $requestParameters['q'];
if (!is_null($order_by)) {
$results = $results
->orderBy($order_by, $sort ?? 'asc');
}
if (!is_null($letter)) {
$query = $query
->where($this->displayNameFieldName, 'like', "{$letter}%");
}
if (empty($q)) {
$results = $results
->orderBy('mal_id');
}
if (empty($q)) {
$query = $query
->orderBy('mal_id');
}
// if search index is disabled, use mongo's full text-search
if (empty($q) && is_null($letter) && !$this->isSearchIndexUsed()) {
$results = $results
->whereRaw([
'$text' => [
'$search' => $query
],
], [
'score' => [
'$meta' => 'textScore'
]
])
->orderBy('score', ['$meta' => 'textScore']);
}
// if search index is disabled, use mongo's full text-search
if (!empty($q) && is_null($letter) && !$this->isSearchIndexUsed()) {
/** @noinspection PhpParamsInspection */
$query = $query
->whereRaw([
'$text' => [
'$search' => $q
],
], [
'score' => [
'$meta' => 'textScore'
]
])
->orderBy('score', ['$meta' => 'textScore']);
}
return $this->buildQuery($requestParameters, $results);
// The ->filter() call is a local model scope function, which applies filters based on the query string
// parameters. This way we can simplify the code base and avoid a bunch of
// "if ($this->request->get("asd")) { }" lines in controllers.
$queryFilteredByQueryStringParams = $query->filter($requestParameters);
return $this->buildQuery($requestParameters, $queryFilteredByQueryStringParams);
});
}
#[ArrayShape(['per_page' => "int", 'total' => "int", 'current_page' => "int", 'last_page' => "int", 'data' => "array"])]
public function paginate(Request $request, \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $results): array
public function paginate(Request $request, \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $results): array
{
$paginated = $this->paginateBuilder($request, $results);
@ -169,9 +195,9 @@ abstract class SearchQueryBuilder implements SearchQueryBuilderService
return compact("page", "limit");
}
public function paginateBuilder(Request $request, \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $results): \Illuminate\Contracts\Pagination\LengthAwarePaginator
public function paginateBuilder(Request $request, \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $results): \Illuminate\Contracts\Pagination\LengthAwarePaginator
{
extract($this->getPaginateParameters($request));
['limit' => $limit, 'page' => $page] = $this->getPaginateParameters($request);
if ($this->isSearchIndexUsed()) {
$paginated = $results

View File

@ -6,11 +6,11 @@ use Illuminate\Http\Request;
interface SearchQueryBuilderService
{
function query(Request $request): \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder;
function query(Request $request): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder;
function paginate(Request $request, \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $results): array;
function paginate(Request $request, \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $results): array;
function paginateBuilder(Request $request, \Laravel\Scout\Builder|\Jenssegers\Mongodb\Eloquent\Builder $results): \Illuminate\Contracts\Pagination\LengthAwarePaginator;
function paginateBuilder(Request $request, \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $results): \Illuminate\Contracts\Pagination\LengthAwarePaginator;
function getIdentifier(): string;

View File

@ -0,0 +1,21 @@
<?php
namespace App\Http\QueryBuilder\Traits;
trait StatusResolver
{
/**
* @param string|null $status
* @return string|null
*/
public function mapStatus(?string $status = null): ?string
{
$status = strtolower($status);
return $this->getStatusMap()[$status] ?? null;
}
protected function getStatusMap(): array
{
return [];
}
}

View File

@ -0,0 +1,21 @@
<?php
namespace App\Http\QueryBuilder\Traits;
trait TypeResolver
{
/**
* @param string|null $type
* @return string|null
*/
public function mapType(?string $type = null): ?string
{
$type = strtolower($type);
return $this->getTypeMap()[$type] ?? null;
}
protected function getTypeMap(): array
{
return [];
}
}

17
app/JikanApiModel.php Normal file
View File

@ -0,0 +1,17 @@
<?php
namespace App;
use App\Filters\FilterQueryString;
class JikanApiModel extends \Jenssegers\Mongodb\Eloquent\Model
{
use FilterQueryString;
/**
* The list of parameters which can be used to filter the resultset from the database.
* The available field names and "order_by" is allowed as values. If "order_by" is specified then
* @var string[]
*/
protected array $filters = [];
}

View File

@ -0,0 +1,44 @@
<?php
namespace App;
use Jenssegers\Mongodb\Eloquent\Model;
use Typesense\LaravelTypesense\Interfaces\TypesenseDocument;
abstract class JikanApiSearchableModel extends JikanApiModel implements TypesenseDocument
{
use JikanSearchable;
/**
* @return string[]
*/
public abstract function typesenseQueryBy(): array;
/**
* The Typesense schema to be created.
*
* @return array
*/
public function getCollectionSchema(): array
{
return [
'name' => $this->searchableAs(),
'fields' => [
[
'name' => '.*',
'type' => 'auto',
]
]
];
}
/**
* Get the name of the index associated with the model.
*
* @return string
*/
public function searchableAs(): string
{
return strtolower($this->table) . '_index';
}
}

View File

@ -1,17 +0,0 @@
<?php
namespace App;
use Jenssegers\Mongodb\Eloquent\Model;
use Typesense\LaravelTypesense\Interfaces\TypesenseDocument;
abstract class JikanModel extends Model implements TypesenseDocument
{
public abstract function typesenseQueryBy(): array;
public function getCollectionSchema(): array
{
// TODO: Implement getCollectionSchema() method.
return [];
}
}

View File

@ -1,6 +1,7 @@
<?php
namespace App;
use Jikan\Helper\Parser;
use Laravel\Scout\Builder;
use Laravel\Scout\Searchable;
@ -8,35 +9,28 @@ trait JikanSearchable
{
use Searchable;
private function flattenArrayWithKeys($array): array {
$result = array();
foreach($array as $key=>$value) {
if(is_array($value)) {
$result = $result + $this->flattenArrayWithKeys($value, $key . '.');
}
else {
$result[$key] = $value;
}
}
return $result;
}
protected function toTypeSenseCompatibleNestedField(string $fieldName): array {
$field = $this->{$fieldName};
if (!is_array($field) && !is_object($field)) {
return $field;
}
return $this->flattenArrayWithKeys($field);
return collect($field)->to2dArrayWithDottedKeys($field, $fieldName.'.');
}
protected function getMalIdsOfField(mixed $field): array {
return array_map(function($elem) {
return $elem->mal_id;
return $elem["mal_id"];
}, $field);
}
public function queryScoutModelsByIds(Builder $builder, array $ids): Builder
protected function convertToTimestamp(?string $datetime): int
{
return $datetime ? Parser::parseDate($datetime)->getTimestamp() : 0;
}
public function queryScoutModelsByIds(Builder $builder, array $ids): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
$query = static::usesSoftDelete()
? $this->withTrashed() : $this->newQuery();
@ -50,7 +44,7 @@ trait JikanSearchable
'whereIn';
return $query->{$whereIn}(
$this->getScoutKeyName(), array_map(function($v) { return (int)$v; }, $ids)
$this->getScoutKeyName(), array_map(fn ($v) => (int)$v, $ids)
);
}
}

View File

@ -0,0 +1,29 @@
<?php
namespace App\Macros;
class To2dArrayWithDottedKeys
{
public function __invoke(): \Closure
{
return function ($prefix = '') {
$iterator = new \RecursiveIteratorIterator(
new \RecursiveArrayIterator($this->toArray()),
\RecursiveIteratorIterator::SELF_FIRST
);
$path = [];
$flatArray = [];
foreach ($iterator as $key => $value) {
$path[$iterator->getDepth()] = $key;
if (!is_array($value)) {
$flatArray[
$prefix . implode('.', array_slice($path, 0, $iterator->getDepth() + 1))
] = $value;
}
}
return $flatArray;
};
}
}

View File

@ -2,19 +2,14 @@
namespace App;
use App\Http\HttpHelper;
use Jenssegers\Mongodb\Eloquent\Model;
use Jikan\Helper\Media;
use Jikan\Helper\Parser;
use Jikan\Jikan;
use Jikan\Model\Common\YoutubeMeta;
use Jikan\Request\Magazine\MagazinesRequest;
/**
* Class Magazine
* @package App
*/
class Magazine extends Model
class Magazine extends JikanApiSearchableModel
{
/**
@ -56,4 +51,21 @@ class Magazine extends Model
true
);
}
}
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'name' => $this->name,
'count' => $this->count
];
}
public function typesenseQueryBy(): array
{
return [
'name'
];
}
}

View File

@ -11,8 +11,11 @@ use Jikan\Model\Common\YoutubeMeta;
use Jikan\Request\Anime\AnimeRequest;
use Jikan\Request\Manga\MangaRequest;
class Manga extends Model
class Manga extends JikanApiSearchableModel
{
// note that here we skip "score", "min_score", "max_score", "rating" and others because they need special logic
// to set the correct filtering on the ORM.
protected array $filters = ["order_by", "status", "type"];
/**
* The attributes that are mass assignable.
@ -60,4 +63,53 @@ class Manga extends Model
)
);
}
}
/**
* Converts the model to an index-able data array.
*
* @return array
*/
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'start_date' => $this->convertToTimestamp($this->published['from']),
'end_date' => $this->convertToTimestamp($this->published['to']),
'title' => $this->title,
'title_english' => $this->title_english,
'title_japanese' => $this->title_japanese,
'title_synonyms' => $this->title_synonyms,
'type' => $this->type,
'chapters' => $this->chapters,
'volumes' => $this->volumes,
'status' => $this->status,
'publishing' => $this->publishing,
'score' => $this->score,
'rank' => $this->rank,
'popularity' => $this->popularity,
'members' => $this->members,
'favorites' => $this->favorites,
'synopsis' => $this->synopsis,
'season' => $this->season,
'magazines' => $this->getMalIdsOfField($this->magazines),
'genres' => $this->getMalIdsOfField($this->genres),
'explicit_genres' => $this->getMalIdsOfField($this->explicit_genres)
];
}
public function getThemesAttribute(): array
{
return [];
}
public function typesenseQueryBy(): array
{
return [
'title',
'title_english',
'title_japanese',
'title_synonyms'
];
}
}

View File

@ -2,17 +2,11 @@
namespace App;
use App\Http\HttpHelper;
use Jenssegers\Mongodb\Eloquent\Model;
use Jikan\Helper\Media;
use Jikan\Helper\Parser;
use Jikan\Jikan;
use Jikan\Model\Common\YoutubeMeta;
use Jikan\Request\Anime\AnimeRequest;
use Jikan\Request\Character\CharacterRequest;
use Jikan\Request\Person\PersonRequest;
use function Symfony\Component\Translation\t;
class Person extends Model
class Person extends JikanApiSearchableModel
{
/**
@ -62,4 +56,31 @@ class Person extends Model
true
);
}
}
/**
* Converts the model to an index-able data array.
*
* @return array
*/
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'name' => $this->name,
'given_name' => $this->given_name,
'family_name' => $this->family_name,
'alternative_names' => $this->alternative_names
];
}
public function typesenseQueryBy(): array
{
return [
"name",
"given_name",
"family_name",
"alternative_names"
];
}
}

View File

@ -5,7 +5,7 @@ namespace App;
use Jenssegers\Mongodb\Eloquent\Model;
use Jikan\Request\User\UserProfileRequest;
class Profile extends Model
class Profile extends JikanApiSearchableModel
{
/**
@ -43,4 +43,25 @@ class Profile extends Model
true
);
}
}
/**
* Converts the model to an index-able data array.
*
* @return array
*/
public function toSearchableArray(): array
{
return [
'id' => (string) $this->mal_id,
'mal_id' => (string) $this->mal_id,
'username' => $this->username
];
}
public function typesenseQueryBy(): array
{
return [
"username"
];
}
}

View File

@ -3,32 +3,79 @@
namespace App\Providers;
use App\Http\QueryBuilder\AnimeSearchQueryBuilder;
use App\Http\QueryBuilder\ClubSearchQueryBuilder;
use App\Http\QueryBuilder\MangaSearchQueryBuilder;
use App\Macros\To2dArrayWithDottedKeys;
use Illuminate\Support\ServiceProvider;
use Illuminate\Support\Collection;
class AppServiceProvider extends ServiceProvider
{
/**
* @throws \ReflectionException
* @throws \Illuminate\Contracts\Container\BindingResolutionException
*/
public function boot(): void
{
$this->registerMacros();
}
/**
* Register any application services.
*
* @throws \ReflectionException
* @throws \Illuminate\Contracts\Container\BindingResolutionException
* @return void
*/
public function register(): void
{
$this->app->singleton(AnimeSearchQueryBuilder::class, function($app) {
$searchIndexesEnabled = $app["config"]->get("scout.driver") != "null";
return new AnimeSearchQueryBuilder($searchIndexesEnabled);
});
$this->app->singleton(AnimeSearchQueryBuilder::class,
$this->getQueryBuilderFactory(AnimeSearchQueryBuilder::class)
);
$this->app->singleton(MangaSearchQueryBuilder::class, function($app) {
$searchIndexesEnabled = $app["config"]->get("scout.driver") != "null";
return new MangaSearchQueryBuilder($searchIndexesEnabled);
});
$this->app->singleton(MangaSearchQueryBuilder::class,
$this->getQueryBuilderFactory(MangaSearchQueryBuilder::class)
);
$this->app->tag([AnimeSearchQueryBuilder::class, MangaSearchQueryBuilder::class], "searchQueryBuilders");
$this->app->singleton(ClubSearchQueryBuilder::class,
$this->getQueryBuilderFactory(ClubSearchQueryBuilder::class)
);
$this->app->tag([
AnimeSearchQueryBuilder::class,
MangaSearchQueryBuilder::class,
ClubSearchQueryBuilder::class
], "searchQueryBuilders");
$this->app->singleton(SearchQueryBuilderProvider::class, function($app) {
return new SearchQueryBuilderProvider($app->tagged("searchQueryBuilders"));
});
}
/**
* @throws \ReflectionException
* @throws \Illuminate\Contracts\Container\BindingResolutionException
* @return void
*/
private function registerMacros(): void
{
Collection::make($this->collectionMacros())
->reject(fn ($class, $macro) => Collection::hasMacro($macro))
->each(fn ($class, $macro) => Collection::macro($macro, app($class)()));
}
private function collectionMacros(): array
{
return [
"to2dArrayWithDottedKeys" => To2dArrayWithDottedKeys::class
];
}
private function getQueryBuilderFactory($queryBuilderClass): \Closure
{
return function($app) use($queryBuilderClass) {
$searchIndexesEnabled = $app["config"]->get("scout.driver") != "null";
return new $queryBuilderClass($searchIndexesEnabled);
};
}
}