Merge pull request #409 from pushrbx/search-improvements-3

Added fixes for searching and various things

app/Anime.php

@@ -385,13 +385,17 @@ class Anime extends JikanApiSearchableModel
     {
         return [
             [
-                "field" => "_text_match",
+                "field" => "_text_match(buckets:" . max_results_per_page() . ")",
                 "direction" => "desc"
             ],
             [
-                "field" => "members",
-                "direction" => "desc"
+                "field" => "popularity",
+                "direction" => "asc"
             ],
+            [
+                "field" => "rank",
+                "direction" => "asc"
+            ]
         ];
     }
 

app/Concerns/ResolvesPaginatorParams.php

@@ -2,11 +2,13 @@
 
 namespace App\Concerns;
 
+use Illuminate\Support\Facades\App;
+
 trait ResolvesPaginatorParams
 {
     private function getPaginatorParams(?int $limit = null, ?int $page = null): array
     {
-        $default_max_results_per_page = env('MAX_RESULTS_PER_PAGE', 25);
+        $default_max_results_per_page = max_results_per_page();
         $limit = $limit ?? $default_max_results_per_page;
         $page = $page ?? 1;
 

app/Contracts/SearchAnalyticsService.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace App\Contracts;
+
+use Illuminate\Support\Collection;
+
+interface SearchAnalyticsService
+{
+    public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void;
+}

app/Dto/Concerns/PreparesData.php

@@ -4,6 +4,7 @@ namespace App\Dto\Concerns;
 
 use Illuminate\Support\Collection;
 use Illuminate\Support\Env;
+use Illuminate\Support\Facades\App;
 use \ReflectionClass;
 use Spatie\LaravelData\Support\DataConfig;
 
@@ -19,8 +20,8 @@ trait PreparesData
         // let's always set the limit parameter to the globally configured default value
         if (property_exists(static::class, "limit") && !$properties->has("limit")) {
             /** @noinspection PhpUndefinedFieldInspection */
-            $properties->put("limit", Env::get("MAX_RESULTS_PER_PAGE",
-                property_exists(static::class, "defaultLimit") ? static::$defaultLimit : 25));
+            $properties->put("limit", max_results_per_page(
+                property_exists(static::class, "defaultLimit") ? static::$defaultLimit : null));
         }
 
         // we want to cast "true" and "false" string values to boolean before validation, so let's take all properties

app/Dto/QueryAnimeSchedulesCommand.php

@@ -17,7 +17,6 @@ use App\Rules\Attributes\EnumValidation;
 use Illuminate\Http\JsonResponse;
 use Spatie\LaravelData\Attributes\WithCast;
 use Spatie\LaravelData\Data;
-use Spatie\LaravelData\Attributes\MapInputName;
 use Spatie\LaravelData\Attributes\MapOutputName;
 
 /**
@@ -30,7 +29,6 @@ final class QueryAnimeSchedulesCommand extends Data implements DataRequest
     #[
         WithCast(EnumCast::class, AnimeScheduleFilterEnum::class),
         EnumValidation(AnimeScheduleFilterEnum::class),
-        MapInputName("filter"),
         MapOutputName("filter")
     ]
     public ?AnimeScheduleFilterEnum $dayFilter;

app/Features/QueryUpcomingAnimeSeasonHandler.php

@@ -18,6 +18,6 @@ final class QueryUpcomingAnimeSeasonHandler extends QueryAnimeSeasonHandlerBase
 
     protected function getSeasonItems($request, ?AnimeTypeEnum $type): Builder
     {
-        return $this->repository->getUpcomingSeasonItems($type, $request->kids, $request->sfw, $request->unapproved);
+        return $this->repository->getUpcomingSeasonItems($type);
     }
 }

app/Manga.php

@@ -302,13 +302,17 @@ class Manga extends JikanApiSearchableModel
     {
         return [
             [
-                "field" => "_text_match",
+                "field" => "_text_match(buckets:" . App::make("jikan-config")->maxResultsPerPage() . ")",
                 "direction" => "desc"
             ],
             [
-                "field" => "members",
-                "direction" => "desc"
+                "field" => "popularity",
+                "direction" => "asc"
             ],
+            [
+                "field" => "rank",
+                "direction" => "asc"
+            ]
         ];
     }
 

app/Profile.php

@@ -19,7 +19,7 @@ class Profile extends JikanApiSearchableModel
     protected $fillable = [
         'mal_id', 'username', 'url', 'images', 'last_online', 'gender', 'birthday', 'location',
         'joined', 'anime_stats', 'manga_stats', 'favorites', 'about',
-        'createdAt', 'modifiedAt'
+        'createdAt', 'modifiedAt', 'internal_username'
     ];
 
     /**

app/Providers/AppServiceProvider.php

@@ -69,6 +69,8 @@ class AppServiceProvider extends ServiceProvider
     public function boot(): void
     {
         $this->registerMacros();
+        // the registration of request handlers should happen after the load of all service providers.
+        $this->registerRequestHandlers();
     }
 
     /**
@@ -81,13 +83,19 @@ class AppServiceProvider extends ServiceProvider
     public function register(): void
     {
         $this->app->singleton(JikanConfig::class, fn() => new JikanConfig(config("jikan")));
+        $this->app->singleton(\App\Contracts\SearchAnalyticsService::class,
+            env("APP_ENV") !== "testing" ? \App\Services\DefaultSearchAnalyticsService::class :
+                \App\Services\DummySearchAnalyticsService::class);
+        $this->app->alias(JikanConfig::class, "jikan-config");
         // cache options class is used to share the request scope level cache settings
         $this->app->singleton(CacheOptions::class);
         $this->app->singleton(CachedScraperService::class, DefaultCachedScraperService::class);
         $this->app->singleton(PrivateFieldMapperService::class, DefaultPrivateFieldMapperService::class);
         $this->app->bind(QueryBuilderPaginatorService::class, DefaultBuilderPaginatorService::class);
+        if (static::getSearchIndexDriver($this->app) === "typesense") {
+            $this->app->singleton(\App\Services\TypesenseCollectionDescriptor::class);
+        }
         $this->registerModelRepositories();
-        $this->registerRequestHandlers();
     }
 
     private function getSearchService(Repository $repository): SearchService
@@ -101,7 +109,9 @@ class AppServiceProvider extends ServiceProvider
                 default => DefaultScoutSearchService::class
             };
 
-            $scoutSearchService = new $serviceClass($repository);
+            $scoutSearchService = $this->app->make($serviceClass, [
+                "repository" => $repository,
+            ]);
             $result = new SearchEngineSearchService($scoutSearchService, $repository);
         }
         else {
@@ -170,10 +180,9 @@ class AppServiceProvider extends ServiceProvider
                 $requestHandlers = [];
                 foreach ($searchRequestHandlersDescriptors as $handlerClass => $repositoryInstance) {
                     $requestHandlers[] = $app->make($handlerClass, [
-                        "queryBuilderService" => new DefaultQueryBuilderService(
-                            $this->getSearchService($repositoryInstance),
-                            $app->make(QueryBuilderPaginatorService::class)
-                        )
+                        "queryBuilderService" => $app->make(DefaultQueryBuilderService::class, [
+                            "searchService" => $this->getSearchService($repositoryInstance)
+                        ]),
                     ]);
                 }
 

app/Rules/Attributes/MaxLimitWithFallback.php

@@ -11,6 +11,6 @@ final class MaxLimitWithFallback extends Rule
 {
     public function __construct()
     {
-        parent::__construct(new MaxResultsPerPageRule());
+        parent::__construct(new MaxResultsPerPageRule(max_results_per_page()));
     }
 }

app/Rules/MaxResultsPerPageRule.php

@@ -4,6 +4,7 @@ namespace App\Rules;
 
 use Illuminate\Contracts\Validation\Rule;
 use Illuminate\Support\Env;
+use Illuminate\Support\Facades\App;
 
 final class MaxResultsPerPageRule implements Rule
 {
@@ -27,7 +28,7 @@ final class MaxResultsPerPageRule implements Rule
             $value = intval($value);
         }
 
-        if ($value > $this->maxResultsPerPage()) {
+        if ($value > max_results_per_page()) {
             return false;
         }
 
@@ -36,11 +37,7 @@ final class MaxResultsPerPageRule implements Rule
 
     public function message(): array|string
     {
-        return "Value {$this->value} is higher than the configured '{$this->maxResultsPerPage()}' max value.";
-    }
-
-    private function maxResultsPerPage(): int
-    {
-        return (int) Env::get("MAX_RESULTS_PER_PAGE", $this->fallbackLimit);
+        $mrpp = max_results_per_page();
+        return "Value {$this->value} is higher than the configured '$mrpp' max value.";
     }
 }

app/SearchMetric.php

@@ -0,0 +1,121 @@
+<?php
+namespace App;
+use MongoDB\BSON\ObjectId;
+use Laravel\Scout\Builder;
+
+class SearchMetric extends JikanApiSearchableModel
+{
+    protected $table = 'search_metrics';
+
+    protected $appends = ['hits'];
+
+    protected $fillable = ["search_term", "request_count", "hits", "hits_count", "index_name"];
+
+    protected $hidden = ["_id"];
+
+    public function searchableAs(): string
+    {
+        return "jikan_search_metrics";
+    }
+
+    public function toSearchableArray()
+    {
+        return [
+            "id" => $this->_id,
+            "search_term" => $this->search_term,
+            "request_count" => $this->request_count,
+            "hits" => $this->hits,
+            "hits_count" => $this->hits_count,
+            "index_name" => $this->index_name
+        ];
+    }
+
+    public function getCollectionSchema(): array
+    {
+        return [
+            "name" => $this->searchableAs(),
+            "fields" => [
+                [
+                    "name" => "id",
+                    "type" => "string",
+                    "optional" => false
+                ],
+                [
+                    "name" => "search_term",
+                    "type" => "string",
+                    "optional" => false,
+                    "sort" => false,
+                    "infix" => true
+                ],
+                [
+                    "name" => "request_count",
+                    "type" => "int64",
+                    "sort" => true,
+                    "optional" => false,
+                ],
+                [
+                    "name" => "hits",
+                    "type" => "int64[]",
+                    "sort" => false,
+                    "optional" => false,
+                ],
+                [
+                    "name" => "hits_count",
+                    "type" => "int64",
+                    "sort" => true,
+                    "optional" => false,
+                ],
+                [
+                    "name" => "index_name",
+                    "type" => "string",
+                    "sort" => false,
+                    "optional" => false,
+                    "facet" => true
+                ]
+            ]
+        ];
+    }
+
+    public function getTypeSenseQueryByWeights(): string|null
+    {
+        return "1";
+    }
+
+    public function getScoutKey(): mixed
+    {
+        return $this->_id;
+    }
+
+    public function getScoutKeyName(): mixed
+    {
+        return '_id';
+    }
+
+    public function getKeyType(): string
+    {
+        return 'string';
+    }
+
+    public function typesenseQueryBy(): array
+    {
+        return ["search_term"];
+    }
+
+    public function queryScoutModelsByIds(Builder $builder, array $ids): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
+    {
+        $query = static::usesSoftDelete()
+            ? $this->withTrashed() : $this->newQuery();
+
+        if ($builder->queryCallback) {
+            call_user_func($builder->queryCallback, $query);
+        }
+
+        $whereIn = in_array($this->getKeyType(), ['int', 'integer']) ?
+            'whereIntegerInRaw' :
+            'whereIn';
+
+        return $query->{$whereIn}(
+            $this->getScoutKeyName(), array_map(fn ($x) => new ObjectId($x), $ids)
+        );
+    }
+}

app/Services/DefaultSearchAnalyticsService.php

@@ -0,0 +1,43 @@
+<?php
+
+namespace App\Services;
+use App\SearchMetric;
+use App\Contracts\SearchAnalyticsService;
+use Illuminate\Support\Collection;
+
+
+/**
+ * The default search analytics service implementation, which saves the stats to the database and indexes it in Typesense.
+ * 
+ * By indexing search terms in Typesense we can use it to provide search suggestions of popular searches.
+ * @package App\Services
+ */
+final class DefaultSearchAnalyticsService implements SearchAnalyticsService
+{
+    public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void
+    {
+        /**
+         * @var \Laravel\Scout\Builder $existingMetrics
+         */
+        $existingMetrics = SearchMetric::search($searchTerm);
+
+        $hitList = $hits->pluck("id")->values()->map(fn($x) => (int)$x)->all();
+
+        if ($existingMetrics->count() > 0) {
+            $metric = $existingMetrics->first();
+            $metric->hits = $hitList;
+            $metric->hits_count = $hitsCount;
+            $metric->request_count = $metric->request_count + 1;
+            $metric->index_name = $indexName;
+            $metric->save();
+        } else {
+            SearchMetric::create([
+                "search_term" => $searchTerm,
+                "request_count" => 1,
+                "hits" => $hitList,
+                "hits_count" => $hitsCount,
+                "index_name" => $indexName
+            ]);
+        }
+    }
+}

app/Services/DummySearchAnalyticsService.php

@@ -0,0 +1,14 @@
+<?php
+
+namespace App\Services;
+
+use App\Contracts\SearchAnalyticsService;
+use Illuminate\Support\Collection;
+
+final class DummySearchAnalyticsService implements SearchAnalyticsService
+{
+    public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void
+    {
+        // noop;
+    }
+}

app/Services/TypeSenseScoutSearchService.php

@@ -4,16 +4,22 @@ namespace App\Services;
 
 use App\Contracts\Repository;
 use App\JikanApiSearchableModel;
+use App\Support\JikanConfig;
 use Illuminate\Support\Str;
+use Laravel\Scout\Builder;
 use Typesense\Documents;
+use App\Contracts\SearchAnalyticsService;
 
 class TypeSenseScoutSearchService implements ScoutSearchService
 {
     private int $maxItemsPerPage;
 
-    public function __construct(private readonly Repository $repository)
+    public function __construct(private readonly Repository $repository,
+                    JikanConfig $config,
+                    private readonly TypesenseCollectionDescriptor $collectionDescriptor,
+                    private readonly SearchAnalyticsService $searchAnalytics)
     {
-        $this->maxItemsPerPage = (int) env('MAX_RESULTS_PER_PAGE', 25);
+        $this->maxItemsPerPage = (int) $config->maxResultsPerPage();
         if ($this->maxItemsPerPage > 250) {
             $this->maxItemsPerPage = 250;
         }
@@ -22,22 +28,29 @@ class TypeSenseScoutSearchService implements ScoutSearchService
     /**
      * Executes a search operation via Laravel Scout on the provided model class.
      * @param string $q
-     * @return \Laravel\Scout\Builder
-     * @throws \Http\Client\Exception
-     * @throws \Typesense\Exceptions\TypesenseClientError
+     * @param string|null $orderByField
+     * @param bool $sortDirectionDescending
+     * @return Builder
      */
     public function search(string $q, ?string $orderByField = null,
                            bool $sortDirectionDescending = false): \Laravel\Scout\Builder
     {
-        return $this->repository->search($q, function (Documents $documents, string $query, array $options) use ($orderByField, $sortDirectionDescending) {
+        return $this->repository->search($q, $this->middleware($orderByField, $sortDirectionDescending));
+    }
+
+    private function middleware(?string $orderByField = null, bool $sortDirectionDescending = false): \Closure
+    {
+        return function (Documents $documents, string $query, array $options) use ($orderByField, $sortDirectionDescending) {
             // let's enable exhaustive search
             // which will make Typesense consider all variations of prefixes and typo corrections of the words
             // in the query exhaustively, without stopping early when enough results are found.
-            $options['exhaustive_search'] = env('TYPESENSE_SEARCH_EXHAUSTIVE', "true");
+            $options['exhaustive_search'] = env('TYPESENSE_SEARCH_EXHAUSTIVE', "false");
             $options['search_cutoff_ms'] = (int) env('TYPESENSE_SEARCH_CUTOFF_MS', 450);
             // this will be ignored together with exhaustive_search set to "true"
-            $options['drop_tokens_threshold'] = (int) env('TYPESENSE_DROP_TOKENS_THRESHOLD', 1);
-            $options['typo_tokens_threshold'] = (int) env('TYPESENSE_TYPO_TOKENS_THRESHOLD', 1);
+            $options['drop_tokens_threshold'] = (int) env('TYPESENSE_DROP_TOKENS_THRESHOLD', $this->maxItemsPerPage);
+            $options['typo_tokens_threshold'] = (int) env('TYPESENSE_TYPO_TOKENS_THRESHOLD', $this->maxItemsPerPage);
+            $options['enable_highlight_v1'] = 'false';
+            $options['infix'] = 'fallback';
             // prevent `Could not parse the filter query: unbalanced `&&` operands.` error
             // this adds support for typesense v0.24.1
             if (array_key_exists('filter_by', $options) && ($options['filter_by'] === ' && ' || $options['filter_by'] === '&&')) {
@@ -48,39 +61,105 @@ class TypeSenseScoutSearchService implements ScoutSearchService
                 $options['per_page'] = min($this->maxItemsPerPage, 250);
             }
 
+            $options = $this->skipTypoCheckingForShortQueries($query, $options);
             $modelInstance = $this->repository->createEntity();
-            // get the weights of the query_by fields, if they are provided by the model.
+
             if ($modelInstance instanceof JikanApiSearchableModel) {
-                $queryByWeights = $modelInstance->getTypeSenseQueryByWeights();
-                if (!is_null($queryByWeights)) {
-                    $options['query_by_weights'] = $queryByWeights;
-                }
-
-                // if the model specifies search index sort order, use it
-                // this is the default sort order for the model
-                $sortByFields = $modelInstance->getSearchIndexSortBy();
-                if (!is_null($sortByFields)) {
-                    $sortBy = "";
-                    foreach ($sortByFields as $f) {
-                        $sortBy .= $f['field'] . ':' . $f['direction'];
-                        $sortBy .= ',';
-                    }
-                    $sortBy = rtrim($sortBy, ',');
-                    $options['sort_by'] = $sortBy;
-                }
-
-                // override ordering field
-                if (!is_null($orderByField)) {
-                    $options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match:desc";
-                }
-
-                // override overall sorting direction
-                if (is_null($orderByField) && $sortDirectionDescending && array_key_exists("sort_by", $options) && Str::contains($options["sort_by"], "asc")) {
-                    $options["sort_by"] = Str::replace("asc", "desc", $options["sort_by"]);
-                }
+                $options = $this->setQueryByWeights($options, $modelInstance);
+                $options = $this->setSortOrder($options, $modelInstance);
+                $options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending);
             }
 
-            return $documents->search($options);
-        });
+            $results = $documents->search($options);
+            $this->recordSearchTelemetry($query, $results);
+
+            return $results;
+        };
+    }
+
+    private function skipTypoCheckingForShortQueries(string $query, array $options): array
+    {
+        if (strlen($query) <= 3) {
+            $options['num_typos'] = 0;
+            $options['typo_tokens_threshold'] = 0;
+            $options['drop_tokens_threshold'] = 0;
+            $options['exhaustive_search'] = 'false';
+            $options['infix'] = 'off';
+            $options['prefix'] = 'false';
+        }
+
+        return $options;
+    }
+
+    private function setQueryByWeights(array $options, JikanApiSearchableModel $modelInstance): array
+    {
+        $queryByWeights = $modelInstance->getTypeSenseQueryByWeights();
+        if (!is_null($queryByWeights)) {
+            $options['query_by_weights'] = $queryByWeights;
+        }
+
+        return $options;
+    }
+
+    private function setSortOrder(array $options, JikanApiSearchableModel $modelInstance): array
+    {
+        $sortByFields = $modelInstance->getSearchIndexSortBy();
+        if (!is_null($sortByFields)) {
+            $sortBy = "";
+            foreach ($sortByFields as $f) {
+                $sortBy .= $f['field'] . ':' . $f['direction'];
+                $sortBy .= ',';
+            }
+            $sortBy = rtrim($sortBy, ',');
+            $options['sort_by'] = $sortBy;
+        }
+
+        return $options;
+    }
+
+    private function overrideSortingOrder(array $options, JikanApiSearchableModel $modelInstance, ?string $orderByField, bool $sortDirectionDescending): array
+    {
+        $modelAttrNames = $this->collectionDescriptor->getSearchableAttributes($modelInstance);
+
+        // fixme: this shouldn't be here, but it's a quick fix for the time being
+        if ($orderByField === "aired.from") {
+            $orderByField = "start_date";
+        }
+
+        if ($orderByField === "aired.to") {
+            $orderByField = "end_date";
+        }
+
+        if ($orderByField === "published.from") {
+            $orderByField = "start_date";
+        }
+
+        if ($orderByField === "published.to") {
+            $orderByField = "end_date";
+        }
+        // fixme end
+
+        // override ordering field
+        if (!is_null($orderByField) && in_array($orderByField, $modelAttrNames)) {
+            $options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->maxItemsPerPage."):desc";
+        }
+
+        // override overall sorting direction
+        if (is_null($orderByField) && $sortDirectionDescending && array_key_exists("sort_by", $options) && Str::contains($options["sort_by"], "asc")) {
+            $options["sort_by"] = Str::replace("asc", "desc", $options["sort_by"]);
+        }
+
+        return $options;
+    }
+
+    private function recordSearchTelemetry(string $query, array $typesenseApiResponse): void
+    {
+        $hits = collect($typesenseApiResponse["hits"]);
+        $this->searchAnalytics->logSearch(
+            $query,
+            $typesenseApiResponse["found"],
+            $hits->pluck('document')->values(),
+            $typesenseApiResponse["request_params"]["collection_name"]
+        );
     }
 }

app/Services/TypesenseCollectionDescriptor.php

@@ -0,0 +1,37 @@
+<?php
+namespace App\Services;
+use Typesense\LaravelTypesense\Typesense;
+use App\JikanApiSearchableModel;
+use Typesense\Collection as TypesenseCollection;
+use Illuminate\Support\Collection;
+
+/**
+ * A service which helps to describe a collection in Typesense based on an eloquent model
+ */
+final class TypesenseCollectionDescriptor
+{
+    private readonly Collection $cache;
+
+    public function __construct(private readonly Typesense $typesense)
+    {
+        $this->cache = collect();   
+    }
+
+    public function getSearchableAttributes(JikanApiSearchableModel $model): array
+    {
+        $modelSearchableAs = $model->searchableAs();
+        if ($this->cache->has($modelSearchableAs)) {
+            return $this->cache->get($modelSearchableAs);
+        }
+        /**
+         * @var TypesenseCollection $collection
+         */
+        $collection = $this->typesense->getCollectionIndex($model);
+        $collectionDetails = $collection->retrieve();
+        $fields = collect($collectionDetails["fields"]);
+        $searchableAttributeNames = $fields->pluck("name")->all();
+        $this->cache->put($modelSearchableAs, $searchableAttributeNames);
+
+        return $searchableAttributeNames;
+    }
+}

app/Support/JikanConfig.php

@@ -18,12 +18,15 @@ final class JikanConfig
 
     private bool $microCachingEnabled;
 
+    private Collection $config;
+
     public function __construct(array $config)
     {
         $config = collect($config);
         $this->perEndpointCacheTtl = $config->get("per_endpoint_cache_ttl", []);
         $this->defaultCacheExpire = $config->get("default_cache_expire", 0);
         $this->microCachingEnabled = in_array($config->get("micro_caching_enabled", false), [true, 1, "1", "true"]);
+        $this->config = $config;
     }
 
     public function cacheTtlForEndpoint(string $endpoint): ?int
@@ -40,4 +43,9 @@ final class JikanConfig
     {
         return $this->microCachingEnabled;
     }
+
+    public function maxResultsPerPage(?int $defaultValue = null): int
+    {
+        return $this->config->get("max_results_per_page", $defaultValue ?? 25);
+    }
 }

app/Support/helpers.php

@@ -54,3 +54,10 @@ if (!function_exists('to_boolean')) {
     }
 }
 
+
+if (!function_exists('max_results_per_page')) {
+    function max_results_per_page(?int $fallbackLimit = null): int
+    {
+        return app()->make("jikan-config")->maxResultsPerPage($fallbackLimit);
+    }
+}

composer.json

@@ -68,7 +68,10 @@
         "phpunit": "@php ./vendor/bin/phpunit --no-coverage",
         "phpunit-cover": "@php ./vendor/bin/phpunit",
         "test": [
-          "@phpunit"
+          "@phpunit --testsuite unit"
+        ],
+        "integration-test": [
+          "@phpunit --testsuite integration"
         ],
         "test-cover": [
           "@phpunit-cover"

config/jikan.php

@@ -1,6 +1,7 @@
 <?php
 
 return [
+    'max_results_per_page' => env('MAX_RESULTS_PER_PAGE', 25),
     'micro_caching_enabled' => env('MICROCACHING', false),
     'default_cache_expire' => env('CACHE_DEFAULT_EXPIRE', 86400),
     'per_endpoint_cache_ttl' => [

config/logging.php

@@ -29,10 +29,7 @@ return [
     |
     */
 
-    'deprecations' => [
-        'channel' => env('LOG_DEPRECATIONS_CHANNEL', 'null'),
-        'trace' => false,
-    ],
+    'deprecations' => env('LOG_DEPRECATIONS_CHANNEL', 'null'),
 
 
     /*
@@ -89,7 +86,7 @@ return [
                 'port' => env('PAPERTRAIL_PORT'),
                 'connectionString' => 'tls://'.env('PAPERTRAIL_URL').':'.env('PAPERTRAIL_PORT'),
             ],
-            'processors' => [PsrLogMessageProcessor::class],
+            'processors' => [Monolog\Processor\PsrLogMessageProcessor::class],
         ],
 
         'syslog' => [

database/factories/AnimeFactory.php

@@ -68,7 +68,7 @@ class AnimeFactory extends JikanMediaModelFactory
             "synopsis" => "test",
             "approved" => true,
             "background" => "test",
-            "premiered" => $this->faker->randomElement(["Winter", "Spring", "Fall", "Summer"]),
+            "premiered" => $this->faker->randomElement(["Winter", "Spring", "Fall", "Summer"]) . " " . $this->faker->year(),
             "broadcast" => [
                 "day" => "",
                 "time" => "",

database/factories/ProfileFactory.php

@@ -21,6 +21,7 @@ final class ProfileFactory extends JikanModelFactory
         return [
             "mal_id" => $mal_id,
             "username" => $username,
+            "internal_username" => $username,
             "url" => $url,
             "request_hash" => sprintf("request:%s:%s", "users", $this->getItemTestUrl("users", $username)),
             "images" => [

tests/Integration/MangaSearchEndpointTest.php

@@ -98,9 +98,9 @@ class MangaSearchEndpointTest extends TestCase
     public function genresParameterCombinationsProvider(): array
     {
         return [
-            [["genres" => "1,2"]],
-            [["genres_exclude" => "4,5", "type" => "tv"]],
-            [["genres" => "1,2", "genres_exclude" => "3", "min_score" => 8, "type" => "tv", "status" => "complete", "page" => 1]],
+            "?genres=1,2" => [["genres" => "1,2"]],
+            "?genres_exclude=4,5&type=manga" => [["genres_exclude" => "4,5", "type" => "manga"]],
+            "?genres=1,2&genres_exclude=3&min_score=8&type=manga&status=complete" => [["genres" => "1,2", "genres_exclude" => "3", "min_score" => 8, "type" => "manga", "status" => "complete", "page" => 1]],
         ];
     }
 

tests/Integration/UserControllerTest.php

@@ -7,8 +7,13 @@ use Illuminate\Support\Carbon;
 use Illuminate\Support\Facades\DB;
 use Jikan\Model\Recommendations\UserRecommendations;
 use Jikan\Model\User\Friends;
+use Jikan\Model\User\AnimeStats;
+use Jikan\Model\User\MangaStats;
+use Jikan\Model\User\LastUpdates;
+use Jikan\Model\User\Profile as JikanProfile;
 use Jikan\Model\User\Reviews\UserReviews;
 use Jikan\MyAnimeList\MalClient;
+use Jikan\Parser\User\Profile\UserProfileParser;
 use Jikan\Request\User\UserRecommendationsRequest;
 use Tests\TestCase;
 
@@ -17,12 +22,159 @@ class UserControllerTest extends TestCase
     use SyntheticMongoDbTransaction;
     use ScoutFlush;
 
-    public function testUserProfile()
+    public function userNameProvider(): array
+    {
+        return [
+            "nekomata1037" => ["nekomata1037"],
+            "ExampleUser123" => ["ExampleUser123"],
+        ];
+    }
+
+    /**
+     * @dataProvider userNameProvider
+     * @param string $username
+     */
+    public function testUserProfile(string $username)
     {
         Profile::factory()->createOne([
-            "username" => "nekomata1037"
+            "username" => $username,
+            "internal_username" => $username
         ]);
-        $this->get('/v4/users/nekomata1037')
+        $this->get('/v4/users/'.$username)
+            ->seeStatusCode(200)
+            ->seeJsonStructure(['data'=>[
+                'mal_id',
+                'username',
+                'url',
+                'images' => [
+                    'jpg' => [
+                        'image_url'
+                    ],
+                    'webp' => [
+                        'image_url'
+                    ]
+                ],
+                'last_online',
+                'gender',
+                'birthday',
+                'location',
+                'joined',
+            ]]);
+    }
+
+    /**
+     * Tests if the scraped item from upstream is inserted correctly in the database.
+     * @dataProvider userNameProvider
+     * @param string $username
+     * @throws \Exception
+     */
+    public function testUserProfileScrapeAndInsert(string $username)
+    {
+        // this is a test for https://github.com/jikan-me/jikan-rest/issues/411
+        $jikanParser = \Mockery::mock(MalClient::class)->makePartial();
+        $userProfileParser = \Mockery::mock(UserProfileParser::class)->makePartial();
+        $userProfileParser->allows()
+            ->getUserId()
+            ->andReturn(1);
+        $userProfileParser->allows()
+            ->getUsername()
+            ->andReturn($username);
+        $userProfileParser->allows()
+            ->getProfileUrl()
+            ->andReturn("https://myanimelist.net/profile/".$username);
+        $userProfileParser->allows()
+            ->getProfileImageUrl()
+            ->andReturn("https://myanimelist.cdn-dena.com/images/userimages/1.jpg");
+        $userProfileParser->allows()
+            ->getLastOnline()
+            ->andReturn(Carbon::now()->toDateTimeImmutable());
+        $userProfileParser->allows()
+            ->getImageUrl()
+            ->andReturn("https://myanimelist.cdn-dena.com/images/userimages/1.jpg");
+        $userProfileParser->allows()
+            ->getGender()
+            ->andReturn(null);
+        $userProfileParser->allows()
+            ->getBirthday()
+            ->andReturn(\DateTimeImmutable::createFromFormat(\DateTimeImmutable::RFC3339, "1990-10-01T00:00:00+00:00"));
+        $userProfileParser->allows()
+            ->getLocation()
+            ->andReturn("Mars");
+        $userProfileParser->allows()
+            ->getJoinDate()
+            ->andReturn(\DateTimeImmutable::createFromFormat(\DateTimeImmutable::RFC3339, "2000-10-01T00:00:00+00:00"));
+        $animeStats = \Mockery::mock(AnimeStats::class)->makePartial();
+        $animeStats->allows([
+            "getDaysWatched" => 0,
+            "getMeanScore" => 0,
+            "getWatching" => 0,
+            "getCompleted" => 0,
+            "getOnHold" => 0,
+            "getDropped" => 0,
+            "getPlanToWatch" => 0,
+            "getTotalEntries" => 0,
+            "getRewatched" => 0,
+            "getEpisodesWatched" => 0
+        ]);
+        $userProfileParser->allows()
+            ->getAnimeStats()
+            ->andReturn($animeStats);
+        $mangaStats = \Mockery::mock(MangaStats::class)->makePartial();
+        $mangaStats->allows([
+            "getDaysRead" => 0,
+            "getMeanScore" => 0,
+            "getReading" => 0,
+            "getCompleted" => 0,
+            "getOnHold" => 0,
+            "getDropped" => 0,
+            "getPlanToRead" => 0,
+            "getTotalEntries" => 0,
+            "getReread" => 0,
+            "getChaptersRead" => 0,
+            "getVolumesRead" => 0
+        ]);
+        $userProfileParser->allows()
+            ->getMangaStats()
+            ->andReturn($mangaStats);
+        $favorites = \Mockery::mock(\Jikan\Model\User\Favorites::class)->makePartial();
+        $favorites->allows([
+            "getAnime" => [],
+            "getManga" => [],
+            "getCharacters" => [],
+            "getPeople" => []
+        ]);
+        $userProfileParser->allows()
+            ->getFavorites()
+            ->andReturn($favorites);
+        $userProfileParser->allows()
+            ->getAbout()
+            ->andReturn(null);
+        $userProfileParser->allows()
+            ->getUserExternalLinks()
+            ->andReturn([]);
+        $userProfileParser->allows()
+            ->getAbout()
+            ->andReturn(null);
+        $lastUpdates = \Mockery::mock(LastUpdates::class)->makePartial();
+        $lastUpdates->allows()
+            ->getAnime()
+            ->andReturn([]);
+        $lastUpdates->allows()
+            ->getManga()
+            ->andReturn([]);
+        $userProfileParser->allows()
+            ->getUserLastUpdates()
+            ->andReturn($lastUpdates);
+
+
+        /** @noinspection PhpParamsInspection */
+        $jikanParser->allows()
+            ->getUserProfile(\Mockery::any())
+            ->andReturn(JikanProfile::fromParser($userProfileParser));
+
+        $this->app->instance('JikanParser', $jikanParser);
+
+        $this->get('/v4/users/'.$username)
             ->seeStatusCode(200)
             ->seeJsonStructure(['data'=>[
                 'mal_id',
@@ -46,8 +198,10 @@ class UserControllerTest extends TestCase
 
     public function testUserStatistics()
     {
+        $username = "nekomata1037";
         Profile::factory()->createOne([
-            "username" => "nekomata1037"
+            "username" => $username,
+            "internal_username" => $username
         ]);
         $this->get('/v4/users/nekomata1037/statistics')
             ->seeStatusCode(200)
@@ -83,8 +237,10 @@ class UserControllerTest extends TestCase
 
     public function testUserAbout()
     {
+        $username = "nekomata1037";
         Profile::factory()->createOne([
-            "username" => "nekomata1037"
+            "username" => $username,
+            "internal_username" => $username
         ]);
         $this->get('/v4/users/nekomata1037/about')
             ->seeStatusCode(200)
@@ -95,8 +251,10 @@ class UserControllerTest extends TestCase
 
     public function testUserFavorites()
     {
+        $username = "nekomata1037";
         Profile::factory()->createOne([
-            "username" => "nekomata1037"
+            "username" => $username,
+            "internal_username" => $username
         ]);
         $this->get('/v4/users/nekomata1037/favorites')
             ->seeStatusCode(200)