Merge pull request #427 from pushrbx/search-improvements-5

Various improvements and hotfixes 2

app/Anime.php

@@ -182,10 +182,14 @@ class Anime extends JikanApiSearchableModel
         }
 
         $producer = (int)$value;
-        return $query
-            ->orWhere('producers.mal_id', $producer)
-            ->orWhere('licensors.mal_id', $producer)
-            ->orWhere('studios.mal_id', $producer);
+        /** @noinspection PhpParamsInspection */
+        return $query->whereRaw([
+            '$or' => [
+                ['producers.mal_id' => $producer],
+                ['licensors.mal_id' => $producer],
+                ['studios.mal_id' => $producer]
+            ]
+        ]);
     }
 
     /** @noinspection PhpUnused */
@@ -195,16 +199,16 @@ class Anime extends JikanApiSearchableModel
             return $query;
         }
 
-        $producers = explode(',', $value);
+        $producers = collect(explode(',', $value))->filter()->toArray();
+        $orFilters = [];
         foreach ($producers as $producer) {
-            if (empty($producer)) {
-                continue;
-            }
-
-            $query = $this->filterByProducer($query, $value);
+            $producer = (int)$producer;
+            $orFilters[] = ['producers.mal_id' => $producer];
+            $orFilters[] = ['licensors.mal_id' => $producer];
+            $orFilters[] = ['studios.mal_id' => $producer];
         }
-
-        return $query;
+        /** @noinspection PhpParamsInspection */
+        return $query->whereRaw(['$or' => $orFilters]);
     }
 
     /** @noinspection PhpUnused */

app/Dto/QueryAnimeSchedulesCommand.php

@@ -28,8 +28,7 @@ final class QueryAnimeSchedulesCommand extends Data implements DataRequest
 
     #[
         WithCast(EnumCast::class, AnimeScheduleFilterEnum::class),
-        EnumValidation(AnimeScheduleFilterEnum::class),
-        MapOutputName("filter")
+        EnumValidation(AnimeScheduleFilterEnum::class)
     ]
-    public ?AnimeScheduleFilterEnum $dayFilter;
+    public ?AnimeScheduleFilterEnum $filter;
 }

app/Dto/SearchCommand.php

@@ -7,7 +7,6 @@ use App\Dto\Concerns\HasLimitParameter;
 use App\Dto\Concerns\HasPageParameter;
 use App\Enums\SortDirection;
 use App\Rules\Attributes\EnumValidation;
-use Spatie\Enum\Laravel\Rules\EnumRule;
 use Spatie\LaravelData\Attributes\Validation\Alpha;
 use Spatie\LaravelData\Attributes\Validation\Max;
 use Spatie\LaravelData\Attributes\Validation\Prohibits;

app/Enums/AnimeOrderByEnum.php

@@ -7,8 +7,6 @@ use Spatie\Enum\Laravel\Enum;
 /**
  * @method static self mal_id()
  * @method static self title()
- * @method static self type()
- * @method static self rating()
  * @method static self start_date()
  * @method static self end_date()
  * @method static self episodes()
@@ -23,7 +21,7 @@ use Spatie\Enum\Laravel\Enum;
  *   schema="anime_search_query_orderby",
  *   description="Available Anime order_by properties",
  *   type="string",
- *   enum={"mal_id", "title", "type", "rating", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" }
+ *   enum={"mal_id", "title", "start_date", "end_date", "episodes", "score", "scored_by", "rank", "popularity", "members", "favorites" }
  * )
  */
 final class AnimeOrderByEnum extends Enum

app/Features/QueryAnimeSchedulesHandler.php

@@ -25,7 +25,7 @@ final class QueryAnimeSchedulesHandler implements RequestHandler
     {
         $requestParams = collect($request->all());
         $limit = $requestParams->get("limit");
-        $results = $this->repository->getCurrentlyAiring($request->dayFilter);
+        $results = $this->repository->getCurrentlyAiring($request->filter);
         // apply sfw, kids and unapproved filters
         /** @noinspection PhpUndefinedMethodInspection */
         $results = $results->filter($requestParams);

app/Features/SearchRequestHandler.php

@@ -10,6 +10,7 @@ use Illuminate\Http\Resources\Json\JsonResource;
 use Illuminate\Http\Resources\Json\ResourceCollection;
 use Illuminate\Http\Response;
 use Illuminate\Support\Collection;
+use Illuminate\Validation\ValidationException;
 use Spatie\Enum\Laravel\Enum;
 
 /**
@@ -30,6 +31,13 @@ abstract class SearchRequestHandler implements RequestHandler
     {
         // note: ->all() doesn't transform the dto, all the parsed data is returned as it was parsed. (and validated)
         $requestData = collect($request->all());
+        $prohibitedSearchCharacters = collect(["\n", "\\n", "\r", "\t", "\0", "%0A"]);
+        if (in_array($requestData->get("q", ""), $prohibitedSearchCharacters->toArray())
+            || $prohibitedSearchCharacters->filter(fn($value) => strpos($requestData->get("q", ""), $value) !== false)->count() > 0) {
+            throw ValidationException::withMessages([
+                "q" => "The q parameter cannot contain any of the following characters: \\n, \\r, \\t, \\0, %0A"
+            ]);
+        }
         $builder = $this->queryBuilderService->query(
             $this->prepareOrderByParam($requestData)
         );

app/Manga.php

@@ -101,7 +101,7 @@ class Manga extends JikanApiSearchableModel
 
         $magazine = (int)$value;
         return $query
-            ->orWhere('serializations.mal_id', $magazine);
+            ->where('serializations.mal_id', $magazine);
     }
 
     /** @noinspection PhpUnused */
@@ -111,16 +111,14 @@ class Manga extends JikanApiSearchableModel
             return $query;
         }
 
-        $magazines = explode(',', $value);
-        foreach ($magazines as $magazine) {
-            if (empty($magazine)) {
-                continue;
-            }
+        $magazines = collect(explode(',', $value))->filter()->map(fn($x) => (int)$x)->toArray();
 
-            $query = $this->filterByMagazine($query, $value);
-        }
-
-        return $query;
+        /** @noinspection PhpParamsInspection */
+        return $query->whereRaw([
+            "serializations.mal_id" => [
+                '$in' => $magazines
+            ]
+        ]);
     }
 
     /** @noinspection PhpUnused */

app/Rules/MaxResultsPerPageRule.php

@@ -3,8 +3,6 @@
 namespace App\Rules;
 
 use Illuminate\Contracts\Validation\Rule;
-use Illuminate\Support\Env;
-use Illuminate\Support\Facades\App;
 
 final class MaxResultsPerPageRule implements Rule
 {
@@ -37,7 +35,7 @@ final class MaxResultsPerPageRule implements Rule
 
     public function message(): array|string
     {
-        $mrpp = max_results_per_page();
+        $mrpp = max_results_per_page($this->fallbackLimit);
         return "Value {$this->value} is higher than the configured '$mrpp' max value.";
     }
 }

app/Services/DefaultQueryBuilderService.php

@@ -24,6 +24,9 @@ final class DefaultQueryBuilderService implements QueryBuilderService
                 ->search($requestParameters->get("q"), $searchEngineOptions["order_by"], $searchEngineOptions["sort_direction_descending"]);
         } else {
             $builder = $this->searchService->setFilterParameters($requestParameters)->query();
+            if (!$requestParameters->has("order_by")) {
+                $builder = $builder->orderBy("mal_id");
+            }
         }
 
         return $builder;

app/Services/DefaultSearchAnalyticsService.php

@@ -4,11 +4,12 @@ namespace App\Services;
 use App\SearchMetric;
 use App\Contracts\SearchAnalyticsService;
 use Illuminate\Support\Collection;
+use Typesense\Documents;
 
 
 /**
  * The default search analytics service implementation, which saves the stats to the database and indexes it in Typesense.
- * 
+ *
  * By indexing search terms in Typesense we can use it to provide search suggestions of popular searches.
  * @package App\Services
  */
@@ -17,13 +18,22 @@ final class DefaultSearchAnalyticsService implements SearchAnalyticsService
     public function logSearch(string $searchTerm, int $hitsCount, Collection $hits, string $indexName): void
     {
         /**
-         * @var \Laravel\Scout\Builder $existingMetrics
+         * @var Collection $existingMetrics
          */
-        $existingMetrics = SearchMetric::search($searchTerm);
+        $existingMetrics = SearchMetric::search($searchTerm, function (Documents $documents, string $query, array $options) {
+            if (strlen($query) <= 3) {
+                $options['prioritize_token_position'] = 'true';
+            }
+
+            return $documents->search($options);
+        })->take(1)->get();
 
         $hitList = $hits->pluck("id")->values()->map(fn($x) => (int)$x)->all();
 
         if ($existingMetrics->count() > 0) {
+            /**
+             * @var SearchMetric $metric
+             */
             $metric = $existingMetrics->first();
             $metric->hits = $hitList;
             $metric->hits_count = $hitsCount;

app/Services/TypeSenseScoutSearchService.php

@@ -61,7 +61,6 @@ class TypeSenseScoutSearchService implements ScoutSearchService
                 $options['per_page'] = min($this->maxItemsPerPage, 250);
             }
 
-            $options = $this->skipTypoCheckingForShortQueries($query, $options);
             $modelInstance = $this->repository->createEntity();
 
             if ($modelInstance instanceof JikanApiSearchableModel) {
@@ -69,6 +68,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService
                 $options = $this->setSortOrder($options, $modelInstance);
                 $options = $this->overrideSortingOrder($options, $modelInstance, $orderByField, $sortDirectionDescending);
             }
+            $options = $this->adaptToShortQueries($query, $options);
 
             $results = $documents->search($options);
             $this->recordSearchTelemetry($query, $results);
@@ -77,7 +77,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService
         };
     }
 
-    private function skipTypoCheckingForShortQueries(string $query, array $options): array
+    private function adaptToShortQueries(string $query, array $options): array
     {
         if (strlen($query) <= 3) {
             $options['num_typos'] = 0;
@@ -85,7 +85,18 @@ class TypeSenseScoutSearchService implements ScoutSearchService
             $options['drop_tokens_threshold'] = 0;
             $options['exhaustive_search'] = 'false';
             $options['infix'] = 'off';
-            $options['prefix'] = 'false';
+            $options['prioritize_token_position'] = 'true';
+
+            if (Str::startsWith($options["sort_by"], "_text_match")) {
+                $options["sort_by"] = "_text_match:desc,title:asc";
+            } else {
+                // move text_match to the beginning if there is an orderby parameter set.
+                $options["sort_by"] = Str::replace("(buckets:". $this->jikanConfig->textMatchBuckets().")", "", $options["sort_by"]);
+                $parts = collect(explode(",", $options["sort_by"]));
+                $last = $parts->pop();
+                $parts = $parts->prepend($last);
+                $options["sort_by"] = $parts->implode(",");
+            }
         }
 
         return $options;
@@ -141,7 +152,7 @@ class TypeSenseScoutSearchService implements ScoutSearchService
 
         // override ordering field
         if (!is_null($orderByField) && in_array($orderByField, $modelAttrNames)) {
-            $options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->maxItemsPerPage."):desc";
+            $options['sort_by'] = "$orderByField:" . ($sortDirectionDescending ? "desc" : "asc") . ",_text_match(buckets:".$this->jikanConfig->textMatchBuckets()."):desc";
         }
 
         // override overall sorting direction

app/Support/JikanConfig.php

@@ -3,6 +3,7 @@
 namespace App\Support;
 
 use Illuminate\Support\Collection;
+use Illuminate\Support\Arr;
 
 /**
  * Jikan behavior config
@@ -32,16 +33,15 @@ final class JikanConfig
 
     public function __construct(array $config)
     {
-        $config = collect($config);
-        $this->perEndpointCacheTtl = $config->get("per_endpoint_cache_ttl", []);
-        $this->defaultCacheExpire = $config->get("default_cache_expire", 0);
-        $this->microCachingEnabled = in_array($config->get("micro_caching_enabled", false), [true, 1, "1", "true"]);
-        $this->textMatchBuckets = $config->get("typesense_options.text_match_buckets", 85);
-        $this->exhaustiveSearch = (string) $config->get("typesense_options.exhaustive_search", "false");
-        $this->config = $config;
-        $this->typoTokensThreshold = $config->get("typesense_options.typo_tokens_threshold", $this->maxResultsPerPage());
-        $this->dropTokensThreshold = $config->get("typesense_options.drop_tokens_threshold", $this->maxResultsPerPage());
-        $this->searchCutOffMs = $config->get("typesense_options.search_cutoff_ms", 450);
+        $this->perEndpointCacheTtl = Arr::get($config, "per_endpoint_cache_ttl", []);
+        $this->defaultCacheExpire = Arr::get($config, "default_cache_expire", 0);
+        $this->microCachingEnabled = in_array(Arr::get($config, "micro_caching_enabled", false), [true, 1, "1", "true"]);
+        $this->textMatchBuckets = Arr::get($config,"typesense_options.text_match_buckets", 1);
+        $this->exhaustiveSearch = (string) Arr::get($config, "typesense_options.exhaustive_search", "false");
+        $this->config = collect($config);
+        $this->typoTokensThreshold = Arr::get($config, "typesense_options.typo_tokens_threshold") ?? $this->maxResultsPerPage();
+        $this->dropTokensThreshold = Arr::get($config, "typesense_options.drop_tokens_threshold") ?? $this->maxResultsPerPage();
+        $this->searchCutOffMs = Arr::get($config, "typesense_options.search_cutoff_ms", 450);
     }
 
     public function cacheTtlForEndpoint(string $endpoint): ?int
@@ -61,7 +61,7 @@ final class JikanConfig
 
     public function maxResultsPerPage(?int $defaultValue = null): int
     {
-        return $this->config->get("max_results_per_page", $defaultValue ?? 25);
+        return (int) $this->config->get("max_results_per_page", $defaultValue ?? 25);
     }
 
     public function textMatchBuckets(): int

config/jikan.php

@@ -1,14 +1,14 @@
 <?php
 
 return [
-    'max_results_per_page' => (int) env('MAX_RESULTS_PER_PAGE', 25),
+    'max_results_per_page' => env('MAX_RESULTS_PER_PAGE', 25),
     'micro_caching_enabled' => env('MICROCACHING', false),
     'default_cache_expire' => env('CACHE_DEFAULT_EXPIRE', 86400),
     'typesense_options' => [
-        'text_match_buckets' => env('TYPESENSE_TEXT_MATCH_BUCKETS', 85),
-        'typo_tokens_threshold' => (int) env('TYPESENSE_TYPO_TOKENS_THRESHOLD'),
-        'drop_tokens_threshold' => (int) env('TYPESENSE_DROP_TOKENS_THRESHOLD'),
-        'search_cutoff_ms' => (int) env('TYPESENSE_SEARCH_CUTOFF_MS', 450),
+        'text_match_buckets' => env('TYPESENSE_TEXT_MATCH_BUCKETS', 1),
+        'typo_tokens_threshold' => env('TYPESENSE_TYPO_TOKENS_THRESHOLD'),
+        'drop_tokens_threshold' => env('TYPESENSE_DROP_TOKENS_THRESHOLD'),
+        'search_cutoff_ms' => env('TYPESENSE_SEARCH_CUTOFF_MS', 450),
         'exhaustive_search' => env('TYPESENSE_ENABLE_EXHAUSTIVE_SEARCH', 'false')
     ],
     'per_endpoint_cache_ttl' => [

routes/web.v4.php

@@ -269,7 +269,7 @@ $router->group(
     }
 );
 
-$router->get('schedules[/{dayFilter:[A-Za-z]+}]', [
+$router->get('schedules[/{filter:[A-Za-z]+}]', [
     'uses' => 'ScheduleController@main'
 ]);