Skip to content

[WIP] Hotfixes for 2025 #569

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pushrbx wants to merge 8 commits into
base: master
Choose a base branch
from
Draft

[WIP] Hotfixes for 2025 #569

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
86ee21d
added hotfixes
pushrbx May 1, 2025
c0231c2
fixed dependencies
pushrbx May 1, 2025
a28e147
removed mongoFsync function because it won't work with the default co…
pushrbx May 1, 2025
1a8fe18
add the ability to set an auth mechanism for mongodb
pushrbx May 1, 2025
8bdcf40
fixed validation rules in incremental indexer
pushrbx May 1, 2025
5e1c588
fixed incremental indexer
pushrbx May 1, 2025
fc288a6
wip: search performance upgrade, dropping elasticsearch support
pushrbx May 2, 2025
8d8910a
refactor: standardize query logic for filters, improve TypeSense comp…
pushrbx Sep 16, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ jobs:
name: PHP v8.1 with MongoDB 6.0
concurrency: "ci-${{ github.ref }}"
env:
extensions: curl,intl,mbstring,mongodb-stable,redis,opcache,sockets,pcntl,xdebug
extensions: curl,intl,mbstring,mongodb-1.21.0,redis,opcache,sockets,pcntl,xdebug
key: jikan-rest-ci-cache-v1
phpversion: 8.1
services:
Expand Down
3 changes: 2 additions & 1 deletion Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ COPY --from=composer /usr/bin/composer /usr/bin/composer
COPY --from=php-ext-installer /usr/bin/install-php-extensions /usr/local/bin/
ENV COMPOSER_HOME="/tmp/composer"
RUN set -x \
&& install-php-extensions intl mbstring mongodb-stable redis opcache sockets pcntl \
&& install-php-extensions intl mbstring mongodb-1.21.0 redis opcache sockets pcntl \
# install xdebug (for testing with code coverage), but do not enable it
&& IPE_DONT_ENABLE=1 install-php-extensions xdebug-3.2.0

Expand All @@ -29,6 +29,7 @@ RUN set -ex \
# enable opcache for CLI and JIT, docs: <https://www.php.net/manual/en/opcache.configuration.php#ini.opcache.jit>
&& echo -e "\nopcache.enable=1\nopcache.enable_cli=1\nopcache.jit_buffer_size=32M\nopcache.jit=1235\n" >> \
${PHP_INI_DIR}/conf.d/docker-php-ext-opcache.ini \
&& echo -e "memory_limit = 256M\n" >> ${PHP_INI_DIR}/conf.d/docker-php-memory-limit-override.ini \
# show php version
&& php -v \
# show installed modules
Expand Down
135 changes: 91 additions & 44 deletions app/Anime.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
use Carbon\CarbonImmutable;
use Database\Factories\AnimeFactory;
use Illuminate\Support\Facades\App;
use Illuminate\Support\Facades\Log;
use Jikan\Helper\Constants;
use Jikan\Jikan;
use Jikan\Request\Anime\AnimeRequest;
Expand Down Expand Up @@ -154,25 +153,32 @@ public function filterByRating(\Laravel\Scout\Builder|\Illuminate\Database\Eloqu
/** @noinspection PhpUnused */
public function filterByStartDate(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query, CarbonImmutable $value): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $query
->where("aired.from", ">=",
$value
->setTime(0, 0)
->setTimezone(new \DateTimeZone('UTC'))
->toAtomString()
);
$compareVal = $value->setTime(0, 0)->setTimezone(new \DateTimeZone('UTC'));
if (is_scout_query_builder($query)) {
$query = $query->where("start_date", [">=", $compareVal->getTimestamp()]);
} else {
$query = $query->where("aired.from", ">=", $compareVal->toAtomString());
}
return $query;
}

/** @noinspection PhpUnused */
public function filterByEndDate(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query, CarbonImmutable $value): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $query
->where("aired.to", "<=",
$value
->setTime(0, 0)
->setTimezone(new \DateTimeZone('UTC'))
->toAtomString()
);
$compareVal = $value->setTime(0, 0)->setTimezone(new \DateTimeZone('UTC'));
if (is_scout_query_builder($query)) {
$query = $query->where("(end_date", [
"<=", $compareVal->getTimestamp(),
" || end_date:=", "null",
")"
]);
} else {
$query = $query->where(function (\Jenssegers\Mongodb\Eloquent\Builder $query) use ($compareVal) {
return $query->where("aired.to", "<=", $compareVal->toAtomString())
->orWhere("aired.to", null);
});
}
return $query;
}

public function filterByProducer(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query, string $value): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
Expand All @@ -182,11 +188,21 @@ public function filterByProducer(\Laravel\Scout\Builder|\Illuminate\Database\Elo
}

$producer = (int)$value;
return $query->where(function (\Jenssegers\Mongodb\Eloquent\Builder $query) use ($producer) {
return $query->where('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer);
});
if (is_scout_query_builder($query)) {
$query = $query->where('(producers', [
"=", $producer,
" || licensors:=", $producer,
" || studios:=", $producer,
")"
]);
} else {
$query = $query->where(function (\Jenssegers\Mongodb\Eloquent\Builder $query) use ($producer) {
return $query->where('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer);
});
}
return $query;
}

/** @noinspection PhpUnused */
Expand All @@ -199,45 +215,74 @@ public function filterByProducers(\Laravel\Scout\Builder|\Illuminate\Database\El
/* @var \Illuminate\Support\Collection $producers */
$producers = collect(explode(',', $value))->filter();

return $query->where(function (\Jenssegers\Mongodb\Eloquent\Builder $query) use ($producers) {
$firstProducer = (int)$producers->first();
$query = $query->where('producers.mal_id', $firstProducer)
->orWhere('licensors.mal_id', $firstProducer)
->orWhere('studios.mal_id', $firstProducer);

foreach ($producers->skip(1) as $producer) {
$producer = (int)$producer;
$query = $query->orWhere('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer);
}
if (is_scout_query_builder($query)) {
$producerFilterList = '[' . $producers->implode(', ') . ']';
$query = $query->where('(producers', [
'=', $producerFilterList,
' || licensors:=', $producerFilterList,
' || studios:=', $producerFilterList,
')'
]);
} else {
$query = $query->where(function (\Jenssegers\Mongodb\Eloquent\Builder $query) use ($producers) {
$firstProducer = (int)$producers->first();
$query = $query->where('producers.mal_id', $firstProducer)
->orWhere('licensors.mal_id', $firstProducer)
->orWhere('studios.mal_id', $firstProducer);

foreach ($producers->skip(1) as $producer) {
$producer = (int)$producer;
$query = $query->orWhere('producers.mal_id', $producer)
->orWhere('licensors.mal_id', $producer)
->orWhere('studios.mal_id', $producer);
}

return $query;
});
}

return $query;
});
return $query;
}

/** @noinspection PhpUnused */
public function scopeExceptItemsWithAdultRating(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $query
->where("demographics.mal_id", "!=", Constants::GENRE_ANIME_HENTAI)
->where("demographics.mal_id", "!=", Constants::GENRE_ANIME_EROTICA)
->where("rating", "!=", AnimeRatingEnum::rx()->label)
->where("genres.mal_id", "!=", Constants::GENRE_ANIME_HENTAI);
if (is_scout_query_builder($query)) {
$query = $query
->where("demographics", ["!=", Constants::GENRE_ANIME_HENTAI])
->where("demographics", ["!=", Constants::GENRE_ANIME_EROTICA])
->where("rating", ["!=", AnimeRatingEnum::rx()->label])
->where("genres", ["!=", Constants::GENRE_ANIME_HENTAI]);
} else {
$query = $query
->where("demographics.mal_id", "!=", Constants::GENRE_ANIME_HENTAI)
->where("demographics.mal_id", "!=", Constants::GENRE_ANIME_EROTICA)
->where("rating", "!=", AnimeRatingEnum::rx()->label)
->where("genres.mal_id", "!=", Constants::GENRE_ANIME_HENTAI);
}
return $query;
}

/** @noinspection PhpUnused */
public function scopeExceptKidsItems(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $query
->where("demographics.mal_id", "!=", Constants::GENRE_ANIME_KIDS);
if (is_scout_query_builder($query)) {
$query = $query->where("demographics", ["!=", Constants::GENRE_ANIME_KIDS]);
} else {
$query = $query->where("demographics.mal_id", "!=", Constants::GENRE_ANIME_KIDS);
}
return $query;
}

/** @noinspection PhpUnused */
public function scopeOnlyKidsItems(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return $query
->where("demographics.mal_id", Constants::GENRE_ANIME_KIDS);
if (is_scout_query_builder($query)) {
$query = $query->where("demographics", Constants::GENRE_ANIME_KIDS);
} else {
$query = $query->where("demographics.mal_id", Constants::GENRE_ANIME_KIDS);
}
return $query;
}

public static function scrape(int $id)
Expand Down Expand Up @@ -295,7 +340,9 @@ public function toSearchableArray(): array
'studios' => $this->getMalIdsOfField($this->studios),
'licensors' => $this->getMalIdsOfField($this->licensors),
'genres' => $this->getMalIdsOfField($this->genres),
'explicit_genres' => $this->getMalIdsOfField($this->explicit_genres)
'explicit_genres' => $this->getMalIdsOfField($this->explicit_genres),
'themes' => $this->getMalIdsOfField($this->themes),
'demographics' => $this->getMalIdsOfField($this->demographics)
];
}

Expand Down
83 changes: 64 additions & 19 deletions app/Concerns/MediaFilters.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,15 @@ public function filterByMaxScore(\Laravel\Scout\Builder|\Illuminate\Database\Elo
if (floatval($value) == 10) {
return $query;
}

if (is_scout_query_builder($query)) {
if (array_key_exists("score", $query->wheres)) {
$minScoreValue = $query->wheres["score"][1];
$maxScoreValue = floatval($value);
return $query->where("score", ["[", $minScoreValue, "..", $maxScoreValue, "]"]);
}
return $query->where("score", ["<=", floatval($value)]);
}
return $query->where("score", "<=", floatval($value));
}

Expand All @@ -23,11 +32,23 @@ public function filterByMinScore(\Laravel\Scout\Builder|\Illuminate\Database\Elo
if (floatval($value) == 0) {
return $query;
}

if (is_scout_query_builder($query)) {
if (array_key_exists("score", $query->wheres)) {
$maxScoreValue = $query->wheres["score"][1];
$minScoreValue = floatval($value);
return $query->where("score", ["[", $minScoreValue, "..", $maxScoreValue, "]"]);
}
return $query->where("score", [">=", floatval($value)]);
}
return $query->where("score", ">=", floatval($value));
}

public function filterByScore(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query, mixed $value): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
if (is_null($value) || $value === "") {
return $query;
}
return $query->where("score", floatval($value));
}

Expand All @@ -43,17 +64,29 @@ public function filterByGenres(\Laravel\Scout\Builder|\Illuminate\Database\Eloqu
}
$genres = explode(',', $value);

foreach ($genres as $genreItem) {
$genre = (int) $genreItem;
// here we need a nested where clause
// this logically looks like: (genre = x OR demographic = x OR theme = x)
// so: (any other where clauses from before) AND (genre = x OR demographic = x OR theme = x)
$query = $query->where(function ($q) use ($genre) {
return $q->where('genres.mal_id', $genre)
->orWhere('demographics.mal_id', $genre)
->orWhere('themes.mal_id', $genre)
->orWhere('explicit_genres.mal_id', $genre);
});
if (is_scout_query_builder($query)) {
// https://typesense.org/docs/0.24.1/api/search.html#filter-parameters
$genreFilterList = '[' . implode(', ', $genres) . ']';
$query = $query->where('(genres', [
'=', $genreFilterList,
' || demographics:=', $genreFilterList,
' || themes:=', $genreFilterList,
' || explicit_genres:=', $genreFilterList,
')'
]);
} else {
foreach ($genres as $genreItem) {
$genre = (int) $genreItem;
// here we need a nested where clause
// this logically looks like: (genre = x OR demographic = x OR theme = x)
// so: (any other where clauses from before) AND (genre = x OR demographic = x OR theme = x)
$query = $query->where(function ($q) use ($genre) {
return $q->where('genres.mal_id', $genre)
->orWhere('demographics.mal_id', $genre)
->orWhere('themes.mal_id', $genre)
->orWhere('explicit_genres.mal_id', $genre);
});
}
}

return $query;
Expand All @@ -66,13 +99,25 @@ public function filterByGenresExclude(\Laravel\Scout\Builder|\Illuminate\Databas
}
$genres = explode(',', $value);

foreach ($genres as $genreItem) {
$genre = (int) $genreItem;
$query = $query
->where('genres.mal_id', '!=', $genre)
->where('demographics.mal_id', '!=', $genre)
->where('themes.mal_id', '!=', $genre)
->where('explicit_genres.mal_id', '!=', $genre);
if (is_scout_query_builder($query)) {
// https://typesense.org/docs/0.24.1/api/search.html#filter-parameters
$genreFilterList = '[' . implode(', ', $genres) . ']';
$query = $query->where('(genres', [
'!=', $genreFilterList,
' && demographics:!=', $genreFilterList,
' && themes:!=', $genreFilterList,
' && explicit_genres:!=', $genreFilterList,
')'
]);
} else {
foreach ($genres as $genreItem) {
$genre = (int) $genreItem;
$query = $query
->where('genres.mal_id', '!=', $genre)
->where('demographics.mal_id', '!=', $genre)
->where('themes.mal_id', '!=', $genre)
->where('explicit_genres.mal_id', '!=', $genre);
}
}

return $query;
Expand All @@ -87,7 +132,7 @@ public function filterBySfw(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent

public function filterByUnapproved(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query, ?bool $value): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
{
return !$value ? $query->where("approved", true) : $query;
return !$value ? $query instanceof \Laravel\Scout\Builder ? $query->where("approved", 'true') : $query->where("approved", true) : $query;
}

public function filterByKids(\Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder $query, ?bool $value): \Laravel\Scout\Builder|\Illuminate\Database\Eloquent\Builder
Expand Down
Loading
Loading