feat: improve search with word_delimiter_graph (#1086)

This commit is contained in:
Kyrch
2026-02-08 01:46:02 -03:00
committed by GitHub
parent f8786cb6bf
commit 7d676d3010
22 changed files with 223 additions and 92 deletions
+1 -1
View File
@@ -26,7 +26,7 @@ class CreateNewUser implements CreatesNewUsers
public function create(array $input): User
{
Validator::make($input, [
User::ATTRIBUTE_NAME => ['required', 'string', 'max:255', 'alpha_dash', Rule::unique(User::class), new ModerationRule()],
User::ATTRIBUTE_NAME => ['required', 'string', 'max:35', 'alpha_dash', Rule::unique(User::class), new ModerationRule()],
User::ATTRIBUTE_EMAIL => ['required', 'string', 'email', 'max:255', 'indisposable', Rule::unique(User::class)],
User::ATTRIBUTE_PASSWORD => Password::required(),
'terms' => ['required'],
@@ -24,7 +24,7 @@ class UpdateUserProfileInformation implements UpdatesUserProfileInformation
public function update(User $user, array $input): void
{
$validated = Validator::make($input, [
User::ATTRIBUTE_NAME => ['required_without:'.User::ATTRIBUTE_EMAIL, 'string', 'max:255', 'alpha_dash', Rule::unique(User::class)->ignore($user->id), new ModerationRule()],
User::ATTRIBUTE_NAME => ['required_without:'.User::ATTRIBUTE_EMAIL, 'string', 'max:35', 'alpha_dash', Rule::unique(User::class)->ignore($user->id), new ModerationRule()],
User::ATTRIBUTE_EMAIL => ['required_without:'.User::ATTRIBUTE_NAME, 'string', 'email', 'max:255', 'indisposable', Rule::unique(User::class)->ignore($user->id)],
])->validate();
@@ -0,0 +1,26 @@
<?php
declare(strict_types=1);
namespace App\Concerns\Elastic;
use Elastic\Adapter\Indices\Settings;
trait ConfiguresTextAnalyzers
{
protected function configureTextAnalyzers(Settings $settings): void
{
$settings->analysis([
'analyzer' => [
'name_search' => [
'type' => 'custom',
'tokenizer' => 'standard',
'filter' => [
'lowercase',
'word_delimiter_graph',
],
],
],
]);
}
}
+10 -3
View File
@@ -136,6 +136,10 @@ class VideoResource extends BaseResource
TextColumn::make(Video::ATTRIBUTE_ID)
->label(__('filament.fields.base.id')),
TextColumn::make(Video::ATTRIBUTE_FILENAME)
->label(__('filament.fields.video.filename.name'))
->copyableWithMessage(),
TextColumn::make(Video::ATTRIBUTE_RESOLUTION)
->label(__('filament.fields.video.resolution.name')),
@@ -163,9 +167,9 @@ class VideoResource extends BaseResource
->label(__('filament.fields.video.source.name'))
->formatStateUsing(fn (VideoSource $state): ?string => $state->localize()),
TextColumn::make(Video::ATTRIBUTE_FILENAME)
->label(__('filament.fields.video.filename.name'))
->copyableWithMessage(),
TextColumn::make(Video::ATTRIBUTE_TAGS)
->label(__('filament.fields.video.tags.name'))
->sortable(false),
])
->searchable();
}
@@ -179,6 +183,9 @@ class VideoResource extends BaseResource
TextEntry::make(Video::ATTRIBUTE_ID)
->label(__('filament.fields.base.id')),
TextEntry::make(Video::ATTRIBUTE_TAGS)
->label(__('filament.fields.video.tags.name')),
TextEntry::make(Video::ATTRIBUTE_OVERLAP)
->label(__('filament.fields.video.overlap.name'))
->formatStateUsing(fn (VideoOverlap $state): ?string => $state->localize()),
@@ -28,6 +28,6 @@ class VideoTagsField extends StringField
*/
public function resolve($root, array $args, $context, ResolveInfo $resolveInfo): mixed
{
return implode('', $root->getAttribute($this->getColumn()));
return $root->getAttribute($this->getColumn());
}
}
@@ -9,7 +9,6 @@ use App\Contracts\Http\Api\Field\UpdatableField;
use App\Http\Api\Field\StringField;
use App\Http\Api\Schema\Schema;
use App\Models\List\Playlist;
use App\Rules\ModerationRule;
use Illuminate\Http\Request;
class PlaylistDescriptionField extends StringField implements CreatableField, UpdatableField
@@ -28,8 +28,6 @@ class VideoTagsField extends Field implements RenderableField
public function render(Model $model): string
{
$tags = $model->getAttribute($this->getColumn());
return implode('', $tags);
return $model->getAttribute($this->getColumn());
}
}
+34 -30
View File
@@ -198,24 +198,26 @@ class Video extends BaseModel implements Auditable, HasAggregateViews, SoftDelet
*/
protected function priority(): Attribute
{
$priority = intval($this->source?->getPriority());
return Attribute::make(function (): int {
$priority = intval($this->source?->getPriority());
// Videos that play over the episode will likely have compressed audio
if ($this->overlap === VideoOverlap::OVER) {
$priority -= 8;
}
// Videos that play over the episode will likely have compressed audio
if ($this->overlap === VideoOverlap::OVER) {
$priority -= 8;
}
// Videos that transition to or from the episode may have compressed audio
if ($this->overlap === VideoOverlap::TRANS) {
$priority -= 5;
}
// Videos that transition to or from the episode may have compressed audio
if ($this->overlap === VideoOverlap::TRANS) {
$priority -= 5;
}
// De-prioritize hardsubbed videos
if ($this->lyrics || $this->subbed) {
$priority--;
}
// De-prioritize hardsubbed videos
if ($this->lyrics || $this->subbed) {
$priority--;
}
return Attribute::make(fn (): int => $priority);
return $priority;
});
}
/**
@@ -223,25 +225,27 @@ class Video extends BaseModel implements Auditable, HasAggregateViews, SoftDelet
*/
protected function tags(): Attribute
{
$tags = [];
return Attribute::make(function (): string {
$tags = [];
if ($this->nc) {
$tags[] = 'NC';
}
if ($this->source === VideoSource::BD || $this->source === VideoSource::DVD) {
$tags[] = $this->source->localize();
}
if (filled($this->resolution) && $this->resolution !== 720) {
$tags[] = strval($this->resolution);
}
if ($this->nc) {
$tags[] = 'NC';
}
if ($this->source === VideoSource::BD || $this->source === VideoSource::DVD) {
$tags[] = $this->source->localize();
}
if (filled($this->resolution) && $this->resolution !== 720) {
$tags[] = strval($this->resolution);
}
if ($this->subbed) {
$tags[] = 'Subbed';
} elseif ($this->lyrics) {
$tags[] = 'Lyrics';
}
if ($this->subbed) {
$tags[] = 'Subbed';
} elseif ($this->lyrics) {
$tags[] = 'Lyrics';
}
return Attribute::make(fn (): array => $tags);
return implode('', $tags);
});
}
/**
@@ -26,6 +26,12 @@ class ImportModelsSeeder extends Seeder
*/
public function run(): void
{
if (Config::get('app.env') === 'staging') {
$this->command->alert('This seeder is not allowed in staging environments.');
return;
}
$driver = Config::get('scout.driver');
if (blank($driver)) {
$this->command->info('No driver configured for Scout. Skipping models importing.');
@@ -33,6 +39,9 @@ class ImportModelsSeeder extends Seeder
return;
}
$this->command->info('Running elastic migrations');
Artisan::call('elastic:migrate');
$this->scoutImport(Playlist::class);
$this->scoutImport(Anime::class);
$this->scoutImport(AnimeSynonym::class);
@@ -2,21 +2,28 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateAnimeIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('anime', function (Mapping $mapping) {
Index::create('anime', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->long('anime_id');
$mapping->date('created_at');
$mapping->text('name', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -45,6 +52,7 @@ final class CreateAnimeIndex implements MigrationInterface
],
'text' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'type' => [
'type' => 'long',
@@ -2,21 +2,28 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateArtistIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('artists', function (Mapping $mapping) {
Index::createIfNotExists('artists', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->long('artist_id');
$mapping->date('created_at');
$mapping->text('name', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -39,13 +46,31 @@ final class CreateArtistIndex implements MigrationInterface
]);
$mapping->nested('performances', [
'properties' => [
'performance_id' => ['type' => 'long'],
'created_at' => ['type' => 'date'],
'updated_at' => ['type' => 'date'],
'alias' => ['type' => 'text'],
'as' => ['type' => 'text'],
'membership_alias' => ['type' => 'text'],
'membership_as' => ['type' => 'text'],
'performance_id' => [
'type' => 'long',
],
'created_at' => [
'type' => 'date',
],
'updated_at' => [
'type' => 'date',
],
'alias' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'as' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'membership_alias' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'membership_as' => [
'type' => 'text',
'analyzer' => 'name_search',
],
],
]);
$mapping->date('updated_at');
@@ -2,18 +2,24 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateEntryIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('anime_theme_entries', function (Mapping $mapping) {
Index::createIfNotExists('anime_theme_entries', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->text('anime_slug');
$mapping->date('created_at');
$mapping->long('entry_id');
@@ -48,6 +54,7 @@ final class CreateEntryIndex implements MigrationInterface
'name' => [
'type' => 'text',
'copy_to' => ['anime_slug'],
'analyzer' => 'name_search',
],
'season' => [
'type' => 'long',
@@ -70,6 +77,7 @@ final class CreateEntryIndex implements MigrationInterface
'text' => [
'type' => 'text',
'copy_to' => ['synonym_slug'],
'analyzer' => 'name_search',
],
'type' => [
'type' => 'long',
@@ -141,6 +149,11 @@ final class CreateEntryIndex implements MigrationInterface
],
'title' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'title_native' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'updated_at' => [
'type' => 'date',
@@ -2,20 +2,27 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateSeriesIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('series', function (Mapping $mapping) {
Index::createIfNotExists('series', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->date('created_at');
$mapping->text('name', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -42,6 +49,7 @@ final class CreateSeriesIndex implements MigrationInterface
'name' => [
'type' => 'text',
'copy_to' => ['anime_slug'],
'analyzer' => 'name_search',
],
'season' => [
'type' => 'long',
@@ -64,6 +72,7 @@ final class CreateSeriesIndex implements MigrationInterface
'text' => [
'type' => 'text',
'copy_to' => ['synonym_slug'],
'analyzer' => 'name_search',
],
'type' => [
'type' => 'long',
@@ -2,21 +2,28 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateSongIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('songs', function (Mapping $mapping) {
Index::createIfNotExists('songs', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->date('created_at');
$mapping->long('song_id');
$mapping->text('title', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -24,6 +31,7 @@ final class CreateSongIndex implements MigrationInterface
],
]);
$mapping->text('title_native', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -2,18 +2,24 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateSynonymIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('anime_synonyms', function (Mapping $mapping) {
Index::createIfNotExists('anime_synonyms', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->nested('anime', [
'properties' => [
'anime_id' => [
@@ -24,6 +30,7 @@ final class CreateSynonymIndex implements MigrationInterface
],
'name' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'media_format' => [
'type' => 'long',
@@ -49,6 +56,7 @@ final class CreateSynonymIndex implements MigrationInterface
$mapping->date('created_at');
$mapping->long('synonym_id');
$mapping->text('text', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -2,18 +2,24 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateThemeIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('anime_themes', function (Mapping $mapping) {
Index::createIfNotExists('anime_themes', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->nested('anime', [
'properties' => [
'anime_id' => [
@@ -25,6 +31,7 @@ final class CreateThemeIndex implements MigrationInterface
'name' => [
'type' => 'text',
'copy_to' => ['anime_slug'],
'analyzer' => 'name_search',
],
'season' => [
'type' => 'long',
@@ -47,6 +54,7 @@ final class CreateThemeIndex implements MigrationInterface
'text' => [
'type' => 'text',
'copy_to' => ['synonym_slug'],
'analyzer' => 'name_search',
],
'type' => [
'type' => 'long',
@@ -112,9 +120,11 @@ final class CreateThemeIndex implements MigrationInterface
],
'title' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'title_keyword' => [
'type' => 'keyword',
'title_native' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'updated_at' => [
'type' => 'date',
@@ -2,20 +2,27 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateVideoIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('videos', function (Mapping $mapping) {
Index::createIfNotExists('videos', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->text('anime_slug');
$mapping->text('basename', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -58,6 +65,7 @@ final class CreateVideoIndex implements MigrationInterface
'name' => [
'type' => 'text',
'copy_to' => ['anime_slug'],
'analyzer' => 'name_search',
],
'season' => [
'type' => 'long',
@@ -80,6 +88,7 @@ final class CreateVideoIndex implements MigrationInterface
'text' => [
'type' => 'text',
'copy_to' => ['synonym_slug'],
'analyzer' => 'name_search',
],
'type' => [
'type' => 'long',
@@ -152,6 +161,11 @@ final class CreateVideoIndex implements MigrationInterface
],
'title' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'title_native' => [
'type' => 'text',
'analyzer' => 'name_search',
],
'updated_at' => [
'type' => 'date',
@@ -190,6 +204,7 @@ final class CreateVideoIndex implements MigrationInterface
],
]);
$mapping->text('filename', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -2,20 +2,27 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateStudioIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('studios', function (Mapping $mapping) {
Index::createIfNotExists('studios', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->date('created_at');
$mapping->text('name', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -2,18 +2,24 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreatePlaylistIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('playlists', function (Mapping $mapping) {
Index::createIfNotExists('playlists', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->long('playlist_id');
$mapping->date('created_at');
$mapping->text('hashids', [
@@ -24,6 +30,7 @@ final class CreatePlaylistIndex implements MigrationInterface
],
]);
$mapping->text('name', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
@@ -2,21 +2,28 @@
declare(strict_types=1);
use App\Concerns\Elastic\ConfiguresTextAnalyzers;
use Elastic\Adapter\Indices\Mapping;
use Elastic\Adapter\Indices\Settings;
use Elastic\Migrations\Facades\Index;
use Elastic\Migrations\MigrationInterface;
final class CreateProfileIndex implements MigrationInterface
{
use ConfiguresTextAnalyzers;
/**
* Run the migration.
*/
public function up(): void
{
Index::createIfNotExists('profiles', function (Mapping $mapping) {
Index::createIfNotExists('profiles', function (Mapping $mapping, Settings $settings) {
$this->configureTextAnalyzers($settings);
$mapping->long('profile_id');
$mapping->date('created_at');
$mapping->text('name', [
'analyzer' => 'name_search',
'fields' => [
'keyword' => [
'type' => 'keyword',
+3
View File
@@ -1002,6 +1002,9 @@ return [
'help' => 'Set if this video has subtitles of dialogue.',
'name' => 'Subbed',
],
'tags' => [
'name' => 'Tags',
],
'uncen' => [
'help' => 'Set if this video is an uncensored version of a censored sequence.',
'name' => 'Uncensored',
@@ -230,35 +230,3 @@ test('created if open ai fails', function () {
$response->assertCreated();
});
test('validation error when flagged by open ai', function () {
Feature::activate(AllowExternalProfileManagement::class);
Config::set(ValidationConstants::MODERATION_SERVICE_QUALIFIED, ModerationService::OPENAI->value);
Http::fake([
'https://api.openai.com/v1/moderations' => Http::response([
'results' => [
0 => [
'flagged' => true,
],
],
]),
]);
$visibility = Arr::random(ExternalProfileVisibility::cases());
$parameters = array_merge(
ExternalProfile::factory()->raw(),
[ExternalProfile::ATTRIBUTE_VISIBILITY => $visibility->localize()],
);
$user = User::factory()->withPermissions(CrudPermission::CREATE->format(ExternalProfile::class))->createOne();
Sanctum::actingAs($user);
$response = post(route('api.externalprofile.store', $parameters));
$response->assertJsonValidationErrors([
ExternalProfile::ATTRIBUTE_NAME,
]);
});