Update HomeFeedPipeline, fix tag filtering

.editorconfig

@@ -1,8 +1,8 @@
 root = true
 
 [*]
+indent_style = space
 indent_size = 4
-indent_style = tab
 end_of_line = lf
 charset = utf-8
 trim_trailing_whitespace = true

app/Jobs/HomeFeedPipeline/HashtagInsertFanoutPipeline.php

@@ -12,6 +12,7 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\Services\HashtagFollowService;
 use App\Services\HomeTimelineService;
+use App\Services\StatusService;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
@@ -72,11 +73,21 @@ class HashtagInsertFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
     public function handle(): void
     {
         $hashtag = $this->hashtag;
+        $sid = $hashtag->status_id;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
 
         $ids = HashtagFollowService::getPidByHid($hashtag->hashtag_id);
 
         if(!$ids || !count($ids)) {
-        	return;
+            return;
         }
 
         foreach($ids as $id) {

app/Jobs/HomeFeedPipeline/HashtagRemoveFanoutPipeline.php

@@ -12,6 +12,7 @@ use App\Hashtag;
 use App\StatusHashtag;
 use App\Services\HashtagFollowService;
 use App\Services\HomeTimelineService;
+use App\Services\StatusService;
 use Illuminate\Queue\Middleware\WithoutOverlapping;
 use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
 
@@ -68,6 +69,15 @@ class HashtagRemoveFanoutPipeline implements ShouldQueue, ShouldBeUniqueUntilPro
     {
         $sid = $this->sid;
         $hid = $this->hid;
+        $status = StatusService::get($sid, false);
+
+        if(!$status) {
+            return;
+        }
+
+        if(!in_array($status['pf_type'], ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            return;
+        }
 
         $ids = HashtagFollowService::getPidByHid($hid);
 

app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php

@@ -48,7 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
     {
         $hid = $this->hid;
         $pid = $this->pid;
-        $slug = $this->slug;
+        $slug = strtolower($this->slug);
 
         $statusIds = HomeTimelineService::get($pid, 0, -1);
 
@@ -59,17 +59,17 @@ class HashtagUnfollowPipeline implements ShouldQueue
 
         foreach($statusIds as $id) {
             $status = StatusService::get($id, false);
-            if(!$status) {
+            if(!$status || empty($status['tags'])) {
                 HomeTimelineService::rem($pid, $id);
                 continue;
             }
-            $following = in_array($status['account']['id'], $followingIds);
-            if($following || !isset($status['tags'])) {
+            $following = in_array((int) $status['account']['id'], $followingIds);
+            if($following === true) {
                 continue;
             }
 
             $tags = collect($status['tags'])->map(function($tag) {
-                return $tag['name'];
+                return strtolower($tag['name']);
             })->filter()->values()->toArray();
 
             if(in_array($slug, $tags)) {

app/Jobs/StatusPipeline/StatusEntityLexer.php

@@ -19,6 +19,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use App\Services\StatusService;
 use App\Services\UserFilterService;
 use App\Services\AdminShadowFilterService;
 use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
@@ -26,87 +27,87 @@ use App\Jobs\HomeFeedPipeline\HashtagInsertFanoutPipeline;
 
 class StatusEntityLexer implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
-
-	protected $status;
-	protected $entities;
-	protected $autolink;
-
-	/**
-	 * Delete the job if its models no longer exist.
-	 *
-	 * @var bool
-	 */
-	public $deleteWhenMissingModels = true;
-
-	/**
-	 * Create a new job instance.
-	 *
-	 * @return void
-	 */
-	public function __construct(Status $status)
-	{
-		$this->status = $status;
-	}
-
-	/**
-	 * Execute the job.
-	 *
-	 * @return void
-	 */
-	public function handle()
-	{
-		$profile = $this->status->profile;
-		$status = $this->status;
-
-		if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
-			$profile->status_count = $profile->status_count + 1;
-			$profile->save();
-		}
-
-		if($profile->no_autolink == false) {
-			$this->parseEntities();
-		}
-	}
-
-	public function parseEntities()
-	{
-		$this->extractEntities();
-	}
-
-	public function extractEntities()
-	{
-		$this->entities = Extractor::create()->extract($this->status->caption);
-		$this->autolinkStatus();
-	}
-
-	public function autolinkStatus()
-	{
-		$this->autolink = Autolink::create()->autolink($this->status->caption);
-		$this->storeEntities();
-	}
-
-	public function storeEntities()
-	{
-		$this->storeHashtags();
-		DB::transaction(function () {
-			$status = $this->status;
-			$status->rendered = nl2br($this->autolink);
-			$status->save();
-		});
-	}
-
-	public function storeHashtags()
-	{
-		$tags = array_unique($this->entities['hashtags']);
-		$status = $this->status;
-
-		foreach ($tags as $tag) {
-			if(mb_strlen($tag) > 124) {
-				continue;
-			}
-			DB::transaction(function () use ($status, $tag) {
-				$slug = str_slug($tag, '-', false);
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $status;
+    protected $entities;
+    protected $autolink;
+
+    /**
+     * Delete the job if its models no longer exist.
+     *
+     * @var bool
+     */
+    public $deleteWhenMissingModels = true;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct(Status $status)
+    {
+        $this->status = $status;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $profile = $this->status->profile;
+        $status = $this->status;
+
+        if(in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])) {
+            $profile->status_count = $profile->status_count + 1;
+            $profile->save();
+        }
+
+        if($profile->no_autolink == false) {
+            $this->parseEntities();
+        }
+    }
+
+    public function parseEntities()
+    {
+        $this->extractEntities();
+    }
+
+    public function extractEntities()
+    {
+        $this->entities = Extractor::create()->extract($this->status->caption);
+        $this->autolinkStatus();
+    }
+
+    public function autolinkStatus()
+    {
+        $this->autolink = Autolink::create()->autolink($this->status->caption);
+        $this->storeEntities();
+    }
+
+    public function storeEntities()
+    {
+        $this->storeHashtags();
+        DB::transaction(function () {
+            $status = $this->status;
+            $status->rendered = nl2br($this->autolink);
+            $status->save();
+        });
+    }
+
+    public function storeHashtags()
+    {
+        $tags = array_unique($this->entities['hashtags']);
+        $status = $this->status;
+
+        foreach ($tags as $tag) {
+            if(mb_strlen($tag) > 124) {
+                continue;
+            }
+            DB::transaction(function () use ($status, $tag) {
+                $slug = str_slug($tag, '-', false);
 
                 $hashtag = Hashtag::firstOrCreate([
                     'slug' => $slug
@@ -114,92 +115,93 @@ class StatusEntityLexer implements ShouldQueue
                     'name' => $tag
                 ]);
 
-				StatusHashtag::firstOrCreate(
-					[
-						'status_id' => $status->id,
-						'hashtag_id' => $hashtag->id,
-						'profile_id' => $status->profile_id,
-						'status_visibility' => $status->visibility,
-					]
-				);
-			});
-		}
-		$this->storeMentions();
-	}
-
-	public function storeMentions()
-	{
-		$mentions = array_unique($this->entities['mentions']);
-		$status = $this->status;
-
-		foreach ($mentions as $mention) {
-			$mentioned = Profile::whereUsername($mention)->first();
-
-			if (empty($mentioned) || !isset($mentioned->id)) {
-				continue;
-			}
+                StatusHashtag::firstOrCreate(
+                    [
+                        'status_id' => $status->id,
+                        'hashtag_id' => $hashtag->id,
+                        'profile_id' => $status->profile_id,
+                        'status_visibility' => $status->visibility,
+                    ]
+                );
+            });
+        }
+        $this->storeMentions();
+    }
+
+    public function storeMentions()
+    {
+        $mentions = array_unique($this->entities['mentions']);
+        $status = $this->status;
+
+        foreach ($mentions as $mention) {
+            $mentioned = Profile::whereUsername($mention)->first();
+
+            if (empty($mentioned) || !isset($mentioned->id)) {
+                continue;
+            }
             $blocks = UserFilterService::blocks($mentioned->id);
             if($blocks && in_array($status->profile_id, $blocks)) {
                 continue;
             }
 
-			DB::transaction(function () use ($status, $mentioned) {
-				$m = new Mention();
-				$m->status_id = $status->id;
-				$m->profile_id = $mentioned->id;
-				$m->save();
-
-				MentionPipeline::dispatch($status, $m);
-			});
-		}
-		$this->fanout();
-	}
-
-	public function fanout()
-	{
-		$status = $this->status;
-
-		if(config('exp.cached_home_timeline')) {
-			if( $status->in_reply_to_id === null &&
-				in_array($status->scope, ['public', 'unlisted', 'private'])
-			) {
-				FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
-			}
-		}
-		$this->deliver();
-	}
-
-	public function deliver()
-	{
-		$status = $this->status;
-		$types = [
-			'photo',
-			'photo:album',
-			'video',
-			'video:album',
-			'photo:video:album'
-		];
-
-		if(config_cache('pixelfed.bouncer.enabled')) {
-			Bouncer::get($status);
-		}
-
-		Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
-		$hideNsfw = config('instance.hide_nsfw_on_public_feeds');
-		if( $status->uri == null &&
-			$status->scope == 'public' &&
-			in_array($status->type, $types) &&
-			$status->in_reply_to_id === null &&
-			$status->reblog_of_id === null &&
-			($hideNsfw ? $status->is_nsfw == false : true)
-		) {
+            DB::transaction(function () use ($status, $mentioned) {
+                $m = new Mention();
+                $m->status_id = $status->id;
+                $m->profile_id = $mentioned->id;
+                $m->save();
+
+                MentionPipeline::dispatch($status, $m);
+            });
+        }
+        $this->fanout();
+    }
+
+    public function fanout()
+    {
+        $status = $this->status;
+        StatusService::refresh($status->id);
+
+        if(config('exp.cached_home_timeline')) {
+            if( $status->in_reply_to_id === null &&
+                in_array($status->scope, ['public', 'unlisted', 'private'])
+            ) {
+                FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+            }
+        }
+        $this->deliver();
+    }
+
+    public function deliver()
+    {
+        $status = $this->status;
+        $types = [
+            'photo',
+            'photo:album',
+            'video',
+            'video:album',
+            'photo:video:album'
+        ];
+
+        if(config_cache('pixelfed.bouncer.enabled')) {
+            Bouncer::get($status);
+        }
+
+        Cache::forget('pf:atom:user-feed:by-id:' . $status->profile_id);
+        $hideNsfw = config('instance.hide_nsfw_on_public_feeds');
+        if( $status->uri == null &&
+            $status->scope == 'public' &&
+            in_array($status->type, $types) &&
+            $status->in_reply_to_id === null &&
+            $status->reblog_of_id === null &&
+            ($hideNsfw ? $status->is_nsfw == false : true)
+        ) {
             if(AdminShadowFilterService::canAddToPublicFeedByProfileId($status->profile_id)) {
-    			PublicTimelineService::add($status->id);
+                PublicTimelineService::add($status->id);
             }
-		}
+        }
 
-		if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
-			StatusActivityPubDeliver::dispatch($status);
-		}
-	}
+        if(config_cache('federation.activitypub.enabled') == true && config('app.env') == 'production') {
+            StatusActivityPubDeliver::dispatch($status);
+        }
+    }
 }

app/Jobs/StatusPipeline/StatusTagsPipeline.php

@@ -20,117 +20,119 @@ use App\Util\ActivityPub\Helpers;
 
 class StatusTagsPipeline implements ShouldQueue
 {
-	use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
-
-	protected $activity;
-	protected $status;
-
-	/**
-	 * Create a new job instance.
-	 *
-	 * @return void
-	 */
-	public function __construct($activity, $status)
-	{
-		$this->activity = $activity;
-		$this->status = $status;
-	}
-
-	/**
-	 * Execute the job.
-	 *
-	 * @return void
-	 */
-	public function handle()
-	{
-		$res = $this->activity;
-		$status = $this->status;
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $activity;
+    protected $status;
+
+    /**
+     * Create a new job instance.
+     *
+     * @return void
+     */
+    public function __construct($activity, $status)
+    {
+        $this->activity = $activity;
+        $this->status = $status;
+    }
+
+    /**
+     * Execute the job.
+     *
+     * @return void
+     */
+    public function handle()
+    {
+        $res = $this->activity;
+        $status = $this->status;
 
         if(isset($res['tag']['type'], $res['tag']['name'])) {
             $res['tag'] = [$res['tag']];
         }
 
-		$tags = collect($res['tag']);
+        $tags = collect($res['tag']);
 
-		// Emoji
-		$tags->filter(function($tag) {
-			return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
-		})
-		->map(function($tag) {
-			CustomEmojiService::import($tag['id'], $this->status->id);
-		});
+        // Emoji
+        $tags->filter(function($tag) {
+            return $tag && isset($tag['id'], $tag['icon'], $tag['name'], $tag['type']) && $tag['type'] == 'Emoji';
+        })
+        ->map(function($tag) {
+            CustomEmojiService::import($tag['id'], $this->status->id);
+        });
 
-		// Hashtags
-		$tags->filter(function($tag) {
-			return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
-		})
-		->map(function($tag) use($status) {
-			$name = substr($tag['name'], 0, 1) == '#' ?
-				substr($tag['name'], 1) : $tag['name'];
+        // Hashtags
+        $tags->filter(function($tag) {
+            return $tag && $tag['type'] == 'Hashtag' && isset($tag['href'], $tag['name']);
+        })
+        ->map(function($tag) use($status) {
+            $name = substr($tag['name'], 0, 1) == '#' ?
+                substr($tag['name'], 1) : $tag['name'];
 
-			$banned = TrendingHashtagService::getBannedHashtagNames();
+            $banned = TrendingHashtagService::getBannedHashtagNames();
 
-			if(count($banned)) {
+            if(count($banned)) {
                 if(in_array(strtolower($name), array_map('strtolower', $banned))) {
-                   	return;
+                    return;
                 }
             }
 
             if(config('database.default') === 'pgsql') {
-            	$hashtag = Hashtag::where('name', 'ilike', $name)
-            		->orWhere('slug', 'ilike', str_slug($name, '-', false))
-            		->first();
-
-				if(!$hashtag) {
-					$hashtag = Hashtag::updateOrCreate([
-						'slug' => str_slug($name, '-', false),
-						'name' => $name
-					]);
-				}
+                $hashtag = Hashtag::where('name', 'ilike', $name)
+                    ->orWhere('slug', 'ilike', str_slug($name, '-', false))
+                    ->first();
+
+                if(!$hashtag) {
+                    $hashtag = Hashtag::updateOrCreate([
+                        'slug' => str_slug($name, '-', false),
+                        'name' => $name
+                    ]);
+                }
             } else {
-				$hashtag = Hashtag::updateOrCreate([
-					'slug' => str_slug($name, '-', false),
-					'name' => $name
-				]);
+                $hashtag = Hashtag::updateOrCreate([
+                    'slug' => str_slug($name, '-', false),
+                    'name' => $name
+                ]);
             }
 
-			StatusHashtag::firstOrCreate([
-				'status_id' => $status->id,
-				'hashtag_id' => $hashtag->id,
-				'profile_id' => $status->profile_id,
-				'status_visibility' => $status->scope
-			]);
-		});
-
-		// Mentions
-		$tags->filter(function($tag) {
-			return $tag &&
-				$tag['type'] == 'Mention' &&
-				isset($tag['href']) &&
-				substr($tag['href'], 0, 8) === 'https://';
-		})
-		->map(function($tag) use($status) {
-			if(Helpers::validateLocalUrl($tag['href'])) {
-				$parts = explode('/', $tag['href']);
-				if(!$parts) {
-					return;
-				}
-				$pid = AccountService::usernameToId(end($parts));
-				if(!$pid) {
-					return;
-				}
-			} else {
-				$acct = Helpers::profileFetch($tag['href']);
-				if(!$acct) {
-					return;
-				}
-				$pid = $acct->id;
-			}
-			$mention = new Mention;
-			$mention->status_id = $status->id;
-			$mention->profile_id = $pid;
-			$mention->save();
-			MentionPipeline::dispatch($status, $mention);
-		});
-	}
+            StatusHashtag::firstOrCreate([
+                'status_id' => $status->id,
+                'hashtag_id' => $hashtag->id,
+                'profile_id' => $status->profile_id,
+                'status_visibility' => $status->scope
+            ]);
+        });
+
+        // Mentions
+        $tags->filter(function($tag) {
+            return $tag &&
+                $tag['type'] == 'Mention' &&
+                isset($tag['href']) &&
+                substr($tag['href'], 0, 8) === 'https://';
+        })
+        ->map(function($tag) use($status) {
+            if(Helpers::validateLocalUrl($tag['href'])) {
+                $parts = explode('/', $tag['href']);
+                if(!$parts) {
+                    return;
+                }
+                $pid = AccountService::usernameToId(end($parts));
+                if(!$pid) {
+                    return;
+                }
+            } else {
+                $acct = Helpers::profileFetch($tag['href']);
+                if(!$acct) {
+                    return;
+                }
+                $pid = $acct->id;
+            }
+            $mention = new Mention;
+            $mention->status_id = $status->id;
+            $mention->profile_id = $pid;
+            $mention->save();
+            MentionPipeline::dispatch($status, $mention);
+        });
+
+        StatusService::refresh($status->id);
+    }
 }