Merge pull request #4770 from pixelfed/staging

Update Experimental Home Feed, fix remote posts, shares and reblogs

app/Http/Controllers/Api/ApiV1Controller.php

@@ -2177,18 +2177,26 @@ class ApiV1Controller extends Controller
 				}
 			}
 
-			$res = collect($res)->take($limit)->map(function($id) use($napi) {
+			$res = collect($res)
+			->map(function($id) use($napi) {
 				return $napi ? StatusService::get($id, false) : StatusService::getMastodon($id, false);
-			})->filter(function($res) {
+			})
+			->filter(function($res) {
 				return $res && isset($res['account']);
-			})->map(function($status) use($pid) {
+			})
+			->filter(function($s) use($includeReblogs) {
+				return $includeReblogs ? true : $s['reblog'] == null;
+			})
+			->take($limit)
+			->map(function($status) use($pid) {
 				if($pid) {
 					$status['favourited'] = (bool) LikeService::liked($pid, $status['id']);
 					$status['reblogged'] = (bool) ReblogService::get($pid, $status['id']);
 					$status['bookmarked'] = (bool) BookmarkService::get($pid, $status['id']);
 				}
 				return $status;
-			});
+			})
+			->values();
 
 			$baseUrl = config('app.url') . '/api/v1/timelines/home?limit=' . $limit . '&';
 			$minId = $res->map(function($s) {
@@ -3823,7 +3831,7 @@ class ApiV1Controller extends Controller
 		if($follows) {
 			HashtagService::unfollow($pid, $tag->id);
 			HashtagFollowService::unfollow($tag->id, $pid);
-			HashtagUnfollowPipeline::dispatch($tag->id, $pid)->onQueue('feed');
+			HashtagUnfollowPipeline::dispatch($tag->id, $pid, $tag->slug)->onQueue('feed');
 			$follows->delete();
 		}
 

app/Jobs/HomeFeedPipeline/FeedInsertRemotePipeline.php

@@ -0,0 +1,73 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\HomeTimelineService;
+
+class FeedInsertRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:insert:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:insert:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::add($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/FeedRemoveRemotePipeline.php

@@ -0,0 +1,74 @@
+<?php
+
+namespace App\Jobs\HomeFeedPipeline;
+
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Queue\Middleware\WithoutOverlapping;
+use Illuminate\Contracts\Queue\ShouldBeUniqueUntilProcessing;
+use App\Services\FollowerService;
+use App\Services\StatusService;
+use App\Services\HomeTimelineService;
+
+class FeedRemoveRemotePipeline implements ShouldQueue, ShouldBeUniqueUntilProcessing
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    protected $sid;
+    protected $pid;
+
+    public $timeout = 900;
+    public $tries = 3;
+    public $maxExceptions = 1;
+    public $failOnTimeout = true;
+
+    /**
+     * The number of seconds after which the job's unique lock will be released.
+     *
+     * @var int
+     */
+    public $uniqueFor = 3600;
+
+    /**
+     * Get the unique ID for the job.
+     */
+    public function uniqueId(): string
+    {
+        return 'hts:feed:remove:remote:sid:' . $this->sid;
+    }
+
+    /**
+     * Get the middleware the job should pass through.
+     *
+     * @return array<int, object>
+     */
+    public function middleware(): array
+    {
+        return [(new WithoutOverlapping("hts:feed:remove:remote:sid:{$this->sid}"))->shared()->dontRelease()];
+    }
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($sid, $pid)
+    {
+        $this->sid = $sid;
+        $this->pid = $pid;
+    }
+
+    /**
+     * Execute the job.
+     */
+    public function handle(): void
+    {
+        $ids = FollowerService::localFollowerIds($this->pid);
+
+        foreach($ids as $id) {
+            HomeTimelineService::rem($id, $this->sid);
+        }
+    }
+}

app/Jobs/HomeFeedPipeline/HashtagUnfollowPipeline.php

@@ -24,6 +24,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
 
     protected $pid;
     protected $hid;
+    protected $slug;
 
     public $timeout = 900;
     public $tries = 3;
@@ -33,10 +34,11 @@ class HashtagUnfollowPipeline implements ShouldQueue
     /**
      * Create a new job instance.
      */
-    public function __construct($hid, $pid)
+    public function __construct($hid, $pid, $slug)
     {
         $this->hid = $hid;
         $this->pid = $pid;
+        $this->slug = $slug;
     }
 
     /**
@@ -46,6 +48,7 @@ class HashtagUnfollowPipeline implements ShouldQueue
     {
         $hid = $this->hid;
         $pid = $this->pid;
+        $slug = $this->slug;
 
         $statusIds = HomeTimelineService::get($pid, 0, -1);
 
@@ -60,7 +63,16 @@ class HashtagUnfollowPipeline implements ShouldQueue
                 HomeTimelineService::rem($pid, $id);
                 continue;
             }
-            if(!in_array($status['account']['id'], $followingIds)) {
+            $following = in_array($status['account']['id'], $followingIds);
+            if($following || !isset($status['tags'])) {
+                continue;
+            }
+
+            $tags = collect($status['tags'])->filter(function($tag) {
+                return $tag['name'];
+            })->toArray();
+
+            if(in_array($slug, $tags)) {
                 HomeTimelineService::rem($pid, $id);
             }
         }

app/Jobs/SharePipeline/SharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedInsertPipeline;
 
 class SharePipeline implements ShouldQueue
 {
@@ -82,6 +83,8 @@ class SharePipeline implements ShouldQueue
 			]
 		);
 
+		FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		return $this->remoteAnnounceDeliver();
 	}
 

app/Jobs/SharePipeline/UndoSharePipeline.php

@@ -17,6 +17,7 @@ use GuzzleHttp\{Pool, Client, Promise};
 use App\Util\ActivityPub\HttpSignature;
 use App\Services\ReblogService;
 use App\Services\StatusService;
+use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
 
 class UndoSharePipeline implements ShouldQueue
 {
@@ -35,6 +36,8 @@ class UndoSharePipeline implements ShouldQueue
 		$actor = $status->profile;
 		$parent = Status::find($status->reblog_of_id);
 
+		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+
 		if($parent) {
 			$target = $parent->profile_id;
 			ReblogService::removePostReblog($parent->profile_id, $status->id);

app/Jobs/StatusPipeline/StatusEntityLexer.php

@@ -161,7 +161,6 @@ class StatusEntityLexer implements ShouldQueue
 
 		if(config('exp.cached_home_timeline')) {
 			if( $status->in_reply_to_id === null &&
-				$status->reblog_of_id === null &&
 				in_array($status->scope, ['public', 'unlisted', 'private'])
 			) {
 				FeedInsertPipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');

app/Observers/HashtagFollowObserver.php

@@ -31,7 +31,6 @@ class HashtagFollowObserver implements ShouldHandleEventsAfterCommit
     public function deleting(HashtagFollow $hashtagFollow): void
     {
         HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
-        HashtagUnfollowPipeline::dispatch($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
     }
 
     /**
@@ -48,6 +47,5 @@ class HashtagFollowObserver implements ShouldHandleEventsAfterCommit
     public function forceDeleted(HashtagFollow $hashtagFollow): void
     {
         HashtagFollowService::unfollow($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
-        HashtagUnfollowPipeline::dispatch($hashtagFollow->hashtag_id, $hashtagFollow->profile_id);
     }
 }

app/Observers/StatusObserver.php

@@ -8,6 +8,7 @@ use Cache;
 use App\Models\ImportPost;
 use App\Services\ImportService;
 use App\Jobs\HomeFeedPipeline\FeedRemovePipeline;
+use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
 
 class StatusObserver
 {
@@ -66,7 +67,11 @@ class StatusObserver
         }
 
         if(config('exp.cached_home_timeline')) {
-        	FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	if($status->uri) {
+        		FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	} else {
+        		FeedRemovePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
+        	}
         }
     }
 

app/Util/ActivityPub/Helpers.php

@@ -35,6 +35,7 @@ use App\Services\MediaStorageService;
 use App\Services\NetworkTimelineService;
 use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
+use App\Jobs\HomeFeedPipeline\FeedInsertRemotePipeline;
 use App\Util\Media\License;
 use App\Models\Poll;
 use Illuminate\Contracts\Cache\LockTimeoutException;
@@ -537,6 +538,8 @@ class Helpers {
 
         IncrementPostCount::dispatch($pid)->onQueue('low');
 
+        FeedInsertRemotePipeline::dispatch($status->id, $pid)->onQueue('feed');
+
         return $status;
     }
 

app/Util/ActivityPub/Inbox.php

@@ -49,6 +49,7 @@ use App\Models\Conversation;
 use App\Models\RemoteReport;
 use App\Jobs\ProfilePipeline\IncrementPostCount;
 use App\Jobs\ProfilePipeline\DecrementPostCount;
+use App\Jobs\HomeFeedPipeline\FeedRemoveRemotePipeline;
 
 class Inbox
 {
@@ -707,6 +708,7 @@ class Inbox
 						if(!$status) {
 							return;
 						}
+						FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
 						RemoteStatusDelete::dispatch($status)->onQueue('high');
 						return;
 					break;
@@ -803,6 +805,7 @@ class Inbox
 				if(!$status) {
 					return;
 				}
+				FeedRemoveRemotePipeline::dispatch($status->id, $status->profile_id)->onQueue('feed');
 				Status::whereProfileId($profile->id)
 					->whereReblogOfId($status->id)
 					->delete();