Add NetworkTimelineService cache

app/Http/Controllers/PublicApiController.php

@@ -32,6 +32,7 @@ use App\Services\{
     LikeService,
     PublicTimelineService,
     ProfileService,
+    NetworkTimelineService,
     ReblogService,
     RelationshipService,
     StatusService,
@@ -608,59 +609,92 @@ class PublicApiController extends Controller
 
         $filtered = $user ? UserFilterService::filters($user->profile_id) : [];
 
-        if($min || $max) {
-            $dir = $min ? '>' : '<';
-            $id = $min ?? $max;
-            $timeline = Status::select(
-                        'id',
-                        'uri',
-                        'type',
-                        'scope',
-                        'created_at',
-                      )
-                      ->where('id', $dir, $id)
-                      ->whereNull(['in_reply_to_id', 'reblog_of_id'])
-                      ->whereNotIn('profile_id', $filtered)
-                      ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
-                      ->whereNotNull('uri')
-                      ->whereScope('public')
-                      ->where('id', '>', $amin)
-                      ->orderBy('created_at', 'desc')
-                      ->limit($limit)
-                      ->get()
-                     ->map(function($s) use ($user) {
-                            $status = StatusService::get($s->id);
-                            $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
-                            $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
-                            $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
-                            return $status;
-                      });
-            $res = $timeline->toArray();
-        } else {
-                $timeline = Status::select(
-                            'id',
-                            'uri',
-                            'type',
-                            'scope',
-                            'created_at',
-                          )
-                      	  ->whereNull(['in_reply_to_id', 'reblog_of_id'])
-                          ->whereNotIn('profile_id', $filtered)
-                          ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
-                          ->whereNotNull('uri')
-                          ->whereScope('public')
-                          ->where('id', '>', $amin)
-                          ->orderBy('created_at', 'desc')
-                          ->limit($limit)
-                          ->get()
-                          ->map(function($s) use ($user) {
-                                $status = StatusService::get($s->id);
-                                $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
-                                $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
-                                $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
-                                return $status;
-                          });
-                $res = $timeline->toArray();
+        if(config('instance.timeline.network.cached') == false) {
+	        if($min || $max) {
+	            $dir = $min ? '>' : '<';
+	            $id = $min ?? $max;
+	            $timeline = Status::select(
+	                        'id',
+	                        'uri',
+	                        'type',
+	                        'scope',
+	                        'created_at',
+	                      )
+	                      ->where('id', $dir, $id)
+	                      ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+	                      ->whereNotIn('profile_id', $filtered)
+	                      ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+	                      ->whereNotNull('uri')
+	                      ->whereScope('public')
+	                      ->where('id', '>', $amin)
+	                      ->orderBy('created_at', 'desc')
+	                      ->limit($limit)
+	                      ->get()
+	                     ->map(function($s) use ($user) {
+	                            $status = StatusService::get($s->id);
+	                            $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
+	                            $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
+	                            $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
+	                            return $status;
+	                      });
+	            $res = $timeline->toArray();
+	        } else {
+	                $timeline = Status::select(
+	                            'id',
+	                            'uri',
+	                            'type',
+	                            'scope',
+	                            'created_at',
+	                          )
+	                      	  ->whereNull(['in_reply_to_id', 'reblog_of_id'])
+	                          ->whereNotIn('profile_id', $filtered)
+	                          ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+	                          ->whereNotNull('uri')
+	                          ->whereScope('public')
+	                          ->where('id', '>', $amin)
+	                          ->orderBy('created_at', 'desc')
+	                          ->limit($limit)
+	                          ->get()
+	                          ->map(function($s) use ($user) {
+	                                $status = StatusService::get($s->id);
+	                                $status['favourited'] = (bool) LikeService::liked($user->profile_id, $s->id);
+	                                $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $s->id);
+	                                $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $s->id);
+	                                return $status;
+	                          });
+	                $res = $timeline->toArray();
+	        }
+	    } else {
+            Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
+                if(NetworkTimelineService::count() == 0) {
+                    NetworkTimelineService::warmCache(true, 400);
+                }
+            });
+
+            if ($max) {
+                $feed = NetworkTimelineService::getRankedMaxId($max, $limit);
+            } else if ($min) {
+                $feed = NetworkTimelineService::getRankedMinId($min, $limit);
+            } else {
+                $feed = NetworkTimelineService::get(0, $limit);
+            }
+
+            $res = collect($feed)
+            ->map(function($k) use($user) {
+                $status = StatusService::get($k);
+                if($status && isset($status['account']) && $user) {
+                    $status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
+                    $status['bookmarked'] = (bool) BookmarkService::get($user->profile_id, $k);
+                    $status['reblogged'] = (bool) ReblogService::get($user->profile_id, $k);
+                    $status['relationship'] = RelationshipService::get($user->profile_id, $status['account']['id']);
+                }
+                return $status;
+            })
+            ->filter(function($s) use($filtered) {
+                return $s && isset($s['account']) && in_array($s['account']['id'], $filtered) == false;
+            })
+            ->values()
+            ->toArray();
         }
 
         return response()->json($res);

app/Services/NetworkTimelineService.php

@@ -0,0 +1,95 @@
+<?php
+
+namespace App\Services;
+
+use Illuminate\Support\Facades\Redis;
+use App\{
+	Profile,
+	Status,
+	UserFilter
+};
+
+class NetworkTimelineService
+{
+	const CACHE_KEY = 'pf:services:timeline:network';
+
+	public static function get($start = 0, $stop = 10)
+	{
+		if($stop > 100) {
+			$stop = 100;
+		}
+
+		return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
+	}
+
+	public static function getRankedMaxId($start = null, $limit = 10)
+	{
+		if(!$start) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
+			'withscores' => true,
+			'limit' => [1, $limit]
+		]));
+	}
+
+	public static function getRankedMinId($end = null, $limit = 10)
+	{
+		if(!$end) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
+			'withscores' => true,
+			'limit' => [0, $limit]
+		]));
+	}
+
+	public static function add($val)
+	{
+		if(self::count() > config('instance.timeline.network.cache_dropoff')) {
+			if(config('database.redis.client') === 'phpredis') {
+				Redis::zpopmin(self::CACHE_KEY);
+			}
+		}
+
+		return Redis::zadd(self::CACHE_KEY, $val, $val);
+	}
+
+	public static function rem($val)
+	{
+		return Redis::zrem(self::CACHE_KEY, $val);
+	}
+
+	public static function del($val)
+	{
+		return self::rem($val);
+	}
+
+	public static function count()
+	{
+		return Redis::zcard(self::CACHE_KEY);
+	}
+
+	public static function warmCache($force = false, $limit = 100)
+	{
+		if(self::count() == 0 || $force == true) {
+			Redis::del(self::CACHE_KEY);
+			$ids = Status::whereNotNull('uri')
+				->whereScope('public')
+				->whereNull('in_reply_to_id')
+				->whereNull('reblog_of_id')
+				->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
+				->where('created_at', '>', now()->subHours(config('instance.timeline.network.max_hours_old')))
+				->orderByDesc('created_at')
+				->limit($limit)
+				->pluck('id');
+			foreach($ids as $id) {
+				self::add($id);
+			}
+			return 1;
+		}
+		return 0;
+	}
+}

app/Util/ActivityPub/Helpers.php

@@ -32,6 +32,7 @@ use App\Services\CustomEmojiService;
 use App\Services\InstanceService;
 use App\Services\MediaPathService;
 use App\Services\MediaStorageService;
+use App\Services\NetworkTimelineService;
 use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Jobs\AvatarPipeline\RemoteAvatarFetch;
 use App\Util\Media\License;
@@ -490,6 +491,16 @@ class Helpers {
 			if(isset($activity['tag']) && is_array($activity['tag']) && !empty($activity['tag'])) {
 				StatusTagsPipeline::dispatch($activity, $status);
 			}
+
+			if( config('instance.timeline.network.cached') &&
+				$status->in_reply_to_id === null &&
+				$status->reblog_of_id === null &&
+				in_array($status->type, ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album']) &&
+				$status->created_at->gt(now()->subHours(config('instance.timeline.network.max_hours_old'))
+			) {
+				NetworkTimelineService::add($status->id);
+			}
+
 			return $status;
 		});
 	}

config/instance.php

@@ -24,6 +24,12 @@ return [
 	'timeline' => [
 		'local' => [
 			'is_public' => env('INSTANCE_PUBLIC_LOCAL_TIMELINE', false)
+		],
+
+		'network' => [
+			'cached' => env('PF_NETWORK_TIMELINE') ? env('INSTANCE_NETWORK_TIMELINE_CACHED', false) : false,
+			'cache_dropoff' => env('INSTANCE_NETWORK_TIMELINE_CACHE_DROPOFF', 100),
+			'max_hours_old' => env('INSTANCE_NETWORK_TIMELINE_CACHE_MAX_HOUR_INGEST', 6)
 		]
 	],