Merge pull request #2842 from pixelfed/staging

API perf improvements

CHANGELOG.md

@@ -41,6 +41,9 @@
 - Updated PublicApiController, add LikeService to Network timeline. ([82895591](https://github.com/pixelfed/pixelfed/commit/82895591))
 - Updated moderator api, expire cached status in StatusService. ([f215ee26](https://github.com/pixelfed/pixelfed/commit/f215ee26))
 - Updated StatusHashtagService, fix null status bug. ([51a277e1](https://github.com/pixelfed/pixelfed/commit/51a277e1))
+- Updated NotificationService, use zrevrangebyscore for api. ([d43e6d8d](https://github.com/pixelfed/pixelfed/commit/d43e6d8d))
+- Updated ApiV1Controller, use PublicTimelineService. ([f67c67bc](https://github.com/pixelfed/pixelfed/commit/f67c67bc))
+- Updated ApiV1Controller, use ProfileService for verify_credentials. ([352aa573](https://github.com/pixelfed/pixelfed/commit/352aa573))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.0 (2021-06-01)](https://github.com/pixelfed/pixelfed/compare/v0.10.10...v0.11.0)

app/Http/Controllers/Api/ApiV1Controller.php

@@ -49,8 +49,11 @@ use App\Jobs\VideoPipeline\{
 	VideoThumbnail
 };
 use App\Services\{
+	LikeService,
 	NotificationService,
 	MediaPathService,
+	PublicTimelineService,
+	ProfileService,
 	SearchApiV2Service,
 	StatusService,
 	MediaBlocklistService
@@ -116,25 +119,13 @@ class ApiV1Controller extends Controller
 	public function verifyCredentials(Request $request)
 	{
 		abort_if(!$request->user(), 403);
-		$id = $request->user()->id;
+		$id = $request->user()->profile_id;
 
-		if($request->user()->last_active_at) {
-			$key = 'user:last_active_at:id:'.$id;
-			$ttl = now()->addMinutes(5);
-			Cache::remember($key, $ttl, function() use($id) {
-				$user = User::findOrFail($id);
-				$user->last_active_at = now();
-				$user->save();
-				return;
-			});
-		}
+		$res = ProfileService::get($id);
 
-		$profile = Profile::whereNull('status')->whereUserId($id)->firstOrFail();
-		$resource = new Fractal\Resource\Item($profile, new AccountTransformer());
-		$res = $this->fractal->createData($resource)->toArray();
 		$res['source'] = [
-			'privacy' => $profile->is_private ? 'private' : 'public',
-			'sensitive' => $profile->cw ? true : false,
+			'privacy' => $res['locked'] ? 'private' : 'public',
+			'sensitive' => false,
 			'language' => null,
 			'note' => '',
 			'fields' => []
@@ -1283,7 +1274,6 @@ class ApiV1Controller extends Controller
 
 		$pid = $request->user()->profile_id;
 		$limit = $request->input('limit', 20);
-		$timeago = now()->subMonths(6);
 
 		$since = $request->input('since_id');
 		$min = $request->input('min_id');
@@ -1293,27 +1283,20 @@ class ApiV1Controller extends Controller
 			$min = 1;
 		}
 
-		$dir = $since ? '>' : ($min ? '>=' : '<');
-		$id = $since ?? $min ?? $max;
-
-		$notifications = Notification::whereProfileId($pid)
-			->where('id', $dir, $id)
-			->whereDate('created_at', '>', $timeago)
-			->orderByDesc('id')
-			->limit($limit)
-			->get();
-
-		$minId = $notifications->min('id');
-		$maxId = $notifications->max('id');
-
-		$resource = new Fractal\Resource\Collection(
-			$notifications,
-			new NotificationTransformer()
-		);
+		$maxId = null;
+		$minId = null;
 
-		$res = $this->fractal
-			->createData($resource)
-			->toArray();
+		if($max) {
+			$res = NotificationService::getMax($pid, $max, $limit);
+			$ids = NotificationService::getRankedMaxId($pid, $max, $limit);
+			$maxId = max($ids);
+			$minId = min($ids);
+		} else {
+			$res = NotificationService::getMin($pid, $min ?? $since, $limit);
+			$ids = NotificationService::getRankedMinId($pid, $min ?? $since, $limit);
+			$maxId = max($ids);
+			$minId = min($ids);
+		}
 
 		$baseUrl = config('app.url') . '/api/v1/notifications?';
 
@@ -1470,90 +1453,37 @@ class ApiV1Controller extends Controller
 	public function timelinePublic(Request $request)
 	{
 		$this->validate($request,[
-		  'page'        => 'nullable|integer|max:40',
 		  'min_id'      => 'nullable|integer|min:0|max:' . PHP_INT_MAX,
 		  'max_id'      => 'nullable|integer|min:0|max:' . PHP_INT_MAX,
 		  'limit'       => 'nullable|integer|max:80'
 		]);
 
-		$page = $request->input('page');
 		$min = $request->input('min_id');
 		$max = $request->input('max_id');
 		$limit = $request->input('limit') ?? 3;
 		$user = $request->user();
 
-		if($user) {
-			$key = 'user:last_active_at:id:'.$user->id;
-			$ttl = now()->addMinutes(5);
-			Cache::remember($key, $ttl, function() use($user) {
-				$user->last_active_at = now();
-				$user->save();
-				return;
-			});
-		}
+		if(PublicTimelineService::count() == 0) {
+        	PublicTimelineService::warmCache(true, 400);
+        }
 
-		if($min || $max) {
-			$dir = $min ? '>' : '<';
-			$id = $min ?? $max;
-			$timeline = Status::select(
-						'id',
-						'uri',
-						'caption',
-						'rendered',
-						'profile_id',
-						'type',
-						'in_reply_to_id',
-						'reblog_of_id',
-						'is_nsfw',
-						'scope',
-						'local',
-						'reply_count',
-						'comments_disabled',
-						'place_id',
-						'likes_count',
-						'reblogs_count',
-						'created_at',
-						'updated_at'
-					  )->whereNull('uri')
-					  ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album'])
-					  ->with('profile', 'hashtags', 'mentions')
-					  ->where('id', $dir, $id)
-					  ->whereScope('public')
-					  ->where('created_at', '>', now()->subDays(14))
-					  ->latest()
-					  ->limit($limit)
-					  ->get();
+        if ($max) {
+			$feed = PublicTimelineService::getRankedMaxId($max, $limit);
+		} else if ($min) {
+			$feed = PublicTimelineService::getRankedMinId($min, $limit);
 		} else {
-			$timeline = Status::select(
-						'id',
-						'uri',
-						'caption',
-						'rendered',
-						'profile_id',
-						'type',
-						'in_reply_to_id',
-						'reblog_of_id',
-						'is_nsfw',
-						'scope',
-						'local',
-						'reply_count',
-						'comments_disabled',
-						'place_id',
-						'likes_count',
-						'reblogs_count',
-						'created_at',
-						'updated_at'
-					  )->whereNull('uri')
-					  ->whereIn('type', ['photo', 'photo:album', 'video', 'video:album'])
-					  ->with('profile', 'hashtags', 'mentions')
-					  ->whereScope('public')
-					  ->where('created_at', '>', now()->subDays(14))
-					  ->latest()
-					  ->simplePaginate($limit);
+			$feed = PublicTimelineService::get(0, $limit);
 		}
 
-		$fractal = new Fractal\Resource\Collection($timeline, new StatusTransformer());
-		$res = $this->fractal->createData($fractal)->toArray();
+		$res = collect($feed)
+            ->map(function($k) use($user) {
+                $status = StatusService::get($k);
+                if($user) {
+                	$status['favourited'] = (bool) LikeService::liked($user->profile_id, $k);
+                }
+                return $status;
+            })
+            ->toArray();
 		return response()->json($res);
 	}
 

app/Services/NotificationService.php

@@ -46,6 +46,60 @@ class NotificationService {
 		return $ids;
 	}
 
+	public static function getMax($id = false, $start, $limit = 10)
+	{
+		$ids = self::getRankedMaxId($id, $start, $limit);
+
+		if(empty($ids)) {
+			return [];
+		}
+
+		$res = collect([]);
+		foreach($ids as $id) {
+			$res->push(self::getNotification($id));
+		}
+		return $res->toArray();
+	}
+
+	public static function getMin($id = false, $start, $limit = 10)
+	{
+		$ids = self::getRankedMinId($id, $start, $limit);
+
+		if(empty($ids)) {
+			return [];
+		}
+
+		$res = collect([]);
+		foreach($ids as $id) {
+			$res->push(self::getNotification($id));
+		}
+		return $res->toArray();
+	}
+
+	public static function getRankedMaxId($id = false, $start = null, $limit = 10)
+	{
+		if(!$start || !$id) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, $start, '-inf', [
+			'withscores' => true,
+			'limit' => [1, $limit]
+		]));
+	}
+
+	public static function getRankedMinId($id = false, $end = null, $limit = 10)
+	{
+		if(!$end || !$id) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY.$id, '+inf', $end, [
+			'withscores' => true,
+			'limit' => [0, $limit]
+		]));
+	}
+
 	public static function set($id, $val)
 	{
 		return Redis::zadd(self::CACHE_KEY . $id, $val, $val);
@@ -53,6 +107,7 @@ class NotificationService {
 
 	public static function del($id, $val)
 	{
+		Cache::forget('service:notification:' . $val);
 		return Redis::zrem(self::CACHE_KEY . $id, $val);
 	}
 
@@ -73,7 +128,7 @@ class NotificationService {
 
 	public static function getNotification($id)
 	{
-		return Cache::remember('service:notification:'.$id, now()->addMonths(3), function() use($id) {
+		return Cache::remember('service:notification:'.$id, now()->addDays(3), function() use($id) {
 			$n = Notification::with('item')->findOrFail($id);
 			$fractal = new Fractal\Manager();
 			$fractal->setSerializer(new ArraySerializer());
@@ -84,7 +139,7 @@ class NotificationService {
 
 	public static function setNotification(Notification $notification)
 	{
-		return Cache::remember('service:notification:'.$notification->id, now()->addMonths(3), function() use($notification) {
+		return Cache::remember('service:notification:'.$notification->id, now()->addDays(3), function() use($notification) {
 			$fractal = new Fractal\Manager();
 			$fractal->setSerializer(new ArraySerializer());
 			$resource = new Fractal\Resource\Item($notification, new NotificationTransformer());
@@ -106,4 +161,4 @@ class NotificationService {
 		}
 		return 0;
 	}
-}
+}

app/Services/PublicTimelineService.php

@@ -18,18 +18,41 @@ class PublicTimelineService {
 		if($stop > 100) {
 			$stop = 100;
 		}
-		$tl = [];
-		$keys = Redis::zrevrange(self::CACHE_KEY, $start, $stop);
-		foreach($keys as $key) {
-			array_push($tl, StatusService::get($key));
+
+		return Redis::zrevrange(self::CACHE_KEY, $start, $stop);
+	}
+
+	public static function getRankedMaxId($start = null, $limit = 10)
+	{
+		if(!$start) {
+			return [];
+		}
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, $start, '-inf', [
+			'withscores' => true,
+			'limit' => [1, $limit]
+		]));
+	}
+
+	public static function getRankedMinId($end = null, $limit = 10)
+	{
+		if(!$end) {
+			return [];
 		}
-		return $tl;
+
+		return array_keys(Redis::zrevrangebyscore(self::CACHE_KEY, '+inf', $end, [
+			'withscores' => true,
+			'limit' => [0, $limit]
+		]));
 	}
 
 	public static function add($val)
 	{
-		// return Redis::zadd(self::CACHE_KEY, $val, $val);
-		return;
+		if(self::count() > 400) {
+			Redis::zpopmin(self::CACHE_KEY);
+		}
+
+		return Redis::zadd(self::CACHE_KEY, $val, $val);
 	}
 
 	public static function rem($val)
@@ -44,7 +67,7 @@ class PublicTimelineService {
 
 	public static function count()
 	{
-		return Redis::zcount(self::CACHE_KEY, '-inf', '+inf');
+		return Redis::zcard(self::CACHE_KEY);
 	}
 
 	public static function warmCache($force = false, $limit = 100)
@@ -55,7 +78,7 @@ class PublicTimelineService {
 				->whereNull('reblog_of_id')
 				->whereIn('type', ['photo', 'photo:album', 'video', 'video:album', 'photo:video:album'])
 				->whereScope('public')
-				->latest()
+				->orderByDesc('id')
 				->limit($limit)
 				->pluck('id');
 			foreach($ids as $id) {

app/Services/StatusService.php

@@ -37,6 +37,7 @@ class StatusService {
 
 	public static function del($id)
 	{
+		PublicTimelineService::rem($id);
 		return Cache::forget(self::key($id));
 	}
-}
+}

resources/views/admin/instances/home.blade.php

@@ -85,6 +85,7 @@
 @endsection
 
 @push('scripts')
+<script type="text/javascript" src="{{mix('js/components.js')}}"></script>
 <script type="text/javascript">
 	$(document).ready(function() {
 		$('.filesize').each(function(k,v) {

resources/views/admin/instances/show.blade.php

@@ -103,6 +103,7 @@
 @endsection
 
 @push('scripts')
+<script type="text/javascript" src="{{mix('js/components.js')}}"></script>
 <script type="text/javascript">
 	$(document).ready(function() {
 		$('.filesize').each(function(k,v) {
@@ -110,4 +111,4 @@
 		});
 	});
 </script>
-@endpush
+@endpush