Refactor total local post count logic, cache value and schedule updates twice daily to eliminate the perf issue on larger instances

app/Console/Commands/InstanceUpdateTotalLocalPosts.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace App\Console\Commands;
+
+use App\Services\ConfigCacheService;
+use Cache;
+use DB;
+use Illuminate\Console\Command;
+use Storage;
+
+class InstanceUpdateTotalLocalPosts extends Command
+{
+    /**
+     * The name and signature of the console command.
+     *
+     * @var string
+     */
+    protected $signature = 'app:instance-update-total-local-posts';
+
+    /**
+     * The console command description.
+     *
+     * @var string
+     */
+    protected $description = 'Update the total number of local statuses/post count';
+
+    /**
+     * Execute the console command.
+     */
+    public function handle()
+    {
+        $cached = $this->checkForCache();
+        if (! $cached) {
+            $this->initCache();
+
+            return;
+        }
+        $cache = $this->getCached();
+        if (! $cache || ! isset($cache['count'])) {
+            $this->error('Problem fetching cache');
+
+            return;
+        }
+        $this->updateAndCache();
+        Cache::forget('api:nodeinfo');
+
+    }
+
+    protected function checkForCache()
+    {
+        return Storage::exists('total_local_posts.json');
+    }
+
+    protected function initCache()
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count,
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+    }
+
+    protected function getCached()
+    {
+        return Storage::json('total_local_posts.json');
+    }
+
+    protected function updateAndCache()
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count,
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES | JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+
+    }
+}

app/Console/Kernel.php

@@ -51,6 +51,7 @@ class Kernel extends ConsoleKernel
         $schedule->command('app:notification-epoch-update')->weeklyOn(1, '2:21')->onOneServer();
         $schedule->command('app:hashtag-cached-count-update')->hourlyAt(25)->onOneServer();
         $schedule->command('app:account-post-count-stat-update')->everySixHours(25)->onOneServer();
+        $schedule->command('app:instance-update-total-local-posts')->twiceDailyAt(1, 13, 45)->onOneServer();
     }
 
     /**

app/Http/Controllers/PixelfedDirectoryController.php

@@ -4,13 +4,13 @@ namespace App\Http\Controllers;
 
 use App\Models\ConfigCache;
 use App\Services\AccountService;
+use App\Services\InstanceService;
 use App\Services\StatusService;
+use App\User;
+use Cache;
 use Illuminate\Http\Request;
 use Illuminate\Support\Str;
-use Cache;
 use Storage;
-use App\Status;
-use App\User;
 
 class PixelfedDirectoryController extends Controller
 {
@@ -140,10 +140,8 @@ class PixelfedDirectoryController extends Controller
             'stories' => (bool) config_cache('instance.stories.enabled'),
         ];
 
-        $statusesCount = Cache::remember('api:nodeinfo:statuses', 21600, function() {
-            return Status::whereLocal(true)->count();
-        });
-        $usersCount = Cache::remember('api:nodeinfo:users', 43200, function() {
+        $statusesCount = InstanceService::totalLocalStatuses();
+        $usersCount = Cache::remember('api:nodeinfo:users', 43200, function () {
             return User::count();
         });
         $res['stats'] = [

app/Services/ConfigCacheService.php

@@ -9,6 +9,7 @@ use Illuminate\Database\QueryException;
 class ConfigCacheService
 {
     const CACHE_KEY = 'config_cache:_v0-key:';
+
     const PROTECTED_KEYS = [
         'filesystems.disks.s3.key',
         'filesystems.disks.s3.secret',
@@ -133,6 +134,8 @@ class ConfigCacheService
                     'filesystems.disks.spaces.url',
                     'filesystems.disks.spaces.endpoint',
                     'filesystems.disks.spaces.use_path_style_endpoint',
+
+                    'instance.stats.total_local_posts',
                     // 'system.user_mode'
                 ];
 
@@ -146,7 +149,7 @@ class ConfigCacheService
 
                 $protect = false;
                 $protected = null;
-                if(in_array($key, self::PROTECTED_KEYS)) {
+                if (in_array($key, self::PROTECTED_KEYS)) {
                     $protect = true;
                 }
 
@@ -154,7 +157,7 @@ class ConfigCacheService
                 $c = ConfigCacheModel::where('k', $key)->first();
 
                 if ($c) {
-                    if($protect) {
+                    if ($protect) {
                         return decrypt($c->v) ?? config($key);
                     } else {
                         return $c->v ?? config($key);
@@ -165,7 +168,7 @@ class ConfigCacheService
                     return;
                 }
 
-                if($protect && $v) {
+                if ($protect && $v) {
                     $protected = encrypt($v);
                 }
 
@@ -176,7 +179,7 @@ class ConfigCacheService
 
                 return $v;
             });
-        } catch (Exception | QueryException $e) {
+        } catch (Exception|QueryException $e) {
             return config($key);
         }
     }
@@ -187,7 +190,7 @@ class ConfigCacheService
 
         $protect = false;
         $protected = null;
-        if(in_array($key, self::PROTECTED_KEYS)) {
+        if (in_array($key, self::PROTECTED_KEYS)) {
             $protect = true;
             $protected = encrypt($val);
         }

app/Services/InstanceService.php

@@ -18,6 +18,8 @@ class InstanceService
 
     const CACHE_KEY_STATS = 'pf:services:instances:stats';
 
+    const CACHE_KEY_TOTAL_POSTS = 'pf:services:instances:self:total-posts';
+
     const CACHE_KEY_BANNER_BLURHASH = 'pf:services:instance:header-blurhash:v1';
 
     const CACHE_KEY_API_PEERS_LIST = 'pf:services:instance:api:peers:list:v0';
@@ -96,6 +98,11 @@ class InstanceService
         return true;
     }
 
+    public static function totalLocalStatuses()
+    {
+        return config_cache('instance.stats.total_local_posts');
+    }
+
     public static function headerBlurhash()
     {
         return Cache::rememberForever(self::CACHE_KEY_BANNER_BLURHASH, function () {

app/Services/LandingService.php

@@ -2,7 +2,6 @@
 
 namespace App\Services;
 
-use App\Status;
 use App\User;
 use App\Util\Site\Nodeinfo;
 use Illuminate\Support\Facades\Cache;
@@ -18,9 +17,7 @@ class LandingService
             return User::count();
         });
 
-        $postCount = Cache::remember('api:nodeinfo:statuses', 21600, function () {
-            return Status::whereLocal(true)->count();
-        });
+        $postCount = InstanceService::totalLocalStatuses();
 
         $contactAccount = Cache::remember('api:v1:instance-data:contact', 604800, function () {
             if (config_cache('instance.admin.pid')) {

app/Services/StatusService.php

@@ -2,15 +2,11 @@
 
 namespace App\Services;
 
-use Illuminate\Support\Facades\Cache;
-use Illuminate\Support\Facades\Redis;
-use DB;
 use App\Status;
 use App\Transformer\Api\StatusStatelessTransformer;
-use App\Transformer\Api\StatusTransformer;
+use Illuminate\Support\Facades\Cache;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use League\Fractal\Pagination\IlluminatePaginatorAdapter;
 
 class StatusService
 {
@@ -19,18 +15,19 @@ class StatusService
     public static function key($id, $publicOnly = true)
     {
         $p = $publicOnly ? 'pub:' : 'all:';
-        return self::CACHE_KEY . $p . $id;
+
+        return self::CACHE_KEY.$p.$id;
     }
 
     public static function get($id, $publicOnly = true, $mastodonMode = false)
     {
-        $res = Cache::remember(self::key($id, $publicOnly), 21600, function() use($id, $publicOnly) {
-            if($publicOnly) {
+        $res = Cache::remember(self::key($id, $publicOnly), 21600, function () use ($id, $publicOnly) {
+            if ($publicOnly) {
                 $status = Status::whereScope('public')->find($id);
             } else {
                 $status = Status::whereIn('scope', ['public', 'private', 'unlisted', 'group'])->find($id);
             }
-            if(!$status) {
+            if (! $status) {
                 return null;
             }
             $fractal = new Fractal\Manager();
@@ -38,32 +35,34 @@ class StatusService
             $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
             $res = $fractal->createData($resource)->toArray();
             $res['_pid'] = isset($res['account']) && isset($res['account']['id']) ? $res['account']['id'] : null;
-            if(isset($res['_pid'])) {
+            if (isset($res['_pid'])) {
                 unset($res['account']);
             }
+
             return $res;
         });
-        if($res && isset($res['_pid'])) {
+        if ($res && isset($res['_pid'])) {
             $res['account'] = $mastodonMode === true ? AccountService::getMastodon($res['_pid'], true) : AccountService::get($res['_pid'], true);
             unset($res['_pid']);
         }
+
         return $res;
     }
 
     public static function getMastodon($id, $publicOnly = true)
     {
         $status = self::get($id, $publicOnly, true);
-        if(!$status) {
+        if (! $status) {
             return null;
         }
 
-        if(!isset($status['account'])) {
+        if (! isset($status['account'])) {
             return null;
         }
 
         $status['replies_count'] = $status['reply_count'];
 
-        if(config('exp.emc') == false) {
+        if (config('exp.emc') == false) {
             return $status;
         }
 
@@ -113,28 +112,29 @@ class StatusService
     {
         $status = self::get($id, false);
 
-        if(!$status) {
+        if (! $status) {
             return [
                 'liked' => false,
                 'shared' => false,
-                'bookmarked' => false
+                'bookmarked' => false,
             ];
         }
 
         return [
             'liked' => LikeService::liked($pid, $id),
             'shared' => self::isShared($id, $pid),
-            'bookmarked' => self::isBookmarked($id, $pid)
+            'bookmarked' => self::isBookmarked($id, $pid),
         ];
     }
 
     public static function getFull($id, $pid, $publicOnly = true)
     {
         $res = self::get($id, $publicOnly);
-        if(!$res || !isset($res['account']) || !isset($res['account']['id'])) {
+        if (! $res || ! isset($res['account']) || ! isset($res['account']['id'])) {
             return $res;
         }
         $res['relationship'] = RelationshipService::get($pid, $res['account']['id']);
+
         return $res;
     }
 
@@ -142,31 +142,33 @@ class StatusService
     {
         $status = Status::whereScope('direct')->find($id);
 
-        if(!$status) {
+        if (! $status) {
             return null;
         }
 
         $fractal = new Fractal\Manager();
         $fractal->setSerializer(new ArraySerializer());
         $resource = new Fractal\Resource\Item($status, new StatusStatelessTransformer());
+
         return $fractal->createData($resource)->toArray();
     }
 
     public static function del($id, $purge = false)
     {
-        if($purge) {
+        if ($purge) {
             $status = self::get($id);
-            if($status && isset($status['account']) && isset($status['account']['id'])) {
-                Cache::forget('profile:embed:' . $status['account']['id']);
+            if ($status && isset($status['account']) && isset($status['account']['id'])) {
+                Cache::forget('profile:embed:'.$status['account']['id']);
             }
-            Cache::forget('status:transformer:media:attachments:' . $id);
+            Cache::forget('status:transformer:media:attachments:'.$id);
             MediaService::del($id);
-            Cache::forget('pf:services:sh:id:' . $id);
+            Cache::forget('pf:services:sh:id:'.$id);
             PublicTimelineService::rem($id);
             NetworkTimelineService::rem($id);
         }
 
         Cache::forget(self::key($id, false));
+
         return Cache::forget(self::key($id));
     }
 
@@ -194,8 +196,6 @@ class StatusService
 
     public static function totalLocalStatuses()
     {
-        return Cache::remember(self::CACHE_KEY . 'totalpub', 14400, function() {
-            return Status::whereNull('url')->count();
-        });
+        return InstanceService::totalLocalStatuses();
     }
 }

app/Util/Site/Nodeinfo.php

@@ -2,12 +2,9 @@
 
 namespace App\Util\Site;
 
-use Illuminate\Support\Facades\Cache;
-use App\Like;
-use App\Profile;
-use App\Status;
+use App\Services\InstanceService;
 use App\User;
-use Illuminate\Support\Str;
+use Illuminate\Support\Facades\Cache;
 
 class Nodeinfo
 {
@@ -17,49 +14,48 @@ class Nodeinfo
             $activeHalfYear = self::activeUsersHalfYear();
             $activeMonth = self::activeUsersMonthly();
 
-            $users = Cache::remember('api:nodeinfo:users', 43200, function() {
+            $users = Cache::remember('api:nodeinfo:users', 43200, function () {
                 return User::count();
             });
 
-            $statuses = Cache::remember('api:nodeinfo:statuses', 21600, function() {
-                return Status::whereLocal(true)->count();
-            });
+            $statuses = InstanceService::totalLocalStatuses();
 
-            $features = [ 'features' => \App\Util\Site\Config::get()['features'] ];
+            $features = ['features' => \App\Util\Site\Config::get()['features']];
 
             return [
                 'metadata' => [
                     'nodeName' => config_cache('app.name'),
                     'software' => [
-                        'homepage'  => 'https://pixelfed.org',
-                        'repo'      => 'https://github.com/pixelfed/pixelfed',
+                        'homepage' => 'https://pixelfed.org',
+                        'repo' => 'https://github.com/pixelfed/pixelfed',
                     ],
-                    'config' => $features
+                    'config' => $features,
                 ],
-                'protocols'         => [
+                'protocols' => [
                     'activitypub',
                 ],
                 'services' => [
-                    'inbound'  => [],
+                    'inbound' => [],
                     'outbound' => [],
                 ],
                 'software' => [
-                    'name'          => 'pixelfed',
-                    'version'       => config('pixelfed.version'),
+                    'name' => 'pixelfed',
+                    'version' => config('pixelfed.version'),
                 ],
                 'usage' => [
-                    'localPosts'    => (int) $statuses,
+                    'localPosts' => (int) $statuses,
                     'localComments' => 0,
-                    'users'         => [
-                        'total'          => (int) $users,
+                    'users' => [
+                        'total' => (int) $users,
                         'activeHalfyear' => (int) $activeHalfYear,
-                        'activeMonth'    => (int) $activeMonth,
+                        'activeMonth' => (int) $activeMonth,
                     ],
                 ],
                 'version' => '2.0',
             ];
         });
         $res['openRegistrations'] = (bool) config_cache('pixelfed.open_registration');
+
         return $res;
     }
 
@@ -69,7 +65,7 @@ class Nodeinfo
             'links' => [
                 [
                     'href' => config('pixelfed.nodeinfo.url'),
-                    'rel'  => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
+                    'rel' => 'http://nodeinfo.diaspora.software/ns/schema/2.0',
                 ],
             ],
         ];
@@ -77,18 +73,18 @@ class Nodeinfo
 
     public static function activeUsersMonthly()
     {
-        return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function() {
+        return Cache::remember('api:nodeinfo:active-users-monthly', 43200, function () {
             return User::withTrashed()
-                    ->select('last_active_at, updated_at')
-                    ->where('updated_at', '>', now()->subWeeks(5))
-                    ->orWhere('last_active_at', '>', now()->subWeeks(5))
-                    ->count();
+                ->select('last_active_at, updated_at')
+                ->where('updated_at', '>', now()->subWeeks(5))
+                ->orWhere('last_active_at', '>', now()->subWeeks(5))
+                ->count();
         });
     }
 
     public static function activeUsersHalfYear()
     {
-        return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function() {
+        return Cache::remember('api:nodeinfo:active-users-half-year', 43200, function () {
             return User::withTrashed()
                 ->select('last_active_at, updated_at')
                 ->where('last_active_at', '>', now()->subMonths(6))

database/migrations/2024_06_19_084835_add_total_local_posts_to_config_cache.php

@@ -0,0 +1,34 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+use Illuminate\Support\Facades\DB;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Storage;
+use App\Services\ConfigCacheService;
+
+return new class extends Migration
+{
+    /**
+     * Run the migrations.
+     */
+    public function up(): void
+    {
+        $count = DB::table('statuses')->whereNull(['url', 'deleted_at'])->count();
+        $res = [
+            'count' => $count
+        ];
+        Storage::put('total_local_posts.json', json_encode($res, JSON_UNESCAPED_SLASHES|JSON_PRETTY_PRINT));
+        ConfigCacheService::put('instance.stats.total_local_posts', $res['count']);
+        Cache::forget('api:nodeinfo');
+    }
+
+    /**
+     * Reverse the migrations.
+     */
+    public function down(): void
+    {
+        //
+    }
+};