Update Status storage, add SanitizerService to fix spacing in html stripped content

app/Http/Controllers/Api/ApiV1Controller.php

@@ -58,6 +58,7 @@ use App\Services\NotificationService;
 use App\Services\PublicTimelineService;
 use App\Services\ReblogService;
 use App\Services\RelationshipService;
+use App\Services\SanitizeService;
 use App\Services\SnowflakeService;
 use App\Services\StatusService;
 use App\Services\UserFilterService;
@@ -87,7 +88,6 @@ use Illuminate\Support\Str;
 use Laravel\Passport\Passport;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
-use Purify;
 use Storage;
 
 class ApiV1Controller extends Controller
@@ -1964,7 +1964,7 @@ class ApiV1Controller extends Controller
             'media:update:'.$user->id,
             10,
             function () use ($media, $request) {
-                $caption = Purify::clean($request->input('description'));
+                $caption = app(SanitizeService::class)->html($request->input('description'));
 
                 if ($caption != $media->caption) {
                     $media->caption = $caption;

app/Http/Controllers/Api/ApiV1Dot1Controller.php

@@ -32,6 +32,7 @@ use App\Services\NotificationAppGatewayService;
 use App\Services\ProfileStatusService;
 use App\Services\PublicTimelineService;
 use App\Services\PushNotificationService;
+use App\Services\SanitizeService;
 use App\Services\StatusService;
 use App\Services\UserStorageService;
 use App\Status;
@@ -50,7 +51,6 @@ use Jenssegers\Agent\Agent;
 use League\Fractal;
 use League\Fractal\Serializer\ArraySerializer;
 use Mail;
-use Purify;
 
 class ApiV1Dot1Controller extends Controller
 {
@@ -1294,7 +1294,8 @@ class ApiV1Dot1Controller extends Controller
             return [];
         }
         $defaultCaption = '';
-        $content = $request->filled('status') ? strip_tags(Purify::clean($request->input('status'))) : $defaultCaption;
+        $cleanedStatus = app(SanitizeService::class)->html($request->input('status', ''));
+        $content = $request->filled('status') ? strip_tags($cleanedStatus) : $defaultCaption;
         $cw = $user->profile->cw == true ? true : $request->boolean('sensitive', false);
         $spoilerText = $cw && $request->filled('spoiler_text') ? $request->input('spoiler_text') : null;
 

app/Http/Controllers/RemoteAuthController.php

@@ -3,9 +3,11 @@
 namespace App\Http\Controllers;
 
 use App\Models\RemoteAuth;
+use App\Rules\PixelfedUsername;
 use App\Services\Account\RemoteAuthService;
 use App\Services\EmailService;
 use App\Services\MediaStorageService;
+use App\Services\SanitizeService;
 use App\User;
 use App\Util\ActivityPub\Helpers;
 use App\Util\Lexer\RestrictedNames;
@@ -14,7 +16,6 @@ use Illuminate\Http\Request;
 use Illuminate\Support\Facades\Auth;
 use Illuminate\Support\Facades\Hash;
 use Illuminate\Support\Str;
-use App\Rules\PixelfedUsername;
 use InvalidArgumentException;
 use Purify;
 
@@ -360,7 +361,7 @@ class RemoteAuthController extends Controller
                 'required',
                 'min:2',
                 'max:30',
-                new PixelfedUsername(),
+                new PixelfedUsername,
             ],
         ]);
         $username = strtolower($request->input('username'));
@@ -544,7 +545,7 @@ class RemoteAuthController extends Controller
         ]);
 
         $profile = $request->user()->profile;
-        $profile->bio = Purify::clean($request->input('bio'));
+        $profile->bio = app(SanitizeService::class)->html($request->input('bio'));
         $profile->save();
 
         return [200];

app/Jobs/ProfilePipeline/HandleUpdateActivity.php

@@ -2,19 +2,17 @@
 
 namespace App\Jobs\ProfilePipeline;
 
+use App\Jobs\AvatarPipeline\RemoteAvatarFetchFromUrl;
+use App\Profile;
+use App\Services\SanitizeService;
+use App\Util\Lexer\Autolink;
+use Cache;
 use Illuminate\Bus\Queueable;
-use Illuminate\Contracts\Queue\ShouldBeUnique;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
-use App\Avatar;
-use App\Profile;
-use App\Util\ActivityPub\Helpers;
-use Cache;
 use Purify;
-use App\Jobs\AvatarPipeline\RemoteAvatarFetchFromUrl;
-use App\Util\Lexer\Autolink;
 
 class HandleUpdateActivity implements ShouldQueue
 {
@@ -34,61 +32,58 @@ class HandleUpdateActivity implements ShouldQueue
 
     /**
      * Execute the job.
-     *
-     * @return void
      */
     public function handle(): void
     {
         $payload = $this->payload;
 
-        if(empty($payload) || !isset($payload['actor'])) {
+        if (empty($payload) || ! isset($payload['actor'])) {
             return;
         }
 
         $profile = Profile::whereRemoteUrl($payload['actor'])->first();
 
-        if(!$profile || $profile->domain === null || $profile->private_key) {
+        if (! $profile || $profile->domain === null || $profile->private_key) {
             return;
         }
 
-        if($profile->sharedInbox == null || $profile->sharedInbox != $payload['object']['endpoints']['sharedInbox']) {
+        if ($profile->sharedInbox == null || $profile->sharedInbox != $payload['object']['endpoints']['sharedInbox']) {
             $profile->sharedInbox = $payload['object']['endpoints']['sharedInbox'];
         }
 
-        if($profile->public_key !== $payload['object']['publicKey']['publicKeyPem']) {
+        if ($profile->public_key !== $payload['object']['publicKey']['publicKeyPem']) {
             $profile->public_key = $payload['object']['publicKey']['publicKeyPem'];
         }
 
-        if($profile->bio !== $payload['object']['summary']) {
+        if ($profile->bio !== $payload['object']['summary']) {
             $len = strlen(strip_tags($payload['object']['summary']));
-            if($len) {
-                if($len > 500) {
+            if ($len) {
+                if ($len > 500) {
                     $updated = strip_tags($payload['object']['summary']);
                     $updated = substr($updated, 0, config('pixelfed.max_bio_length'));
                     $profile->bio = Autolink::create()->autolink($updated);
                 } else {
-                    $profile->bio = Purify::clean($payload['object']['summary']);
+                    $profile->bio = app(SanitizeService::class)->html($payload['object']['summary']);
                 }
             } else {
                 $profile->bio = null;
             }
         }
 
-        if($profile->name !== $payload['object']['name']) {
+        if ($profile->name !== $payload['object']['name']) {
             $profile->name = Purify::clean(substr($payload['object']['name'], 0, config('pixelfed.max_name_length')));
         }
 
-        if($profile->isDirty()) {
+        if ($profile->isDirty()) {
             $profile->save();
         }
 
-        if(isset($payload['object']['icon']) && isset($payload['object']['icon']['url'])) {
+        if (isset($payload['object']['icon']) && isset($payload['object']['icon']['url'])) {
             RemoteAvatarFetchFromUrl::dispatch($profile, $payload['object']['icon']['url'])->onQueue('low');
         } else {
             $profile->avatar->update(['remote_url' => null]);
-            Cache::forget('avatar:' . $profile->id);
+            Cache::forget('avatar:'.$profile->id);
         }
 
-        return;
     }
 }

app/Jobs/RemoteFollowPipeline/RemoteFollowPipeline.php

@@ -3,7 +3,8 @@
 namespace App\Jobs\RemoteFollowPipeline;
 
 use App\Jobs\AvatarPipeline\CreateAvatar;
-use App\{Profile};
+use App\Profile;
+use App\Services\SanitizeService;
 use GuzzleHttp\Client;
 use HttpSignatures\Context;
 use HttpSignatures\GuzzleHttpSignatures;
@@ -19,7 +20,9 @@ class RemoteFollowPipeline implements ShouldQueue
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
     protected $url;
+
     protected $follower;
+
     protected $response;
 
     /**
@@ -55,15 +58,15 @@ class RemoteFollowPipeline implements ShouldQueue
     public function discover($url)
     {
         $context = new Context([
-            'keys'      => ['examplekey' => 'secret-key-here'],
+            'keys' => ['examplekey' => 'secret-key-here'],
             'algorithm' => 'hmac-sha256',
-            'headers'   => ['(request-target)', 'date'],
+            'headers' => ['(request-target)', 'date'],
         ]);
 
         $handlerStack = GuzzleHttpSignatures::defaultHandlerFromContext($context);
         $client = new Client(['handler' => $handlerStack]);
         $response = Zttp::withHeaders([
-            'Accept'     => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
+            'Accept' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"',
             'User-Agent' => 'PixelfedBot v0.1 - https://pixelfed.org',
         ])->get($url);
         $this->response = $response->json();
@@ -78,12 +81,12 @@ class RemoteFollowPipeline implements ShouldQueue
         $username = $res['preferredUsername'];
         $remoteUsername = "@{$username}@{$domain}";
 
-        $profile = new Profile();
+        $profile = new Profile;
         $profile->user_id = null;
         $profile->domain = $domain;
         $profile->username = $remoteUsername;
         $profile->name = $res['name'];
-        $profile->bio = Purify::clean($res['summary']);
+        $profile->bio = app(SanitizeService::class)->html($res['summary']);
         $profile->sharedInbox = $res['endpoints']['sharedInbox'];
         $profile->remote_url = $res['url'];
         $profile->save();
@@ -98,7 +101,7 @@ class RemoteFollowPipeline implements ShouldQueue
         $url = $res['inbox'];
 
         $activity = Zttp::withHeaders(['Content-Type' => 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'])->post($url, [
-            'type'   => 'Follow',
+            'type' => 'Follow',
             'object' => $this->follower->url(),
         ]);
     }

app/Jobs/StatusPipeline/StatusRemoteUpdatePipeline.php

@@ -6,6 +6,7 @@ use App\Media;
 use App\Models\StatusEdit;
 use App\ModLog;
 use App\Profile;
+use App\Services\SanitizeService;
 use App\Services\StatusService;
 use App\Status;
 use Illuminate\Bus\Queueable;
@@ -120,7 +121,8 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
     protected function updateImmediateAttributes($status, $activity)
     {
         if (isset($activity['content'])) {
-            $status->caption = strip_tags(Purify::clean($activity['content']));
+            $cleanedCaption = app(SanitizeService::class)->html($activity['content']);
+            $status->caption = strip_tags($cleanedCaption);
         }
 
         if (isset($activity['sensitive'])) {
@@ -143,7 +145,7 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
         }
 
         if (isset($activity['summary'])) {
-            $status->cw_summary = Purify::clean($activity['summary']);
+            $status->cw_summary = app(SanitizeService::class)->html($activity['summary']);
         } else {
             $status->cw_summary = null;
         }
@@ -155,8 +157,8 @@ class StatusRemoteUpdatePipeline implements ShouldQueue
 
     protected function createEdit($status, $activity)
     {
-        $cleaned = isset($activity['content']) ? Purify::clean($activity['content']) : null;
-        $spoiler_text = isset($activity['summary']) ? Purify::clean($activity['summary']) : null;
+        $cleaned = isset($activity['content']) ? app(SanitizeService::class)->html($activity['content']) : null;
+        $spoiler_text = isset($activity['summary']) ? app(SanitizeService::class)->html($activity['summary']) : null;
         $sensitive = isset($activity['sensitive']) ? $activity['sensitive'] : null;
         $mids = $status->media()->count() ? $status->media()->orderBy('order')->pluck('id')->toArray() : null;
         StatusEdit::create([

app/Services/SanitizeService.php

@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Services;
+
+use Stevebauman\Purify\Facades\Purify;
+
+class SanitizeService
+{
+    public function purify($html)
+    {
+        $cleaned = Purify::clean($html);
+
+        return $cleaned;
+    }
+
+    public function html($html)
+    {
+        return $this->cleanHtmlWithSpacing($html);
+    }
+
+    public function cleanHtmlWithSpacing($html)
+    {
+        $blockTags = ['p', 'img', 'div', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'li', 'blockquote', 'br'];
+
+        foreach ($blockTags as $tag) {
+            $html = preg_replace("/<\/{$tag}>/i", "</{$tag}> ", $html);
+        }
+
+        $html = preg_replace("/<br\s*\/?>/i", '<br /> ', $html);
+
+        $cleaned = Purify::clean($html);
+
+        $cleaned = preg_replace('/\s+/', ' ', $cleaned);
+        $cleaned = trim($cleaned);
+
+        return $cleaned;
+    }
+}

app/Util/ActivityPub/Helpers.php

@@ -19,6 +19,7 @@ use App\Services\DomainService;
 use App\Services\InstanceService;
 use App\Services\MediaPathService;
 use App\Services\NetworkTimelineService;
+use App\Services\SanitizeService;
 use App\Services\UserFilterService;
 use App\Status;
 use App\Util\Media\License;
@@ -175,7 +176,7 @@ class Helpers
                 return false;
             }
 
-            if (!$disableDNSCheck && ! self::passesSecurityChecks($host, $disableDNSCheck, $forceBanCheck)) {
+            if (! $disableDNSCheck && ! self::passesSecurityChecks($host, $disableDNSCheck, $forceBanCheck)) {
                 return false;
             }
 
@@ -666,8 +667,11 @@ class Helpers
         bool $commentsDisabled
     ): Status {
         $caption = isset($activity['content']) ?
-            Purify::clean($activity['content']) :
+            app(SanitizeService::class)->html($activity['content']) :
             '';
+        $cwSummary = ($cw && isset($activity['summary'])) ?
+            app(SanitizeService::class)->html($activity['summary']) :
+            null;
 
         return Status::updateOrCreate(
             ['uri' => $url],
@@ -683,9 +687,7 @@ class Helpers
                 'is_nsfw' => $cw,
                 'scope' => $scope,
                 'visibility' => $scope,
-                'cw_summary' => ($cw && isset($activity['summary'])) ?
-                    Purify::clean(strip_tags($activity['summary'])) :
-                    null,
+                'cw_summary' => $cwSummary ? strip_tags($cwSummary) : null,
                 'comments_disabled' => $commentsDisabled,
             ]
         );
@@ -823,12 +825,15 @@ class Helpers
         })->toArray();
 
         $defaultCaption = '';
+        $cleanedCaption = ! empty($res['content']) ?
+            app(SanitizeService::class)->html($res['content']) :
+            null;
         $status = new Status;
         $status->profile_id = $profile->id;
         $status->url = isset($res['url']) ? $res['url'] : $url;
         $status->uri = isset($res['url']) ? $res['url'] : $url;
         $status->object_url = $id;
-        $status->caption = strip_tags(Purify::clean($res['content'])) ?? $defaultCaption;
+        $status->caption = $cleanedCaption ? strip_tags($cleanedCaption) : $defaultCaption;
         $status->rendered = Purify::clean($res['content'] ?? $defaultCaption);
         $status->created_at = Carbon::parse($ts)->tz('UTC');
         $status->in_reply_to_id = null;
@@ -1261,7 +1266,7 @@ class Helpers
             'key_id' => $res['publicKey']['id'],
             'remote_url' => $res['id'],
             'name' => isset($res['name']) ? Purify::clean($res['name']) : 'user',
-            'bio' => isset($res['summary']) ? Purify::clean($res['summary']) : null,
+            'bio' => isset($res['summary']) ? app(SanitizeService::class)->html($res['summary']) : null,
             'sharedInbox' => $res['endpoints']['sharedInbox'] ?? null,
             'inbox_url' => $res['inbox'],
             'outbox_url' => $res['outbox'] ?? null,

app/Util/ActivityPub/Inbox.php

@@ -33,6 +33,7 @@ use App\Services\PollService;
 use App\Services\PushNotificationService;
 use App\Services\ReblogService;
 use App\Services\RelationshipService;
+use App\Services\SanitizeService;
 use App\Services\StoryIndexService;
 use App\Services\UserFilterService;
 use App\Status;
@@ -50,7 +51,6 @@ use Cache;
 use Illuminate\Support\Facades\Bus;
 use Illuminate\Support\Facades\Log;
 use Illuminate\Support\Str;
-use Purify;
 use Storage;
 use Throwable;
 
@@ -423,7 +423,7 @@ class Inbox
             return;
         }
 
-        $msg = Purify::clean($activity['content']);
+        $msg = app(SanitizeService::class)->html($activity['content']);
         $msgText = strip_tags($msg);
 
         if (Str::startsWith($msgText, '@'.$profile->username)) {
@@ -1064,7 +1064,7 @@ class Inbox
         $actor = $this->payload['actor'];
         $storyUrl = $this->payload['inReplyTo'];
         $to = $this->payload['to'];
-        $text = Purify::clean($this->payload['content']);
+        $text = app(SanitizeService::class)->html($this->payload['content']);
 
         if (parse_url($id, PHP_URL_HOST) !== parse_url($actor, PHP_URL_HOST)) {
             return;
@@ -1184,7 +1184,7 @@ class Inbox
         $actor = $this->payload['actor'];
         $storyUrl = $this->payload['inReplyTo'];
         $to = $this->payload['to'];
-        $text = Purify::clean($this->payload['content']);
+        $text = app(SanitizeService::class)->html($this->payload['content']);
 
         if (parse_url($id, PHP_URL_HOST) !== parse_url($actor, PHP_URL_HOST)) {
             return;
@@ -1310,9 +1310,9 @@ class Inbox
         $content = null;
         if (isset($this->payload['content'])) {
             if (strlen($this->payload['content']) > 5000) {
-                $content = Purify::clean(substr($this->payload['content'], 0, 5000).' ... (truncated message due to exceeding max length)');
+                $content = app(SanitizeService::class)->html(substr($this->payload['content'], 0, 5000).' ... (truncated message due to exceeding max length)');
             } else {
-                $content = Purify::clean($this->payload['content']);
+                $content = app(SanitizeService::class)->html($this->payload['content']);
             }
         }
         $object = $this->payload['object'];