Update media storage pipeline, improve support for non-local filesystems

app/Jobs/ImageOptimizePipeline/ImageOptimize.php

@@ -8,6 +8,7 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Storage;
 
 class ImageOptimize implements ShouldQueue
 {
@@ -40,20 +41,32 @@ class ImageOptimize implements ShouldQueue
     public function handle()
     {
         $media = $this->media;
-        if(!$media) {
+        if (! $media) {
             return;
         }
-        $path = storage_path('app/'.$media->media_path);
-        if (!is_file($path) || $media->skip_optimize) {
-            return;
+
+        $localFs = config('filesystems.default') === 'local';
+
+        if ($localFs) {
+            $path = storage_path('app/'.$media->media_path);
+            if (! is_file($path) || $media->skip_optimize) {
+                return;
+            }
+        } else {
+            $disk = Storage::disk(config('filesystems.default'));
+            if (! $disk->exists($media->media_path) || $media->skip_optimize) {
+                return;
+            }
         }
 
-        if((bool) config_cache('pixelfed.optimize_image') == false) {
-        	ImageThumbnail::dispatch($media)->onQueue('mmo');
-    		return;
-    	} else {
-        	ImageResize::dispatch($media)->onQueue('mmo');
-    		return;
-    	}
+        if ((bool) config_cache('pixelfed.optimize_image') == false) {
+            ImageThumbnail::dispatch($media)->onQueue('mmo');
+
+            return;
+        } else {
+            ImageResize::dispatch($media)->onQueue('mmo');
+
+            return;
+        }
     }
 }

app/Jobs/ImageOptimizePipeline/ImageResize.php

@@ -10,6 +10,7 @@ use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
 use Log;
+use Storage;
 
 class ImageResize implements ShouldQueue
 {
@@ -23,7 +24,7 @@ class ImageResize implements ShouldQueue
      * @var bool
      */
     public $deleteWhenMissingModels = true;
-    
+
     /**
      * Create a new job instance.
      *
@@ -42,24 +43,37 @@ class ImageResize implements ShouldQueue
     public function handle()
     {
         $media = $this->media;
-        if(!$media) {
+        if (! $media) {
             return;
         }
-        $path = storage_path('app/'.$media->media_path);
-        if (!is_file($path) || $media->skip_optimize) {
-            Log::info('Tried to optimize media that does not exist or is not readable. ' . $path);
-            return;
+
+        $localFs = config('filesystems.default') === 'local';
+
+        if ($localFs) {
+            $path = storage_path('app/'.$media->media_path);
+            if (! is_file($path) || $media->skip_optimize) {
+                return;
+            }
+        } else {
+            $disk = Storage::disk(config('filesystems.default'));
+            if (! $disk->exists($media->media_path) || $media->skip_optimize) {
+                return;
+            }
         }
 
-        if((bool) config_cache('pixelfed.optimize_image') === false) {
-        	ImageThumbnail::dispatch($media)->onQueue('mmo');
-        	return;
+        if ((bool) config_cache('pixelfed.optimize_image') === false) {
+            ImageThumbnail::dispatch($media)->onQueue('mmo');
+
+            return;
         }
+
         try {
-            $img = new Image();
+            $img = new Image;
             $img->resizeImage($media);
-        } catch (Exception $e) {
-            Log::error($e);
+        } catch (\Exception $e) {
+            if (config('app.dev_log')) {
+                Log::error('Image resize failed: '.$e->getMessage());
+            }
         }
 
         ImageThumbnail::dispatch($media)->onQueue('mmo');

app/Jobs/ImageOptimizePipeline/ImageThumbnail.php

@@ -10,6 +10,8 @@ use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Log;
+use Storage;
 
 class ImageThumbnail implements ShouldQueue
 {
@@ -23,7 +25,7 @@ class ImageThumbnail implements ShouldQueue
      * @var bool
      */
     public $deleteWhenMissingModels = true;
-    
+
     /**
      * Create a new job instance.
      *
@@ -42,18 +44,31 @@ class ImageThumbnail implements ShouldQueue
     public function handle()
     {
         $media = $this->media;
-        if(!$media) {
+        if (! $media) {
             return;
         }
-        $path = storage_path('app/'.$media->media_path);
-        if (!is_file($path)) {
-            return;
+
+        $localFs = config('filesystems.default') === 'local';
+
+        if ($localFs) {
+            $path = storage_path('app/'.$media->media_path);
+            if (! is_file($path)) {
+                return;
+            }
+        } else {
+            $disk = Storage::disk(config('filesystems.default'));
+            if (! $disk->exists($media->media_path)) {
+                return;
+            }
         }
 
         try {
-            $img = new Image();
+            $img = new Image;
             $img->resizeThumbnail($media);
-        } catch (Exception $e) {
+        } catch (\Exception $e) {
+            if (config('app.dev_log')) {
+                Log::error('Thumbnail generation failed: '.$e->getMessage());
+            }
         }
 
         $media->processed_at = Carbon::now();

app/Jobs/ImageOptimizePipeline/ImageUpdate.php

@@ -2,17 +2,17 @@
 
 namespace App\Jobs\ImageOptimizePipeline;
 
-use Storage;
+use App\Jobs\MediaPipeline\MediaStoragePipeline;
 use App\Media;
 use Illuminate\Bus\Queueable;
 use Illuminate\Contracts\Queue\ShouldQueue;
 use Illuminate\Foundation\Bus\Dispatchable;
 use Illuminate\Queue\InteractsWithQueue;
 use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Str;
 use ImageOptimizer;
-use Illuminate\Http\File;
-use App\Services\MediaPathService;
-use App\Jobs\MediaPipeline\MediaStoragePipeline;
+use Log;
+use Storage;
 
 class ImageUpdate implements ShouldQueue
 {
@@ -24,7 +24,7 @@ class ImageUpdate implements ShouldQueue
         'image/jpeg',
         'image/png',
         'image/webp',
-        'image/avif'
+        'image/avif',
     ];
 
     /**
@@ -52,35 +52,124 @@ class ImageUpdate implements ShouldQueue
     public function handle()
     {
         $media = $this->media;
-        if(!$media) {
+        if (! $media) {
             return;
         }
-        $path = storage_path('app/'.$media->media_path);
-        $thumb = storage_path('app/'.$media->thumbnail_path);
 
-        if (!is_file($path)) {
-            return;
+        $disk = Storage::disk(config('filesystems.default'));
+        $localFs = config('filesystems.default') === 'local';
+        $mediaPath = $media->media_path;
+        $fileSize = 0;
+
+        if ($localFs) {
+            $path = storage_path('app/'.$media->media_path);
+            $thumbPath = storage_path('app/'.$media->thumbnail_path);
+            if (! is_file($path)) {
+                return;
+            }
+            $mediaPath = $path;
+        } else {
+            if (! $disk->exists($media->media_path)) {
+                return;
+            }
         }
 
-        if((bool) config_cache('pixelfed.optimize_image')) {
+        if ((bool) config_cache('pixelfed.optimize_image') && $localFs) {
             if (in_array($media->mime, $this->protectedMimes) == true) {
-                ImageOptimizer::optimize($thumb);
-                if(!$media->skip_optimize) {
-                    ImageOptimizer::optimize($path);
+                try {
+                    $thumbPath = storage_path('app/'.$media->thumbnail_path);
+                    if (file_exists($thumbPath)) {
+                        ImageOptimizer::optimize($thumbPath);
+                    }
+
+                    if (! $media->skip_optimize) {
+                        $mediaPath = storage_path('app/'.$media->media_path);
+                        ImageOptimizer::optimize($mediaPath);
+                    }
+                } catch (\Exception $e) {
+                    if (config('app.dev_log')) {
+                        Log::error('Image optimization failed: '.$e->getMessage());
+                    }
                 }
             }
+        } elseif ((bool) config_cache('pixelfed.optimize_image') && ! $localFs) {
+            if (in_array($media->mime, $this->protectedMimes) == true) {
+                $this->optimizeRemoteImages($media, $disk);
+            }
         }
 
-        if (!is_file($path) || !is_file($thumb)) {
-            return;
+        try {
+            $photo_size = $this->getFileSize($media->media_path);
+            $thumb_size = $media->thumbnail_path ? $this->getFileSize($media->thumbnail_path) : 0;
+            $total = ($photo_size + $thumb_size);
+            $media->size = $total;
+            $media->save();
+        } catch (\Exception $e) {
+            if (config('app.dev_log')) {
+                Log::error('Failed to calculate media sizes: '.$e->getMessage());
+            }
         }
 
-        $photo_size = filesize($path);
-        $thumb_size = filesize($thumb);
-        $total = ($photo_size + $thumb_size);
-        $media->size = $total;
-        $media->save();
-
         MediaStoragePipeline::dispatch($media);
     }
+
+    protected function getFileSize($path)
+    {
+        $disk = Storage::disk(config('filesystems.default'));
+        $localFs = config('filesystems.default') === 'local';
+
+        if (! $path || empty($path)) {
+            return 0;
+        }
+
+        if ($localFs) {
+            return filesize(storage_path('app/'.$path)) ?? 0;
+        } else {
+            return $disk->size($path) ?? 0;
+        }
+    }
+
+    /**
+     * Optimize images stored on remote storage (S3, etc)
+     */
+    protected function optimizeRemoteImages($media, $disk)
+    {
+        try {
+            $tempDir = sys_get_temp_dir().'/pixelfed_optimize_'.Str::random(18);
+            mkdir($tempDir, 0755, true);
+
+            if ($media->thumbnail_path) {
+                $tempThumb = $tempDir.'/thumb_'.basename($media->thumbnail_path);
+                $thumbContents = $disk->get($media->thumbnail_path);
+                file_put_contents($tempThumb, $thumbContents);
+
+                ImageOptimizer::optimize($tempThumb);
+
+                $disk->put($media->thumbnail_path, file_get_contents($tempThumb));
+                unlink($tempThumb);
+            }
+
+            if (! $media->skip_optimize) {
+                $tempMedia = $tempDir.'/media_'.basename($media->media_path);
+                $mediaContents = $disk->get($media->media_path);
+                file_put_contents($tempMedia, $mediaContents);
+
+                ImageOptimizer::optimize($tempMedia);
+
+                $disk->put($media->media_path, file_get_contents($tempMedia));
+                unlink($tempMedia);
+            }
+
+            rmdir($tempDir);
+
+        } catch (\Exception $e) {
+            if (isset($tempDir) && is_dir($tempDir)) {
+                array_map('unlink', glob($tempDir.'/*'));
+                rmdir($tempDir);
+            }
+            if (config('app.dev_log')) {
+                Log::error('Remote image optimization failed: '.$e->getMessage());
+            }
+        }
+    }
 }

app/Services/MediaStorageService.php

@@ -18,10 +18,9 @@ class MediaStorageService
 {
     public static function store(Media $media)
     {
-        if ((bool) config_cache('pixelfed.cloud_storage') == true) {
+        if ((bool) config_cache('pixelfed.cloud_storage') == true && config('filesystems.default') === 'local') {
             (new self)->cloudStore($media);
         }
-
     }
 
     public static function move(Media $media)
@@ -30,7 +29,7 @@ class MediaStorageService
             return;
         }
 
-        if ((bool) config_cache('pixelfed.cloud_storage') == true) {
+        if ((bool) config_cache('pixelfed.cloud_storage') == true && config('filesystems.default') === 'local') {
             return (new self)->cloudMove($media);
         }
 

app/Util/Media/Blurhash.php

@@ -2,40 +2,45 @@
 
 namespace App\Util\Media;
 
-use App\Util\Blurhash\Blurhash as BlurhashEngine;
 use App\Media;
+use App\Util\Blurhash\Blurhash as BlurhashEngine;
 
-class Blurhash {
-
+class Blurhash
+{
     const DEFAULT_HASH = 'U4Rfzst8?bt7ogayj[j[~pfQ9Goe%Mj[WBay';
 
-    public static function generate(Media $media)
+    public static function generate(Media $media, $path = false)
     {
-        if(!in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) {
+        if (! in_array($media->mime, ['image/png', 'image/jpeg', 'image/jpg', 'video/mp4'])) {
             return self::DEFAULT_HASH;
         }
 
-        if($media->thumbnail_path == null) {
+        if ($media->thumbnail_path == null) {
             return self::DEFAULT_HASH;
         }
 
-        $file  = storage_path('app/' . $media->thumbnail_path);
+        if ($path) {
+            $file = $path;
+        } else {
+            $localFs = config('filesystems.default') === 'local';
+            $file = storage_path('app/'.$media->thumbnail_path);
+        }
 
-        if(!is_file($file)) {
+        if (! is_file($file)) {
             return self::DEFAULT_HASH;
         }
 
         $image = imagecreatefromstring(file_get_contents($file));
-        if(!$image) {
+        if (! $image) {
             return self::DEFAULT_HASH;
         }
         $width = imagesx($image);
         $height = imagesy($image);
 
         $pixels = [];
-        for ($y = 0; $y < $height; ++$y) {
+        for ($y = 0; $y < $height; $y++) {
             $row = [];
-            for ($x = 0; $x < $width; ++$x) {
+            for ($x = 0; $x < $width; $x++) {
                 $index = imagecolorat($image, $x, $y);
                 $colors = imagecolorsforindex($image, $index);
 
@@ -49,10 +54,10 @@ class Blurhash {
         $components_x = 4;
         $components_y = 4;
         $blurhash = BlurhashEngine::encode($pixels, $components_x, $components_y);
-        if(strlen($blurhash) > 191) {
+        if (strlen($blurhash) > 191) {
             return self::DEFAULT_HASH;
         }
+
         return $blurhash;
     }
-
 }

app/Util/Media/Image.php

@@ -3,22 +3,27 @@
 namespace App\Util\Media;
 
 use App\Media;
-use Intervention\Image\ImageManager;
+use App\Services\StatusService;
+use Cache;
 use Intervention\Image\Encoders\JpegEncoder;
-use Intervention\Image\Encoders\WebpEncoder;
-use Intervention\Image\Encoders\AvifEncoder;
 use Intervention\Image\Encoders\PngEncoder;
-use Cache, Log, Storage;
-use App\Util\Media\Blurhash;
-use App\Services\StatusService;
+use Intervention\Image\Encoders\WebpEncoder;
+use Intervention\Image\ImageManager;
+use Log;
+use Storage;
 
 class Image
 {
     public $square;
+
     public $landscape;
+
     public $portrait;
+
     public $thumbnail;
+
     public $orientation;
+
     public $acceptedMimes = [
         'image/png',
         'image/jpeg',
@@ -30,6 +35,8 @@ class Image
 
     protected $imageManager;
 
+    protected $defaultDisk;
+
     public function __construct()
     {
         ini_set('memory_limit', config('pixelfed.memory_limit', '1024M'));
@@ -38,12 +45,14 @@ class Image
         $this->landscape = $this->orientations()['landscape'];
         $this->portrait = $this->orientations()['portrait'];
         $this->thumbnail = [
-            'width'  => 640,
+            'width' => 640,
             'height' => 640,
         ];
         $this->orientation = null;
 
-        $driver = match(config('image.driver')) {
+        $this->defaultDisk = config('filesystems.default');
+
+        $driver = match (config('image.driver')) {
             'imagick' => \Intervention\Image\Drivers\Imagick\Driver::class,
             'vips' => \Intervention\Image\Drivers\Vips\Driver::class,
             default => \Intervention\Image\Drivers\Gd\Driver::class
@@ -62,15 +71,15 @@ class Image
     {
         return [
             'square' => [
-                'width'  => 1080,
+                'width' => 1080,
                 'height' => 1080,
             ],
             'landscape' => [
-                'width'  => 1920,
+                'width' => 1920,
                 'height' => 1080,
             ],
             'portrait' => [
-                'width'  => 1080,
+                'width' => 1080,
                 'height' => 1350,
             ],
         ];
@@ -80,7 +89,7 @@ class Image
     {
         if ($isThumbnail) {
             return [
-                'dimensions'  => $this->thumbnail,
+                'dimensions' => $this->thumbnail,
                 'orientation' => 'thumbnail',
             ];
         }
@@ -91,7 +100,7 @@ class Image
         $this->orientation = $orientation;
 
         return [
-            'dimensions'  => $this->orientations()[$orientation],
+            'dimensions' => $this->orientations()[$orientation],
             'orientation' => $orientation,
             'width_original' => $width,
             'height_original' => $height,
@@ -121,48 +130,68 @@ class Image
     public function handleImageTransform(Media $media, $thumbnail = false)
     {
         $path = $media->media_path;
-        $file = storage_path('app/'.$path);
-        if (!in_array($media->mime, $this->acceptedMimes)) {
+        $localFs = config('filesystems.default') === 'local';
+
+        if (! in_array($media->mime, $this->acceptedMimes)) {
             return;
         }
 
         try {
-            $fileInfo = pathinfo($file);
+            $fileContents = null;
+            $tempFile = null;
+
+            if ($this->defaultDisk === 'local') {
+                $filePath = storage_path('app/'.$path);
+                $fileContents = file_get_contents($filePath);
+            } else {
+                $fileContents = Storage::disk($this->defaultDisk)->get($path);
+            }
+
+            $fileInfo = pathinfo($path);
             $extension = strtolower($fileInfo['extension'] ?? 'jpg');
             $outputExtension = $extension;
 
             $metadata = null;
-            if (!$thumbnail && config('media.exif.database', false) == true) {
+            if (! $thumbnail && config('media.exif.database', false) == true) {
                 try {
-                    $exif = @exif_read_data($file);
+                    if ($this->defaultDisk !== 'local') {
+                        $tempFile = tempnam(sys_get_temp_dir(), 'exif_');
+                        file_put_contents($tempFile, $fileContents);
+                        $exifPath = $tempFile;
+                    } else {
+                        $exifPath = storage_path('app/'.$path);
+                    }
+
+                    $exif = @exif_read_data($exifPath);
+
                     if ($exif) {
                         $meta = [];
                         $keys = [
-                            "FileName",
-                            "FileSize",
-                            "FileType",
-                            "Make",
-                            "Model",
-                            "MimeType",
-                            "ColorSpace",
-                            "ExifVersion",
-                            "Orientation",
-                            "UserComment",
-                            "XResolution",
-                            "YResolution",
-                            "FileDateTime",
-                            "SectionsFound",
-                            "ExifImageWidth",
-                            "ResolutionUnit",
-                            "ExifImageLength",
-                            "FlashPixVersion",
-                            "Exif_IFD_Pointer",
-                            "YCbCrPositioning",
-                            "ComponentsConfiguration",
-                            "ExposureTime",
-                            "FNumber",
-                            "ISOSpeedRatings",
-                            "ShutterSpeedValue"
+                            'FileName',
+                            'FileSize',
+                            'FileType',
+                            'Make',
+                            'Model',
+                            'MimeType',
+                            'ColorSpace',
+                            'ExifVersion',
+                            'Orientation',
+                            'UserComment',
+                            'XResolution',
+                            'YResolution',
+                            'FileDateTime',
+                            'SectionsFound',
+                            'ExifImageWidth',
+                            'ResolutionUnit',
+                            'ExifImageLength',
+                            'FlashPixVersion',
+                            'Exif_IFD_Pointer',
+                            'YCbCrPositioning',
+                            'ComponentsConfiguration',
+                            'ExposureTime',
+                            'FNumber',
+                            'ISOSpeedRatings',
+                            'ShutterSpeedValue',
                         ];
                         foreach ($exif as $k => $v) {
                             if (in_array($k, $keys)) {
@@ -171,12 +200,22 @@ class Image
                         }
                         $media->metadata = json_encode($meta);
                     }
+
+                    if ($tempFile && file_exists($tempFile)) {
+                        unlink($tempFile);
+                        $tempFile = null;
+                    }
                 } catch (\Exception $e) {
-                    Log::info('EXIF extraction failed: ' . $e->getMessage());
+                    if ($tempFile && file_exists($tempFile)) {
+                        unlink($tempFile);
+                    }
+                    if (config('app.dev_log')) {
+                        Log::info('EXIF extraction failed: '.$e->getMessage());
+                    }
                 }
             }
 
-            $img = $this->imageManager->read($file);
+            $img = $this->imageManager->read($fileContents);
 
             $ratio = $this->getAspect($img->width(), $img->height(), $thumbnail);
             $aspect = $ratio['dimensions'];
@@ -209,7 +248,7 @@ class Image
                     $outputExtension = 'jpg';
                     break;
                 case 'png':
-                    $encoder = new PngEncoder();
+                    $encoder = new PngEncoder;
                     $outputExtension = 'png';
                     break;
                 case 'webp':
@@ -230,11 +269,17 @@ class Image
             }
 
             $converted = $this->setBaseName($path, $thumbnail, $outputExtension);
-            $newPath = storage_path('app/'.$converted['path']);
-
             $encoded = $encoder->encode($img);
 
-            file_put_contents($newPath, $encoded->toString());
+            if ($localFs) {
+                $newPath = storage_path('app/'.$converted['path']);
+                file_put_contents($newPath, $encoded->toString());
+            } else {
+                Storage::disk($this->defaultDisk)->put(
+                    $converted['path'],
+                    $encoded->toString()
+                );
+            }
 
             if ($thumbnail == true) {
                 $media->thumbnail_path = $converted['path'];
@@ -244,7 +289,7 @@ class Image
                 $media->height = $img->height();
                 $media->orientation = $orientation;
                 $media->media_path = $converted['path'];
-                $media->mime = 'image/' . $outputExtension;
+                $media->mime = 'image/'.$outputExtension;
             }
 
             $media->save();
@@ -253,7 +298,7 @@ class Image
                 $this->generateBlurhash($media);
             }
 
-            if($media->status_id) {
+            if ($media->status_id) {
                 Cache::forget('status:transformer:media:attachments:'.$media->status_id);
                 Cache::forget('status:thumb:'.$media->status_id);
                 StatusService::del($media->status_id);
@@ -262,7 +307,9 @@ class Image
         } catch (\Exception $e) {
             $media->processed_at = now();
             $media->save();
-            Log::info('MediaResizeException: ' . $e->getMessage() . ' | Could not process media id: ' . $media->id);
+            if (config('app.dev_log')) {
+                Log::info('MediaResizeException: '.$e->getMessage().' | Could not process media id: '.$media->id);
+            }
         }
     }
 
@@ -277,10 +324,28 @@ class Image
 
     protected function generateBlurhash($media)
     {
-        $blurhash = Blurhash::generate($media);
-        if ($blurhash) {
-            $media->blurhash = $blurhash;
-            $media->save();
+        try {
+            if ($this->defaultDisk === 'local') {
+                $thumbnailPath = storage_path('app/'.$media->thumbnail_path);
+                $blurhash = Blurhash::generate($media, $thumbnailPath);
+            } else {
+                $tempFile = tempnam(sys_get_temp_dir(), 'blurhash_');
+                $contents = Storage::disk($this->defaultDisk)->get($media->thumbnail_path);
+                file_put_contents($tempFile, $contents);
+
+                $blurhash = Blurhash::generate($media, $tempFile);
+
+                unlink($tempFile);
+            }
+
+            if ($blurhash) {
+                $media->blurhash = $blurhash;
+                $media->save();
+            }
+        } catch (\Exception $e) {
+            if (config('app.dev_log')) {
+                Log::info('Blurhash generation failed: '.$e->getMessage());
+            }
         }
     }
 }