Add Resilient Media Storage

app/Services/MediaStorageService.php

@@ -86,12 +86,11 @@ class MediaStorageService {
 		$thumbname = array_pop($pt);
 		$storagePath = implode('/', $p);
 
-		$disk = Storage::disk(config('filesystems.cloud'));
-		$file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
-		$url = $disk->url($file);
-		$thumbFile = $disk->putFileAs($storagePath, new File($thumb), $thumbname, 'public');
-		$thumbUrl = $disk->url($thumbFile);
-		$media->thumbnail_url = $thumbUrl;
+		$url = ResilientMediaStorageService::store($storagePath, $path, $name);
+		if($thumb) {
+			$thumbUrl = ResilientMediaStorageService::store($storagePath, $thumb, $thumbname);
+			$media->thumbnail_url = $thumbUrl;
+		}
 		$media->cdn_url = $url;
 		$media->optimized_url = $url;
 		$media->replicated_at = now();

app/Services/ResilientMediaStorageService.php

@@ -0,0 +1,66 @@
+<?php
+
+namespace App\Services;
+
+use Storage;
+use Illuminate\Http\File;
+use Exception;
+use GuzzleHttp\Exception\ClientException;
+use Aws\S3\Exception\S3Exception;
+use GuzzleHttp\Exception\ConnectException;
+use League\Flysystem\UnableToWriteFile;
+
+class ResilientMediaStorageService
+{
+    static $attempts = 0;
+
+    public static function store($storagePath, $path, $name)
+    {
+        return (bool) config_cache('pixelfed.cloud_storage') && (bool) config('media.storage.remote.resilient_mode') ?
+            self::handleResilientStore($storagePath, $path, $name) :
+            self::handleStore($storagePath, $path, $name);
+    }
+
+    public static function handleStore($storagePath, $path, $name)
+    {
+        return retry(3, function() use($storagePath, $path, $name) {
+            $baseDisk = (bool) config_cache('pixelfed.cloud_storage') ? config('filesystems.cloud') : 'local';
+            $disk = Storage::disk($baseDisk);
+            $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            return $disk->url($file);
+        }, random_int(100, 500));
+    }
+
+    public static function handleResilientStore($storagePath, $path, $name)
+    {
+        $attempts = 0;
+        return retry(4, function() use($storagePath, $path, $name, $attempts) {
+            self::$attempts++;
+            usleep(100000);
+            $baseDisk = self::$attempts > 1 ? self::getAltDriver() : config('filesystems.cloud');
+            try {
+                $disk = Storage::disk($baseDisk);
+                $file = $disk->putFileAs($storagePath, new File($path), $name, 'public');
+            } catch (S3Exception | ClientException | ConnectException | UnableToWriteFile | Exception $e) {}
+            return $disk->url($file);
+        }, function (int $attempt, Exception $exception) {
+            return $attempt * 200;
+        });
+    }
+
+    public static function getAltDriver()
+    {
+        $drivers = [];
+        if(config('filesystems.disks.alt-primary.enabled')) {
+            $drivers[] = 'alt-primary';
+        }
+        if(config('filesystems.disks.alt-secondary.enabled')) {
+            $drivers[] = 'alt-secondary';
+        }
+        if(empty($drivers)) {
+            return false;
+        }
+        $key = array_rand($drivers, 1);
+        return $drivers[$key];
+    }
+}

config/filesystems.php

@@ -79,6 +79,34 @@ return [
             'throw' => true,
         ],
 
+        'alt-primary' => [
+            'enabled'  => env('ALT_PRI_ENABLED', false),
+            'driver'   => 's3',
+            'key'      => env('ALT_PRI_AWS_ACCESS_KEY_ID'),
+            'secret'   => env('ALT_PRI_AWS_SECRET_ACCESS_KEY'),
+            'region'   => env('ALT_PRI_AWS_DEFAULT_REGION'),
+            'bucket'   => env('ALT_PRI_AWS_BUCKET'),
+            'visibility' => 'public',
+            'url'      => env('ALT_PRI_AWS_URL'),
+            'endpoint' => env('ALT_PRI_AWS_ENDPOINT'),
+            'use_path_style_endpoint' => env('ALT_PRI_AWS_USE_PATH_STYLE_ENDPOINT', false),
+            'throw' => true,
+        ],
+
+        'alt-secondary' => [
+            'enabled'  => env('ALT_SEC_ENABLED', false),
+            'driver'   => 's3',
+            'key'      => env('ALT_SEC_AWS_ACCESS_KEY_ID'),
+            'secret'   => env('ALT_SEC_AWS_SECRET_ACCESS_KEY'),
+            'region'   => env('ALT_SEC_AWS_DEFAULT_REGION'),
+            'bucket'   => env('ALT_SEC_AWS_BUCKET'),
+            'visibility' => 'public',
+            'url'      => env('ALT_SEC_AWS_URL'),
+            'endpoint' => env('ALT_SEC_AWS_ENDPOINT'),
+            'use_path_style_endpoint' => env('ALT_SEC_AWS_USE_PATH_STYLE_ENDPOINT', false),
+            'throw' => true,
+        ],
+
         'spaces' => [
             'driver' => 's3',
             'key' => env('DO_SPACES_KEY'),

config/media.php

@@ -1,24 +1,26 @@
 <?php
 
 return [
-	'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
+    'delete_local_after_cloud' => env('MEDIA_DELETE_LOCAL_AFTER_CLOUD', true),
 
-	'exif' => [
-		'database' => env('MEDIA_EXIF_DATABASE', false),
-	],
+    'exif' => [
+        'database' => env('MEDIA_EXIF_DATABASE', false),
+    ],
 
-	'storage' => [
-		'remote' => [
-			/*
-		    |--------------------------------------------------------------------------
-		    | Store remote media on cloud/S3
-		    |--------------------------------------------------------------------------
-		    |
-		    | Set this to cache remote media on cloud/S3 filesystem drivers.
-		    | Disabled by default.
-		    |
-		    */
-			'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false)
-		],
-	]
+    'storage' => [
+        'remote' => [
+            /*
+            |--------------------------------------------------------------------------
+            | Store remote media on cloud/S3
+            |--------------------------------------------------------------------------
+            |
+            | Set this to cache remote media on cloud/S3 filesystem drivers.
+            | Disabled by default.
+            |
+            */
+            'cloud' => env('MEDIA_REMOTE_STORE_CLOUD', false),
+
+            'resilient_mode' => env('ALT_PRI_ENABLED', false) || env('ALT_SEC_ENABLED', false),
+        ],
+    ]
 ];