123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195 |
- <?php
- namespace App\Console\Commands;
- use Illuminate\Console\Command;
- use App\Media;
- use App\Status;
- use Illuminate\Support\Facades\Log;
- use Illuminate\Support\Facades\Storage;
- use App\Services\MediaService;
- use App\Services\StatusService;
- use Symfony\Component\HttpFoundation\File\Exception\FileNotFoundException;
- use Symfony\Component\HttpKernel\Exception\NotFoundHttpException;
- class MediaS3GarbageCollector extends Command
- {
- /**
- * The name and signature of the console command.
- *
- * @var string
- */
- protected $signature = 'media:s3gc {--limit=200} {--huge} {--log-errors}';
- /**
- * The console command description.
- *
- * @var string
- */
- protected $description = 'Delete (local) media uploads that exist on S3';
- /**
- * Create a new command instance.
- *
- * @return void
- */
- public function __construct()
- {
- parent::__construct();
- }
- /**
- * Execute the console command.
- *
- * @return int
- */
- public function handle()
- {
- $enabled = (bool) config_cache('pixelfed.cloud_storage');
- if(!$enabled) {
- $this->error('Cloud storage not enabled. Exiting...');
- return;
- }
- $deleteEnabled = config('media.delete_local_after_cloud');
- if(!$deleteEnabled) {
- $this->error('Delete local storage after cloud upload is not enabled');
- return;
- }
- $limit = $this->option('limit');
- $hugeMode = $this->option('huge');
- $log = $this->option('log-errors');
- if($limit > 2000 && !$hugeMode) {
- $this->error('Limit exceeded, please use a limit under 2000 or run again with the --huge flag');
- return;
- }
- $minId = Media::orderByDesc('id')->where('created_at', '<', now()->subHours(12))->first();
- if(!$minId) {
- return;
- } else {
- $minId = $minId->id;
- }
- return $hugeMode ?
- $this->hugeMode($minId, $limit, $log) :
- $this->regularMode($minId, $limit, $log);
- }
- protected function regularMode($minId, $limit, $log)
- {
- $gc = Media::whereRemoteMedia(false)
- ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
- ->whereNot('version', '4')
- ->where('id', '<', $minId)
- ->inRandomOrder()
- ->take($limit)
- ->get();
- $totalSize = 0;
- $bar = $this->output->createProgressBar($gc->count());
- $bar->start();
- $cloudDisk = Storage::disk(config('filesystems.cloud'));
- $localDisk = Storage::disk('local');
- foreach($gc as $media) {
- try {
- if(
- $cloudDisk->exists($media->media_path)
- ) {
- if( $localDisk->exists($media->media_path)) {
- $localDisk->delete($media->media_path);
- $media->version = 4;
- $media->save();
- $totalSize = $totalSize + $media->size;
- MediaService::del($media->status_id);
- StatusService::del($media->status_id, false);
- if($localDisk->exists($media->thumbnail_path)) {
- $localDisk->delete($media->thumbnail_path);
- }
- } else {
- $media->version = 4;
- $media->save();
- }
- } else {
- if($log) {
- Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
- }
- }
- $bar->advance();
- } catch (FileNotFoundException $e) {
- $bar->advance();
- continue;
- } catch (NotFoundHttpException $e) {
- $bar->advance();
- continue;
- } catch (\Exception $e) {
- $bar->advance();
- continue;
- }
- }
- $bar->finish();
- $this->line(' ');
- $this->info('Finished!');
- if($totalSize) {
- $this->info('Cleared ' . $totalSize . ' bytes of media from local disk!');
- }
- return 0;
- }
- protected function hugeMode($minId, $limit, $log)
- {
- $cloudDisk = Storage::disk(config('filesystems.cloud'));
- $localDisk = Storage::disk('local');
- $bar = $this->output->createProgressBar($limit);
- $bar->start();
- Media::whereRemoteMedia(false)
- ->whereNotNull(['status_id', 'cdn_url', 'replicated_at'])
- ->whereNot('version', '4')
- ->where('id', '<', $minId)
- ->chunk(50, function($medias) use($cloudDisk, $localDisk, $bar, $log) {
- foreach($medias as $media) {
- try {
- if($cloudDisk->exists($media->media_path)) {
- if( $localDisk->exists($media->media_path)) {
- $localDisk->delete($media->media_path);
- $media->version = 4;
- $media->save();
- MediaService::del($media->status_id);
- StatusService::del($media->status_id, false);
- if($localDisk->exists($media->thumbnail_path)) {
- $localDisk->delete($media->thumbnail_path);
- }
- } else {
- $media->version = 4;
- $media->save();
- }
- } else {
- if($log) {
- Log::channel('media')->info('[GC] Local media not properly persisted to cloud storage', ['media_id' => $media->id]);
- }
- }
- $bar->advance();
- } catch (FileNotFoundException $e) {
- $bar->advance();
- continue;
- } catch (NotFoundHttpException $e) {
- $bar->advance();
- continue;
- } catch (\Exception $e) {
- $bar->advance();
- continue;
- }
- }
- });
- $bar->finish();
- $this->line(' ');
- $this->info('Finished!');
- }
- }
|