Merge pull request #3545 from pixelfed/staging

Staging

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### New Features
 - Custom content warnings/spoiler text ([d4864213](https://github.com/pixelfed/pixelfed/commit/d4864213))
+- Add NetworkTimelineService cache ([1310d95c](https://github.com/pixelfed/pixelfed/commit/1310d95c))
 
 ### Breaking
 - Replaced `predis` with `phpredis` as default redis driver due to predis being deprecated, install [phpredis](https://github.com/phpredis/phpredis/blob/develop/INSTALL.markdown) if you're still using predis.
@@ -27,6 +28,13 @@
 - Refactor AP profileFetch logic to fix race conditions and improve updating fields and avatars ([505261da](https://github.com/pixelfed/pixelfed/commit/505261da))
 - Update network timeline api, limit falloff to 2 days ([13a66303](https://github.com/pixelfed/pixelfed/commit/13a66303))
 - Update Inbox, store follow request activity ([c82f2085](https://github.com/pixelfed/pixelfed/commit/c82f2085))
+- Update UserFilterService, improve cache strategy by using in-memory state via UserFilterObserver for empty lists with a ttl of 90 days ([9c17def4](https://github.com/pixelfed/pixelfed/commit/9c17def4))
+- Update ApiV1Controller, add network timeline support via NetworkTimelineService ([f54fd6e9](https://github.com/pixelfed/pixelfed/commit/f54fd6e9))
+- Bump max_collection_length default to 100 from 18 ([65cf9cca](https://github.com/pixelfed/pixelfed/commit/65cf9cca))
+- Improve follow request flow, federate rejections and delete rejections from database to properly handle future follow requests from same actor ([4470981a](https://github.com/pixelfed/pixelfed/commit/4470981a))
+- Update follower counts on follow_request approval ([e97900a0](https://github.com/pixelfed/pixelfed/commit/e97900a0))
+- Update ApiV1Controller, improve local/remote logic in public timeline endpoint ([4ff179ad](https://github.com/pixelfed/pixelfed/commit/4ff179ad))
+- Update ApiV1Controller, fix network timeline ([11e99d78](https://github.com/pixelfed/pixelfed/commit/11e99d78))
 -  ([](https://github.com/pixelfed/pixelfed/commit/))
 
 ## [v0.11.3 (2022-05-09)](https://github.com/pixelfed/pixelfed/compare/v0.11.2...v0.11.3)

app/Http/Controllers/Api/ApiV1Controller.php

@@ -1973,17 +1973,18 @@ class ApiV1Controller extends Controller
 		  'min_id'      => 'nullable|integer|min:0|max:' . PHP_INT_MAX,
 		  'max_id'      => 'nullable|integer|min:0|max:' . PHP_INT_MAX,
 		  'limit'       => 'nullable|integer|max:100',
-		  'remote'		=> 'sometimes'
+		  'remote'		=> 'sometimes',
+		  'local'		=> 'sometimes'
 		]);
 
 		$min = $request->input('min_id');
 		$max = $request->input('max_id');
 		$limit = $request->input('limit') ?? 20;
 		$user = $request->user();
-		$remote = $request->has('remote');
+		$remote = ($request->has('remote') && $request->input('remote') == true) || ($request->filled('local') && $request->input('local') != true);
         $filtered = $user ? UserFilterService::filters($user->profile_id) : [];
 
-        if($remote && config('instance.timeline.network.cached')) {
+        if((!$request->has('local') || $remote) && config('instance.timeline.network.cached')) {
 			Cache::remember('api:v1:timelines:network:cache_check', 10368000, function() {
 				if(NetworkTimelineService::count() == 0) {
 					NetworkTimelineService::warmCache(true, config('instance.timeline.network.cache_dropoff'));

app/Http/Controllers/LiveStreamController.php

@@ -12,225 +12,304 @@ use App\Services\LiveStreamService;
 
 class LiveStreamController extends Controller
 {
-    public function createStream(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	if(config('livestreaming.broadcast.limits.enabled')) {
-    		if($request->user()->is_admin) {
-
-    		} else {
-    			$limits = config('livestreaming.broadcast.limits');
-    			$user = $request->user();
-    			abort_if($limits['admins_only'] && $user->is_admin == false, 401, 'LSE:003');
-    			if($limits['min_account_age']) {
-    				abort_if($user->created_at->gt(now()->subDays($limits['min_account_age'])), 403, 'LSE:005');
-    			}
-
-    			if($limits['min_follower_count']) {
-    				$account = AccountService::get($user->profile_id);
-    				abort_if($account['followers_count'] < $limits['min_follower_count'], 403, 'LSE:008');
-    			}
-    		}
-    	}
-
-    	$this->validate($request, [
-    		'name' => 'nullable|string|max:80',
-    		'description' => 'nullable|string|max:240',
-    		'visibility' => 'required|in:public,private'
-    	]);
-
-    	$stream = new LiveStream;
-    	$stream->name = $request->input('name');
-    	$stream->description = $request->input('description');
-    	$stream->visibility = $request->input('visibility');
-    	$stream->profile_id = $request->user()->profile_id;
-    	$stream->stream_id = Str::random(40);
-    	$stream->stream_key = Str::random(64);
-    	$stream->save();
-
-    	return [
-    		'url' => $stream->getStreamKeyUrl(),
-    		'id' => $stream->stream_id
-    	];
-    }
-
-    public function getUserStream(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	$stream = LiveStream::whereProfileId($request->input('profile_id'))->first();
-
-    	if(!$stream) {
-    		return [];
-    	}
-
-    	$res = [];
-    	$owner = $stream->profile_id == $request->user()->profile_id;
-
-    	if($stream->visibility === 'private') {
-    		abort_if(!$owner && !FollowerService::follows($request->user()->profile_id, $stream->profile_id), 403, 'LSE:011');
-    	}
-
-    	if($owner) {
-    		$res['stream_key'] = $stream->stream_key;
-    		$res['stream_id'] = $stream->stream_id;
-    		$res['stream_url'] = $stream->getStreamKeyUrl();
-    	}
-
-    	if($stream->live_at == null) {
-    		$res['hls_url'] = null;
-	    	$res['name'] = $stream->name;
-	    	$res['description'] = $stream->description;
-	    	return $res;
-    	}
-
-    	$res = [
-    		'hls_url' => $stream->getHlsUrl(),
-    		'name' => $stream->name,
-    		'description' => $stream->description
-    	];
-
-    	return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
-    }
-
-    public function deleteStream(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	LiveStream::whereProfileId($request->user()->profile_id)
-    		->get()
-    		->each(function($stream) {
-    			Storage::deleteDirectory("public/live-hls/{$stream->stream_id}");
-    			$stream->delete();
-    		});
-
-    	return [200];
-    }
-
-    public function getActiveStreams(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	return LiveStream::whereVisibility('local')->whereNotNull('live_at')->get()->map(function($stream) {
-    		return [
-    			'account' => AccountService::get($stream->profile_id),
-    			'stream_id' => $stream->stream_id
-    		];
-    	});
-    }
-
-    public function getLatestChat(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	$stream = LiveStream::whereProfileId($request->input('profile_id'))
-    		->whereNotNull('live_at')
-    		->first();
-
-    	if(!$stream) {
-    		return [];
-    	}
-
-    	$owner = $stream->profile_id == $request->user()->profile_id;
-    	if($stream->visibility === 'private') {
-    		abort_if(!$owner && !FollowerService::follows($request->user()->profile_id, $stream->profile_id), 403, 'LSE:021');
-    	}
-
-    	$res = collect(LiveStreamService::getComments($stream->profile_id))
-    		->map(function($r) {
-    			return json_decode($r);
-    		});
-
-    	return $res;
-    }
-
-    public function addChatComment(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	$this->validate($request, [
-    		'profile_id' => 'required|exists:profiles,id',
-    		'message' => 'required|max:140'
-    	]);
-
-    	$stream = LiveStream::whereProfileId($request->input('profile_id'))->firstOrFail();
-
-    	$owner = $stream->profile_id == $request->user()->profile_id;
-    	if($stream->visibility === 'private') {
-    		abort_if(!$owner && !FollowerService::follows($request->user()->profile_id, $stream->profile_id), 403, 'LSE:022');
-    	}
-
-    	$res = [
-    		'pid' => (string) $request->user()->profile_id,
-    		'username' => $request->user()->username,
-    		'text' => $request->input('message'),
-    		'ts' => now()->timestamp
-    	];
-
-    	LiveStreamService::addComment($stream->profile_id, json_encode($res, JSON_UNESCAPED_SLASHES));
-
-    	return $res;
-    }
-
-    public function editStream(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	$this->validate($request, [
-    		'name' => 'nullable|string|max:80',
-    		'description' => 'nullable|string|max:240'
-    	]);
-
-    	$stream = LiveStream::whereProfileId($request->user()->profile_id)->firstOrFail();
-    	$stream->name = $request->input('name');
-    	$stream->description = $request->input('description');
-    	$stream->save();
-
-    	return;
-    }
-
-    public function deleteChatComment(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	$this->validate($request, [
-    		'profile_id' => 'required|exists:profiles,id',
-    		'message' => 'required'
-    	]);
-
-    	abort_if($request->user()->profile_id != $request->input('profile_id'), 403);
-
-    	$stream = LiveStream::whereProfileId($request->user()->profile_id)->firstOrFail();
-
-    	$payload = $request->input('message');
-    	$payload = json_encode($payload, JSON_UNESCAPED_SLASHES);
-    	LiveStreamService::deleteComment($stream->profile_id, $payload);
-
-    	return;
-    }
-
-    public function getConfig(Request $request)
-    {
-    	abort_if(!config('livestreaming.enabled'), 400);
-    	abort_if(!$request->user(), 403);
-
-    	$res = [
-    		'enabled' => config('livestreaming.enabled'),
-    		'broadcast' => [
-    			'sources' => config('livestreaming.broadcast.sources'),
-    			'limits' => config('livestreaming.broadcast.limits')
-    		],
-    	];
-
-    	return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
-    }
+	public function createStream(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		if(config('livestreaming.broadcast.limits.enabled')) {
+			if($request->user()->is_admin) {
+
+			} else {
+				$limits = config('livestreaming.broadcast.limits');
+				$user = $request->user();
+				abort_if($limits['admins_only'] && $user->is_admin == false, 401, 'LSE:003');
+				if($limits['min_account_age']) {
+					abort_if($user->created_at->gt(now()->subDays($limits['min_account_age'])), 403, 'LSE:005');
+				}
+
+				if($limits['min_follower_count']) {
+					$account = AccountService::get($user->profile_id);
+					abort_if($account['followers_count'] < $limits['min_follower_count'], 403, 'LSE:008');
+				}
+			}
+		}
+
+		$this->validate($request, [
+			'name' => 'nullable|string|max:80',
+			'description' => 'nullable|string|max:240',
+			'visibility' => 'required|in:public,private'
+		]);
+
+		$stream = new LiveStream;
+		$stream->name = $request->input('name');
+		$stream->description = $request->input('description');
+		$stream->visibility = $request->input('visibility');
+		$stream->profile_id = $request->user()->profile_id;
+		$stream->stream_id = Str::random(40) . '_' . $stream->profile_id;
+		$stream->stream_key = 'streamkey-' . Str::random(64);
+		$stream->save();
+
+		return [
+			'host' => $stream->getStreamServer(),
+			'key' => $stream->stream_key,
+			'url' => $stream->getStreamKeyUrl(),
+			'id' => $stream->stream_id
+		];
+	}
+
+	public function getUserStream(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		$stream = LiveStream::whereProfileId($request->input('profile_id'))->first();
+
+		if(!$stream) {
+			return [];
+		}
+
+		$res = [];
+		$owner = $stream->profile_id == $request->user()->profile_id;
+
+		if($stream->visibility === 'private') {
+			abort_if(!$owner && !FollowerService::follows($request->user()->profile_id, $stream->profile_id), 403, 'LSE:011');
+		}
+
+		if($owner) {
+			$res['stream_key'] = $stream->stream_key;
+			$res['stream_id'] = $stream->stream_id;
+			$res['stream_url'] = $stream->getStreamKeyUrl();
+		}
+
+		if($stream->live_at == null) {
+			$res['hls_url'] = null;
+			$res['name'] = $stream->name;
+			$res['description'] = $stream->description;
+			return $res;
+		}
+
+		$res = [
+			'hls_url' => $stream->getHlsUrl(),
+			'name' => $stream->name,
+			'description' => $stream->description
+		];
+
+		return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
+	}
+
+	public function deleteStream(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		LiveStream::whereProfileId($request->user()->profile_id)
+			->get()
+			->each(function($stream) {
+				Storage::deleteDirectory("public/live-hls/{$stream->stream_id}");
+				$stream->delete();
+			});
+
+		return [200];
+	}
+
+	public function getActiveStreams(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		return LiveStream::whereVisibility('local')->whereNotNull('live_at')->get()->map(function($stream) {
+			return [
+				'account' => AccountService::get($stream->profile_id),
+				'stream_id' => $stream->stream_id
+			];
+		});
+	}
+
+	public function getLatestChat(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		$stream = LiveStream::whereProfileId($request->input('profile_id'))
+			->whereNotNull('live_at')
+			->first();
+
+		if(!$stream) {
+			return [];
+		}
+
+		$owner = $stream->profile_id == $request->user()->profile_id;
+		if($stream->visibility === 'private') {
+			abort_if(!$owner && !FollowerService::follows($request->user()->profile_id, $stream->profile_id), 403, 'LSE:021');
+		}
+
+		$res = collect(LiveStreamService::getComments($stream->profile_id))
+			->map(function($res) {
+				return json_decode($res);
+			});
+
+		return $res;
+	}
+
+	public function addChatComment(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		$this->validate($request, [
+			'profile_id' => 'required|exists:profiles,id',
+			'message' => 'required|max:140'
+		]);
+
+		$stream = LiveStream::whereProfileId($request->input('profile_id'))->firstOrFail();
+
+		$owner = $stream->profile_id == $request->user()->profile_id;
+		if($stream->visibility === 'private') {
+			abort_if(!$owner && !FollowerService::follows($request->user()->profile_id, $stream->profile_id), 403, 'LSE:022');
+		}
+
+		$res = [
+			'pid' => (string) $request->user()->profile_id,
+			'username' => $request->user()->username,
+			'text' => $request->input('message'),
+			'ts' => now()->timestamp
+		];
+
+		LiveStreamService::addComment($stream->profile_id, json_encode($res, JSON_UNESCAPED_SLASHES));
+
+		return $res;
+	}
+
+	public function editStream(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		$this->validate($request, [
+			'name' => 'nullable|string|max:80',
+			'description' => 'nullable|string|max:240'
+		]);
+
+		$stream = LiveStream::whereProfileId($request->user()->profile_id)->firstOrFail();
+		$stream->name = $request->input('name');
+		$stream->description = $request->input('description');
+		$stream->save();
+
+		return;
+	}
+
+	public function deleteChatComment(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		$this->validate($request, [
+			'profile_id' => 'required|exists:profiles,id',
+			'message' => 'required'
+		]);
+
+		abort_if($request->user()->profile_id != $request->input('profile_id'), 403);
+
+		$stream = LiveStream::whereProfileId($request->user()->profile_id)->firstOrFail();
+
+		$payload = $request->input('message');
+		$payload = json_encode($payload, JSON_UNESCAPED_SLASHES);
+		LiveStreamService::deleteComment($stream->profile_id, $payload);
+
+		return;
+	}
+
+	public function getConfig(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->user(), 403);
+
+		$res = [
+			'enabled' => config('livestreaming.enabled'),
+			'broadcast' => [
+				'sources' => config('livestreaming.broadcast.sources'),
+				'limits' => config('livestreaming.broadcast.limits')
+			],
+		];
+
+		return response()->json($res, 200, [], JSON_UNESCAPED_SLASHES);
+	}
+
+	public function clientBroadcastPublish(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		$key = $request->input('name');
+		$name = $request->input('name');
+
+		abort_if(!$name, 400);
+
+		if(empty($key)) {
+			abort_if(!$request->filled('tcurl'), 400);
+			$url = $this->parseStreamUrl($request->input('tcurl'));
+			$key = $request->filled('name') ? $request->input('name') : $url['name'];
+		}
+
+		$token = substr($name, 0, 10) === 'streamkey-';
+
+		if($token) {
+			$stream = LiveStream::whereStreamKey($key)->firstOrFail();
+			return redirect($stream->getStreamRtmpUrl(), 301);
+		} else {
+			$stream = LiveStream::whereStreamId($key)->firstOrFail();
+		}
+
+		if($request->filled('name') && $token == false) {
+			$stream->live_at = now();
+			$stream->save();
+			return [];
+		} else {
+			abort(400);
+		}
+
+		abort(400);
+	}
+
+	public function clientBroadcastFinish(Request $request)
+	{
+		abort_if(!config('livestreaming.enabled'), 400);
+		abort_if(!$request->filled('tcurl'), 400);
+		$url = $this->parseStreamUrl($request->input('tcurl'));
+		$name = $url['name'] ?? $request->input('name');
+
+		$stream = LiveStream::whereStreamId($name)->whereStreamKey($url['key'])->firstOrFail();
+
+		if(config('livestreaming.broadcast.delete_token_after_finished')) {
+			$stream->delete();
+		} else {
+			$stream->live_at = null;
+			$stream->save();
+		}
+
+		return [];
+	}
+
+	protected function parseStreamUrl($url)
+	{
+		$name = null;
+		$key = null;
+		$query = parse_url($url, PHP_URL_QUERY);
+		$parts = explode('&', $query);
+		foreach($parts as $part) {
+			if (!strlen(trim($part))) {
+				continue;
+			}
+			$s = explode('=', $part);
+			if(in_array($s[0], ['name', 'key'])) {
+				if($s[0] === 'name') {
+					$name = $s[1];
+				}
+				if($s[0] === 'key') {
+					$key = $s[1];
+				}
+			}
+		}
+
+		return ['name' => $name, 'key' => $key];
+	}
 }

app/Models/LiveStream.php

@@ -8,40 +8,34 @@ use Storage;
 
 class LiveStream extends Model
 {
-    use HasFactory;
-
-    public function getHlsUrl()
-    {
-    	$path = Storage::url("live-hls/{$this->stream_id}/index.m3u8");
-    	return url($path);
-    }
-
-    public function getStreamKeyUrl()
-    {
-    	$proto = 'rtmp://';
-    	$host = config('livestreaming.server.host');
-    	$port = ':' . config('livestreaming.server.port');
-    	$path = '/' . config('livestreaming.server.path') . '?';
-    	$query = http_build_query([
-    		'name' => $this->stream_id,
-    		'key' => $this->stream_key,
-    		'ts' => time()
-    	]);
-
-    	return $proto . $host . $port . $path . $query;
-    }
-
-    public function getStreamRtmpUrl()
-    {
-    	$proto = 'rtmp://';
-    	$host = config('livestreaming.server.host');
-    	$port = ':' . config('livestreaming.server.port');
-    	$path = '/' . config('livestreaming.server.path') . '/'. $this->stream_id . '?';
-    	$query = http_build_query([
-    		'key' => $this->stream_key,
-    		'ts' => time()
-    	]);
-
-    	return $proto . $host . $port . $path . $query;
-    }
+	use HasFactory;
+
+	public function getHlsUrl()
+	{
+		$path = Storage::url("live-hls/{$this->stream_id}/index.m3u8");
+		return url($path);
+	}
+
+	public function getStreamServer()
+	{
+		$proto = 'rtmp://';
+		$host = config('livestreaming.server.host');
+		$port = ':' . config('livestreaming.server.port');
+		$path = '/' . config('livestreaming.server.path');
+		return $proto . $host . $port . $path;
+	}
+
+	public function getStreamKeyUrl()
+	{
+		$path = $this->getStreamServer() . '?';
+		$query = http_build_query([
+			'name' => $this->stream_key,
+		]);
+		return $path . $query;
+	}
+
+	public function getStreamRtmpUrl()
+	{
+		return $this->getStreamServer() . '/' . $this->stream_id;
+	}
 }

routes/api.php

@@ -106,5 +106,7 @@ Route::group(['prefix' => 'api'], function() use($middleware) {
 		Route::post('chat/message', 'LiveStreamController@addChatComment')->middleware($middleware);
 		Route::post('chat/delete', 'LiveStreamController@deleteChatComment')->middleware($middleware);
 		Route::get('config', 'LiveStreamController@getConfig')->middleware($middleware);
+		Route::post('broadcast/publish', 'LiveStreamController@clientBroadcastPublish');
+		Route::post('broadcast/finish', 'LiveStreamController@clientBroadcastFinish');
 	});
 });

storage/app/public/.gitignore

@@ -5,3 +5,4 @@
 !emoji/
 !textimg/
 !headers/
+!live-hls/

storage/app/public/live-hls/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore