diff --git a/lib/models/playback/playback_model.dart b/lib/models/playback/playback_model.dart
index 738a64dfa..ad7a0a90d 100644
--- a/lib/models/playback/playback_model.dart
+++ b/lib/models/playback/playback_model.dart
@@ -246,6 +246,8 @@ class PlaybackModelHelper {
 
       final firstItemIsSynced = syncedItem != null && syncedItem.status == TaskStatus.complete;
 
+      final actualStartPosition = startPosition ?? fullItem.userData.playBackPosition;
+
       final options = {
         PlaybackType.directStream,
         PlaybackType.transcode,
@@ -269,7 +271,7 @@ class PlaybackModelHelper {
               forcedPlaybackType ?? playbackType,
               oldModel: oldModel,
               libraryQueue: queue,
-              startPosition: startPosition,
+              startPosition: actualStartPosition,
             ),
           PlaybackType.offline => await _createOfflinePlaybackModel(
               fullItem,
@@ -283,7 +285,7 @@ class PlaybackModelHelper {
               fullItem,
               item.streamModel,
               forcedPlaybackType ?? PlaybackType.directStream,
-              startPosition: startPosition,
+              startPosition: actualStartPosition,
               oldModel: oldModel,
               libraryQueue: queue,
             )) ??
diff --git a/lib/providers/video_player_provider.dart b/lib/providers/video_player_provider.dart
index 5c7037f61..8e46f0e50 100644
--- a/lib/providers/video_player_provider.dart
+++ b/lib/providers/video_player_provider.dart
@@ -126,24 +126,12 @@ class VideoPlayerNotifier extends StateNotifier<MediaControlsWrapper> {
     PlaybackModel? newPlaybackModel = model;
 
     if (media != null) {
-      await state.loadVideo(model, startPosition, false);
+      await state.loadVideo(model, startPosition, true);
       await state.setVolume(ref.read(videoPlayerSettingsProvider).volume);
 
-      state.stateStream?.takeWhile((event) => event.buffering == true).listen(
-        null,
-        onDone: () async {
-          final start = startPosition;
-          if (start != Duration.zero) {
-            await state.seek(start);
-          }
-          await state.setAudioTrack(null, model);
-          await state.setSubtitleTrack(null, model);
-          state.play();
-          ref.read(playBackModel.notifier).update((state) => newPlaybackModel);
-        },
-      );
-
-      ref.read(playBackModel.notifier).update((state) => model);
+      await state.setAudioTrack(null, model);
+      await state.setSubtitleTrack(null, model);
+      ref.read(playBackModel.notifier).update((state) => newPlaybackModel);
 
       return true;
     }
diff --git a/lib/stubs/web/lib_mdk_web.dart b/lib/stubs/web/lib_mdk_web.dart
index e8cd01c43..8048c1fdf 100644
--- a/lib/stubs/web/lib_mdk_web.dart
+++ b/lib/stubs/web/lib_mdk_web.dart
@@ -24,7 +24,7 @@ class LibMDK extends BasePlayer {
   Future<void> dispose() async {}
 
   @override
-  Future<void> loadVideo(String url, bool play) async {}
+  Future<void> loadVideo(String url, bool play, {Duration startPosition = Duration.zero}) async {}
 
   void setState(PlayerState state) {}
 
diff --git a/lib/util/item_base_model/play_item_helpers.dart b/lib/util/item_base_model/play_item_helpers.dart
index 63989110e..a7788f7c7 100644
--- a/lib/util/item_base_model/play_item_helpers.dart
+++ b/lib/util/item_base_model/play_item_helpers.dart
@@ -222,7 +222,9 @@ extension ItemBaseModelExtensions on ItemBaseModel? {
       return;
     }
 
-    await _playVideo(context, startPosition: startPosition, current: model, ref: ref, cancelOperation: op);
+    final actualStartPosition = startPosition ?? await model.startDuration() ?? Duration.zero;
+
+    await _playVideo(context, startPosition: actualStartPosition, current: model, ref: ref, cancelOperation: op);
   }
 }
 
diff --git a/lib/wrappers/media_control_wrapper.dart b/lib/wrappers/media_control_wrapper.dart
index 1e77f482c..8dbef29e2 100644
--- a/lib/wrappers/media_control_wrapper.dart
+++ b/lib/wrappers/media_control_wrapper.dart
@@ -118,7 +118,10 @@ class MediaControlsWrapper extends BaseAudioHandler implements VideoPlayerContro
       final context = ref.read(localizationContextProvider);
       await (_player as NativePlayer).sendPlaybackDataToNative(context, model, startPosition);
     }
-    await _player?.loadVideo(model.media?.url ?? "", play);
+    await _player?.loadVideo(model.media?.url ?? "", play, startPosition: startPosition);
+    if (play) {
+      ref.read(playBackModel)?.playbackStarted(startPosition, ref);
+    }
     _player?.applySubtitleSettings(ref.read(subtitleSettingsProvider));
   }
 
@@ -223,8 +226,12 @@ class MediaControlsWrapper extends BaseAudioHandler implements VideoPlayerContro
   Future<void> play() async {
     WakelockPlus.enable();
     _player?.play();
+
     final currentPosition = await ref.read(playBackModel.select((value) => value?.startDuration()));
-    ref.read(playBackModel)?.playbackStarted(currentPosition ?? Duration.zero, ref);
+    final isPlaying = playbackState.value.playing;
+    if (!isPlaying) {
+      ref.read(playBackModel)?.playbackStarted(currentPosition ?? Duration.zero, ref);
+    }
 
     final playBackItem = ref.read(playBackModel.select((value) => value?.item));
     if (playBackItem == null) return;
diff --git a/lib/wrappers/players/base_player.dart b/lib/wrappers/players/base_player.dart
index 4f2b334ee..5fe417ab4 100644
--- a/lib/wrappers/players/base_player.dart
+++ b/lib/wrappers/players/base_player.dart
@@ -26,7 +26,7 @@ abstract class BasePlayer {
   });
   Future<void> dispose();
   Future<void> open(BuildContext context);
-  Future<void> loadVideo(String url, bool play);
+  Future<void> loadVideo(String url, bool play, {Duration startPosition = Duration.zero});
   Future<void> seek(Duration position);
   Future<void> play();
   Future<void> setVolume(double volume);
diff --git a/lib/wrappers/players/lib_mdk.dart b/lib/wrappers/players/lib_mdk.dart
index f01fe6beb..8a2289b6f 100644
--- a/lib/wrappers/players/lib_mdk.dart
+++ b/lib/wrappers/players/lib_mdk.dart
@@ -68,7 +68,7 @@ class LibMDK extends BasePlayer {
   }
 
   @override
-  Future<void> loadVideo(String url, bool play) async {
+  Future<void> loadVideo(String url, bool play, {Duration startPosition = Duration.zero}) async {
     _controller?.dispose();
 
     final validUrl = isValidUrl(url);
@@ -81,6 +81,10 @@ class LibMDK extends BasePlayer {
     await _controller?.initialize();
     _controller?.addListener(() => updateState());
 
+    if (startPosition != Duration.zero) {
+      await _controller?.seekTo(startPosition);
+    }
+
     if (play) {
       await _controller?.play();
     }
diff --git a/lib/wrappers/players/lib_mpv.dart b/lib/wrappers/players/lib_mpv.dart
index e6acc8e66..5d45226f7 100644
--- a/lib/wrappers/players/lib_mpv.dart
+++ b/lib/wrappers/players/lib_mpv.dart
@@ -94,11 +94,14 @@ class LibMPV extends BasePlayer {
   }
 
   @override
-  Future<void> loadVideo(String url, bool play) async {
+  Future<void> loadVideo(String url, bool play, {Duration startPosition = Duration.zero}) async {
     _loadCompleter = Completer<void>();
-    await _player?.open(mpv.Media(url), play: play);
     _firstLoadAttempt = DateTime.now();
 
+    await setStartPosition(startPosition);
+
+    await _player?.open(mpv.Media(url), play: play);
+
     _retryTimer?.cancel();
     _retryTimer = null;
 
@@ -111,27 +114,56 @@ class LibMPV extends BasePlayer {
           _retryTimer?.cancel();
           _retryTimer = null;
         } else {
-          if (lastState.buffering == false) {
-            _finishedLoading();
-          } else {
-            log("Retrying to load video $url");
-            _player?.open(mpv.Media(url), play: play);
-            _retryTimer?.reset();
-          }
+          log("Retrying to load video $url");
+          await setStartPosition(startPosition);
+          await _player?.open(mpv.Media(url), play: play);
+          _retryTimer?.reset();
         }
       },
     );
+
+    // Wait for the player to be ready
+    if (_loadCompleter?.isCompleted == false) {
+      StreamSubscription? subBuffering;
+      StreamSubscription? subDuration;
+
+      void onReady() {
+        if (_loadCompleter?.isCompleted == true) return;
+        _finishedLoading();
+        subBuffering?.cancel();
+        subDuration?.cancel();
+      }
+
+      subBuffering = _player?.stream.buffering.listen((event) {
+        if (event == false && (_player?.state.duration ?? Duration.zero) > Duration.zero) {
+          onReady();
+        }
+      });
+      subDuration = _player?.stream.duration.listen((event) {
+        if (event > Duration.zero) onReady();
+      });
+    }
+
     _loadCompleter?.future.then(
       (value) async {
-        await Future.delayed(const Duration(milliseconds: 150));
-        if (play && !lastState.playing) {
-          await _player?.play();
+        // Backup seek in case property didn't work
+        if (startPosition != Duration.zero && (_player?.state.position.inSeconds ?? 0) < startPosition.inSeconds - 5) {
+          await _player?.seek(startPosition);
         }
       },
     );
     return setState(lastState.update(buffering: true));
   }
 
+  Future<void> setStartPosition(Duration position) async {
+    if (_player?.platform case final mpv.NativePlayer platform) {
+      await platform.setProperty(
+        'start',
+        '${position.inMilliseconds / 1000}',
+      );
+    }
+  }
+
   void _finishedLoading() {
     _loadCompleter?.complete();
     _retryTimer?.cancel();
diff --git a/lib/wrappers/players/native_player.dart b/lib/wrappers/players/native_player.dart
index a3a6b8aae..50b24cd9d 100644
--- a/lib/wrappers/players/native_player.dart
+++ b/lib/wrappers/players/native_player.dart
@@ -35,7 +35,8 @@ class NativePlayer extends BasePlayer implements VideoPlayerListenerCallback {
   }
 
   @override
-  Future<void> loadVideo(String url, bool play) async => player.open(url, play);
+  Future<void> loadVideo(String url, bool play, {Duration startPosition = Duration.zero}) async =>
+      player.open(url, play);
 
   @override
   Future<StartResult> open(BuildContext newContext) async {