diff --git a/src/Decoders/NativeObjectDecoder.php b/src/Decoders/NativeObjectDecoder.php
index 2b0b67c..7229881 100644
--- a/src/Decoders/NativeObjectDecoder.php
+++ b/src/Decoders/NativeObjectDecoder.php
@@ -6,6 +6,7 @@
 
 use Intervention\Image\Drivers\SpecializableDecoder;
 use Intervention\Image\Drivers\Vips\Core;
+use Intervention\Image\Drivers\Vips\Modifiers\AlignRotationModifier;
 use Intervention\Image\Exceptions\DecoderException;
 use Intervention\Image\Exceptions\RuntimeException;
 use Intervention\Image\Image;
@@ -35,17 +36,17 @@ public function decode(mixed $input): ImageInterface|ColorInterface
             throw new DecoderException('Unable to decode input');
         }
 
-        // auto-rotate
-        if ($this->driver()->config()->autoOrientation === true) {
-            $input = $input->autorot();
-        }
-
         // build image instance
         $image = new Image(
             $this->driver(),
             new Core($input)
         );
 
+        // auto-rotate
+        if ($this->driver()->config()->autoOrientation === true && $this->exifRotation($input) > 1) {
+            $image->modify(new AlignRotationModifier());
+        }
+
         // set media type on origin
         if ($mediaType = $this->vipsMediaType($input)) {
             $image->origin()->setMediaType($mediaType);
@@ -96,4 +97,22 @@ protected function vipsMediaType(VipsImage $vips): ?MediaType
             default => null
         };
     }
+
+    /**
+     * Return the exif rotation of the given image or null if there isn't any
+     */
+    protected function exifRotation(VipsImage $vips): ?int
+    {
+        if (!in_array('orientation', $vips->getFields())) {
+            return null;
+        }
+
+        try {
+            $orientation = $vips->get('orientation');
+        } catch (VipsException) {
+            return null;
+        }
+
+        return is_int($orientation) ? $orientation : null;
+    }
 }
diff --git a/src/Modifiers/AlignRotationModifier.php b/src/Modifiers/AlignRotationModifier.php
index d0440d7..0ddb437 100644
--- a/src/Modifiers/AlignRotationModifier.php
+++ b/src/Modifiers/AlignRotationModifier.php
@@ -4,6 +4,7 @@
 
 namespace Intervention\Image\Drivers\Vips\Modifiers;
 
+use Intervention\Image\Drivers\Vips\Core;
 use Intervention\Image\Interfaces\ImageInterface;
 use Intervention\Image\Interfaces\SpecializedInterface;
 use Intervention\Image\Modifiers\AlignRotationModifier as GenericAlignRotationModifier;
@@ -18,7 +19,8 @@ class AlignRotationModifier extends GenericAlignRotationModifier implements Spec
     public function apply(ImageInterface $image): ImageInterface
     {
         $image->core()->setNative(
-            $image->core()->native()->autorot()
+            // autorot() does not seem to work with the default sequential access of this library
+            Core::ensureInMemory($image->core())->native()->autorot()
         );
 
         return $image;
diff --git a/tests/Unit/Modifiers/AlignRotationModifierTest.php b/tests/Unit/Modifiers/AlignRotationModifierTest.php
index 1c9ecaa..f76c3f5 100644
--- a/tests/Unit/Modifiers/AlignRotationModifierTest.php
+++ b/tests/Unit/Modifiers/AlignRotationModifierTest.php
@@ -15,9 +15,29 @@ public function testApply(): void
         $image = (new ImageManager(Driver::class, autoOrientation: false))->read(
             $this->getTestResourcePath('orientation.jpg')
         );
+
         $this->assertColor(250, 2, 3, 255, $image->pickColor(3, 3));
         $result = $image->orient();
         $this->assertColor(1, 0, 254, 255, $image->pickColor(3, 3));
         $this->assertColor(1, 0, 254, 255, $result->pickColor(3, 3));
     }
+
+    /**
+     * According to libvips/libvips#4475 you can't use autorot with
+     * orientated JPEGs and sequential access. This test checks the negative
+     * scenario, before the fix.
+     */
+    public function testAutoOrientationSave(): void
+    {
+        $tmpFile = sys_get_temp_dir() . '/out.jpg';
+
+        (new ImageManager(Driver::class, autoOrientation: true))->read(
+            $this->getTestResourcePath('orientation.jpg')
+        )->save($tmpFile);
+
+        $this->assertFileExists($tmpFile);
+
+        // remove tmp file
+        unlink($tmpFile);
+    }
 }