diff --git a/lib/extras/enc/jpegli.cc b/lib/extras/enc/jpegli.cc
index 14a7a398..461cfbe3 100644
--- a/lib/extras/enc/jpegli.cc
+++ b/lib/extras/enc/jpegli.cc
@@ -34,6 +34,7 @@
 #include "lib/extras/xyb_transform.h"
 #include "lib/jpegli/common.h"
 #include "lib/jpegli/encode.h"
+#include "lib/jpegli/encode_internal.h"
 #include "lib/jpegli/types.h"
 
 namespace jxl {
@@ -434,30 +435,49 @@ Status EncodeJpeg(const PackedPixelFile& ppf, const JpegSettings& jpeg_settings,
     }
     jpegli_set_cicp_transfer_function(&cinfo, cicp_tf);
     jpegli_set_defaults(&cinfo);
+    // All factors need to be specified to subsample the blue channel
+    // for XYB. H and V are swapped between YCbCr and XYB.
     if (!jpeg_settings.chroma_subsampling.empty()) {
+      cinfo.master->chroma_subsampling_set_by_cli = true;
       if (jpeg_settings.chroma_subsampling == "444") {
         cinfo.comp_info[0].h_samp_factor = 1;
         cinfo.comp_info[0].v_samp_factor = 1;
       } else if (jpeg_settings.chroma_subsampling == "440") {
         cinfo.comp_info[0].h_samp_factor = 1;
         cinfo.comp_info[0].v_samp_factor = 2;
+        if (jpeg_settings.xyb) {
+          cinfo.comp_info[0].h_samp_factor = 2;
+          cinfo.comp_info[0].v_samp_factor = 2;
+          cinfo.comp_info[1].h_samp_factor = 2;
+          cinfo.comp_info[1].v_samp_factor = 2;
+          cinfo.comp_info[2].h_samp_factor = 2;
+          cinfo.comp_info[2].v_samp_factor = 1;
+        }
       } else if (jpeg_settings.chroma_subsampling == "422") {
         cinfo.comp_info[0].h_samp_factor = 2;
         cinfo.comp_info[0].v_samp_factor = 1;
+        if (jpeg_settings.xyb) {
+          cinfo.comp_info[0].h_samp_factor = 2;
+          cinfo.comp_info[0].v_samp_factor = 2;
+          cinfo.comp_info[1].h_samp_factor = 2;
+          cinfo.comp_info[1].v_samp_factor = 2;
+          cinfo.comp_info[2].h_samp_factor = 1;
+          cinfo.comp_info[2].v_samp_factor = 2;
+        }
       } else if (jpeg_settings.chroma_subsampling == "420") {
         cinfo.comp_info[0].h_samp_factor = 2;
         cinfo.comp_info[0].v_samp_factor = 2;
+        if (jpeg_settings.xyb) {
+          cinfo.comp_info[0].h_samp_factor = 2;
+          cinfo.comp_info[0].v_samp_factor = 2;
+          cinfo.comp_info[1].h_samp_factor = 2;
+          cinfo.comp_info[1].v_samp_factor = 2;
+          cinfo.comp_info[2].h_samp_factor = 1;
+          cinfo.comp_info[2].v_samp_factor = 1;
+        }
       } else {
         return false;
       }
-      for (int i = 1; i < cinfo.num_components; ++i) {
-        cinfo.comp_info[i].h_samp_factor = 1;
-        cinfo.comp_info[i].v_samp_factor = 1;
-      }
-    } else if (!jpeg_settings.xyb) {
-      // Default is no chroma subsampling.
-      cinfo.comp_info[0].h_samp_factor = 1;
-      cinfo.comp_info[0].v_samp_factor = 1;
     }
     jpegli_enable_adaptive_quantization(
         &cinfo, TO_JXL_BOOL(jpeg_settings.use_adaptive_quantization));
diff --git a/lib/jpegli/encode.cc b/lib/jpegli/encode.cc
index ac4917d2..af3c8cbf 100644
--- a/lib/jpegli/encode.cc
+++ b/lib/jpegli/encode.cc
@@ -783,8 +783,11 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     default:
       JPEGLI_ERROR("Unsupported jpeg colorspace %d", colorspace);
   }
-  // Adobe marker is only needed to distinguish CMYK and YCCK JPEGs.
-  cinfo->write_Adobe_marker = TO_JXL_BOOL(cinfo->jpeg_color_space == JCS_YCCK);
+  // Adobe marker is needed to distinguish CMYK, YCCK and RGB(XYB) JPEGs.
+  cinfo->write_Adobe_marker =
+      TO_JXL_BOOL((cinfo->jpeg_color_space == JCS_CMYK ||
+                   cinfo->jpeg_color_space == JCS_YCCK ||
+                   cinfo->jpeg_color_space == JCS_RGB));
   if (cinfo->comp_info == nullptr) {
     cinfo->comp_info =
         jpegli::Allocate<jpeg_component_info>(cinfo, MAX_COMPONENTS);
@@ -795,6 +798,7 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     jpeg_component_info* comp = &cinfo->comp_info[c];
     comp->component_index = c;
     comp->component_id = c + 1;
+    // Default is no chroma subsampling.
     comp->h_samp_factor = 1;
     comp->v_samp_factor = 1;
     comp->quant_tbl_no = 0;
@@ -806,10 +810,6 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     cinfo->comp_info[1].component_id = 'G';
     cinfo->comp_info[2].component_id = 'B';
     if (cinfo->master->xyb_mode) {
-      // Subsample blue channel.
-      cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
-      cinfo->comp_info[1].h_samp_factor = cinfo->comp_info[1].v_samp_factor = 2;
-      cinfo->comp_info[2].h_samp_factor = cinfo->comp_info[2].v_samp_factor = 1;
       // Use separate quantization tables for each component
       cinfo->comp_info[1].quant_tbl_no = 1;
       cinfo->comp_info[2].quant_tbl_no = 2;
@@ -825,11 +825,6 @@ void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
     cinfo->comp_info[2].quant_tbl_no = 1;
     cinfo->comp_info[1].dc_tbl_no = cinfo->comp_info[1].ac_tbl_no = 1;
     cinfo->comp_info[2].dc_tbl_no = cinfo->comp_info[2].ac_tbl_no = 1;
-    // Use chroma subsampling by default
-    cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
-    if (colorspace == JCS_YCCK) {
-      cinfo->comp_info[3].h_samp_factor = cinfo->comp_info[3].v_samp_factor = 2;
-    }
   }
 }
 
@@ -837,12 +832,28 @@ void jpegli_set_distance(j_compress_ptr cinfo, float distance,
                          boolean force_baseline) {
   CheckState(cinfo, jpegli::kEncStart);
   cinfo->master->force_baseline = FROM_JXL_BOOL(force_baseline);
+  if (distance >= 1.9f && !(cinfo->master->xyb_mode) &&
+      !cinfo->master->chroma_subsampling_set_by_cli) {
+    // At medium qualities, 420 subsampling begins to outperform 444.
+    cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
+    if (cinfo->jpeg_color_space == JCS_YCCK) {
+      cinfo->comp_info[3].h_samp_factor = cinfo->comp_info[3].v_samp_factor = 2;
+    }
+  }
+  // Disable adaptive quantization at high qualities.
+  if (distance <= 1.0f && !(cinfo->master->xyb_mode)) {
+    cinfo->master->use_adaptive_quantization = false;
+  }
+  // At quality 100 (distance 0) auto select RGB colorspace.
+  if (distance <= 0.01f && cinfo->in_color_space == JCS_RGB) {
+    jpegli_set_colorspace(cinfo, JCS_RGB);
+  }
   float distances[NUM_QUANT_TBLS] = {distance, distance, distance};
   jpegli::SetQuantMatrices(cinfo, distances, /*add_two_chroma_tables=*/true);
 }
 
 float jpegli_quality_to_distance(int quality) {
-  return (quality >= 100  ? 0.01f
+  return (quality >= 100  ? 0.00f
           : quality >= 30 ? 0.1f + (100 - quality) * 0.09f
                           : 53.0f / 3000.0f * quality * quality -
                                 23.0f / 20.0f * quality + 25.0f);
@@ -862,6 +873,22 @@ void jpegli_set_quality(j_compress_ptr cinfo, int quality,
   CheckState(cinfo, jpegli::kEncStart);
   cinfo->master->force_baseline = FROM_JXL_BOOL(force_baseline);
   float distance = jpegli_quality_to_distance(quality);
+  if (distance >= 1.9f && !(cinfo->master->xyb_mode) &&
+      !cinfo->master->chroma_subsampling_set_by_cli) {
+    // At medium qualities, 420 subsampling begins to outperform 444.
+    cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
+    if (cinfo->jpeg_color_space == JCS_YCCK) {
+      cinfo->comp_info[3].h_samp_factor = cinfo->comp_info[3].v_samp_factor = 2;
+    }
+  }
+  // Disable adaptive quantization at high qualities.
+  if (distance <= 1.0f && !(cinfo->master->xyb_mode)) {
+    cinfo->master->use_adaptive_quantization = false;
+  }
+  // At quality 100 (distance 0) auto select RGB colorspace.
+  if (distance <= 0.01f && cinfo->in_color_space == JCS_RGB) {
+    jpegli_set_colorspace(cinfo, JCS_RGB);
+  }
   float distances[NUM_QUANT_TBLS] = {distance, distance, distance};
   jpegli::SetQuantMatrices(cinfo, distances, /*add_two_chroma_tables=*/false);
 }
diff --git a/lib/jpegli/encode_internal.h b/lib/jpegli/encode_internal.h
index 3f5d1a76..b0236a6a 100644
--- a/lib/jpegli/encode_internal.h
+++ b/lib/jpegli/encode_internal.h
@@ -75,6 +75,7 @@ struct jpeg_comp_master {
   uint8_t cicp_transfer_function;
   bool use_std_tables;
   bool use_adaptive_quantization;
+  bool chroma_subsampling_set_by_cli = false;
   int progressive_level;
   size_t xsize_blocks;
   size_t ysize_blocks;