[ruby/openssl] Fix potential UAF in ossl_crypto_fixed_length_secure_compare

StringValue() can invoke an object's #to_str method, which may execute
arbitrary Ruby code. If #to_str mutates the other string argument during
comparison, its buffer may be reallocated, leaving a previously obtained
RSTRING_PTR pointing to freed memory.

This patch calls StringValue() on both arguments before retrieving their
data pointers to prevent a potential use-after-free.

ruby/openssl@c82c28c663

Author: apocalypse9949

ext/openssl/ossl.c

@@ -527,10 +527,18 @@ ossl_fips_mode_set(VALUE self, VALUE enabled)
 static VALUE
 ossl_crypto_fixed_length_secure_compare(VALUE dummy, VALUE str1, VALUE str2)
 {
-    const unsigned char *p1 = (const unsigned char *)StringValuePtr(str1);
-    const unsigned char *p2 = (const unsigned char *)StringValuePtr(str2);
-    long len1 = RSTRING_LEN(str1);
-    long len2 = RSTRING_LEN(str2);
+    const unsigned char *p1;
+    const unsigned char *p2;
+    long len1;
+    long len2;
+
+    StringValue(str1);
+    StringValue(str2);
+
+    p1 = (const unsigned char *)RSTRING_PTR(str1);
+    p2 = (const unsigned char *)RSTRING_PTR(str2);
+    len1 = RSTRING_LEN(str1);
+    len2 = RSTRING_LEN(str2);
 
     if (len1 != len2) {
         ossl_raise(rb_eArgError, "inputs must be of equal length");

test/openssl/test_ossl.rb

@@ -24,6 +24,16 @@ def test_fixed_length_secure_compare
     assert_raise(ArgumentError) { OpenSSL.fixed_length_secure_compare("aaa", "bbbb") }
   end
 
+  def test_fixed_length_secure_compare_uaf
+    str1 = "A" * 1000000
+    evil_obj = Object.new
+    evil_obj.define_singleton_method(:to_str) do
+      str1.replace("C" * 1000000)
+      "B" * 1000000
+    end
+    assert_false(OpenSSL.fixed_length_secure_compare(str1, evil_obj))
+  end
+
   def test_secure_compare
     assert_false(OpenSSL.secure_compare("aaa", "a"))
     assert_false(OpenSSL.secure_compare("aaa", "aa"))