8377512: AOT cache creation fails with invalid native pointer

src/hotspot/share/cds/aotConstantPoolResolver.cpp

@@ -81,6 +81,7 @@ bool AOTConstantPoolResolver::is_resolution_deterministic(ConstantPool* cp, int
 bool AOTConstantPoolResolver::is_class_resolution_deterministic(InstanceKlass* cp_holder, Klass* resolved_class) {
   assert(!is_in_archivebuilder_buffer(cp_holder), "sanity");
   assert(!is_in_archivebuilder_buffer(resolved_class), "sanity");
+  assert_at_safepoint(); // try_add_candidate() is called below and requires to be at safepoint.
 
   if (resolved_class->is_instance_klass()) {
     InstanceKlass* ik = InstanceKlass::cast(resolved_class);
@@ -285,7 +286,15 @@ void AOTConstantPoolResolver::maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m
     break;
 
   case Bytecodes::_invokehandle:
-    InterpreterRuntime::cds_resolve_invokehandle(raw_index, cp, CHECK);
+    if (CDSConfig::is_dumping_method_handles()) {
+      ResolvedMethodEntry* method_entry = cp->resolved_method_entry_at(raw_index);
+      int cp_index = method_entry->constant_pool_index();
+      Symbol* sig = cp->uncached_signature_ref_at(cp_index);
+      Klass* k;
+      if (check_methodtype_signature(cp(), sig, &k, true)) {
+        InterpreterRuntime::cds_resolve_invokehandle(raw_index, cp, CHECK);
+      }
+    }
     break;
 
   default:
@@ -339,7 +348,7 @@ void AOTConstantPoolResolver::preresolve_indy_cp_entries(JavaThread* current, In
 // Check the MethodType signatures used by parameters to the indy BSMs. Make sure we don't
 // use types that have been excluded, or else we might end up creating MethodTypes that cannot be stored
 // in the AOT cache.
-bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret) {
+bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret, bool is_invokehandle) {
   ResourceMark rm;
   for (SignatureStream ss(sig); !ss.is_done(); ss.next()) {
     if (ss.is_reference()) {
@@ -352,11 +361,18 @@ bool AOTConstantPoolResolver::check_methodtype_signature(ConstantPool* cp, Symbo
       if (SystemDictionaryShared::should_be_excluded(k)) {
         if (log_is_enabled(Warning, aot, resolve)) {
           ResourceMark rm;
-          log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is excluded", k->external_name());
+          log_warning(aot, resolve)("Cannot aot-resolve %s because %s is excluded",
+                                    is_invokehandle ? "invokehandle" : "Lambda proxy",
+                                    k->external_name());
         }
         return false;
       }
 
+      // cp->pool_holder() must be able to resolve k in production run
+      precond(CDSConfig::is_dumping_aot_linked_classes());
+      precond(SystemDictionaryShared::is_builtin_loader(cp->pool_holder()->class_loader_data()));
+      precond(SystemDictionaryShared::is_builtin_loader(k->class_loader_data()));
+
       if (ss.at_return_type() && return_type_ret != nullptr) {
         *return_type_ret = k;
       }
@@ -414,11 +430,44 @@ bool AOTConstantPoolResolver::check_lambda_metafactory_methodhandle_arg(Constant
     return false;
   }
 
+  // klass and sigature of the method (no need to check the method name)
   Symbol* sig = cp->method_handle_signature_ref_at(mh_index);
+  Symbol* klass_name = cp->klass_name_at(cp->method_handle_klass_index_at(mh_index));
+
   if (log_is_enabled(Debug, aot, resolve)) {
     ResourceMark rm;
     log_debug(aot, resolve)("Checking MethodType of MethodHandle for LambdaMetafactory BSM arg %d: %s", arg_i, sig->as_C_string());
   }
+
+  {
+    Klass* k = find_loaded_class(Thread::current(), cp->pool_holder()->class_loader(), klass_name);
+    if (k == nullptr) {
+      // Dumping AOT cache: all classes should have been loaded by FinalImageRecipes::load_all_classes(). k must have
+      // been a class that was excluded when FinalImageRecipes recorded all classes at the end of the training run.
+      //
+      // Dumping static CDS archive: all classes in the classlist have already been loaded, before we resolve
+      // constants. k must have been a class that was excluded when the classlist was written
+      // at the end of the training run.
+      if (log_is_enabled(Warning, aot, resolve)) {
+        ResourceMark rm;
+        log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is not loaded", klass_name->as_C_string());
+      }
+      return false;
+    }
+    if (SystemDictionaryShared::should_be_excluded(k)) {
+      if (log_is_enabled(Warning, aot, resolve)) {
+        ResourceMark rm;
+        log_warning(aot, resolve)("Cannot aot-resolve Lambda proxy because %s is excluded", k->external_name());
+      }
+      return false;
+    }
+
+    // cp->pool_holder() must be able to resolve k in production run
+    precond(CDSConfig::is_dumping_aot_linked_classes());
+    precond(SystemDictionaryShared::is_builtin_loader(cp->pool_holder()->class_loader_data()));
+    precond(SystemDictionaryShared::is_builtin_loader(k->class_loader_data()));
+  }
+
   return check_methodtype_signature(cp, sig);
 }
 

src/hotspot/share/cds/aotConstantPoolResolver.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -74,7 +74,10 @@ class AOTConstantPoolResolver :  AllStatic {
   static void maybe_resolve_fmi_ref(InstanceKlass* ik, Method* m, Bytecodes::Code bc, int raw_index,
                                     GrowableArray<bool>* resolve_fmi_list, TRAPS);
 
-  static bool check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret = nullptr);
+public:
+  static bool check_methodtype_signature(ConstantPool* cp, Symbol* sig, Klass** return_type_ret = nullptr, bool is_invokehandle = false);
+
+private:
   static bool check_lambda_metafactory_signature(ConstantPool* cp, Symbol* sig);
   static bool check_lambda_metafactory_methodtype_arg(ConstantPool* cp, int bsms_attribute_index, int arg_i);
   static bool check_lambda_metafactory_methodhandle_arg(ConstantPool* cp, int bsms_attribute_index, int arg_i);

src/hotspot/share/cds/archiveHeapWriter.cpp

@@ -754,8 +754,15 @@ void ArchiveHeapWriter::compute_ptrmap(ArchiveHeapInfo* heap_info) {
     Metadata** buffered_field_addr = requested_addr_to_buffered_addr(requested_field_addr);
     Metadata* native_ptr = *buffered_field_addr;
     guarantee(native_ptr != nullptr, "sanity");
-    guarantee(ArchiveBuilder::current()->has_been_buffered((address)native_ptr),
-              "Metadata %p should have been archived", native_ptr);
+
+    if (!ArchiveBuilder::current()->has_been_archived((address)native_ptr)) {
+      ResourceMark rm;
+      LogStreamHandle(Error, aot) log;
+      log.print("Marking native pointer for oop %p (type = %s, offset = %d)",
+                cast_from_oop<void*>(src_obj), src_obj->klass()->external_name(), field_offset);
+      src_obj->print_on(&log);
+      fatal("Metadata %p should have been archived", native_ptr);
+    }
 
     address buffered_native_ptr = ArchiveBuilder::current()->get_buffered_addr((address)native_ptr);
     address requested_native_ptr = ArchiveBuilder::current()->to_requested(buffered_native_ptr);

src/hotspot/share/oops/cpCache.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1998, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1998, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -550,16 +550,36 @@ bool ConstantPoolCache::can_archive_resolved_method(ConstantPool* src_cp, Resolv
     return false;
   }
 
+  int cp_index = method_entry->constant_pool_index();
+
   if (!method_entry->is_resolved(Bytecodes::_invokevirtual)) {
     if (method_entry->method() == nullptr) {
       return false;
     }
     if (method_entry->method()->is_continuation_native_intrinsic()) {
       return false; // FIXME: corresponding stub is generated on demand during method resolution (see LinkResolver::resolve_static_call).
     }
+    if (method_entry->is_resolved(Bytecodes::_invokehandle)) {
+      if (!CDSConfig::is_dumping_method_handles()) {
+        return false;
+      }
+
+      Symbol* sig = constant_pool()->uncached_signature_ref_at(cp_index);
+      Klass* k;
+      if (!AOTConstantPoolResolver::check_methodtype_signature(constant_pool(), sig, &k, true)) {
+        // invokehandles that were resolved in the training run should have been filtered in
+        // AOTConstantPoolResolver::maybe_resolve_fmi_ref so we shouldn't come to here.
+        //
+        // If we come here it's because the AOT assembly phase has executed an invokehandle
+        // that uses an excluded type like jdk.jfr.Event. This should not happen because the
+        // AOT assembly phase should execute only a very limited set of Java code.
+        ResourceMark rm;
+        fatal("AOT assembly phase must not resolve any invokehandles whose signatures include an excluded type");
+      }
+      return true;
+    }
   }
 
-  int cp_index = method_entry->constant_pool_index();
   assert(src_cp->tag_at(cp_index).is_method() || src_cp->tag_at(cp_index).is_interface_method(), "sanity");
 
   if (!AOTConstantPoolResolver::is_resolution_deterministic(src_cp, cp_index)) {
@@ -570,13 +590,6 @@ bool ConstantPoolCache::can_archive_resolved_method(ConstantPool* src_cp, Resolv
       method_entry->is_resolved(Bytecodes::_invokevirtual) ||
       method_entry->is_resolved(Bytecodes::_invokespecial)) {
     return true;
-  } else if (method_entry->is_resolved(Bytecodes::_invokehandle)) {
-    if (CDSConfig::is_dumping_method_handles()) {
-      // invokehandle depends on archived MethodType and LambdaForms.
-      return true;
-    } else {
-      return false;
-    }
   } else {
     return false;
   }

test/hotspot/jtreg/runtime/cds/appcds/aotCache/ExcludedClasses.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2023, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
  * @run driver jdk.test.lib.helpers.ClassFileInstaller -jar app.jar
  *                 TestApp
  *                 TestApp$Foo
+ *                 TestApp$Foo$A
  *                 TestApp$Foo$Bar
  *                 TestApp$Foo$ShouldBeExcluded
  *                 TestApp$Foo$ShouldBeExcludedChild
@@ -43,6 +44,8 @@
  */
 
 import java.io.File;
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
 import java.lang.reflect.Array;
 import java.lang.reflect.InvocationHandler;
 import java.lang.reflect.Method;
@@ -166,6 +169,8 @@ static int hotSpot() {
             long start = System.currentTimeMillis();
             while (System.currentTimeMillis() - start < 1000) {
                 lambdaHotSpot();
+                lambdaHotSpot2();
+                invokeHandleHotSpot();
                 s.hotSpot2();
                 b.hotSpot3();
                 Taz.hotSpot4();
@@ -208,12 +213,52 @@ static void lambdaHotSpot() {
             }
         }
 
+        interface A {
+            Object get();
+        }
+
+        // Lambdas that refer to excluded classes should not be AOT-resolved
+        static void lambdaHotSpot2() {
+            long start = System.currentTimeMillis();
+            A a = ShouldBeExcluded::new;
+            while (System.currentTimeMillis() - start < 20) {
+                Object obj = (ShouldBeExcluded)a.get();
+            }
+        }
+
+        static void invokeHandleHotSpot() {
+            try {
+                invokeHandleHotSpotImpl();
+            } catch (Throwable t) {
+                throw new RuntimeException("Unexpected", t);
+            }
+        }
+
+        static void invokeHandleHotSpotImpl() throws Throwable {
+            MethodHandle constructorHandle =
+                MethodHandles.lookup().unreflectConstructor(ShouldBeExcluded.class.getConstructor());
+            long start = System.currentTimeMillis();
+            while (System.currentTimeMillis() - start < 20) {
+                // The JVM rewrites this to:
+                // invokehandle  <java/lang/invoke/MethodHandle.invoke()LShouldBeExcluded;>
+                //
+                // The AOT cache must not contain a java.lang.invoke.MethodType that refers to the
+                // ShouldBeExcluded class.
+                ShouldBeExcluded o = (ShouldBeExcluded)constructorHandle.invoke();
+                if (o.getClass() != ShouldBeExcluded.class) {
+                    throw new RuntimeException("Unexpected object: " + o);
+                }
+            }
+        }
+
         static void doit(Runnable r) {
             r.run();
         }
 
         // All subclasses of jdk.jfr.Event are excluded from the CDS archive.
         static class ShouldBeExcluded extends jdk.jfr.Event {
+            public ShouldBeExcluded() {}
+
             int f = (int)(System.currentTimeMillis()) + 123;
             int m() {
                 return f + 456;
@@ -281,4 +326,3 @@ static void hotSpot4() {
         }
     }
 }
-