Readded ldap plugin, added instrumentation module opening support (#2977)

Author: JonasKunz

CHANGELOG.asciidoc

@@ -34,11 +34,11 @@ endif::[]
 ===== Features
 * Add experimental log sending from the agent with `log_sending` - {pull}2694[#2694]
 * Add bootstrap checks that enable <<jvm-filtering>> on startup - {pull}2951[#2951]
+* Added support for LDAP - {pull}2977[#2977]
 
 [float]
 ===== Bug fixes
-
-* Use `127.0.0.1` as defaut for `server_url` to prevent ipv6 ambiguity - {pull}2927[#2927]
+* Use `127.0.0.1` as default for `server_url` to prevent ipv6 ambiguity - {pull}2927[#2927]
 * Fix some span-compression concurrency issues - {pull}2865[#2865]
 * Add warning when agent is accidentally started on a JVM/JDK command-line tool - {pull}2924[#2924]
 * Fix `NullPointerException` caused by the Elasticsearch REST client instrumentation when collecting dropped span metrics - {pull}2959[#2959]

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/ElasticApmAgent.java

@@ -30,6 +30,7 @@
 import co.elastic.apm.agent.bci.bytebuddy.RootPackageCustomLocator;
 import co.elastic.apm.agent.bci.bytebuddy.SimpleMethodSignatureOffsetMappingFactory;
 import co.elastic.apm.agent.bci.classloading.ExternalPluginClassLoader;
+import co.elastic.apm.agent.bci.modules.ModuleOpener;
 import co.elastic.apm.agent.common.ThreadUtils;
 import co.elastic.apm.agent.common.util.SystemStandardOutputLogger;
 import co.elastic.apm.agent.configuration.CoreConfiguration;
@@ -83,8 +84,10 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -132,7 +135,7 @@ public class ElasticApmAgent {
  * with the corresponding instrumentation class.
  */
  private static final ConcurrentMap<String, ClassLoader> adviceClassName2instrumentationClassLoader = new ConcurrentHashMap<>();
- private static final ConcurrentMap<String, Collection<String>> pluginPackages2pluginClassLoaderRootPackages = new ConcurrentHashMap<>();
+ private static final ConcurrentMap<String, PluginClassLoaderCustomizations> pluginPackages2pluginClassLoaderCustomizations = new ConcurrentHashMap<>();
 
  /**
  * Called reflectively by {@code co.elastic.apm.agent.premain.AgentMain} to initialize the agent
@@ -252,9 +255,14 @@ private static synchronized void initInstrumentation(final ElasticApmTracer trac
  PluginClassLoaderRootPackageCustomizer.class,
  getAgentClassLoader());
  for (PluginClassLoaderRootPackageCustomizer rootPackageCustomizer : rootPackageCustomizers) {
- Collection<String> previous = pluginPackages2pluginClassLoaderRootPackages.put(
+ PluginClassLoaderCustomizations customizations = new PluginClassLoaderCustomizations(
+ rootPackageCustomizer.pluginClassLoaderRootPackages(),
+ rootPackageCustomizer.requiredModuleOpens()
+ );
+ PluginClassLoaderCustomizations previous = pluginPackages2pluginClassLoaderCustomizations.put(
  rootPackageCustomizer.getPluginPackage(),
- Collections.unmodifiableList(new ArrayList<>(rootPackageCustomizer.pluginClassLoaderRootPackages())));
+ customizations
+ );
  if (previous != null) {
  throw new IllegalStateException("Only one PluginClassLoaderRootPackageCustomizer is allowed per plugin package: "
  + rootPackageCustomizer.getPluginPackage());
@@ -316,6 +324,18 @@ public void run() {
  }
  }
 
+ public static boolean areModulesSupported() {
+ return ModuleOpener.areModulesSupported();
+ }
+ 
+ public static boolean openModule(Class<?> classFromTargetModule, ClassLoader openTo, Collection<String> packagesToOpen) {
+ if (instrumentation == null) {
+ throw new IllegalStateException("Can't open modules before the agent has been initialized");
+ }
+ return ModuleOpener.getInstance().openModuleTo(instrumentation, classFromTargetModule, openTo, packagesToOpen);
+ }
+
+
  static synchronized void doReInitInstrumentation(Iterable<ElasticApmInstrumentation> instrumentations) {
  Logger logger = getLogger();
  logger.info("Re initializing instrumentation");
@@ -651,7 +671,7 @@ public static synchronized void reset() {
  instrumentation = null;
  IndyPluginClassLoaderFactory.clear();
  adviceClassName2instrumentationClassLoader.clear();
- pluginPackages2pluginClassLoaderRootPackages.clear();
+ pluginPackages2pluginClassLoaderCustomizations.clear();
  }
 
  private static AgentBuilder getAgentBuilder(final ByteBuddy byteBuddy, final CoreConfiguration coreConfiguration, final Logger logger,
@@ -922,10 +942,39 @@ public static ClassLoader getInstrumentationClassLoader(String adviceClass) {
  }
 
  public static Collection<String> getPluginClassLoaderRootPackages(String pluginPackage) {
- Collection<String> pluginPackages = pluginPackages2pluginClassLoaderRootPackages.get(pluginPackage);
- if (pluginPackages != null) {
- return pluginPackages;
+ PluginClassLoaderCustomizations customizations = pluginPackages2pluginClassLoaderCustomizations.get(pluginPackage);
+ if (customizations != null) {
+ return customizations.packages;
  }
  return Collections.singleton(pluginPackage);
  }
+
+ public static Map<String, List<String>> getRequiredPluginModuleOpens(String pluginPackage) {
+ PluginClassLoaderCustomizations customizations = pluginPackages2pluginClassLoaderCustomizations.get(pluginPackage);
+ if (customizations != null) {
+ return customizations.requiredOpens;
+ }
+ return Collections.emptyMap();
+ }
+
+ private static class PluginClassLoaderCustomizations {
+ final List<String> packages;
+
+ final Map<String, List<String>> requiredOpens;
+
+ private PluginClassLoaderCustomizations(Collection<String> packages, Map<String, ? extends Collection<String>> requiredOpens) {
+ this.packages = Collections.unmodifiableList(new ArrayList<>(packages));
+ if (!requiredOpens.isEmpty()) {
+ Map<String, List<String>> opens = new HashMap<>();
+ for (Map.Entry<String, ? extends Collection<String>> open : requiredOpens.entrySet()) {
+ if (!open.getValue().isEmpty()) {
+ opens.put(open.getKey(), Collections.unmodifiableList(new ArrayList<>(open.getValue())));
+ }
+ }
+ this.requiredOpens = Collections.unmodifiableMap(opens);
+ } else {
+ this.requiredOpens = Collections.emptyMap();
+ }
+ }
+ }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/IndyBootstrap.java

@@ -49,6 +49,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 
@@ -201,11 +202,6 @@ public class IndyBootstrap {
  */
  public static final String LOOKUP_EXPOSER_CLASS_NAME = "co.elastic.apm.agent.bci.classloading.LookupExposer";
 
- /**
- * The root package name prefix that all embedded plugins classes should start with
- */
- private static final String EMBEDDED_PLUGINS_PACKAGE_PREFIX = "co.elastic.apm.agent.";
-
  /**
  * Caches the names of classes that are defined within a package and it's subpackages
  */
@@ -394,6 +390,7 @@ private static ConstantCallSite internalBootstrap(MethodHandles.Lookup lookup, S
  ClassLoader targetClassLoader = lookup.lookupClass().getClassLoader();
  ClassFileLocator classFileLocator;
  List<String> pluginClasses = new ArrayList<>();
+ Map<String, List<String>> requiredModuleOpens = Collections.emptyMap();
  if (instrumentationClassLoader instanceof ExternalPluginClassLoader) {
  List<String> externalPluginClasses = ((ExternalPluginClassLoader) instrumentationClassLoader).getClassNames();
  for (String externalPluginClass : externalPluginClasses) {
@@ -413,7 +410,9 @@ private static ConstantCallSite internalBootstrap(MethodHandles.Lookup lookup, S
  ClassFileLocator.ForJarFile.of(agentJarFile)
  );
  } else {
- pluginClasses.addAll(getClassNamesFromBundledPlugin(adviceClassName, instrumentationClassLoader));
+ String pluginPackage = PluginClassLoaderRootPackageCustomizer.getPluginPackageFromClassName(adviceClassName);
+ pluginClasses.addAll(getClassNamesFromBundledPlugin(pluginPackage, instrumentationClassLoader));
+ requiredModuleOpens = ElasticApmAgent.getRequiredPluginModuleOpens(pluginPackage);
  classFileLocator = ClassFileLocator.ForClassLoader.of(instrumentationClassLoader);
  }
  pluginClasses.add(LOOKUP_EXPOSER_CLASS_NAME);
@@ -428,6 +427,14 @@ private static ConstantCallSite internalBootstrap(MethodHandles.Lookup lookup, S
  // if config classes would be loaded from the plugin CL,
  // tracer.getConfig(Config.class) would return null when called from an advice as the classes are not the same
  .or(nameContains("Config").and(hasSuperType(is(ConfigurationOptionProvider.class)))));
+ if (ElasticApmAgent.areModulesSupported() && !requiredModuleOpens.isEmpty()) {
+ boolean success = addRequiredModuleOpens(requiredModuleOpens, targetClassLoader, pluginClassLoader);
+ if (!success) {
+ // must not be a static field as it would initialize logging before it's ready
+ LoggerFactory.getLogger(IndyBootstrap.class).error("Cannot bootstrap advice because required modules could not be opened!");
+ return null;
+ }
+ }
  Class<?> adviceInPluginCL = pluginClassLoader.loadClass(adviceClassName);
  Class<LookupExposer> lookupExposer = (Class<LookupExposer>) pluginClassLoader.loadClass(LOOKUP_EXPOSER_CLASS_NAME);
  // can't use MethodHandle.lookup(), see also https://github.com/elastic/apm-agent-java/issues/1450
@@ -444,11 +451,36 @@ private static ConstantCallSite internalBootstrap(MethodHandles.Lookup lookup, S
  }
  }
 
- private static List<String> getClassNamesFromBundledPlugin(String adviceClassName, ClassLoader adviceClassLoader) throws IOException, URISyntaxException {
- if (!adviceClassName.startsWith(EMBEDDED_PLUGINS_PACKAGE_PREFIX)) {
- throw new IllegalArgumentException("invalid advice class location : " + adviceClassName);
+ /**
+ * Attempts to open required modules for a given plugin classloader.
+ *
+ * @param requiredModuleOpens A map of FQNs of "witness" classes from modules and the corresponding package names to be opened.
+ * @param targetClassLoader the Classloader in which the class instrumented by the plugin classloader lives.
+ * Will be used to lookup "witness" classes from modules.
+ * @param pluginClassLoader the instrumentation plugin classloader which will be given module access.
+ * @return true if all required modules could be opened. On java < 17, true may be returned
+ * even if not all modules could be opened because in these versions module boundaries are not enforced.
+ * If any of the provided witness classes is not found, false is always returned.
+ */
+ private static boolean addRequiredModuleOpens(Map<String, List<String>> requiredModuleOpens, ClassLoader targetClassLoader, ClassLoader pluginClassLoader) {
+ try {
+ for (Map.Entry<String, List<String>> requiredOpen : requiredModuleOpens.entrySet()) {
+ String witnessClassName = requiredOpen.getKey();
+ List<String> packagesToOpen = requiredOpen.getValue();
+ Class<?> witness = Class.forName(witnessClassName, false, targetClassLoader);
+ if (!ElasticApmAgent.openModule(witness, pluginClassLoader, packagesToOpen)) {
+ return false;
+ }
+ }
+ } catch (ClassNotFoundException e) {
+ // must not be a static field as it would initialize logging before it's ready
+ LoggerFactory.getLogger(IndyBootstrap.class).error("Cannot open module because witness class is not found", e);
+ return false;
  }
- String pluginPackage = adviceClassName.substring(0, adviceClassName.indexOf('.', EMBEDDED_PLUGINS_PACKAGE_PREFIX.length()));
+ return true;
+ }
+
+ private static List<String> getClassNamesFromBundledPlugin(String pluginPackage, ClassLoader adviceClassLoader) throws IOException, URISyntaxException {
  List<String> pluginClasses = classesByPackage.get(pluginPackage);
  if (pluginClasses == null) {
  pluginClasses = new ArrayList<>();
@@ -461,5 +493,4 @@ private static List<String> getClassNamesFromBundledPlugin(String adviceClassNam
  }
  return pluginClasses;
  }
-
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/PluginClassLoaderRootPackageCustomizer.java

@@ -18,7 +18,11 @@
  */
 package co.elastic.apm.agent.bci;
 
+import java.lang.instrument.Instrumentation;
 import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * This class must be provided at most once per {@linkplain #getPluginPackage() plugin package}.
@@ -34,10 +38,14 @@ public abstract class PluginClassLoaderRootPackageCustomizer {
 
  public PluginClassLoaderRootPackageCustomizer() {
  String className = getClass().getName();
+ pluginPackage = getPluginPackageFromClassName(className);
+ }
+
+ public static String getPluginPackageFromClassName(String className) {
  if (!className.startsWith(EMBEDDED_PLUGINS_PACKAGE_PREFIX)) {
  throw new IllegalArgumentException("invalid instrumentation class location : " + className);
  }
- this.pluginPackage = className.substring(0, className.indexOf('.', EMBEDDED_PLUGINS_PACKAGE_PREFIX.length()));
+ return className.substring(0, className.indexOf('.', EMBEDDED_PLUGINS_PACKAGE_PREFIX.length()));
  }
 
  public final String getPluginPackage() {
@@ -51,4 +59,34 @@ public final String getPluginPackage() {
  * If the {@linkplain #getPluginPackage() plugin package} should be part of the root packages, implementations need to explicitly add it.
  */
  public abstract Collection<String> pluginClassLoaderRootPackages();
+
+ /**
+ * Starting with Java 9, Java added a module system which allows restricting access to code within modules.
+ * From Java 9 to 16, illegal access print a warning on a console.
+ * Starting with Java 17, Illegal access cause {@link IllegalAccessException}s.
+ * <p>
+ * Instrumentation plugins are loaded in an isolated classloader and therefore in an unnamed module.
+ * This module by default cannot access anything "private" within other modules, including
+ * the module containing the instrumented class. If such an access is required for the plugin,
+ * access can be granted using this method. Before anything from the plugin is invoked,
+ * {@link Instrumentation#redefineModule(Module, Set, Map, Map, Set, Map) will be used to give the classloader
+ * full access ("open") to the target module.}
+ * <p>
+ * <p>
+ * Each entry within the result map corresponds to a module which needs to be made accessible.
+ * Each module is looked up based on a "witness" class which resides within the respective module.
+ * Therefore, the keys of the return value from this map are the fully qualified names of these witness classes.
+ * The values of the map is a set of package names to open for the instrumentation plugin (like the
+ * packages passed to the --add-opens command line argument).
+ * <p>
+ * IMPORTANT: The "witness" classes are looked up when an advice from the plugin is invoked for the first time.
+ * It is best to provide classes which are known to be already loaded at that point in time in order
+ * to not mess with class initialization order. For example, the instrumented class which invoked the
+ * advice would be a safe choice.
+ *
+ * @return the map of "witness" FQNs to a collection of package names to open
+ */
+ public Map<String, ? extends Collection<String>> requiredModuleOpens() {
+ return Collections.emptyMap();
+ }
 }

apm-agent-core/src/main/java/co/elastic/apm/agent/bci/modules/ModuleOpener.java

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package co.elastic.apm.agent.bci.modules;
+
+import co.elastic.apm.agent.common.JvmRuntimeInfo;
+import co.elastic.apm.agent.sdk.logging.Logger;
+import co.elastic.apm.agent.sdk.logging.LoggerFactory;
+import org.codehaus.mojo.animal_sniffer.IgnoreJRERequirement;
+
+import javax.annotation.Nullable;
+import java.lang.instrument.Instrumentation;
+import java.util.Collection;
+
+@IgnoreJRERequirement
+public abstract class ModuleOpener {
+
+ private static final Logger logger = LoggerFactory.getLogger(ModuleOpener.class);
+ @Nullable
+ private static ModuleOpener instance;
+
+ private static final String IMPL_NAME = "co.elastic.apm.agent.bci.modules.ModuleOpenerImpl";
+
+ public static boolean areModulesSupported() {
+ return JvmRuntimeInfo.ofCurrentVM().getMajorVersion() >= 9;
+ }
+
+ public abstract boolean openModuleTo(Instrumentation instrumentation, Class<?> classFromTargetModule, ClassLoader openTo, Collection<String> packagesToOpen);
+
+ public static ModuleOpener getInstance() {
+ if (instance == null) {
+ synchronized (ModuleOpener.class) {
+ if (instance == null) {
+ if (areModulesSupported()) {
+ try {
+ instance = (ModuleOpener) Class.forName(IMPL_NAME).getDeclaredConstructor().newInstance();
+ } catch (Exception e) {
+ logger.error("Failed to initialize ModuleOpener", e);
+ instance = new NoOp();
+ }
+ } else {
+ instance = new NoOp();
+ }
+ }
+ }
+ }
+ return instance;
+ }
+
+ private static class NoOp extends ModuleOpener {
+
+ @Override
+ public boolean openModuleTo(Instrumentation instrumentation, Class<?> classFromTargetModule, ClassLoader openTo, Collection<String> packagesToOpen) {
+ return true;
+ }
+ }
+}