Skip to content

Commit c9b5279

Browse files
SylvainJugeSylvainJuge
authored
ECS reformat support for Tomcat (#2839)
* first working version * add simple integration test * wip testing * disable IT by default to avoid {flaky,slow}ness * update doc * reuse common tests + fix one replace bug * cleanup * cleanup cleanup * post-review changes * update changelog * update doc * remove duplication in doc * cleanup doc again
1 parent 0c8173e commit c9b5279

File tree

26 files changed

+890
-168
lines changed

26 files changed

+890
-168
lines changed

‎CHANGELOG.asciidoc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ endif::[]
3232
* Add support for spring-kafka batch listeners - {pull}2815[#2815]
3333
* Improved instrumentation for legacy Apache HttpClient (when not using an `HttpUriRequest`, such as `BasicHttpRequest`)
3434
* Prevented exclusion of agent-packages via `classes_excluded_from_instrumentation` to avoid unintended side effects
35+
* Add Tomcat support for log reformatting - {pull}2839[#2839]
3536
3637
[float]
3738
===== Bug fixes

‎apm-agent-common/src/test/java/co/elastic/apm/agent/test/AgentFileAccessor.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,11 +78,17 @@ public static Path getArtifactPath(Path modulePath, String artifactSuffix, Strin
7878
String artifactName = modulePath.getFileName().toString(); // by convention artifact name the last part of the path
7979
try {
8080
Path targetFolder = moduleRoot.resolve("target");
81+
82+
String errorMsg = String.format("unable to find artifact '%s%s-{version}%s' in folder '%s', make sure to run 'mvn package' in folder '%s' first", artifactName, artifactSuffix, extension, targetFolder.toAbsolutePath(), moduleRoot);
83+
if (!Files.isDirectory(targetFolder)) {
84+
throw new IllegalStateException(errorMsg);
85+
}
86+
8187
return Files.find(targetFolder, 1, (path, attr) -> path.getFileName().toString()
8288
.matches(artifactName + "-\\d\\.\\d+\\.\\d+(\\.RC\\d+)?(-SNAPSHOT)?" + artifactSuffix + extension))
8389
.findFirst()
8490
.map(Path::toAbsolutePath)
85-
.orElseThrow(() -> new IllegalStateException(String.format("unable to find artifact %s%s-{version}%s in folder %s, make sure to run 'mvn package' in folder '%s' first", artifactName, artifactSuffix, extension, targetFolder.toAbsolutePath(), moduleRoot)));
91+
.orElseThrow(() -> new IllegalStateException(errorMsg));
8692
} catch (IOException e) {
8793
throw new IllegalStateException(e);
8894
}

‎apm-agent-core/src/main/java/co/elastic/apm/agent/logging/LoggingConfiguration.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,7 @@ static LogLevel mapLogLevel(LogLevel original) {
212212
WildcardMatcher.valueOf("org.apache.log4j.SimpleLayout"),
213213
WildcardMatcher.valueOf("ch.qos.logback.core.encoder.EchoEncoder"),
214214
WildcardMatcher.valueOf("java.util.logging.SimpleFormatter"),
215+
WildcardMatcher.valueOf("org.apache.juli.OneLineFormatter"),
215216
WildcardMatcher.valueOf("org.springframework.boot.logging.java.SimpleFormatter")
216217
));
217218

‎apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/pom.xml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,17 @@
4444
<argLine>--add-opens java.logging/java.util.logging=ALL-UNNAMED</argLine>
4545
</configuration>
4646
</plugin>
47+
<!-- packaging test code so tomcat logging plugin can reuse them -->
48+
<plugin>
49+
<artifactId>maven-jar-plugin</artifactId>
50+
<executions>
51+
<execution>
52+
<goals>
53+
<goal>test-jar</goal>
54+
</goals>
55+
</execution>
56+
</executions>
57+
</plugin>
4758
</plugins>
4859
</build>
4960
</project>

‎apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/reformatting/AbstractJulEcsReformattingHelper.java‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package co.elastic.apm.agent.jul.reformatting;
20+
21+
import co.elastic.apm.agent.loginstr.correlation.CorrelationIdMapAdapter;
22+
import co.elastic.apm.agent.loginstr.reformatting.AbstractEcsReformattingHelper;
23+
import co.elastic.apm.agent.sdk.logging.Logger;
24+
import co.elastic.apm.agent.sdk.logging.LoggerFactory;
25+
import co.elastic.apm.agent.util.LoggerUtils;
26+
import co.elastic.logging.AdditionalField;
27+
import co.elastic.logging.jul.EcsFormatter;
28+
29+
import javax.annotation.Nullable;
30+
import java.io.IOException;
31+
import java.util.ArrayList;
32+
import java.util.List;
33+
import java.util.Map;
34+
import java.util.logging.FileHandler;
35+
import java.util.logging.Formatter;
36+
import java.util.logging.Handler;
37+
import java.util.logging.StreamHandler;
38+
39+
public abstract class AbstractJulEcsReformattingHelper extends AbstractEcsReformattingHelper<Handler, Formatter> {
40+
41+
private static final Logger logger = LoggerFactory.getLogger(AbstractJulEcsReformattingHelper.class);
42+
private static final Logger oneTimeLogFileLimitWarningLogger = LoggerUtils.logOnce(logger);
43+
44+
@Nullable
45+
@Override
46+
protected Formatter getFormatterFrom(Handler handler) {
47+
return handler.getFormatter();
48+
}
49+
50+
@Override
51+
protected void setFormatter(Handler handler, Formatter formatter) {
52+
handler.setFormatter(formatter);
53+
}
54+
55+
@Override
56+
protected void closeShadeAppender(Handler handler) {
57+
handler.close();
58+
}
59+
60+
@Nullable
61+
@Override
62+
public Formatter createEcsFormatter(String eventDataset, @Nullable String serviceName, @Nullable String serviceVersion,
63+
@Nullable String serviceNodeName, @Nullable Map<String, String> additionalFields,
64+
Formatter originalFormatter) {
65+
EcsFormatter ecsFormatter = new EcsFormatter() {
66+
@Override
67+
protected Map<String, String> getMdcEntries() {
68+
// using internal tracer state as ECS formatter is not instrumented within the agent plugin
69+
return CorrelationIdMapAdapter.get();
70+
}
71+
};
72+
ecsFormatter.setServiceName(serviceName);
73+
ecsFormatter.setServiceVersion(serviceVersion);
74+
ecsFormatter.setServiceNodeName(serviceNodeName);
75+
ecsFormatter.setEventDataset(eventDataset);
76+
if (additionalFields != null && !additionalFields.isEmpty()) {
77+
List<AdditionalField> additionalFieldList = new ArrayList<>();
78+
for (Map.Entry<String, String> keyValuePair : additionalFields.entrySet()) {
79+
additionalFieldList.add(new AdditionalField(keyValuePair.getKey(), keyValuePair.getValue()));
80+
}
81+
ecsFormatter.setAdditionalFields(additionalFieldList);
82+
}
83+
ecsFormatter.setIncludeOrigin(false);
84+
ecsFormatter.setStackTraceAsArray(false);
85+
return ecsFormatter;
86+
}
87+
88+
protected abstract String getShadeFilePatternAndCreateDir() throws IOException;
89+
90+
@Nullable
91+
@Override
92+
protected Handler createAndStartEcsAppender(Handler originalHandler, String ecsAppenderName, Formatter ecsFormatter) {
93+
StreamHandler shadeHandler = null;
94+
if (isFileHandler(originalHandler)) {
95+
try {
96+
String pattern = getShadeFilePatternAndCreateDir();
97+
// In earlier versions, there is only constructor with log file limit given as int, whereas in later ones there are
98+
// overloads for both either int or long. Typically, this should be enough, but not necessarily
99+
int maxLogFileSize = (int) getMaxLogFileSize();
100+
if ((long) maxLogFileSize != getMaxLogFileSize()) {
101+
maxLogFileSize = (int) getDefaultMaxLogFileSize();
102+
oneTimeLogFileLimitWarningLogger.warn("Configured log max size ({} bytes) is too big for JUL settings, which " +
103+
"use int to configure the file size limit. Consider reducing the log max size configuration to a value below " +
104+
"Integer#MAX_VALUE. Defaulting to {} bytes.", getMaxLogFileSize(), maxLogFileSize);
105+
}
106+
shadeHandler = new FileHandler(pattern, maxLogFileSize, 2, true);
107+
shadeHandler.setFormatter(ecsFormatter);
108+
} catch (Exception e) {
109+
logger.error("Failed to create Log shading FileAppender. Auto ECS reformatting will not work.", e);
110+
}
111+
}
112+
return shadeHandler;
113+
}
114+
115+
protected abstract boolean isFileHandler(Handler originalHandler);
116+
117+
}

‎apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/reformatting/JulConsoleHandlerPublishAdvice.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import net.bytebuddy.implementation.bytecode.assign.Assigner;
2323

2424
import java.util.logging.ConsoleHandler;
25+
import java.util.logging.Handler;
2526
import java.util.logging.LogRecord;
2627
import java.util.logging.StreamHandler;
2728

@@ -40,7 +41,7 @@ public static boolean initializeReformatting(@Advice.This(typing = Assigner.Typi
4041
public static void reformatLoggingEvent(@Advice.Argument(value = 0, typing = Assigner.Typing.DYNAMIC) final LogRecord logRecord,
4142
@Advice.This(typing = Assigner.Typing.DYNAMIC) ConsoleHandler thisHandler) {
4243

43-
StreamHandler shadeAppender = helper.onAppendExit(thisHandler);
44+
Handler shadeAppender = helper.onAppendExit(thisHandler);
4445
if (shadeAppender != null) {
4546
shadeAppender.publish(logRecord);
4647
}

‎apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/reformatting/JulEcsReformattingHelper.java‎

Lines changed: 13 additions & 84 deletions
Original file line numberDiff line numberDiff line change
@@ -18,32 +18,18 @@
1818
*/
1919
package co.elastic.apm.agent.jul.reformatting;
2020

21-
import co.elastic.apm.agent.loginstr.correlation.CorrelationIdMapAdapter;
22-
import co.elastic.apm.agent.loginstr.reformatting.AbstractEcsReformattingHelper;
2321
import co.elastic.apm.agent.loginstr.reformatting.Utils;
24-
import co.elastic.apm.agent.sdk.logging.Logger;
25-
import co.elastic.apm.agent.sdk.logging.LoggerFactory;
26-
import co.elastic.apm.agent.util.LoggerUtils;
27-
import co.elastic.logging.AdditionalField;
28-
import co.elastic.logging.jul.EcsFormatter;
2922

3023
import javax.annotation.Nullable;
3124
import java.io.File;
3225
import java.io.IOException;
3326
import java.nio.file.Files;
3427
import java.nio.file.Path;
35-
import java.util.ArrayList;
36-
import java.util.List;
37-
import java.util.Map;
3828
import java.util.logging.ConsoleHandler;
3929
import java.util.logging.FileHandler;
40-
import java.util.logging.Formatter;
41-
import java.util.logging.StreamHandler;
30+
import java.util.logging.Handler;
4231

43-
class JulEcsReformattingHelper extends AbstractEcsReformattingHelper<StreamHandler, Formatter> {
44-
45-
private static final Logger logger = LoggerFactory.getLogger(JulEcsReformattingHelper.class);
46-
private static final Logger oneTimeLogFileLimitWarningLogger = LoggerUtils.logOnce(logger);
32+
class JulEcsReformattingHelper extends AbstractJulEcsReformattingHelper {
4733

4834
private static final ThreadLocal<String> currentPattern = new ThreadLocal<>();
4935
private static final ThreadLocal<Path> currentExampleLogFile = new ThreadLocal<>();
@@ -61,19 +47,8 @@ public boolean onAppendEnter(FileHandler fileHandler, String pattern, File examp
6147
}
6248
}
6349

64-
@Nullable
65-
@Override
66-
protected Formatter getFormatterFrom(StreamHandler handler) {
67-
return handler.getFormatter();
68-
}
69-
70-
@Override
71-
protected void setFormatter(StreamHandler handler, Formatter formatter) {
72-
handler.setFormatter(formatter);
73-
}
74-
7550
@Override
76-
protected String getAppenderName(StreamHandler handler) {
51+
protected String getAppenderName(Handler handler) {
7752
if (handler instanceof FileHandler) {
7853
return "FILE";
7954
} else if (handler instanceof ConsoleHandler) {
@@ -84,60 +59,18 @@ protected String getAppenderName(StreamHandler handler) {
8459
}
8560

8661
@Override
87-
protected Formatter createEcsFormatter(String eventDataset, @Nullable String serviceName, @Nullable String serviceVersion,
88-
@Nullable String serviceNodeName, @Nullable Map<String, String> additionalFields,
89-
Formatter originalFormatter) {
90-
EcsFormatter ecsFormatter = new EcsFormatter() {
91-
@Override
92-
protected Map<String, String> getMdcEntries() {
93-
// using internal tracer state as ECS formatter is not instrumented within the agent plugin
94-
return CorrelationIdMapAdapter.get();
95-
}
96-
};
97-
ecsFormatter.setServiceName(serviceName);
98-
ecsFormatter.setServiceVersion(serviceVersion);
99-
ecsFormatter.setServiceNodeName(serviceNodeName);
100-
ecsFormatter.setEventDataset(eventDataset);
101-
if (additionalFields != null && !additionalFields.isEmpty()) {
102-
List<AdditionalField> additionalFieldList = new ArrayList<>();
103-
for (Map.Entry<String, String> keyValuePair : additionalFields.entrySet()) {
104-
additionalFieldList.add(new AdditionalField(keyValuePair.getKey(), keyValuePair.getValue()));
105-
}
106-
ecsFormatter.setAdditionalFields(additionalFieldList);
107-
}
108-
ecsFormatter.setIncludeOrigin(false);
109-
ecsFormatter.setStackTraceAsArray(false);
110-
return ecsFormatter;
62+
protected boolean isFileHandler(Handler originalHandler) {
63+
return originalHandler instanceof FileHandler;
11164
}
11265

113-
@Nullable
11466
@Override
115-
protected StreamHandler createAndStartEcsAppender(StreamHandler originalHandler, String ecsAppenderName, Formatter ecsFormatter) {
116-
StreamHandler shadeHandler = null;
117-
if (originalHandler instanceof FileHandler) {
118-
try {
119-
String pattern = computeEcsFileHandlerPattern(
120-
currentPattern.get(),
121-
currentExampleLogFile.get(),
122-
getConfiguredReformattingDir(),
123-
true
124-
);
125-
// In earlier versions, there is only constructor with log file limit given as int, whereas in later ones there are
126-
// overloads for both either int or long. Typically, this should be enough, but not necessarily
127-
int maxLogFileSize = (int) getMaxLogFileSize();
128-
if ((long) maxLogFileSize != getMaxLogFileSize()) {
129-
maxLogFileSize = (int) getDefaultMaxLogFileSize();
130-
oneTimeLogFileLimitWarningLogger.warn("Configured log max size ({} bytes) is too big for JUL settings, which " +
131-
"use int to configure the file size limit. Consider reducing the log max size configuration to a value below " +
132-
"Integer#MAX_VALUE. Defaulting to {} bytes.", getMaxLogFileSize(), maxLogFileSize);
133-
}
134-
shadeHandler = new FileHandler(pattern, maxLogFileSize, 2, true);
135-
shadeHandler.setFormatter(ecsFormatter);
136-
} catch (Exception e) {
137-
logger.error("Failed to create Log shading FileAppender. Auto ECS reformatting will not work.", e);
138-
}
139-
}
140-
return shadeHandler;
67+
protected String getShadeFilePatternAndCreateDir() throws IOException {
68+
return computeEcsFileHandlerPattern(
69+
currentPattern.get(),
70+
currentExampleLogFile.get(),
71+
getConfiguredReformattingDir(),
72+
true
73+
);
14174
}
14275

14376
static String computeEcsFileHandlerPattern(String pattern, Path originalFilePath, @Nullable String configuredReformattingDir,
@@ -148,7 +81,7 @@ static String computeEcsFileHandlerPattern(String pattern, Path originalFilePath
14881
pattern = pattern + ".%g";
14982
}
15083
int lastPathSeparatorIndex = pattern.lastIndexOf('/');
151-
if (lastPathSeparatorIndex > 0 && pattern.length() > lastPathSeparatorIndex) {
84+
if (lastPathSeparatorIndex > 0 && lastPathSeparatorIndex < pattern.length() - 1) {
15285
pattern = pattern.substring(lastPathSeparatorIndex + 1);
15386
}
15487
Path logReformattingDir = Utils.computeLogReformattingDir(originalFilePath, configuredReformattingDir);
@@ -161,8 +94,4 @@ static String computeEcsFileHandlerPattern(String pattern, Path originalFilePath
16194
return pattern;
16295
}
16396

164-
@Override
165-
protected void closeShadeAppender(StreamHandler shadeHandler) {
166-
shadeHandler.close();
167-
}
16897
}

‎apm-agent-plugins/apm-logging-plugin/apm-jul-plugin/src/main/java/co/elastic/apm/agent/jul/reformatting/JulFileHandlerPublishAdvice.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323

2424
import java.io.File;
2525
import java.util.logging.FileHandler;
26+
import java.util.logging.Handler;
2627
import java.util.logging.LogRecord;
2728
import java.util.logging.StreamHandler;
2829

@@ -43,7 +44,7 @@ public static boolean initializeReformatting(@Advice.This(typing = Assigner.Typi
4344
public static void reformatLoggingEvent(@Advice.Argument(value = 0, typing = Assigner.Typing.DYNAMIC) final LogRecord logRecord,
4445
@Advice.This(typing = Assigner.Typing.DYNAMIC) FileHandler thisHandler) {
4546

46-
StreamHandler shadeAppender = helper.onAppendExit(thisHandler);
47+
Handler shadeAppender = helper.onAppendExit(thisHandler);
4748
if (shadeAppender != null) {
4849
shadeAppender.publish(logRecord);
4950
}

0 commit comments

Comments
 (0)