Skip to content

Commit 5fad53b

Browse files
felixbarnySylvainJuge
felixbarny
and
SylvainJuge
authored
Send logs via intake API (#2694)
 Co-authored-by: Sylvain Juge <sylvain.juge@elastic.co>
1 parent 835ec87 commit 5fad53b

File tree

47 files changed

+1324
-166
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+1324
-166
lines changed

‎CHANGELOG.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@ endif::[]
2323
[[release-notes-x.x.x]]
2424
==== x.x.x - YYYY/MM/DD
2525
26+
[float]
27+
===== Features
28+
* Add experimental log sending from the agent with `log_sending` - {pull}2694[#2694]
29+
2630
[float]
2731
===== Bug fixes
2832
* Use `127.0.0.1` as defaut for `server_url` to prevent ipv6 ambiguity - {pull}2927[#2927]

‎apm-agent-core/src/main/java/co/elastic/apm/agent/bci/ElasticApmAgent.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import co.elastic.apm.agent.impl.ElasticApmTracer;
3636
import co.elastic.apm.agent.impl.ElasticApmTracerBuilder;
3737
import co.elastic.apm.agent.impl.GlobalTracer;
38+
import co.elastic.apm.agent.logging.ApmServerLogAppender;
3839
import co.elastic.apm.agent.matcher.MethodMatcher;
3940
import co.elastic.apm.agent.sdk.ElasticApmInstrumentation;
4041
import co.elastic.apm.agent.sdk.logging.Logger;
@@ -151,7 +152,10 @@ public static void initialize(@Nullable final String agentArguments, final Instr
151152
}
152153
}
153154

154-
ElasticApmTracer tracer = new ElasticApmTracerBuilder(configSources).build();
155+
ElasticApmTracer tracer = new ElasticApmTracerBuilder(configSources)
156+
// server log appender requires buffering log events before the config and reporter are ready.
157+
.withLifecycleListener(ApmServerLogAppender.getInstance().getInitListener())
158+
.build();
155159
initInstrumentation(tracer, instrumentation, premain);
156160
tracer.start(premain);
157161
}

‎apm-agent-core/src/main/java/co/elastic/apm/agent/logging/ApmServerLogAppender.java‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package co.elastic.apm.agent.logging;
20+
21+
import co.elastic.apm.agent.context.AbstractLifecycleListener;
22+
import co.elastic.apm.agent.context.LifecycleListener;
23+
import co.elastic.apm.agent.impl.ElasticApmTracer;
24+
import co.elastic.apm.agent.report.Reporter;
25+
import co.elastic.logging.log4j2.EcsLayout;
26+
import org.apache.logging.log4j.core.Appender;
27+
import org.apache.logging.log4j.core.Core;
28+
import org.apache.logging.log4j.core.Layout;
29+
import org.apache.logging.log4j.core.LogEvent;
30+
import org.apache.logging.log4j.core.appender.AbstractAppender;
31+
import org.apache.logging.log4j.core.config.plugins.Plugin;
32+
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
33+
import org.apache.logging.log4j.core.config.plugins.PluginElement;
34+
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
35+
36+
import javax.annotation.Nullable;
37+
import java.util.ArrayList;
38+
import java.util.Objects;
39+
40+
@Plugin(name = Log4j2ConfigurationFactory.APM_SERVER_PLUGIN_NAME, category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true)
41+
public class ApmServerLogAppender extends AbstractAppender {
42+
43+
private static final int MAX_BUFFER_SIZE = 1024;
44+
45+
@Nullable
46+
private static ApmServerLogAppender INSTANCE;
47+
48+
@Nullable
49+
private LoggingConfiguration config;
50+
@Nullable
51+
private Reporter reporter;
52+
53+
private final ArrayList<LogEvent> buffer;
54+
55+
private ApmServerLogAppender(String name, Layout<?> layout) {
56+
// recursive calls filtering is done through a filter on the appender ref, not on the appender itself
57+
super(name, null , layout, true, null);
58+
this.buffer = new ArrayList<>();
59+
}
60+
61+
public static ApmServerLogAppender getInstance() {
62+
return Objects.requireNonNull(INSTANCE);
63+
}
64+
65+
@SuppressWarnings("unused")
66+
@PluginFactory
67+
public static ApmServerLogAppender createAppender(@PluginAttribute("name") String name,
68+
@PluginElement("Layout") Layout<?> layout) {
69+
70+
if (!(layout instanceof EcsLayout)) {
71+
throw new IllegalArgumentException("invalid layout " + layout);
72+
}
73+
74+
if (INSTANCE == null) {
75+
INSTANCE = new ApmServerLogAppender(name, layout);
76+
}
77+
78+
return new ApmServerLogAppender(name, layout);
79+
}
80+
81+
@Override
82+
public void append(LogEvent event) {
83+
84+
boolean bufferBeforeInit = !isAgentInitialized();
85+
if (bufferBeforeInit) {
86+
synchronized (buffer) {
87+
bufferBeforeInit = !isAgentInitialized();
88+
89+
// buffering before the configuration is known
90+
if (bufferBeforeInit) {
91+
if (buffer.size() < MAX_BUFFER_SIZE) {
92+
buffer.add(event.toImmutable());
93+
}
94+
return;
95+
}
96+
}
97+
}
98+
99+
sendLogEvent(event);
100+
}
101+
102+
public LifecycleListener getInitListener() {
103+
return new AbstractLifecycleListener() {
104+
@Override
105+
public void init(ElasticApmTracer tracer) throws Exception {
106+
initStreaming(tracer.getConfig(LoggingConfiguration.class), tracer.getReporter());
107+
}
108+
};
109+
}
110+
111+
private void initStreaming(LoggingConfiguration config, Reporter reporter) {
112+
if (isAgentInitialized()) {
113+
throw new IllegalStateException("streaming already initialized");
114+
}
115+
116+
synchronized (buffer) {
117+
this.config = config;
118+
this.reporter = reporter;
119+
120+
for (LogEvent logEvent : buffer) {
121+
sendLogEvent(logEvent);
122+
}
123+
buffer.clear();
124+
buffer.trimToSize();
125+
}
126+
}
127+
128+
private void sendLogEvent(LogEvent event) {
129+
if (!config.getSendLogs()) {
130+
return;
131+
}
132+
Objects.requireNonNull(reporter).reportAgentLog(getLayout().toByteArray(event));
133+
}
134+
135+
private boolean isAgentInitialized() {
136+
return this.config != null && this.reporter != null;
137+
}
138+
139+
}

‎apm-agent-core/src/main/java/co/elastic/apm/agent/logging/ApmServerLogFilter.java‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
/*
2+
* Licensed to Elasticsearch B.V. under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch B.V. licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package co.elastic.apm.agent.logging;
20+
21+
import org.apache.logging.log4j.Level;
22+
import org.apache.logging.log4j.Marker;
23+
import org.apache.logging.log4j.core.Core;
24+
import org.apache.logging.log4j.core.Filter;
25+
import org.apache.logging.log4j.core.LogEvent;
26+
import org.apache.logging.log4j.core.Logger;
27+
import org.apache.logging.log4j.core.config.plugins.Plugin;
28+
import org.apache.logging.log4j.core.config.plugins.PluginAttribute;
29+
import org.apache.logging.log4j.core.config.plugins.PluginFactory;
30+
import org.apache.logging.log4j.core.filter.AbstractFilter;
31+
import org.apache.logging.log4j.message.Message;
32+
33+
/**
34+
* Log4j log event filter that allows to filter-out recursive calls to the logger.
35+
*/
36+
@SuppressWarnings("unused")
37+
@Plugin(name = Log4j2ConfigurationFactory.APM_SERVER_FILTER_PLUGIN_NAME, category = Core.CATEGORY_NAME, elementType = Filter.ELEMENT_TYPE, printObject = true)
38+
public class ApmServerLogFilter extends AbstractFilter {
39+
40+
private final String ignoreLoggerPrefix;
41+
42+
private ApmServerLogFilter(String ignoreLoggerPrefix) {
43+
this.ignoreLoggerPrefix = ignoreLoggerPrefix;
44+
}
45+
46+
@PluginFactory
47+
public static ApmServerLogFilter createFilter(@PluginAttribute("onMatch") Result matchIgnored,
48+
@PluginAttribute("onMismatch") Result mismatchIgnored,
49+
@PluginAttribute("ignoreLoggerPrefix") String ignoreLoggerPrefix) {
50+
51+
return new ApmServerLogFilter(ignoreLoggerPrefix);
52+
}
53+
54+
private Result filter(String loggerName) {
55+
if (loggerName.startsWith(ignoreLoggerPrefix)) {
56+
return Result.DENY;
57+
}
58+
return Result.NEUTRAL;
59+
}
60+
61+
@Override
62+
public Result filter(LogEvent event) {
63+
return filter(event.getLoggerName());
64+
}
65+
66+
@Override
67+
public Result filter(Logger logger, Level level, Marker marker, Message msg, Throwable t) {
68+
return filter(logger.getName());
69+
}
70+
71+
@Override
72+
public Result filter(Logger logger, Level level, Marker marker, Object msg, Throwable t) {
73+
return filter(logger.getName());
74+
}
75+
76+
@Override
77+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object... params) {
78+
return filter(logger.getName());
79+
}
80+
81+
@Override
82+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0) {
83+
return filter(logger.getName());
84+
}
85+
86+
@Override
87+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1) {
88+
return filter(logger.getName());
89+
}
90+
91+
@Override
92+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2) {
93+
return filter(logger.getName());
94+
}
95+
96+
@Override
97+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3) {
98+
return filter(logger.getName());
99+
}
100+
101+
@Override
102+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3, Object p4) {
103+
return filter(logger.getName());
104+
}
105+
106+
@Override
107+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5) {
108+
return filter(logger.getName());
109+
}
110+
111+
@Override
112+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6) {
113+
return filter(logger.getName());
114+
}
115+
116+
@Override
117+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6, Object p7) {
118+
return filter(logger.getName());
119+
}
120+
121+
@Override
122+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6, Object p7, Object p8) {
123+
return filter(logger.getName());
124+
}
125+
126+
@Override
127+
public Result filter(Logger logger, Level level, Marker marker, String msg, Object p0, Object p1, Object p2, Object p3, Object p4, Object p5, Object p6, Object p7, Object p8, Object p9) {
128+
return filter(logger.getName());
129+
}
130+
131+
}

0 commit comments

Comments
 (0)