Skip to content

Commit c2f40df

Browse files
author
Simon Zeltser
authored
fix: add default time range filter for ListLogEntries API (#304)
* feat!: listLogEntries now appends default 24h filter BREAKING CHANGE: previously if the user called listLogEntries API with no parameters, the query would try to retrieve all log entries, taking up time and eventually crashing on projects with high volume logs. The new behavior complies with gcloud: if no timestamp filter was specified, it appends a filter covering last 24 hours.
1 parent e97dda7 commit c2f40df

File tree

5 files changed

+206
-28
lines changed

5 files changed

+206
-28
lines changed

‎google-cloud-logging/src/main/java/com/google/cloud/logging/ITimestampDefaultFilter.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.logging;
18+
19+
/**
20+
* Encapsulates implementation of default time filter. This is needed for testing since we can't
21+
* mock static classes with EasyMock
22+
*/
23+
public interface ITimestampDefaultFilter {
24+
25+
/**
26+
* Creates a default filter with timestamp to query Cloud Logging
27+
*
28+
* @return The filter using timestamp field
29+
*/
30+
String createDefaultTimestampFilter();
31+
}

‎google-cloud-logging/src/main/java/com/google/cloud/logging/LoggingImpl.java‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,7 @@
7777
import com.google.logging.v2.WriteLogEntriesRequest;
7878
import com.google.logging.v2.WriteLogEntriesResponse;
7979
import com.google.protobuf.Empty;
80-
import java.util.ArrayList;
81-
import java.util.List;
82-
import java.util.Map;
80+
import java.util.*;
8381
import java.util.concurrent.ConcurrentHashMap;
8482
import java.util.concurrent.ExecutionException;
8583
import java.util.concurrent.TimeUnit;
@@ -111,6 +109,9 @@ public Void apply(WriteLogEntriesResponse input) {
111109
};
112110
private static final ThreadLocal<Boolean> inWriteCall = new ThreadLocal<>();
113111

112+
@VisibleForTesting
113+
static ITimestampDefaultFilter defaultTimestampFilterCreator = new TimestampDefaultFilter();
114+
114115
LoggingImpl(LoggingOptions options) {
115116
super(options);
116117
rpc = options.getLoggingRpcV2();
@@ -705,8 +706,7 @@ public void write(Iterable<LogEntry> logEntries, WriteOption... options) {
705706
public void flush() {
706707
// BUG(1795): We should force batcher to issue RPC call for buffered messages,
707708
// so the code below doesn't wait uselessly.
708-
ArrayList<ApiFuture<Void>> writesToFlush = new ArrayList<>();
709-
writesToFlush.addAll(pendingWrites.values());
709+
ArrayList<ApiFuture<Void>> writesToFlush = new ArrayList<>(pendingWrites.values());
710710

711711
try {
712712
ApiFutures.allAsList(writesToFlush).get(FLUSH_WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS);
@@ -779,9 +779,21 @@ static ListLogEntriesRequest listLogEntriesRequest(
779779
builder.setOrderBy(orderBy);
780780
}
781781
String filter = FILTER.get(options);
782+
// Make sure timestamp filter is either explicitly specified or we add a default time filter
783+
// of 24 hours back to be inline with gcloud behavior for the same API
782784
if (filter != null) {
785+
if (!filter.toLowerCase().contains("timestamp")) {
786+
filter =
787+
String.format(
788+
"%s AND %s", filter, defaultTimestampFilterCreator.createDefaultTimestampFilter());
789+
}
783790
builder.setFilter(filter);
791+
} else {
792+
// If filter is not specified, default filter is looking back 24 hours in line with gcloud
793+
// behavior
794+
builder.setFilter(defaultTimestampFilterCreator.createDefaultTimestampFilter());
784795
}
796+
785797
return builder.build();
786798
}
787799

@@ -794,16 +806,16 @@ private static ApiFuture<AsyncPage<LogEntry>> listLogEntriesAsync(
794806
list,
795807
new Function<ListLogEntriesResponse, AsyncPage<LogEntry>>() {
796808
@Override
797-
public AsyncPage<LogEntry> apply(ListLogEntriesResponse listLogEntrysResponse) {
809+
public AsyncPage<LogEntry> apply(ListLogEntriesResponse listLogEntriesResponse) {
798810
List<LogEntry> entries =
799-
listLogEntrysResponse.getEntriesList() == null
811+
listLogEntriesResponse.getEntriesList() == null
800812
? ImmutableList.<LogEntry>of()
801813
: Lists.transform(
802-
listLogEntrysResponse.getEntriesList(), LogEntry.FROM_PB_FUNCTION);
814+
listLogEntriesResponse.getEntriesList(), LogEntry.FROM_PB_FUNCTION);
803815
String cursor =
804-
listLogEntrysResponse.getNextPageToken().equals("")
816+
listLogEntriesResponse.getNextPageToken().equals("")
805817
? null
806-
: listLogEntrysResponse.getNextPageToken();
818+
: listLogEntriesResponse.getNextPageToken();
807819
return new AsyncPageImpl<>(
808820
new LogEntryPageFetcher(serviceOptions, cursor, options), cursor, entries);
809821
}

‎google-cloud-logging/src/main/java/com/google/cloud/logging/TimestampDefaultFilter.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2020 Google LLC
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.google.cloud.logging;
18+
19+
import java.text.DateFormat;
20+
import java.text.SimpleDateFormat;
21+
import java.util.Calendar;
22+
import java.util.Date;
23+
import java.util.TimeZone;
24+
25+
public class TimestampDefaultFilter implements ITimestampDefaultFilter {
26+
@Override
27+
public String createDefaultTimestampFilter() {
28+
DateFormat rfcDateFormat = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
29+
rfcDateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
30+
return "timestamp>=\"" + rfcDateFormat.format(yesterday()) + "\"";
31+
}
32+
33+
private Date yesterday() {
34+
TimeZone timeZone = TimeZone.getTimeZone("UTC");
35+
Calendar calendar = Calendar.getInstance(timeZone);
36+
calendar.add(Calendar.DATE, -1);
37+
38+
return calendar.getTime();
39+
}
40+
}

‎google-cloud-logging/src/test/java/com/google/cloud/logging/LoggingImplTest.java‎

Lines changed: 58 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -154,7 +154,7 @@ public com.google.api.MonitoredResourceDescriptor apply(
154154
private static final String NEXT_CURSOR = "nextCursor";
155155
private static final Boolean DISABLED = Boolean.FALSE;
156156
private static final Timestamp EXCLUSION_CREATED_TIME = fromMillis(System.currentTimeMillis());
157-
private static final Timestamp EXCLUSION_UPDATED_TIME = fromMillis(System.currentTimeMillis());;
157+
private static final Timestamp EXCLUSION_UPDATED_TIME = fromMillis(System.currentTimeMillis());
158158
private static final Exclusion EXCLUSION =
159159
Exclusion.newBuilder(EXCLUSION_NAME, EXCLUSION_FILTER)
160160
.setDisabled(DISABLED)
@@ -185,6 +185,18 @@ public void setUp() {
185185
.setServiceRpcFactory(rpcFactoryMock)
186186
.setRetrySettings(ServiceOptions.getNoRetrySettings())
187187
.build();
188+
189+
// By default when calling ListLogEntries, we append a filter of last 24 hours.
190+
// However when testing, the time when it was called by the test and by the method
191+
// implementation might differ by microseconds so we use the same time filter implementation
192+
// for test and in "real" method
193+
LoggingImpl.defaultTimestampFilterCreator =
194+
new ITimestampDefaultFilter() {
195+
@Override
196+
public String createDefaultTimestampFilter() {
197+
return "timestamp>=\"2020-10-30T15:39:09Z\"";
198+
}
199+
};
188200
}
189201

190202
@After
@@ -1585,7 +1597,7 @@ public void testDeleteLog_Null() {
15851597
}
15861598

15871599
@Test
1588-
public void testDeleteLogAync() throws ExecutionException, InterruptedException {
1600+
public void testDeleteLogAsync() throws ExecutionException, InterruptedException {
15891601
DeleteLogRequest request = DeleteLogRequest.newBuilder().setLogName(LOG_NAME_PB).build();
15901602
ApiFuture<Empty> response = ApiFutures.immediateFuture(Empty.getDefaultInstance());
15911603
EasyMock.expect(loggingRpcMock.delete(request)).andReturn(response);
@@ -1621,7 +1633,7 @@ public void testWriteLogEntries() {
16211633
}
16221634

16231635
@Test
1624-
public void testWriteLogEntriesDoesnotEnableFlushByDefault() {
1636+
public void testWriteLogEntriesDoesNotEnableFlushByDefault() {
16251637
WriteLogEntriesRequest request =
16261638
WriteLogEntriesRequest.newBuilder()
16271639
.addAllEntries(
@@ -1679,7 +1691,7 @@ public void testWriteLogEntriesWithOptions() {
16791691
}
16801692

16811693
@Test
1682-
public void testWriteLogEntriesAsync() throws ExecutionException, InterruptedException {
1694+
public void testWriteLogEntriesAsync() {
16831695
WriteLogEntriesRequest request =
16841696
WriteLogEntriesRequest.newBuilder()
16851697
.addAllEntries(
@@ -1723,16 +1735,16 @@ public void testListLogEntries() {
17231735
String cursor = "cursor";
17241736
EasyMock.replay(rpcFactoryMock);
17251737
logging = options.getService();
1726-
ListLogEntriesRequest request =
1727-
ListLogEntriesRequest.newBuilder().addResourceNames(PROJECT_PB).build();
1738+
17281739
List<LogEntry> entriesList = ImmutableList.of(LOG_ENTRY1, LOG_ENTRY2);
17291740
ListLogEntriesResponse response =
17301741
ListLogEntriesResponse.newBuilder()
17311742
.setNextPageToken(cursor)
17321743
.addAllEntries(Lists.transform(entriesList, LogEntry.toPbFunction(PROJECT)))
17331744
.build();
17341745
ApiFuture<ListLogEntriesResponse> futureResponse = ApiFutures.immediateFuture(response);
1735-
EasyMock.expect(loggingRpcMock.list(request)).andReturn(futureResponse);
1746+
EasyMock.expect(loggingRpcMock.list(EasyMock.anyObject(ListLogEntriesRequest.class)))
1747+
.andReturn(futureResponse);
17361748
EasyMock.replay(loggingRpcMock);
17371749
Page<LogEntry> page = logging.listLogEntries();
17381750
assertEquals(cursor, page.getNextPageToken());
@@ -1744,11 +1756,18 @@ public void testListLogEntriesNextPage() throws ExecutionException, InterruptedE
17441756
String cursor1 = "cursor";
17451757
EasyMock.replay(rpcFactoryMock);
17461758
logging = options.getService();
1759+
1760+
String defaultTimeFilter =
1761+
LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter();
17471762
ListLogEntriesRequest request1 =
1748-
ListLogEntriesRequest.newBuilder().addResourceNames(PROJECT_PB).build();
1763+
ListLogEntriesRequest.newBuilder()
1764+
.addResourceNames(PROJECT_PB)
1765+
.setFilter(defaultTimeFilter)
1766+
.build();
17491767
ListLogEntriesRequest request2 =
17501768
ListLogEntriesRequest.newBuilder()
17511769
.addResourceNames(PROJECT_PB)
1770+
.setFilter(defaultTimeFilter)
17521771
.setPageToken(cursor1)
17531772
.build();
17541773
List<LogEntry> descriptorList1 = ImmutableList.of(LOG_ENTRY1, LOG_ENTRY2);
@@ -1785,7 +1804,11 @@ public void testListLogEntriesEmpty() {
17851804
EasyMock.replay(rpcFactoryMock);
17861805
logging = options.getService();
17871806
ListLogEntriesRequest request =
1788-
ListLogEntriesRequest.newBuilder().addResourceNames(PROJECT_PB).build();
1807+
ListLogEntriesRequest.newBuilder()
1808+
.addResourceNames(PROJECT_PB)
1809+
.setFilter(LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter())
1810+
.build();
1811+
17891812
List<LogEntry> entriesList = ImmutableList.of();
17901813
ListLogEntriesResponse response =
17911814
ListLogEntriesResponse.newBuilder()
@@ -1809,7 +1832,10 @@ public void testListLogEntriesWithOptions() {
18091832
ListLogEntriesRequest.newBuilder()
18101833
.addResourceNames(PROJECT_PB)
18111834
.setOrderBy("timestamp desc")
1812-
.setFilter("logName:syslog")
1835+
.setFilter(
1836+
String.format(
1837+
"logName:syslog AND %s",
1838+
LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter()))
18131839
.build();
18141840
List<LogEntry> entriesList = ImmutableList.of(LOG_ENTRY1, LOG_ENTRY2);
18151841
ListLogEntriesResponse response =
@@ -1834,7 +1860,10 @@ public void testListLogEntriesAsync() throws ExecutionException, InterruptedExce
18341860
EasyMock.replay(rpcFactoryMock);
18351861
logging = options.getService();
18361862
ListLogEntriesRequest request =
1837-
ListLogEntriesRequest.newBuilder().addResourceNames(PROJECT_PB).build();
1863+
ListLogEntriesRequest.newBuilder()
1864+
.addResourceNames(PROJECT_PB)
1865+
.setFilter(LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter())
1866+
.build();
18381867
List<LogEntry> entriesList = ImmutableList.of(LOG_ENTRY1, LOG_ENTRY2);
18391868
ListLogEntriesResponse response =
18401869
ListLogEntriesResponse.newBuilder()
@@ -1855,10 +1884,14 @@ public void testListLogEntriesAsyncNextPage() {
18551884
EasyMock.replay(rpcFactoryMock);
18561885
logging = options.getService();
18571886
ListLogEntriesRequest request1 =
1858-
ListLogEntriesRequest.newBuilder().addResourceNames(PROJECT_PB).build();
1887+
ListLogEntriesRequest.newBuilder()
1888+
.addResourceNames(PROJECT_PB)
1889+
.setFilter(LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter())
1890+
.build();
18591891
ListLogEntriesRequest request2 =
18601892
ListLogEntriesRequest.newBuilder()
18611893
.addResourceNames(PROJECT_PB)
1894+
.setFilter(LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter())
18621895
.setPageToken(cursor1)
18631896
.build();
18641897
List<LogEntry> descriptorList1 = ImmutableList.of(LOG_ENTRY1, LOG_ENTRY2);
@@ -1890,12 +1923,15 @@ public void testListLogEntriesAsyncNextPage() {
18901923
}
18911924

18921925
@Test
1893-
public void testListLogEntriesAyncEmpty() throws ExecutionException, InterruptedException {
1926+
public void testListLogEntriesAsyncEmpty() throws ExecutionException, InterruptedException {
18941927
String cursor = "cursor";
18951928
EasyMock.replay(rpcFactoryMock);
18961929
logging = options.getService();
18971930
ListLogEntriesRequest request =
1898-
ListLogEntriesRequest.newBuilder().addResourceNames(PROJECT_PB).build();
1931+
ListLogEntriesRequest.newBuilder()
1932+
.addResourceNames(PROJECT_PB)
1933+
.setFilter(LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter())
1934+
.build();
18991935
List<LogEntry> entriesList = ImmutableList.of();
19001936
ListLogEntriesResponse response =
19011937
ListLogEntriesResponse.newBuilder()
@@ -1915,11 +1951,15 @@ public void testListLogEntriesAsyncWithOptions() throws ExecutionException, Inte
19151951
String cursor = "cursor";
19161952
EasyMock.replay(rpcFactoryMock);
19171953
logging = options.getService();
1954+
String filter =
1955+
String.format(
1956+
"logName:syslog AND %s",
1957+
LoggingImpl.defaultTimestampFilterCreator.createDefaultTimestampFilter());
19181958
ListLogEntriesRequest request =
19191959
ListLogEntriesRequest.newBuilder()
19201960
.addResourceNames(PROJECT_PB)
19211961
.setOrderBy("timestamp desc")
1922-
.setFilter("logName:syslog")
1962+
.setFilter(filter)
19231963
.build();
19241964
List<LogEntry> entriesList = ImmutableList.of(LOG_ENTRY1, LOG_ENTRY2);
19251965
ListLogEntriesResponse response =
@@ -1933,7 +1973,7 @@ public void testListLogEntriesAsyncWithOptions() throws ExecutionException, Inte
19331973
AsyncPage<LogEntry> page =
19341974
logging
19351975
.listLogEntriesAsync(
1936-
EntryListOption.filter("logName:syslog"),
1976+
EntryListOption.filter(filter),
19371977
EntryListOption.sortOrder(SortingField.TIMESTAMP, Logging.SortingOrder.DESCENDING))
19381978
.get();
19391979
assertEquals(cursor, page.getNextPageToken());
@@ -2011,8 +2051,8 @@ public void run() {
20112051
};
20122052
threads[i].start();
20132053
}
2014-
for (int i = 0; i < threads.length; i++) {
2015-
threads[i].join();
2054+
for (Thread thread : threads) {
2055+
thread.join();
20162056
}
20172057
assertSame(0, exceptions.get());
20182058
}

0 commit comments

Comments
 (0)