Skip to content

Commit 3bce87a

Browse files
BenWhiteheadBenWhitehead
authored
test: integration StorageControl and HNS bucket into integration runner (#2739)
`StorageControlClient` is now able to be `@Inject`ed to a test class (both PROD and TEST_BENCH) New `BucketType.HNS` added and able to be used with `@BucketFixture` in all the usual places. Update `BucketCleaner` to be HNS and managed folders aware. Remove dependency from google-cloud-storage-control -> google-cloud-storage. Instead, move ITFolderTest into google-cloud-storage and have google-cloud-storage depend on google-cloud-storage-control with scope: test.
1 parent 795f2c3 commit 3bce87a

File tree

11 files changed

+433
-120
lines changed

11 files changed

+433
-120
lines changed

‎google-cloud-storage-control/pom.xml‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -74,37 +74,6 @@
7474
<dependency>
7575
<groupId>com.google.api.grpc</groupId>
7676
<artifactId>grpc-google-cloud-storage-control-v2</artifactId>
77-
<scope>test</scope>
78-
</dependency>
79-
<dependency>
80-
<groupId>com.google.apis</groupId>
81-
<artifactId>google-api-services-storage</artifactId>
82-
<scope>test</scope>
83-
</dependency>
84-
<dependency>
85-
<groupId>com.google.cloud</groupId>
86-
<artifactId>google-cloud-storage</artifactId>
87-
<scope>test</scope>
88-
</dependency>
89-
<dependency>
90-
<groupId>com.google.auth</groupId>
91-
<artifactId>google-auth-library-oauth2-http</artifactId>
92-
<scope>test</scope>
93-
</dependency>
94-
<dependency>
95-
<groupId>com.google.http-client</groupId>
96-
<artifactId>google-http-client-jackson2</artifactId>
97-
<scope>test</scope>
98-
</dependency>
99-
<dependency>
100-
<groupId>com.google.http-client</groupId>
101-
<artifactId>google-http-client</artifactId>
102-
<scope>test</scope>
103-
</dependency>
104-
<dependency>
105-
<groupId>com.google.cloud</groupId>
106-
<artifactId>google-cloud-core-http</artifactId>
107-
<scope>test</scope>
10877
</dependency>
10978
</dependencies>
11079

‎google-cloud-storage-control/src/test/java/com/google/storage/control/v2/ITFoldersTest.java‎

Lines changed: 0 additions & 70 deletions
This file was deleted.

‎google-cloud-storage/pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,16 @@
262262
</exclusion>
263263
</exclusions>
264264
</dependency>
265+
<dependency>
266+
<groupId>com.google.api.grpc</groupId>
267+
<artifactId>proto-google-cloud-storage-control-v2</artifactId>
268+
<scope>test</scope>
269+
</dependency>
270+
<dependency>
271+
<groupId>com.google.cloud</groupId>
272+
<artifactId>google-cloud-storage-control</artifactId>
273+
<scope>test</scope>
274+
</dependency>
265275
<dependency>
266276
<groupId>org.mockito</groupId>
267277
<artifactId>mockito-core</artifactId>

‎google-cloud-storage/src/test/java/com/google/cloud/storage/it/BucketCleaner.java‎

Lines changed: 176 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,35 @@
1616

1717
package com.google.cloud.storage.it;
1818

19+
import com.google.api.gax.grpc.GrpcCallContext;
1920
import com.google.api.gax.paging.Page;
21+
import com.google.api.gax.rpc.ApiException;
22+
import com.google.api.gax.rpc.FailedPreconditionException;
2023
import com.google.cloud.storage.Blob;
2124
import com.google.cloud.storage.Storage;
25+
import com.google.cloud.storage.Storage.BlobField;
2226
import com.google.cloud.storage.Storage.BlobListOption;
2327
import com.google.cloud.storage.Storage.BlobSourceOption;
2428
import com.google.cloud.storage.Storage.BucketSourceOption;
29+
import com.google.common.collect.ImmutableList;
30+
import com.google.common.collect.ImmutableMap;
31+
import com.google.storage.control.v2.BucketName;
32+
import com.google.storage.control.v2.DeleteFolderRequest;
33+
import com.google.storage.control.v2.DeleteManagedFolderRequest;
34+
import com.google.storage.control.v2.Folder;
35+
import com.google.storage.control.v2.GetStorageLayoutRequest;
36+
import com.google.storage.control.v2.ListFoldersRequest;
37+
import com.google.storage.control.v2.ListManagedFoldersRequest;
38+
import com.google.storage.control.v2.StorageControlClient;
39+
import com.google.storage.control.v2.StorageLayout;
40+
import com.google.storage.control.v2.StorageLayoutName;
41+
import java.util.Collections;
42+
import java.util.Comparator;
2543
import java.util.List;
2644
import java.util.logging.Level;
2745
import java.util.logging.Logger;
2846
import java.util.stream.Collectors;
47+
import java.util.stream.Stream;
2948
import java.util.stream.StreamSupport;
3049

3150
public final class BucketCleaner {
@@ -48,12 +67,9 @@ public static void doCleanup(String bucketName, Storage s) {
4867
b.getName(),
4968
s.delete(b.getBlobId(), BlobSourceOption.userProject(projectId))))
5069
.collect(Collectors.toList());
51-
List<DeleteResult> failedDeletes =
52-
deleteResults.stream().filter(r -> !r.success).collect(Collectors.toList());
53-
failedDeletes.forEach(
54-
r -> LOGGER.warning(String.format("Failed to delete object %s/%s", bucketName, r.name)));
70+
boolean anyFailedObjectDeletes = getIfAnyFailedAndReport(bucketName, deleteResults, "object");
5571

56-
if (failedDeletes.isEmpty()) {
72+
if (!anyFailedObjectDeletes) {
5773
s.delete(bucketName, BucketSourceOption.userProject(projectId));
5874
} else {
5975
LOGGER.warning("Unable to delete bucket due to previous failed object deletes");
@@ -64,6 +80,161 @@ public static void doCleanup(String bucketName, Storage s) {
6480
}
6581
}
6682

83+
public static void doCleanup(String bucketName, Storage s, StorageControlClient ctrl) {
84+
LOGGER.warning("Starting bucket cleanup: " + bucketName);
85+
String projectId = s.getOptions().getProjectId();
86+
try {
87+
// TODO: probe bucket existence, a bad test could have deleted the bucket
88+
Page<Blob> page1 =
89+
s.list(
90+
bucketName,
91+
BlobListOption.userProject(projectId),
92+
BlobListOption.versions(true),
93+
BlobListOption.fields(BlobField.NAME));
94+
95+
List<DeleteResult> objectResults =
96+
StreamSupport.stream(page1.iterateAll().spliterator(), false)
97+
.map(
98+
b ->
99+
new DeleteResult(
100+
b.getName(),
101+
s.delete(b.getBlobId(), BlobSourceOption.userProject(projectId))))
102+
.collect(Collectors.toList());
103+
boolean anyFailedObjectDelete = getIfAnyFailedAndReport(bucketName, objectResults, "object");
104+
boolean anyFailedFolderDelete = false;
105+
boolean anyFailedManagedFolderDelete = false;
106+
107+
GrpcCallContext grpcCallContext =
108+
GrpcCallContext.createDefault()
109+
.withExtraHeaders(
110+
ImmutableMap.of("x-goog-user-project", ImmutableList.of(projectId)));
111+
if (!anyFailedObjectDelete) {
112+
BucketName parent = BucketName.of("_", bucketName);
113+
StorageLayout storageLayout =
114+
ctrl.getStorageLayoutCallable()
115+
.call(
116+
GetStorageLayoutRequest.newBuilder()
117+
.setName(
118+
StorageLayoutName.of(parent.getProject(), parent.getBucket())
119+
.toString())
120+
.build(),
121+
grpcCallContext);
122+
123+
List<DeleteResult> folderDeletes;
124+
if (storageLayout.hasHierarchicalNamespace()
125+
&& storageLayout.getHierarchicalNamespace().getEnabled()) {
126+
folderDeletes =
127+
StreamSupport.stream(
128+
ctrl.listFoldersPagedCallable()
129+
.call(
130+
ListFoldersRequest.newBuilder().setParent(parent.toString()).build(),
131+
grpcCallContext)
132+
.iterateAll()
133+
.spliterator(),
134+
false)
135+
.collect(Collectors.toList())
136+
.stream()
137+
.sorted(Collections.reverseOrder(Comparator.comparing(Folder::getName)))
138+
.map(
139+
folder -> {
140+
String formatted = String.format("folder = %s", folder.getName());
141+
LOGGER.warning(formatted);
142+
boolean success = true;
143+
try {
144+
ctrl.deleteFolderCallable()
145+
.call(
146+
DeleteFolderRequest.newBuilder()
147+
.setName(folder.getName())
148+
.build(),
149+
grpcCallContext);
150+
} catch (ApiException e) {
151+
success = false;
152+
}
153+
return new DeleteResult(folder.getName(), success);
154+
})
155+
.collect(Collectors.toList());
156+
} else {
157+
folderDeletes = ImmutableList.of();
158+
}
159+
160+
List<DeleteResult> managedFolderDeletes;
161+
try {
162+
managedFolderDeletes =
163+
StreamSupport.stream(
164+
ctrl.listManagedFoldersPagedCallable()
165+
.call(
166+
ListManagedFoldersRequest.newBuilder()
167+
.setParent(parent.toString())
168+
.build(),
169+
grpcCallContext)
170+
.iterateAll()
171+
.spliterator(),
172+
false)
173+
.map(
174+
managedFolder -> {
175+
String formatted =
176+
String.format("managedFolder = %s", managedFolder.getName());
177+
LOGGER.warning(formatted);
178+
boolean success = true;
179+
try {
180+
ctrl.deleteManagedFolderCallable()
181+
.call(
182+
DeleteManagedFolderRequest.newBuilder()
183+
.setName(managedFolder.getName())
184+
.build(),
185+
grpcCallContext);
186+
} catch (ApiException e) {
187+
success = false;
188+
}
189+
return new DeleteResult(managedFolder.getName(), success);
190+
})
191+
.collect(Collectors.toList());
192+
} catch (FailedPreconditionException fpe) {
193+
// FAILED_PRECONDITION: Uniform bucket-level access is required to be enabled on the
194+
// bucket in order to perform this operation. Read more at
195+
// https://cloud.google.com/storage/docs/uniform-bucket-level-access
196+
managedFolderDeletes = ImmutableList.of();
197+
}
198+
199+
anyFailedFolderDelete = getIfAnyFailedAndReport(bucketName, folderDeletes, "folder");
200+
anyFailedManagedFolderDelete =
201+
getIfAnyFailedAndReport(bucketName, managedFolderDeletes, "managed folder");
202+
}
203+
204+
List<String> failed =
205+
Stream.of(
206+
anyFailedObjectDelete ? "object" : "",
207+
anyFailedFolderDelete ? "folder" : "",
208+
anyFailedManagedFolderDelete ? "managed folder" : "")
209+
.filter(ss -> !ss.isEmpty())
210+
.collect(Collectors.toList());
211+
212+
if (!anyFailedObjectDelete && !anyFailedFolderDelete && !anyFailedManagedFolderDelete) {
213+
s.delete(bucketName, BucketSourceOption.userProject(projectId));
214+
} else {
215+
LOGGER.warning(
216+
String.format(
217+
"Unable to delete bucket %s due to previous failed %s deletes",
218+
bucketName, failed));
219+
}
220+
221+
LOGGER.warning("Bucket cleanup complete: " + bucketName);
222+
} catch (Exception e) {
223+
LOGGER.log(Level.SEVERE, e, () -> "Error during bucket cleanup.");
224+
}
225+
}
226+
227+
private static boolean getIfAnyFailedAndReport(
228+
String bucketName, List<DeleteResult> deleteResults, String resourceType) {
229+
List<DeleteResult> failedDeletes =
230+
deleteResults.stream().filter(r -> !r.success).collect(Collectors.toList());
231+
failedDeletes.forEach(
232+
r ->
233+
LOGGER.warning(
234+
String.format("Failed to delete %s %s/%s", resourceType, bucketName, r.name)));
235+
return !failedDeletes.isEmpty();
236+
}
237+
67238
private static final class DeleteResult {
68239
private final String name;
69240
private final boolean success;

0 commit comments

Comments
 (0)