Skip to content

HDDS-16650. Reuse bucket metadata in S3 lifecycle PUT and DELETE #11372

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
adoroszlai merged 6 commits into from
Oct 2, 2026
Merged

HDDS-16650. Reuse bucket metadata in S3 lifecycle PUT and DELETE #11372

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8d62d82
HDDS-16650. Reuse bucket metadata after owner verification in S3 Dele…
rich7420 Sep 30, 2026
537acdc
HDDS-16649. Reuse bucket metadata after owner verification in S3 PutB…
rich7420 Sep 30, 2026
b730db3
Merge remote-tracking branch 'upstream/master' into HDDS-16650
rich7420 Oct 1, 2026
d5eec88
Merge branch 'HDDS-16649' into HDDS-16650
rich7420 Oct 1, 2026
ab3271f
HDDS-16650. Reuse request-context bucket for lifecycle PUT and DELETE
rich7420 Oct 1, 2026
8f9343c
HDDS-16650. Use cached bucket name in request context
rich7420 Oct 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions ...e/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketLifecycleHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public Response deleteBucketLifecycleConfiguration(S3RequestContext context, Str
protected void deleteLifecycleConfiguration(S3RequestContext context, String bucketName)
throws IOException, OS3Exception {
try {
context.getVolume().getBucket(bucketName).deleteLifecycleConfiguration();
context.getBucket(bucketName).deleteLifecycleConfiguration();
} catch (OMException ex) {
// DeleteBucketLifecycle is idempotent: deleting a missing config
// must still return 204, not 404 — same as normal key deletion.
Expand All @@ -114,7 +114,7 @@ private void verifyBucketOwner(S3RequestContext context, String bucketName) thro
}

try {
String actualOwner = context.getVolume().getBucket(bucketName).getOwner();
String actualOwner = context.getBucket(bucketName).getOwner();
if (actualOwner != null && !actualOwner.equals(expectedBucketOwner)) {
LOG.debug("Bucket: {}, ExpectedBucketOwner: {}, ActualBucketOwner: {}",
bucketName, expectedBucketOwner, actualOwner);
Expand All @@ -130,7 +130,7 @@ public Response putBucketLifecycleConfiguration(S3RequestContext context, String
throws IOException, OS3Exception {
verifyBucketOwner(context, bucketName);
S3LifecycleConfiguration s3LifecycleConfiguration;
OzoneBucket ozoneBucket = context.getVolume().getBucket(bucketName);
OzoneBucket ozoneBucket = context.getBucket(bucketName);
OmLifecycleConfiguration lcc;
try {
s3LifecycleConfiguration = new PutBucketLifecycleConfigurationUnmarshaller().readFrom(body);
Expand Down
12 changes: 12 additions & 0 deletions ...p-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/S3RequestContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,18 @@
import java.io.IOException;
import org.apache.hadoop.ozone.audit.AuditLogger.PerformanceStringBuilder;
import org.apache.hadoop.ozone.audit.S3GAction;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.util.Time;
import org.apache.ratis.util.Preconditions;

class S3RequestContext {
private final long startNanos;
private final PerformanceStringBuilder perf;
private final EndpointBase endpoint;
private S3GAction action;
private OzoneVolume volume;
private OzoneBucket bucket;

S3RequestContext(EndpointBase endpoint, S3GAction action) {
this.endpoint = endpoint;
Expand All @@ -54,6 +57,15 @@ OzoneVolume getVolume() throws IOException {
return volume;
}

OzoneBucket getBucket(String name) throws IOException {
if (bucket == null) {
bucket = getVolume().getBucket(name);
} else {
Preconditions.assertEquals(bucket.getName(), name, "Multiple buckets in one request are not supported");
}
return bucket;
}

S3GAction getAction() {
return action;
}
Expand Down
125 changes: 125 additions & 0 deletions ...src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationDelete.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,42 @@

import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_NO_CONTENT;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_LIFECYCLE_CONFIGURATION;
import static org.apache.hadoop.ozone.s3.util.S3Consts.EXPECTED_BUCKET_OWNER_HEADER;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.NullAndEmptySource;
import org.junit.jupiter.params.provider.ValueSource;

/**
* Testing for DeleteBucketLifecycleConfiguration.
Expand Down Expand Up @@ -93,4 +114,108 @@ private static InputStream getBody() {

return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
}

@ParameterizedTest
@NullAndEmptySource
@ValueSource(strings = "owner")
public void testDeleteLifecycleLookupCount(String expectedOwner) throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers);

try (Response response = endpoint.delete("bucket1")) {
assertThat(response.getStatus()).isEqualTo(HTTP_NO_CONTENT);
}

verify(proxy).getBucketDetails("volume1", "bucket1");
verify(proxy).deleteLifecycleConfiguration("volume1", "bucket1");
}

@Test
public void testDeleteLifecycleWithoutHeaders() throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, null);
try (Response response = endpoint.delete("bucket1")) {
assertThat(response.getStatus()).isEqualTo(HTTP_NO_CONTENT);
}
verify(proxy).deleteLifecycleConfiguration("volume1", "bucket1");
}

@ParameterizedTest
@NullAndEmptySource
@ValueSource(strings = "owner")
public void testDeleteLifecycleMissingConfigurationIsIdempotent(String expectedOwner) throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers);
doThrow(new OMException("Missing configuration", ResultCodes.LIFECYCLE_CONFIGURATION_NOT_FOUND))
.when(proxy).deleteLifecycleConfiguration("volume1", "bucket1");
try (Response response = endpoint.delete("bucket1")) {
assertThat(response.getStatus()).isEqualTo(HTTP_NO_CONTENT);
}
}

@ParameterizedTest
@CsvSource({", BUCKET_NOT_FOUND, NO_SUCH_BUCKET", "owner, BUCKET_NOT_FOUND, NO_SUCH_BUCKET",
", VOLUME_NOT_FOUND, NO_SUCH_BUCKET", "owner, VOLUME_NOT_FOUND, NO_SUCH_BUCKET",
", PERMISSION_DENIED, ACCESS_DENIED", "owner, PERMISSION_DENIED, ACCESS_DENIED",
", INTERNAL_ERROR, INTERNAL_ERROR", "owner, INTERNAL_ERROR, INTERNAL_ERROR"})
public void testDeleteLifecycleProtocolErrors(
String expectedOwner, ResultCodes code, S3ErrorTable expected) throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers);
doThrow(new OMException("Delete failed", code)).when(proxy).deleteLifecycleConfiguration("volume1", "bucket1");
assertErrorResponse(expected, () -> endpoint.delete("bucket1"));
}

@Test
public void testDeleteLifecycleRejectsOwnerBeforeDeleting() throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn("other-owner");
BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers);
assertErrorResponse(S3ErrorTable.ACCESS_DENIED, () -> endpoint.delete("bucket1"));
verify(proxy, never()).deleteLifecycleConfiguration("volume1", "bucket1");
}

@ParameterizedTest
@CsvSource({", BUCKET_NOT_FOUND, NO_SUCH_BUCKET", "owner, BUCKET_NOT_FOUND, ACCESS_DENIED",
", VOLUME_NOT_FOUND, NO_SUCH_BUCKET", "owner, VOLUME_NOT_FOUND, ACCESS_DENIED",
", PERMISSION_DENIED, ACCESS_DENIED", "owner, PERMISSION_DENIED, ACCESS_DENIED",
", INTERNAL_ERROR, INTERNAL_ERROR", "owner, INTERNAL_ERROR, ACCESS_DENIED",
"'', PERMISSION_DENIED, ACCESS_DENIED"})
public void testDeleteLifecycleLookupFailurePreventsDeletion(
String expectedOwner, ResultCodes code, S3ErrorTable expected) throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders headers = mock(HttpHeaders.class);
when(headers.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, headers);
when(proxy.getBucketDetails("volume1", "bucket1")).thenThrow(new OMException("Lookup failed", code));
assertErrorResponse(expected, () -> endpoint.delete("bucket1"));
verify(proxy, never()).deleteLifecycleConfiguration("volume1", "bucket1");
}

private BucketEndpoint newProtocolEndpoint(ClientProtocol proxy, HttpHeaders requestHeaders) throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
OzoneVolume volume = OzoneVolume.newBuilder(conf, proxy)
.setName("volume1").setAcls(Collections.emptyList()).build();
OzoneBucket bucket = OzoneBucket.newBuilder(conf, proxy)
.setVolumeName("volume1").setName("bucket1").setOwner("owner").build();
when(proxy.getBucketDetails("volume1", "bucket1")).thenReturn(bucket);
ObjectStore store = mock(ObjectStore.class);
when(store.getS3Volume()).thenReturn(volume);
when(store.getClientProxy()).thenReturn(proxy);
OzoneClient client = mock(OzoneClient.class);
when(client.getObjectStore()).thenReturn(store);
when(client.getProxy()).thenReturn(proxy);
BucketEndpoint endpoint = EndpointBuilder.newBucketEndpointBuilder()
.setClient(client).setHeaders(requestHeaders).build();
endpoint.queryParamsForTest().set(S3Consts.QueryParams.LIFECYCLE, "");
return endpoint;
}

}
90 changes: 90 additions & 0 deletions ...ay/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestS3LifecycleConfigurationPut.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
import static java.net.HttpURLConnection.HTTP_OK;
import static org.apache.hadoop.ozone.s3.endpoint.EndpointTestUtils.assertErrorResponse;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.ACCESS_DENIED;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INTERNAL_ERROR;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
Expand All @@ -30,21 +31,26 @@
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.NO_SUCH_BUCKET;
import static org.apache.hadoop.ozone.s3.exception.S3ErrorTable.QUOTA_EXCEEDED;
import static org.apache.hadoop.ozone.s3.util.S3Consts.EXPECTED_BUCKET_OWNER_HEADER;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;

import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.Collections;
import java.util.function.Supplier;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.Response;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
Expand All @@ -53,14 +59,19 @@
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.protocol.ClientProtocol;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmLCExpiration;
import org.apache.hadoop.ozone.om.helpers.OmLifecycleConfiguration;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;

/**
Expand Down Expand Up @@ -820,4 +831,83 @@ private static InputStream withAbortDaysAfterInitiationOverflow() {

return new ByteArrayInputStream(xml.getBytes(StandardCharsets.UTF_8));
}

@ParameterizedTest
@CsvSource({", OBJECT_STORE", "'', OBJECT_STORE", "owner, OBJECT_STORE",
", FILE_SYSTEM_OPTIMIZED", "owner, FILE_SYSTEM_OPTIMIZED"})
public void testPutLifecycleReusesBucket(String expectedOwner, BucketLayout layout) throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders requestHeaders = mock(HttpHeaders.class);
when(requestHeaders.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, requestHeaders, layout);

try (Response response = endpoint.put("bucket1", onePrefix())) {
assertThat(response.getStatus()).isEqualTo(HTTP_OK);
}

verify(proxy).getBucketDetails("volume1", "bucket1");
ArgumentCaptor<OmLifecycleConfiguration> config = ArgumentCaptor.forClass(OmLifecycleConfiguration.class);
verify(proxy).setLifecycleConfiguration(config.capture());
assertThat(config.getValue().getVolume()).isEqualTo("volume1");
assertThat(config.getValue().getBucket()).isEqualTo("bucket1");
assertThat(config.getValue().getBucketLayout()).isEqualTo(layout);
}

@Test
public void testPutLifecycleOwnerMismatchPrecedesXmlValidation() throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders requestHeaders = mock(HttpHeaders.class);
when(requestHeaders.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn("other-owner");
BucketEndpoint endpoint = newProtocolEndpoint(proxy, requestHeaders, BucketLayout.OBJECT_STORE);
assertErrorResponse(ACCESS_DENIED, () -> endpoint.put("bucket1", null));
verify(proxy, never()).setLifecycleConfiguration(any());
}

@ParameterizedTest
@CsvSource({", BUCKET_NOT_FOUND, NO_SUCH_BUCKET", "owner, BUCKET_NOT_FOUND, ACCESS_DENIED",
", INTERNAL_ERROR, INTERNAL_ERROR", "owner, INTERNAL_ERROR, ACCESS_DENIED",
", PERMISSION_DENIED, ACCESS_DENIED", "owner, PERMISSION_DENIED, ACCESS_DENIED",
"'', PERMISSION_DENIED, ACCESS_DENIED", ", VOLUME_NOT_FOUND, NO_SUCH_BUCKET",
"owner, VOLUME_NOT_FOUND, ACCESS_DENIED"})
public void testPutLifecycleLookupFailurePrecedesXmlValidation(
String expectedOwner, ResultCodes code, S3ErrorTable expected) throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders requestHeaders = mock(HttpHeaders.class);
when(requestHeaders.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn(expectedOwner);
BucketEndpoint endpoint = newProtocolEndpoint(proxy, requestHeaders, BucketLayout.OBJECT_STORE);
when(proxy.getBucketDetails("volume1", "bucket1")).thenThrow(new OMException("Lookup failed", code));
assertErrorResponse(expected, () -> endpoint.put("bucket1", null));
verify(proxy, never()).setLifecycleConfiguration(any());
}

@Test
public void testPutLifecycleMatchingOwnerStillValidatesXml() throws Exception {
ClientProtocol proxy = mock(ClientProtocol.class);
HttpHeaders requestHeaders = mock(HttpHeaders.class);
when(requestHeaders.getHeaderString(EXPECTED_BUCKET_OWNER_HEADER)).thenReturn("owner");
BucketEndpoint endpoint = newProtocolEndpoint(proxy, requestHeaders, BucketLayout.OBJECT_STORE);
assertErrorResponse(MALFORMED_XML, () -> endpoint.put("bucket1", null));
verify(proxy, never()).setLifecycleConfiguration(any());
}

private BucketEndpoint newProtocolEndpoint(ClientProtocol proxy, HttpHeaders requestHeaders, BucketLayout layout)
throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
OzoneVolume volume = OzoneVolume.newBuilder(conf, proxy)
.setName("volume1").setAcls(Collections.emptyList()).build();
OzoneBucket bucket = OzoneBucket.newBuilder(conf, proxy)
.setVolumeName("volume1").setName("bucket1").setOwner("owner").setBucketLayout(layout).build();
when(proxy.getBucketDetails("volume1", "bucket1")).thenReturn(bucket);
ObjectStore store = mock(ObjectStore.class);
when(store.getS3Volume()).thenReturn(volume);
when(store.getClientProxy()).thenReturn(proxy);
OzoneClient client = mock(OzoneClient.class);
when(client.getObjectStore()).thenReturn(store);
when(client.getProxy()).thenReturn(proxy);
BucketEndpoint endpoint = EndpointBuilder.newBucketEndpointBuilder()
.setClient(client).setHeaders(requestHeaders).build();
endpoint.queryParamsForTest().set(S3Consts.QueryParams.LIFECYCLE, "");
return endpoint;
}

}
Loading
Loading