Skip to content

HDDS-14209. Reduce parameter count in ObjectEndpoint #9532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
adoroszlai wants to merge 2 commits into
base: master
Choose a base branch
from
Draft

HDDS-14209. Reduce parameter count in ObjectEndpoint #9532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8280b29
HDDS-14209. Reduce parameter count in ObjectEndpoint
adoroszlai Dec 18, 2025
6269ead
set default value for upload ID in completeMultipartUpload
adoroszlai Dec 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,11 @@ protected String getQueryParam(String key) {
return getQueryParameters().getFirst(key);
}

protected String getQueryParam(String key, String defaultValue) {
final String value = getQueryParam(key);
return value != null ? value : defaultValue;
}

public MultivaluedMap<String, String> getQueryParameters() {
return context.getUriInfo().getQueryParameters();
}
Expand Down
45 changes: 23 additions & 22 deletions hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -168,7 +168,7 @@ public class ObjectEndpoint extends EndpointBase {

/*FOR the feature Overriding Response Header
https://docs.aws.amazon.com/de_de/AmazonS3/latest/API/API_GetObject.html */
private Map<String, String> overrideQueryParameter;
private final Map<String, String> overrideQueryParameter;
private int bufferSize;
private int chunkSize;
private boolean datastreamEnabled;
Expand Down Expand Up @@ -209,17 +209,18 @@ public void init() {
* See: https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html for
* more details.
*/
@SuppressWarnings({"checkstyle:MethodLength", "checkstyle:ParameterNumber"})
@SuppressWarnings("checkstyle:MethodLength")
@PUT
public Response put(
@PathParam(BUCKET) String bucketName,
@PathParam(PATH) String keyPath,
@HeaderParam(HttpHeaders.CONTENT_LENGTH) long length,
@QueryParam(QueryParams.PART_NUMBER) int partNumber,
@QueryParam(QueryParams.UPLOAD_ID) @DefaultValue("") String uploadID,
@QueryParam(QueryParams.TAGGING) String taggingMarker,
@QueryParam(QueryParams.ACL) String aclMarker,
final InputStream body) throws IOException, OS3Exception {
@QueryParam(QueryParams.PART_NUMBER) int partNumber,
final InputStream body
) throws IOException, OS3Exception {
final String aclMarker = getQueryParam(QueryParams.ACL);
final String taggingMarker = getQueryParam(QueryParams.TAGGING);
final String uploadID = getQueryParam(QueryParams.UPLOAD_ID);
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.CREATE_KEY;
boolean auditSuccess = true;
Expand Down Expand Up @@ -403,17 +404,17 @@ public Response put(
* https://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadListParts.html
* for more details.
*/
@SuppressWarnings({"checkstyle:MethodLength", "checkstyle:ParameterNumber"})
@SuppressWarnings("checkstyle:MethodLength")
@GET
public Response get(
@PathParam(BUCKET) String bucketName,
@PathParam(PATH) String keyPath,
@QueryParam(QueryParams.PART_NUMBER) int partNumber,
@QueryParam(QueryParams.UPLOAD_ID) String uploadId,
@QueryParam(QueryParams.MAX_PARTS) @DefaultValue("1000") int maxParts,
@QueryParam(QueryParams.PART_NUMBER_MARKER) String partNumberMarker,
@QueryParam(QueryParams.TAGGING) String taggingMarker)
throws IOException, OS3Exception {
@QueryParam(QueryParams.MAX_PARTS) @DefaultValue("1000") int maxParts
) throws IOException, OS3Exception {
final String uploadId = getQueryParam(QueryParams.UPLOAD_ID);
final String partNumberMarker = getQueryParam(QueryParams.PART_NUMBER_MARKER);
final String taggingMarker = getQueryParam(QueryParams.TAGGING);
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.GET_KEY;
PerformanceStringBuilder perf = new PerformanceStringBuilder();
Expand Down Expand Up @@ -720,10 +721,11 @@ private Response abortMultipartUpload(OzoneVolume volume, String bucket,
@SuppressWarnings("emptyblock")
public Response delete(
@PathParam(BUCKET) String bucketName,
@PathParam(PATH) String keyPath,
@QueryParam(QueryParams.UPLOAD_ID) @DefaultValue("") String uploadId,
@QueryParam(QueryParams.TAGGING) String taggingMarker) throws
IOException, OS3Exception {
@PathParam(PATH) String keyPath
) throws IOException, OS3Exception {
final String taggingMarker = getQueryParam(QueryParams.TAGGING);
final String uploadId = getQueryParam(QueryParams.UPLOAD_ID);

long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.DELETE_KEY;

Expand Down Expand Up @@ -798,8 +800,7 @@ public Response delete(
public Response initializeMultipartUpload(
@PathParam(BUCKET) String bucket,
@PathParam(PATH) String key
)
throws IOException, OS3Exception {
) throws IOException, OS3Exception {
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.INIT_MULTIPART_UPLOAD;

Expand Down Expand Up @@ -863,9 +864,9 @@ private ReplicationConfig getReplicationConfig(OzoneBucket ozoneBucket,
public Response completeMultipartUpload(
@PathParam(BUCKET) String bucket,
@PathParam(PATH) String key,
@QueryParam(QueryParams.UPLOAD_ID) @DefaultValue("") String uploadID,
CompleteMultipartUploadRequest multipartUploadRequest)
throws IOException, OS3Exception {
CompleteMultipartUploadRequest multipartUploadRequest
) throws IOException, OS3Exception {
final String uploadID = getQueryParam(QueryParams.UPLOAD_ID, "");
long startNanos = Time.monotonicNowNanos();
S3GAction s3GAction = S3GAction.COMPLETE_MULTIPART_UPLOAD;
OzoneVolume volume = getVolume();
Expand Down
8 changes: 5 additions & 3 deletions ...s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestAbortMultipartUpload.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.Test;

/**
Expand Down Expand Up @@ -63,15 +64,16 @@ public void testAbortMultipartUpload() throws Exception {
assertNotNull(multipartUploadInitiateResponse.getUploadID());
String uploadID = multipartUploadInitiateResponse.getUploadID();


// Abort multipart upload
response = rest.delete(bucket, key, uploadID, null);
rest.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
response = rest.delete(bucket, key);

assertEquals(204, response.getStatus());

// test with unknown upload Id.
try {
rest.delete(bucket, key, "random", null);
rest.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, "random");
rest.delete(bucket, key);
} catch (OS3Exception ex) {
assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getCode(), ex.getCode());
assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getErrorMessage(),
Expand Down
31 changes: 17 additions & 14 deletions hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestListParts.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import org.apache.hadoop.ozone.client.OzoneClientStub;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

Expand All @@ -44,7 +45,6 @@
public class TestListParts {

private ObjectEndpoint rest;
private String uploadID;

@BeforeEach
public void setUp() throws Exception {
Expand All @@ -67,34 +67,35 @@ public void setUp() throws Exception {
OzoneConsts.KEY);
MultipartUploadInitiateResponse multipartUploadInitiateResponse =
(MultipartUploadInitiateResponse) response.getEntity();
assertNotNull(multipartUploadInitiateResponse.getUploadID());
uploadID = multipartUploadInitiateResponse.getUploadID();
String uploadID = multipartUploadInitiateResponse.getUploadID();
assertNotNull(uploadID);
rest.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);

assertEquals(200, response.getStatus());

String content = "Multipart Upload";
ByteArrayInputStream body =
new ByteArrayInputStream(content.getBytes(UTF_8));
response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
content.length(), 1, uploadID, null, null, body);
content.length(), 1, body);

assertNotNull(response.getHeaderString(OzoneConsts.ETAG));

response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
content.length(), 2, uploadID, null, null, body);
content.length(), 2, body);

assertNotNull(response.getHeaderString(OzoneConsts.ETAG));

response = rest.put(OzoneConsts.S3_BUCKET, OzoneConsts.KEY,
content.length(), 3, uploadID, null, null, body);
content.length(), 3, body);

assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
}

@Test
public void testListParts() throws Exception {
Response response = rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0,
uploadID, 3, "0", null);
rest.getQueryParameters().putSingle(S3Consts.QueryParams.PART_NUMBER_MARKER, "0");
Response response = rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0, 3);

ListPartsResponse listPartsResponse =
(ListPartsResponse) response.getEntity();
Expand All @@ -106,17 +107,18 @@ public void testListParts() throws Exception {

@Test
public void testListPartsContinuation() throws Exception {
Response response = rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0,
uploadID, 2, "0", null);
rest.getQueryParameters().putSingle(S3Consts.QueryParams.PART_NUMBER_MARKER, "0");
Response response = rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0, 2);
ListPartsResponse listPartsResponse =
(ListPartsResponse) response.getEntity();

assertTrue(listPartsResponse.getTruncated());
assertEquals(2, listPartsResponse.getPartList().size());

// Continue
response = rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0, uploadID, 2,
Integer.toString(listPartsResponse.getNextPartNumberMarker()), null);
rest.getQueryParameters().putSingle(S3Consts.QueryParams.PART_NUMBER_MARKER,
Integer.toString(listPartsResponse.getNextPartNumberMarker()));
response = rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0, 2);
listPartsResponse = (ListPartsResponse) response.getEntity();

assertFalse(listPartsResponse.getTruncated());
Expand All @@ -126,9 +128,10 @@ public void testListPartsContinuation() throws Exception {

@Test
public void testListPartsWithUnknownUploadID() throws Exception {
rest.getQueryParameters().putSingle(S3Consts.QueryParams.PART_NUMBER_MARKER, "0");
rest.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, "no-such-upload");
try {
rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0,
uploadID, 2, "0", null);
rest.get(OzoneConsts.S3_BUCKET, OzoneConsts.KEY, 0, 2);
} catch (OS3Exception ex) {
assertEquals(S3ErrorTable.NO_SUCH_UPLOAD.getErrorMessage(),
ex.getErrorMessage());
Expand Down
7 changes: 5 additions & 2 deletions ...ateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadComplete.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

Expand Down Expand Up @@ -106,8 +107,9 @@ private Part uploadPart(String key, String uploadID, int partNumber, String
content) throws IOException, OS3Exception {
ByteArrayInputStream body =
new ByteArrayInputStream(content.getBytes(UTF_8));
rest.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
Response response = rest.put(OzoneConsts.S3_BUCKET, key, content.length(),
partNumber, uploadID, null, null, body);
partNumber, body);
assertEquals(200, response.getStatus());
assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
Part part = new Part();
Expand All @@ -120,8 +122,9 @@ private Part uploadPart(String key, String uploadID, int partNumber, String
private void completeMultipartUpload(String key,
CompleteMultipartUploadRequest completeMultipartUploadRequest,
String uploadID) throws IOException, OS3Exception {
rest.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
Response response = rest.completeMultipartUpload(OzoneConsts.S3_BUCKET, key,
uploadID, completeMultipartUploadRequest);
completeMultipartUploadRequest);

assertEquals(200, response.getStatus());

Expand Down
13 changes: 9 additions & 4 deletions ...ateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestMultipartUploadWithCopy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
import org.apache.hadoop.ozone.s3.endpoint.CompleteMultipartUploadRequest.Part;
import org.apache.hadoop.ozone.s3.exception.OS3Exception;
import org.apache.hadoop.ozone.s3.exception.S3ErrorTable;
import org.apache.hadoop.ozone.s3.util.S3Consts;
import org.apache.hadoop.ozone.web.utils.OzoneUtils;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -330,8 +331,9 @@ private Part uploadPart(String key, String uploadID, int partNumber, String
setHeaders();
ByteArrayInputStream body =
new ByteArrayInputStream(content.getBytes(UTF_8));
endpoint.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, content.length(),
partNumber, uploadID, null, null, body);
partNumber, body);
assertEquals(200, response.getStatus());
assertNotNull(response.getHeaderString(OzoneConsts.ETAG));
Part part = new Part();
Expand Down Expand Up @@ -375,8 +377,9 @@ private Part uploadPartWithCopy(String key, String uploadID, int partNumber,
setHeaders(additionalHeaders);

ByteArrayInputStream body = new ByteArrayInputStream("".getBytes(UTF_8));
endpoint.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
Response response = endpoint.put(OzoneConsts.S3_BUCKET, key, 0, partNumber,
uploadID, null, null, body);
body);
assertEquals(200, response.getStatus());

CopyPartResult result = (CopyPartResult) response.getEntity();
Expand All @@ -403,7 +406,8 @@ public void testUploadWithRangeCopyContentLength()
OzoneConsts.S3_BUCKET + "/" + EXISTING_KEY);
additionalHeaders.put(COPY_SOURCE_HEADER_RANGE, "bytes=0-3");
setHeaders(additionalHeaders);
endpoint.put(OzoneConsts.S3_BUCKET, KEY, 0, 1, uploadID, null, null, body);
endpoint.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
endpoint.put(OzoneConsts.S3_BUCKET, KEY, 0, 1, body);
OzoneMultipartUploadPartListParts parts =
client.getObjectStore().getS3Bucket(OzoneConsts.S3_BUCKET)
.listParts(KEY, uploadID, 0, 100);
Expand All @@ -415,8 +419,9 @@ private void completeMultipartUpload(String key,
CompleteMultipartUploadRequest completeMultipartUploadRequest,
String uploadID) throws IOException, OS3Exception {
setHeaders();
endpoint.getQueryParameters().putSingle(S3Consts.QueryParams.UPLOAD_ID, uploadID);
Response response = endpoint.completeMultipartUpload(OzoneConsts.S3_BUCKET, key,
uploadID, completeMultipartUploadRequest);
completeMultipartUploadRequest);

assertEquals(200, response.getStatus());

Expand Down
2 changes: 1 addition & 1 deletion ...p-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectDelete.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public void delete() throws IOException, OS3Exception {
.build();

//WHEN
rest.delete("b1", "key1", null, null);
rest.delete("b1", "key1");

//THEN
assertFalse(bucket.listKeys("").hasNext(),
Expand Down
Loading
Loading