diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index b76e5aa52629..849ae08efd1b 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1095,11 +1095,11 @@ public ListKeysResult listKeys(String volumeName, String bucketName, } else { // This allows us to seek directly to the first key with the right prefix. seekKey = getOzoneKey(volumeName, bucketName, - StringUtils.isNotBlank(keyPrefix) ? keyPrefix : OM_KEY_PREFIX); + (keyPrefix != null && !keyPrefix.isEmpty()) ? keyPrefix : OM_KEY_PREFIX); } String seekPrefix; - if (StringUtils.isNotBlank(keyPrefix)) { + if (keyPrefix != null && !keyPrefix.isEmpty()) { seekPrefix = getOzoneKey(volumeName, bucketName, keyPrefix); } else { seekPrefix = getBucketKey(volumeName, bucketName) + OM_KEY_PREFIX; diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java index 092bc4ba509a..337d3886c948 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/ObjectKeyNameAdapter.java @@ -36,7 +36,7 @@ public EncodingTypeObject unmarshal(String s) { public String marshal(EncodingTypeObject s) throws UnsupportedEncodingException { if (s.getEncodingType() != null && s.getEncodingType().equals("url")) { - return S3Utils.urlEncode(s.getName()) + return S3Utils.s3urlEncode(s.getName()) .replaceAll("%2F", "/"); } return s.getName(); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java index 85ff5fae535a..127abe871211 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/commontypes/RequestParameters.java @@ -47,6 +47,12 @@ default int getInt(String key, int defaultValue) { } } + /** + * @return true if the query parameter is present, even when its value is + * an empty string (eg. {@code delimiter=}). + */ + boolean containsKey(String key); + /** Additional methods for tests. */ interface Mutable extends RequestParameters { @@ -72,6 +78,11 @@ public String get(String key) { return params.getFirst(key); } + @Override + public boolean containsKey(String key) { + return params.containsKey(key); + } + @Override public void set(String key, String value) { params.putSingle(key, value); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java index d35da257cd0b..49c70302c7c2 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/BucketEndpoint.java @@ -106,11 +106,13 @@ public Response get( @Override Response handleGetRequest(S3RequestContext context, String bucketName) throws IOException, OS3Exception { final String continueToken = queryParams().get(QueryParams.CONTINUATION_TOKEN); - final String delimiter = queryParams().get(QueryParams.DELIMITER); + final String delimiter = queryParams().containsKey(QueryParams.DELIMITER) ? + queryParams().get(QueryParams.DELIMITER) : null; final String encodingType = queryParams().get(QueryParams.ENCODING_TYPE); final String marker = queryParams().get(QueryParams.MARKER); int maxKeys = queryParams().getInt(QueryParams.MAX_KEYS, 1000); - String prefix = queryParams().get(QueryParams.PREFIX, ""); + final boolean prefixSpecified = queryParams().containsKey(QueryParams.PREFIX); + final String prefix = prefixSpecified ? queryParams().get(QueryParams.PREFIX) : ""; String startAfter = queryParams().get(QueryParams.START_AFTER); Iterator ozoneKeyIterator = null; @@ -157,17 +159,21 @@ Response handleGetRequest(S3RequestContext context, String bucketName) throws IO if (encodingType != null && !encodingType.equals(ENCODING_TYPE)) { throw S3ErrorTable.newError(S3ErrorTable.INVALID_ARGUMENT, encodingType); } - // If you specify the encoding-type request parameter,should return - // encoded key name values in the following response elements: - // Delimiter, Prefix, Key, and StartAfter. + // encoded key name values in the following response elements: Delimiter, + // Key, and StartAfter. The echoed Prefix request parameter is returned without URL encoding. // // For detail refer: // https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html#AmazonS3-ListObjectsV2-response-EncodingType ListObjectResponse response = new ListObjectResponse(); - response.setDelimiter(EncodingTypeObject.createNullable(delimiter, encodingType)); + // AWS omits Delimiter from the response when the client passes delimiter= or does not specify delimiter at all. + if (delimiter != null && !delimiter.isEmpty()) { + response.setDelimiter(EncodingTypeObject.createNullable(delimiter, encodingType)); + } response.setName(bucketName); - response.setPrefix(EncodingTypeObject.createNullable(prefix, encodingType)); + if (prefixSpecified) { + response.setPrefix(EncodingTypeObject.createNullable(prefix, null)); + } response.setMarker(marker == null ? "" : marker); response.setMaxKeys(maxKeys); response.setEncodingType(encodingType); diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java index 29c556e4d31d..4f9fe6c2a277 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java @@ -62,6 +62,17 @@ public static String urlEncode(String str) return URLEncoder.encode(str, UTF_8.name()); } + /** + * Percent-encode a string for S3 {@code encoding-type=url} responses. + * + *

Unlike {@link URLEncoder} (application/x-www-form-urlencoded), AWS S3 + * uses percent-encoding where spaces are {@code %20}, not {@code +}. + */ + public static String s3urlEncode(String str) + throws UnsupportedEncodingException { + return urlEncode(str).replace("+", "%20"); + } + private S3Utils() { // no instances } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java index f2b10ac4e1bf..c98d0a27981a 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/commontypes/TestObjectKeyNameAdapter.java @@ -31,7 +31,7 @@ public class TestObjectKeyNameAdapter { public void testEncodeResult() throws Exception { assertEquals("abc/", getAdapter() .marshal(EncodingTypeObject.createNullable("abc/", ENCODING_TYPE))); - assertEquals("a+b+c/", getAdapter() + assertEquals("a%20b%20c/", getAdapter() .marshal(EncodingTypeObject.createNullable("a b c/", ENCODING_TYPE))); assertEquals("a%2Bb%2Bc/", getAdapter() .marshal(EncodingTypeObject.createNullable("a+b+c/", ENCODING_TYPE))); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java index d2a47c0c5862..4d97e92720c9 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestBucketList.java @@ -34,6 +34,7 @@ import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.s3.commontypes.EncodingTypeObject; +import org.apache.hadoop.ozone.s3.commontypes.ObjectKeyNameAdapter; import org.apache.hadoop.ozone.s3.exception.OS3Exception; import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.hadoop.ozone.s3.util.S3Consts.QueryParams; @@ -214,6 +215,7 @@ public void listWithPrefixAndEmptyStrDelimiter() assertEquals(0, getBucketResponse.getCommonPrefixes().size()); assertEquals(4, getBucketResponse.getContents().size()); + assertNull(getBucketResponse.getDelimiter()); assertEquals("dir1/", getBucketResponse.getContents().get(0).getKey().getName()); assertEquals("dir1/dir2/", @@ -401,7 +403,7 @@ public void testEncodingType() throws IOException, OS3Exception { ... - data%3D + data= data%3D %3D url @@ -415,7 +417,8 @@ public void testEncodingType() throws IOException, OS3Exception { - if encodingType == null , the = will not be encoded to "%3D" + Echoed Prefix is not URL-encoded. Delimiter, StartAfter, Key, and + CommonPrefixes are encoded when encodingType == url. * */ OzoneClient ozoneClient = @@ -437,9 +440,8 @@ public void testEncodingType() throws IOException, OS3Exception { // The Object name will be encoded by ObjectKeyNameAdapter // if encodingType == url assertEncodingTypeObject(delimiter, encodingType, response.getDelimiter()); - assertEncodingTypeObject(prefix, encodingType, response.getPrefix()); - assertEncodingTypeObject(startAfter, encodingType, - response.getStartAfter()); + assertEncodingTypeObject(prefix, null, response.getPrefix()); + assertEncodingTypeObject(startAfter, encodingType, response.getStartAfter()); assertNotNull(response.getCommonPrefixes()); assertNotNull(response.getContents()); assertEncodingTypeObject(prefix + delimiter, encodingType, @@ -566,6 +568,56 @@ public void testListObjectsRespectsConfiguredMaxKeysLimit() throws Exception { assertEquals(Integer.parseInt(configuredMaxKeysLimit), response.getContents().size()); } + @Test + public void testListObjectsUrlEncodingUsesPercentTwentyForSpaces() + throws Exception { + OzoneClient client = createClientWithKeys( + "foo+1/bar", "foo/bar/xyzzy", "quux ab/thud", "asdf+b"); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, "/"); + endpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, ENCODING_TYPE); + ListObjectResponse response = (ListObjectResponse) endpoint.get("b1").getEntity(); + + ObjectKeyNameAdapter adapter = new ObjectKeyNameAdapter(); + assertEquals("asdf%2Bb", adapter.marshal(response.getContents().get(0).getKey())); + assertEquals(3, response.getCommonPrefixes().size()); + assertEquals("foo%2B1/", adapter.marshal(response.getCommonPrefixes().get(0).getPrefix())); + assertEquals("foo/", adapter.marshal(response.getCommonPrefixes().get(1).getPrefix())); + assertEquals("quux%20ab/", adapter.marshal(response.getCommonPrefixes().get(2).getPrefix())); + } + + @Test + public void testListObjectsOmitsDelimiterWhenEmpty() throws Exception { + OzoneClient client = createClientWithKeys("bar", "baz", "cab", "foo"); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + + endpoint.queryParamsForTest().set(QueryParams.DELIMITER, ""); + ListObjectResponse response = (ListObjectResponse) endpoint.get("b1").getEntity(); + + assertNull(response.getDelimiter()); + assertEquals(4, response.getContents().size()); + assertEquals(0, response.getCommonPrefixes().size()); + } + + @Test + public void testListObjectsPrefixWithNewline() throws Exception { + OzoneClient client = createClientWithKeys("foo/bar", "foo/baz", "quux"); + BucketEndpoint endpoint = newBucketEndpointBuilder().setClient(client).build(); + + String prefix = String.valueOf((char) 0x0a); + endpoint.queryParamsForTest().set(QueryParams.PREFIX, prefix); + endpoint.queryParamsForTest().set(QueryParams.ENCODING_TYPE, ENCODING_TYPE); + ListObjectResponse response = (ListObjectResponse) endpoint.get("b1").getEntity(); + + assertNotNull(response.getPrefix()); + assertEquals(prefix, response.getPrefix().getName()); + assertNull(response.getPrefix().getEncodingType()); + assertEquals(prefix, new ObjectKeyNameAdapter().marshal(response.getPrefix())); + assertEquals(0, response.getContents().size()); + assertEquals(0, response.getCommonPrefixes().size()); + } + private void assertEncodingTypeObject( String exceptName, String exceptEncodingType, EncodingTypeObject object) { assertEquals(exceptName, object.getName());