From 33f1061a9fcc0bfecd4d70494f2ce2cdcb292100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rog=C3=A9rio=20Lino?= Date: Tue, 14 Nov 2023 16:27:16 -0300 Subject: [PATCH] Fix S3Session for pagination According with AWS SDK doc the nextMarker field is only returned when defining the delimiter query param: > NextMarker When the response is truncated (the IsTruncated element value in the response is true), you can use the key name in this field as the marker parameter in the subsequent request to get the next set of objects. Amazon S3 lists objects in alphabetical order. Note This element is returned only if you have the delimiter request parameter specified. If the response does not include the NextMarker element and it is truncated, you can use the value of the last Key element in the response as the marker parameter in the subsequent request to get the next set of object keys. https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjects.html#API_ListObjects_ResponseElements After upgrading my project to newest version of `spring-integration-aws` I've started facing this loop issue for buckets with more than `1000` items, where `isTruncated` is true and `nextMarker` is `null`. * Update the `S3Session` code to use the `key` from last item as `nextMarker` in next the request when response `isTruncated`, avoiding infinite loop. --- .../integration/aws/support/S3Session.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/springframework/integration/aws/support/S3Session.java b/src/main/java/org/springframework/integration/aws/support/S3Session.java index 87a8ad5..95f28e3 100644 --- a/src/main/java/org/springframework/integration/aws/support/S3Session.java +++ b/src/main/java/org/springframework/integration/aws/support/S3Session.java @@ -48,6 +48,7 @@ import org.springframework.util.StringUtils; * @author Jim Krygowski * @author Anwar Chirakkattil * @author Xavier François + * @author Rogerio Lino */ public class S3Session implements Session { @@ -82,8 +83,11 @@ public class S3Session implements Session { List objectSummaries = new ArrayList<>(); do { objectListing = this.amazonS3.listObjects(listObjectsRequest.build()); - objectSummaries.addAll(objectListing.contents()); - listObjectsRequest.marker(objectListing.nextMarker()); + List contents = objectListing.contents(); + objectSummaries.addAll(contents); + if (objectListing.isTruncated()) { + listObjectsRequest.marker(contents.get(contents.size() - 1).key()); + } } while (objectListing.isTruncated()); @@ -108,10 +112,13 @@ public class S3Session implements Session { List names = new ArrayList<>(); do { objectListing = this.amazonS3.listObjects(listObjectsRequest.build()); - for (S3Object objectSummary : objectListing.contents()) { + List contents = objectListing.contents(); + for (S3Object objectSummary : contents) { names.add(objectSummary.key()); } - listObjectsRequest.marker(objectListing.nextMarker()); + if (objectListing.isTruncated()) { + listObjectsRequest.marker(contents.get(contents.size() - 1).key()); + } } while (objectListing.isTruncated());