Fix #1222: Chunk request with large number of matched resources (#1223)

* Fix #1222: Chunk request with large number of matched resources
* CAPI, with v3 API, is limited to returning responses with max 5000 max matched resources; so we chunk them into request with max 5000 matched resources
* first try

* Fix #1222: Chunk request with large number of matched resources
* CAPI, with v3 API, is limited to returning responses with max 5000 max matched resources; so we chunk them into request with max 5000 matched resources
* update code that calls the request instead of the request code itself

Author: anthonydahanne

cloudfoundry-util/src/main/java/org/cloudfoundry/util/ResourceMatchingUtilsV3.java

@@ -24,6 +24,7 @@
 import java.util.Collection;
 import java.util.Enumeration;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
 import org.apache.commons.compress.archivers.zip.ZipFile;
 import org.cloudfoundry.client.CloudFoundryClient;
@@ -46,6 +47,7 @@ public final class ResourceMatchingUtilsV3 {
 
     private static final Logger LOGGER =
             LoggerFactory.getLogger("cloudfoundry-client.resource-matching-v3");
+    public static final int MAX_RESOURCES_SIZE = 5000;
 
     private ResourceMatchingUtilsV3() {}
 
@@ -55,9 +57,9 @@ public static Mono<List<MatchedResource>> getMatchedResources(
                         ? getArtifactMetadataFromDirectory(application)
                         : getArtifactMetadataFromZip(application))
                 .collectList()
-                .flatMap(
-                        (List<ArtifactMetadata> artifactMetadatas) ->
-                                requestListMatchingResources(cloudFoundryClient, artifactMetadatas))
+                .flatMapMany(Flux::fromIterable)
+                .buffer(MAX_RESOURCES_SIZE)
+                .flatMap(chunk -> requestListMatchingResources(cloudFoundryClient, chunk))
                 .map(ListMatchingResourcesResponse::getResources)
                 .doOnNext(
                         matched ->
@@ -68,6 +70,8 @@ public static Mono<List<MatchedResource>> getMatchedResources(
                                                 matched.stream()
                                                         .mapToInt(MatchedResource::getSize)
                                                         .sum())))
+                .collectList()
+                .map(lists -> lists.stream().flatMap(List::stream).collect(Collectors.toList()))
                 .subscribeOn(Schedulers.boundedElastic());
     }
 

cloudfoundry-util/src/test/java/org/cloudfoundry/util/ResourceMatchingUtilsV3Test.java

@@ -0,0 +1,57 @@
+package org.cloudfoundry.util;
+
+import static org.cloudfoundry.util.ResourceMatchingUtilsV3.getMatchedResources;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import org.cloudfoundry.client.CloudFoundryClient;
+import org.cloudfoundry.client.v3.resourcematch.ListMatchingResourcesResponse;
+import org.cloudfoundry.client.v3.resourcematch.MatchedResource;
+import org.junit.jupiter.api.Test;
+import org.springframework.core.io.ClassPathResource;
+import reactor.core.publisher.Mono;
+
+class ResourceMatchingUtilsV3Test {
+
+    @Test
+    void requestListMatchingResources2() throws IOException {
+        CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
+        when(cloudFoundryClient.resourceMatchV3())
+                .thenReturn(
+                        request ->
+                                Mono.just(
+                                        ListMatchingResourcesResponse.builder()
+                                                .addAllResources(request.getResources())
+                                                .build()));
+
+        Path testApplication = new ClassPathResource("test-application.zip").getFile().toPath();
+
+        List<MatchedResource> result =
+                getMatchedResources(cloudFoundryClient, testApplication).block();
+        assertNotNull(result);
+        assertEquals(2, result.size());
+    }
+
+    @Test
+    void requestListMatchingResources15001() throws IOException {
+        CloudFoundryClient cloudFoundryClient = mock(CloudFoundryClient.class);
+        when(cloudFoundryClient.resourceMatchV3())
+                .thenReturn(
+                        request ->
+                                Mono.just(
+                                        ListMatchingResourcesResponse.builder()
+                                                .addAllResources(request.getResources())
+                                                .build()));
+        Path testApplication = new ClassPathResource("15001_files.zip").getFile().toPath();
+
+        List<MatchedResource> result =
+                getMatchedResources(cloudFoundryClient, testApplication).block();
+        assertNotNull(result);
+        assertEquals(15001, result.size());
+    }
+}