diff --git a/README.md b/README.md index 814d9027c..a11f26d05 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ To utilize the GitLab API for Java in your project, simply add the following dep ```java dependencies { ... - compile group: 'org.gitlab4j', name: 'gitlab4j-api', version: '4.9.15' + compile group: 'org.gitlab4j', name: 'gitlab4j-api', version: '4.9.16' } ``` @@ -22,7 +22,7 @@ dependencies { org.gitlab4j gitlab4j-api - 4.9.15 + 4.9.16 ``` @@ -141,34 +141,41 @@ List allProjects = projectPager.all(); --- ## Java 8 Stream Support -As of GitLab4J-API 4.9.2, you can also stream list based items in a Java 8 Stream using a Pager instance as follows: -```java -// Get a Pager instance to get a Stream instance. -Pager projectPager = gitlabApi.getProjectsApi().getProjects(10); +As of GitLab4J-API 4.9.2, all GitLabJ-API methods that return a List result also similarlly named method returns a Java 8 Stream. The Stream returning methods use the following naming convention: ```getXxxxxStream()```. + + +**IMPORTANT** +The built-in methods that return a Stream do so using ___eager evaluation___, meaning all items are pre-fetched from the GitLab server and a Stream is returned which will stream those items. **Eager evaluation does NOT support paralell reading of data from ther server, it does however allow for paralell processing of the Stream post data fetch.** + +To stream using ___lazy evaluation___, use the GitLab4J-API methods that return a ```Pager``` instance, and then call the ```lazyStream()``` method on the ```Pager``` instance to create a lazy evaluation Stream. The Stream utilizes the ```Pager``` instance to page through the available items. **A lazy Stream does NOT support parallel operations or skipping.** + + +**Eager evaluation example usage:** -// Stream the Projects printing out the project name. -projectPager.stream().map(Project::getName).forEach(name -> System.out.println(name)); -``` -The following API classes also include ```getXxxxxStream()``` methods which return a Java 8 Stream: -``` -GroupApi -IssuesApi -NotesApi -ProjectApi -RepositoryApi -TagsApi -UserApi -``` -Example usage: ```java -// Stream the visible Projects printing out the project name. -gitlabApi.getProjectsApi().getProjectsStream().map(Project::getName).forEach(name -> System.out.println(name)); +// Stream the visible projects printing out the project name. +Stream projectStream = gitlabApi.getProjectApi().getProjectsStream(); +projectStream.map(Project::getName).forEach(name -> System.out.println(name)); // Operate on the stream in parallel, this example sorts User instances by username +// NOTE: Fetching of the users is not done in paralell, +// only the soprting of the users is a paralell operation. Stream stream = new UserApi(gitLabApi).getUsersStream(); -List sortedUsers = stream.parallel().sorted(comparing(User::getUsername)).collect(toList()); +List users = stream.parallel().sorted(comparing(User::getUsername)).collect(toList()); +``` + +**Lazy evaluation example usage:** + +```java +// Get a Pager instance to that will be used to lazily stream Project instances. +// In this example, 10 Projects per page will be pre-fetched. +Pager projectPager = gitlabApi.getProjectApi().getProjects(10); + +// Lazily stream the Projects, printing out each project name, limit the output to 5 project names +projectPager.lazyStream().limit(5).map(Project::getName).forEach(name -> System.out.println(name)); ``` + --- ## Java 8 Optional<T> Support GitLab4J-API supports Java 8 Optional<T> for API calls that result in the return of a single item. Here is an example on how to use the Java 8 Optional<T> API calls: diff --git a/src/main/java/org/gitlab4j/api/GitLabApi.java b/src/main/java/org/gitlab4j/api/GitLabApi.java index 5dd44f14e..551c06f8d 100644 --- a/src/main/java/org/gitlab4j/api/GitLabApi.java +++ b/src/main/java/org/gitlab4j/api/GitLabApi.java @@ -27,7 +27,7 @@ public class GitLabApi { private final static Logger LOGGER = Logger.getLogger(GitLabApi.class.getName()); /** GitLab4J default per page. GitLab will ignore anything over 100. */ - public static final int DEFAULT_PER_PAGE = 100; + public static final int DEFAULT_PER_PAGE = 96; /** Specifies the version of the GitLab API to communicate with. */ public enum ApiVersion { @@ -1393,7 +1393,7 @@ public static final T orElseThrow(Optional optional) throws GitLabApiExce return (optional.get()); } - + /** * Gets the SnippetsApi instance owned by this GitLabApi instance. The SnippetsApi is used * to perform all snippet related API calls. diff --git a/src/main/java/org/gitlab4j/api/Pager.java b/src/main/java/org/gitlab4j/api/Pager.java index bb73d6514..f8ef261c4 100644 --- a/src/main/java/org/gitlab4j/api/Pager.java +++ b/src/main/java/org/gitlab4j/api/Pager.java @@ -18,12 +18,12 @@ import com.fasterxml.jackson.databind.ObjectMapper; /** - *

This class defines an Iterator implementation that is used as a paging iterator for all API methods that + *

This class defines an Iterator implementation that is used as a paging iterator for all API methods that * return a List of objects. It hides the details of interacting with the GitLab API when paging is involved * simplifying accessing large lists of objects.

- * + * *

Example usage:

- * + * * 
  *   // Get a Pager instance that will page through the projects with 10 projects per page
  *   Pager<Project> projectPager = gitlabApi.getProjectsApi().getProjectsPager(10);
@@ -35,8 +35,8 @@
  *           System.out.println(project.getName() + " : " + project.getDescription());
  *       }
  *   }
- *
- * + * + * * @param the GitLab4J type contained in the List. */ public class Pager implements Iterator>, Constants { @@ -48,6 +48,7 @@ public class Pager implements Iterator>, Constants { private List pageParam = new ArrayList<>(1); private List currentItems; + private Stream pagerStream = null; private AbstractApi api; private MultivaluedMap queryParams; @@ -59,7 +60,7 @@ public class Pager implements Iterator>, Constants { /** * Creates a Pager instance to access the API through the specified path and query parameters. - * + * * @param api the AbstractApi implementation to communicate through * @param type the GitLab4J type that will be contained in the List * @param itemsPerPage items per page @@ -114,7 +115,7 @@ public class Pager implements Iterator>, Constants { /** * Get the specified integer header value from the Response instance. - * + * * @param response the Response instance to get the value from * @param key the HTTP header key to get the value for * @return the specified integer header value from the Response instance @@ -136,7 +137,7 @@ private int getHeaderValue(Response response, String key) throws GitLabApiExcept /** * Sets the "page" query parameter. - * + * * @param page the value for the "page" query parameter */ private void setPageParam(int page) { @@ -204,7 +205,7 @@ public List next() { /** * This method is not implemented and will throw an UnsupportedOperationException if called. - * + * * @throws UnsupportedOperationException when invoked */ @Override @@ -289,7 +290,7 @@ public List page(int pageNumber) { throw new RuntimeException(e); } } - + /** * Gets all the items from each page as a single List instance. * @@ -312,34 +313,62 @@ public List all() throws GitLabApiException { } /** - * Builds and returns a Stream instance for streaming all the items from each page. + * Builds and returns a Stream instance which is pre-populated with all items from all pages. * - * @return a Stream instance for streaming all the items from each pag - * @throws GitLabApiException if any error occurs + * @return a Stream instance which is pre-populated with all items from all pages + * @throws IllegalStateException if Stream has already been issued + * @throws GitLabApiException if any other error occurs */ - public Stream stream() throws GitLabApiException { + public Stream stream() throws GitLabApiException, IllegalStateException { - // Make sure that current page is 0, this will ensure the whole list is streamed - // regardless of what page the instance is currently on. - currentPage = 0; + if (pagerStream == null) { + synchronized (this) { + if (pagerStream == null) { - // Create a Stream.Builder to contain all the items. This is more efficient than - // getting a List with all() and streaming that List - Stream.Builder streamBuilder = Stream.builder(); + // Make sure that current page is 0, this will ensure the whole list is streamed + // regardless of what page the instance is currently on. + currentPage = 0; - // Iterate through the pages and append each page of items to the stream builder - while (hasNext()) { - next().forEach(streamBuilder); - } + // Create a Stream.Builder to contain all the items. This is more efficient than + // getting a List with all() and streaming that List + Stream.Builder streamBuilder = Stream.builder(); - return (streamBuilder.build()); + // Iterate through the pages and append each page of items to the stream builder + while (hasNext()) { + next().forEach(streamBuilder); + } + + pagerStream = streamBuilder.build(); + return (pagerStream); + } + } + } + + throw new IllegalStateException("Stream already issued"); } - public Stream lazyStream() { - // Make sure that current page is 0, this will ensure the whole list is streamed - // regardless of what page the instance is currently on. - currentPage = 0; + /** + * Creates a Stream instance for lazily streaming items from the GitLab server. + * + * @return a Stream instance for lazily streaming items from the GitLab server + * @throws IllegalStateException if Stream has already been issued + */ + public Stream lazyStream() throws IllegalStateException { + + if (pagerStream == null) { + synchronized (this) { + if (pagerStream == null) { + + // Make sure that current page is 0, this will ensure the whole list is streamed + // regardless of what page the instance is currently on. + currentPage = 0; + + pagerStream = StreamSupport.stream(new PagerSpliterator(this), false); + return (pagerStream); + } + } + } - return StreamSupport.stream(new PagerSpliterator(this), false); + throw new IllegalStateException("Stream already issued"); } } diff --git a/src/main/java/org/gitlab4j/api/ProjectApi.java b/src/main/java/org/gitlab4j/api/ProjectApi.java index 3a43f8448..63879d3e3 100644 --- a/src/main/java/org/gitlab4j/api/ProjectApi.java +++ b/src/main/java/org/gitlab4j/api/ProjectApi.java @@ -818,7 +818,7 @@ public Project createProject(Project project, String importUrl) throws GitLabApi if (isApiVersion(ApiVersion.V3)) { boolean isPublic = (project.getPublic() != null ? project.getPublic() : project.getVisibility() == Visibility.PUBLIC); formData.withParam("public", isPublic); - + if (project.getTagList() != null && !project.getTagList().isEmpty()) { throw new IllegalArgumentException("GitLab API v3 does not support tag lists when creating projects"); } @@ -826,7 +826,7 @@ public Project createProject(Project project, String importUrl) throws GitLabApi Visibility visibility = (project.getVisibility() != null ? project.getVisibility() : project.getPublic() == Boolean.TRUE ? Visibility.PUBLIC : null); formData.withParam("visibility", visibility); - + if (project.getTagList() != null && !project.getTagList().isEmpty()) { formData.withParam("tag_list", String.join(",", project.getTagList())); } @@ -1057,7 +1057,7 @@ public Project updateProject(Project project) throws GitLabApiException { formData.withParam("visibility_level", project.getVisibilityLevel()); boolean isPublic = (project.getPublic() != null ? project.getPublic() : project.getVisibility() == Visibility.PUBLIC); formData.withParam("public", isPublic); - + if (project.getTagList() != null && !project.getTagList().isEmpty()) { throw new IllegalArgumentException("GitLab API v3 does not support tag lists when updating projects"); } @@ -1065,7 +1065,7 @@ public Project updateProject(Project project) throws GitLabApiException { Visibility visibility = (project.getVisibility() != null ? project.getVisibility() : project.getPublic() == Boolean.TRUE ? Visibility.PUBLIC : null); formData.withParam("visibility", visibility); - + if (project.getTagList() != null && !project.getTagList().isEmpty()) { formData.withParam("tag_list", String.join(",", project.getTagList())); } @@ -1090,7 +1090,7 @@ public void deleteProject(Object projectIdOrPath) throws GitLabApiException { /** * Forks a project into the user namespace of the authenticated user or the one provided. - * The forking operation for a project is asynchronous and is completed in a background job. + * The forking operation for a project is asynchronous and is completed in a background job. * The request will return immediately. * * 
POST /projects/:id/fork
@@ -1109,7 +1109,7 @@ public Project forkProject(Object projectIdOrPath, String namespace) throws GitL /** * Forks a project into the user namespace of the authenticated user or the one provided. - * The forking operation for a project is asynchronous and is completed in a background job. + * The forking operation for a project is asynchronous and is completed in a background job. * The request will return immediately. * * 
POST /projects/:id/fork
@@ -1130,7 +1130,7 @@ public Project forkProject(Object projectIdOrPath, Integer namespaceId) throws G * Create a forked from/to relation between existing projects. * * 
POST /projects/:id/fork/:forkFromId
- * + * * * @param projectIdOrPath projectIdOrPath the project in the form of an Integer(ID), String(path), or Project instance * @param forkedFromId the ID of the project that was forked from @@ -1455,7 +1455,7 @@ public List getProjectUsers(Object projectIdOrPath, String search) } /** - * Get a Pager of project users matching the specified search string. This Pager includes + * Get a Pager of project users matching the specified search string. This Pager includes * all project members and all users assigned to project parent groups. * * 
GET /projects/:id/users
@@ -1644,7 +1644,7 @@ public Optional getOptionalHook(Object projectIdOrPath, Integer hoo * @return the added ProjectHook instance * @throws GitLabApiException if any exception occurs */ - public ProjectHook addHook(String projectName, String url, ProjectHook enabledHooks, boolean enableSslVerification, String secretToken) + public ProjectHook addHook(String projectName, String url, ProjectHook enabledHooks, boolean enableSslVerification, String secretToken) throws GitLabApiException { if (projectName == null) { @@ -2266,9 +2266,9 @@ public void deletePushRules(Object projectIdOrPath) throws GitLabApiException { /** * Get a list of projects that were forked from the specified project. - * + * * 
GET /projects/:id/forks
- * + * * @param projectIdOrPath projectIdOrPath the project in the form of an Integer(ID), String(path), or Project instance, required * @return a List of forked projects * @throws GitLabApiException if any exception occurs @@ -2382,7 +2382,7 @@ public Project transferProject(Object projectIdOrPath, String namespace) throws } /** - * Uploads and sets the project avatar for the specified project. + * Uploads and sets the project avatar for the specified project. * * 
PUT /projects/:id/uploads
* @@ -2395,4 +2395,4 @@ public Project setProjectAvatar(Object projectIdOrPath, File avatarFile) throws Response response = putUpload(Response.Status.OK, "avatar", avatarFile, "projects", getProjectIdOrPath(projectIdOrPath)); return (response.readEntity(Project.class)); } -} \ No newline at end of file +} diff --git a/src/test/java/org/gitlab4j/api/TestStreams.java b/src/test/java/org/gitlab4j/api/TestStreams.java index 55614b991..86aa0e055 100644 --- a/src/test/java/org/gitlab4j/api/TestStreams.java +++ b/src/test/java/org/gitlab4j/api/TestStreams.java @@ -5,6 +5,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.initMocks; @@ -85,4 +86,36 @@ public void testParallelStream() throws Exception { assertTrue(compareJson(sortedUsers.get(i), users.get(i))); } } + + @Test + public void testLazyStream() throws Exception { + + // Arrange + Pager pager = new UserApi(gitLabApi).getUsers(10); + Stream stream = pager.lazyStream(); + + // Assert + assertNotNull(stream); + List usernames = stream.map(User::getUsername).collect(toList()); + assertNotNull(usernames); + + assertEquals(usernames.size(), sortedUsers.size()); + for (int i = 0; i < sortedUsers.size(); i++) { + assertTrue(usernames.contains(sortedUsers.get(i).getUsername())); + } + } + + @Test + public void testStreamLazyLimit() throws Exception { + + // Arrange and only continue if there are more than 3 users + Pager pager = new UserApi(gitLabApi).getUsers(2); + assumeTrue(pager != null && pager.getTotalItems() > 3); + Stream stream = pager.lazyStream(); + + // Assert + List users = stream.limit(3).collect(toList()); + assertNotNull(users); + assertEquals(3, users.size()); + } }