Spring/spring-data-elasticsearch
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DATAES-766 - Replace CloseableIterator with SearchHitsIterator in stream operations.

Browse Source 
Original pull request: #407
This commit is contained in:
xhaggi
2020-03-20 14:09:08 +01:00
parent db28d93676
commit f354f986ca
16 changed files with 518 additions and 276 deletions
Download Patch File Download Diff File Expand all files Collapse all files

93
src/test/java/org/springframework/data/elasticsearch/core/ElasticsearchTemplateTests.java
View File

@@ -28,17 +28,12 @@ import lombok.Data;
import lombok.EqualsAndHashCode;
import lombok.NoArgsConstructor;
import java.lang.Double;
import java.lang.Integer;
import java.lang.Long;
import java.lang.Object;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.UUID;
import java.util.stream.Collectors;
@@ -290,7 +285,7 @@ public abstract class ElasticsearchTemplateTests {
// then
assertThat(searchHits).isNotNull();
assertThat(searchHits.getTotalHits()).isEqualTo(1);
assertThat(searchHits.getTotalHitsRelation()).isEqualByComparingTo(SearchHits.TotalHitsRelation.EQUAL_TO);
assertThat(searchHits.getTotalHitsRelation()).isEqualByComparingTo(TotalHitsRelation.EQUAL_TO);
}
@Test // DATAES-595
@@ -1055,11 +1050,11 @@ public abstract class ElasticsearchTemplateTests {
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
criteriaQuery.setPageable(PageRequest.of(0, 10));
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
criteriaQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}
@@ -1082,11 +1077,11 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery())
.withPageable(PageRequest.of(0, 10)).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}
@@ -1109,12 +1104,12 @@ public abstract class ElasticsearchTemplateTests {
criteriaQuery.addFields("message");
criteriaQuery.setPageable(PageRequest.of(0, 10));
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
criteriaQuery, SampleEntity.class, index);
String scrollId = scroll.getScrollId();
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scrollId = scroll.getScrollId();
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scrollId, 1000, SampleEntity.class);
}
@@ -1136,12 +1131,12 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery()).withFields("message")
.withQuery(matchAllQuery()).withPageable(PageRequest.of(0, 10)).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
String scrollId = scroll.getScrollId();
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scrollId = scroll.getScrollId();
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scrollId, 1000, SampleEntity.class);
}
@@ -1163,12 +1158,12 @@ public abstract class ElasticsearchTemplateTests {
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
criteriaQuery.setPageable(PageRequest.of(0, 10));
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
criteriaQuery, SampleEntity.class, index);
String scrollId = scroll.getScrollId();
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scrollId = scroll.getScrollId();
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scrollId, 1000, SampleEntity.class);
}
@@ -1190,12 +1185,12 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery())
.withPageable(PageRequest.of(0, 10)).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
String scrollId = scroll.getScrollId();
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scrollId = scroll.getScrollId();
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scrollId, 1000, SampleEntity.class);
}
@@ -1217,12 +1212,12 @@ public abstract class ElasticsearchTemplateTests {
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria());
criteriaQuery.setPageable(PageRequest.of(0, 10));
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
criteriaQuery, SampleEntity.class, index);
String scrollId = scroll.getScrollId();
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scrollId = scroll.getScrollId();
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scrollId, 1000, SampleEntity.class);
}
@@ -1244,12 +1239,12 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery())
.withPageable(PageRequest.of(0, 10)).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
String scrollId = scroll.getScrollId();
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scrollId = scroll.getScrollId();
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scrollId, 1000, SampleEntity.class);
}
@@ -1529,16 +1524,16 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery())
.withIndicesOptions(IndicesOptions.lenientExpandOpen()).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations)
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations)
.searchScrollStart(scrollTimeInMillis, searchQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> entities = new ArrayList<>(scroll.getContent());
List<SearchHit<SampleEntity>> entities = new ArrayList<>(scroll.getSearchHits());
while (scroll.hasContent()) {
while (scroll.hasSearchHits()) {
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(),
scrollTimeInMillis, SampleEntity.class);
entities.addAll(scroll.getContent());
entities.addAll(scroll.getSearchHits());
}
// then
@@ -2431,11 +2426,11 @@ public abstract class ElasticsearchTemplateTests {
CriteriaQuery criteriaQuery = new CriteriaQuery(new Criteria("message").contains("message"));
criteriaQuery.setPageable(PageRequest.of(0, 10));
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
criteriaQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}
@@ -2469,11 +2464,11 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchQuery("message", "message"))
.withPageable(PageRequest.of(0, 10)).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}
@@ -2502,11 +2497,11 @@ public abstract class ElasticsearchTemplateTests {
NativeSearchQuery searchQuery = new NativeSearchQueryBuilder().withQuery(matchAllQuery())
.withPageable(PageRequest.of(0, 10)).withSourceFilter(sourceFilter).build();
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}
@@ -2549,11 +2544,11 @@ public abstract class ElasticsearchTemplateTests {
.withSort(new FieldSortBuilder("message").order(SortOrder.DESC)).withPageable(PageRequest.of(0, 10)).build();
// when
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}
@@ -2598,11 +2593,11 @@ public abstract class ElasticsearchTemplateTests {
.build();
// when
ScrolledPage<SearchHit<SampleEntity>> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
SearchScrollHits<SampleEntity> scroll = ((AbstractElasticsearchTemplate) operations).searchScrollStart(1000,
searchQuery, SampleEntity.class, index);
List<SearchHit<SampleEntity>> sampleEntities = new ArrayList<>();
while (scroll.hasContent()) {
sampleEntities.addAll(scroll.getContent());
while (scroll.hasSearchHits()) {
sampleEntities.addAll(scroll.getSearchHits());
scroll = ((AbstractElasticsearchTemplate) operations).searchScrollContinue(scroll.getScrollId(), 1000,
SampleEntity.class);
}

53
src/test/java/org/springframework/data/elasticsearch/core/StreamQueriesTest.java
View File

@@ -19,12 +19,14 @@ import static org.assertj.core.api.Assertions.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.elasticsearch.search.aggregations.Aggregations;
import org.junit.jupiter.api.Test;
import org.springframework.data.domain.PageImpl;
import org.springframework.data.util.CloseableIterator;
import org.springframework.data.domain.Pageable;
import org.springframework.lang.Nullable;
/**
@@ -36,42 +38,51 @@ public class StreamQueriesTest {
public void shouldCallClearScrollOnIteratorClose() {
// given
List<String> results = new ArrayList<>();
results.add("one");
List<SearchHit<String>> hits = new ArrayList<>();
hits.add(new SearchHit<String>(null, 0, null, null, "one"));
ScrolledPage<String> page = new ScrolledPageImpl("1234", results);
SearchScrollHits<String> searchHits = newSearchScrollHits(hits);
AtomicBoolean clearScrollCalled = new AtomicBoolean(false);
// when
CloseableIterator<String> closeableIterator = StreamQueries.streamResults( //
page, //
scrollId -> new ScrolledPageImpl(scrollId, Collections.emptyList()), //
SearchHitsIterator<String> iterator = StreamQueries.streamResults( //
searchHits, //
scrollId -> newSearchScrollHits(Collections.emptyList()), //
scrollId -> clearScrollCalled.set(true));
while (closeableIterator.hasNext()) {
closeableIterator.next();
while (iterator.hasNext()) {
iterator.next();
}
closeableIterator.close();
iterator.close();
// then
assertThat(clearScrollCalled).isTrue();
}
private static class ScrolledPageImpl extends PageImpl<String> implements ScrolledPage<String> {
@Test // DATAES-766
public void shouldReturnTotalHits() {
private String scrollId;
// given
List<SearchHit<String>> hits = new ArrayList<>();
hits.add(new SearchHit<String>(null, 0, null, null, "one"));
public ScrolledPageImpl(String scrollId, List<String> content) {
super(content);
this.scrollId = scrollId;
}
SearchScrollHits<String> searchHits = newSearchScrollHits(hits);
@Override
@Nullable
public String getScrollId() {
return scrollId;
}
// when
SearchHitsIterator<String> iterator = StreamQueries.streamResults( //
searchHits, //
scrollId -> newSearchScrollHits(Collections.emptyList()), //
scrollId -> {
});
// then
assertThat(iterator.getTotalHits()).isEqualTo(1);
}
private SearchScrollHits<String> newSearchScrollHits(List<SearchHit<String>> hits) {
return new SearchHitsImpl<String>(hits.size(), TotalHitsRelation.EQUAL_TO, 0, "1234", hits, null);
}
}