Add support for bulk inserts in MongoItemWriter

This commit adds support for bulk inserts in the
MongoItemWriter. It introduces a new enumeration
to allow the user to choose the operation to apply
to items (INSERT, UPSERT, DELETE).

Resolves #4149
This commit is contained in:
Claiton Grings
2023-04-12 19:13:58 -03:00
committed by Mahmoud Ben Hassine
parent d84057c2b1
commit 34e3e0e152
3 changed files with 205 additions and 10 deletions

View File

@@ -58,6 +58,10 @@ import org.springframework.util.StringUtils;
*/
public class MongoItemWriter<T> implements ItemWriter<T>, InitializingBean {
public enum Mode {
INSERT, UPSERT, REMOVE;
}
private static final String ID_KEY = "_id";
private MongoOperations template;
@@ -66,7 +70,7 @@ public class MongoItemWriter<T> implements ItemWriter<T>, InitializingBean {
private String collection;
private boolean delete = false;
private Mode mode = Mode.UPSERT;
public MongoItemWriter() {
super();
@@ -78,9 +82,19 @@ public class MongoItemWriter<T> implements ItemWriter<T>, InitializingBean {
* the data store. If set to false (default), the items will be saved. If set to true,
* the items will be removed.
* @param delete removal indicator
* @deprecated use {@link MongoItemWriter#setMode(Mode)}
*/
@Deprecated
public void setDelete(boolean delete) {
this.delete = delete;
this.mode = (delete) ? Mode.REMOVE : Mode.UPSERT;
}
/**
* Set the operating {@link Mode} to be applied by this writer.
* @param mode the mode to be used.
*/
public void setMode(final Mode mode) {
this.mode = mode;
}
/**
@@ -133,15 +147,31 @@ public class MongoItemWriter<T> implements ItemWriter<T>, InitializingBean {
*/
protected void doWrite(Chunk<? extends T> chunk) {
if (!CollectionUtils.isEmpty(chunk.getItems())) {
if (this.delete) {
delete(chunk);
}
else {
saveOrUpdate(chunk);
switch (this.mode) {
case INSERT:
save(chunk);
break;
case REMOVE:
delete(chunk);
break;
default:
saveOrUpdate(chunk);
break;
}
}
}
private void save(final Chunk<? extends T> chunk) {
final BulkOperations bulkOperations = initBulkOperations(BulkMode.ORDERED, chunk.getItems().get(0));
final MongoConverter mongoConverter = this.template.getConverter();
for (final Object item : chunk) {
final Document document = new Document();
mongoConverter.write(item, document);
bulkOperations.insert(document);
}
bulkOperations.execute();
}
private void delete(Chunk<? extends T> chunk) {
BulkOperations bulkOperations = initBulkOperations(BulkMode.ORDERED, chunk.getItems().get(0));
MongoConverter mongoConverter = this.template.getConverter();

View File

@@ -17,6 +17,7 @@
package org.springframework.batch.item.data.builder;
import org.springframework.batch.item.data.MongoItemWriter;
import org.springframework.batch.item.data.MongoItemWriter.Mode;
import org.springframework.data.mongodb.core.MongoOperations;
import org.springframework.util.Assert;
@@ -33,7 +34,7 @@ public class MongoItemWriterBuilder<T> {
private String collection;
private boolean delete = false;
private Mode mode = Mode.UPSERT;
/**
* Indicates if the items being passed to the writer are to be saved or removed from
@@ -42,9 +43,23 @@ public class MongoItemWriterBuilder<T> {
* @param delete removal indicator
* @return The current instance of the builder
* @see MongoItemWriter#setDelete(boolean)
* @deprecated use {@link MongoItemWriterBuilder#mode(Mode)}
*/
@Deprecated
public MongoItemWriterBuilder<T> delete(boolean delete) {
this.delete = delete;
this.mode = (delete) ? Mode.REMOVE : Mode.UPSERT;
return this;
}
/**
* Set the operating {@link Mode} to be applied by this writer.
* @param mode the mode to be used.
* @return The current instance of the builder
* @see MongoItemWriter#setMode(Mode)
*/
public MongoItemWriterBuilder<T> mode(final Mode mode) {
this.mode = mode;
return this;
}
@@ -83,7 +98,7 @@ public class MongoItemWriterBuilder<T> {
MongoItemWriter<T> writer = new MongoItemWriter<>();
writer.setTemplate(this.template);
writer.setDelete(this.delete);
writer.setMode(this.mode);
writer.setCollection(this.collection);
return writer;

View File

@@ -26,6 +26,7 @@ import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.data.MongoItemWriter.Mode;
import org.springframework.batch.support.transaction.ResourcelessTransactionManager;
import org.springframework.data.mapping.context.MappingContext;
import org.springframework.data.mongodb.core.BulkOperations;
@@ -297,6 +298,155 @@ class MongoItemWriterTests {
}
}
// BATCH-4149
@Test
void testInsertModeNoTransactionNoCollection() throws Exception {
Chunk<Item> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setMode(Mode.INSERT);
writer.write(items);
verify(template).bulkOps(any(), any(Class.class));
verify(bulkOperations, times(2)).insert(any(Object.class));
}
@Test
void testInsertModeNoTransactionWithCollection() throws Exception {
Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setMode(Mode.INSERT);
writer.setCollection("collection");
writer.write(items);
verify(template).bulkOps(any(), eq("collection"));
verify(bulkOperations, times(2)).insert(any(Object.class));
}
@Test
void testInsertModeNoTransactionNoItems() throws Exception {
writer.setMode(Mode.INSERT);
writer.write(new Chunk<>());
verifyNoInteractions(template);
verifyNoInteractions(bulkOperations);
}
@Test
void testInsertModeTransactionNoCollection() {
final Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setMode(Mode.INSERT);
new TransactionTemplate(transactionManager).execute((TransactionCallback<Void>) status -> {
assertDoesNotThrow(() -> writer.write(items));
return null;
});
verify(template).bulkOps(any(), any(Class.class));
verify(bulkOperations, times(2)).insert(any(Object.class));
}
@Test
void testInsertModeTransactionWithCollection() {
final Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setMode(Mode.INSERT);
writer.setCollection("collection");
new TransactionTemplate(transactionManager).execute((TransactionCallback<Void>) status -> {
assertDoesNotThrow(() -> writer.write(items));
return null;
});
verify(template).bulkOps(any(), eq("collection"));
verify(bulkOperations, times(2)).insert(any(Object.class));
}
@Test
void testInsertModeTransactionFails() {
final Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setMode(Mode.INSERT);
writer.setCollection("collection");
Exception exception = assertThrows(RuntimeException.class,
() -> new TransactionTemplate(transactionManager).execute((TransactionCallback<Void>) status -> {
assertDoesNotThrow(() -> writer.write(items));
throw new RuntimeException("force rollback");
}));
assertEquals(exception.getMessage(), "force rollback");
verifyNoInteractions(template);
verifyNoInteractions(bulkOperations);
}
@Test
void testInsertModeTransactionReadOnly() {
final Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setMode(Mode.INSERT);
writer.setCollection("collection");
TransactionTemplate transactionTemplate = new TransactionTemplate(transactionManager);
transactionTemplate.setReadOnly(true);
transactionTemplate.execute((TransactionCallback<Void>) status -> {
assertDoesNotThrow(() -> writer.write(items));
return null;
});
verifyNoInteractions(template);
verifyNoInteractions(bulkOperations);
}
@Test
void testRemoveModeNoObjectIdNoCollection() throws Exception {
writer.setMode(Mode.REMOVE);
Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.write(items);
verify(template).bulkOps(any(), any(Class.class));
verify(bulkOperations, never()).remove(any(Query.class));
}
@Test
void testRemoveModeNoObjectIdWithCollection() throws Exception {
writer.setMode(Mode.REMOVE);
Chunk<Object> items = Chunk.of(new Item("Foo"), new Item("Bar"));
writer.setCollection("collection");
writer.write(items);
verify(template).bulkOps(any(), eq("collection"));
verify(bulkOperations, never()).remove(any(Query.class));
}
@Test
void testRemoveModeNoTransactionNoCollection() throws Exception {
writer.setMode(Mode.REMOVE);
Chunk<Object> items = Chunk.of(new Item(1), new Item(2));
writer.write(items);
verify(template).bulkOps(any(), any(Class.class));
verify(bulkOperations, times(2)).remove(any(Query.class));
}
@Test
void testRemoveModeNoTransactionWithCollection() throws Exception {
writer.setMode(Mode.REMOVE);
Chunk<Object> items = Chunk.of(new Item(1), new Item(2));
writer.setCollection("collection");
writer.write(items);
verify(template).bulkOps(any(), eq("collection"));
verify(bulkOperations, times(2)).remove(any(Query.class));
}
static class Item {
Integer id;