Spring/spring-batch-extensions
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bq] Break hard dependency on Jackson for JSON processing

Browse Source 
Signed-off-by: Volodymyr Perebykivskyi <vova235@gmail.com>
This commit is contained in:
Volodymyr
2025-06-12 21:26:13 +03:00
committed by GitHub
parent 4114825743
commit 49b250dd62
6 changed files with 52 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
spring-batch-bigquery/pom.xml
View File

@@ -100,7 +100,7 @@
<dependency>
<groupId>org.wiremock</groupId>
<artifactId>wiremock-standalone</artifactId>
<version>3.13.0</version>
<version>3.13.1</version>
<scope>test</scope>
</dependency>
<dependency>
@@ -108,7 +108,6 @@
<artifactId>jul-to-slf4j</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>

43
spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/BigQueryJsonItemWriter.java
View File

@@ -16,12 +16,9 @@
package org.springframework.batch.extensions.bigquery.writer;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.Table;
import org.springframework.core.convert.converter.Converter;
import org.springframework.batch.item.json.JsonObjectMarshaller;
import org.springframework.util.Assert;
import org.springframework.util.ObjectUtils;
@@ -42,41 +39,27 @@ public class BigQueryJsonItemWriter<T> extends BigQueryBaseItemWriter<T> {
private static final String LF = "\n";
private Converter<T, String> rowMapper;
private ObjectWriter objectWriter;
private Class<?> itemClass;
private JsonObjectMarshaller<T> marshaller;
@Override
protected void doInitializeProperties(List<? extends T> items) {
if (this.itemClass == null) {
T firstItem = items.stream().findFirst().orElseThrow(() -> {
logger.warn("Class type was not found");
return new IllegalStateException("Class type was not found");
});
this.itemClass = firstItem.getClass();
if (this.rowMapper == null) {
this.objectWriter = new ObjectMapper().writerFor(this.itemClass);
}
logger.debug("Writer setup is completed");
}
// Unused
}
/**
* Converter that transforms a single row into a {@link String}.
*
* @param rowMapper your JSON row mapper
* @param marshaller your JSON mapper
*/
public void setRowMapper(Converter<T, String> rowMapper) {
this.rowMapper = rowMapper;
public void setMarshaller(JsonObjectMarshaller<T> marshaller) {
this.marshaller = marshaller;
}
@Override
protected List<byte[]> convertObjectsToByteArrays(List<? extends T> items) {
return items
.stream()
.map(this::mapItemToJson)
.map(marshaller::marshal)
.filter(Predicate.not(ObjectUtils::isEmpty))
.map(this::convertToNdJson)
.map(row -> row.getBytes(StandardCharsets.UTF_8))
@@ -85,6 +68,8 @@ public class BigQueryJsonItemWriter<T> extends BigQueryBaseItemWriter<T> {
@Override
protected void performFormatSpecificChecks() {
Assert.notNull(this.marshaller, "Marshaller is mandatory");
Table table = getTable();
if (Boolean.TRUE.equals(writeChannelConfig.getAutodetect())) {
@@ -106,16 +91,6 @@ public class BigQueryJsonItemWriter<T> extends BigQueryBaseItemWriter<T> {
Assert.isTrue(Objects.equals(format, super.writeChannelConfig.getFormat()), "Only %s format is allowed".formatted(format));
}
private String mapItemToJson(T t) {
try {
return rowMapper == null ? objectWriter.writeValueAsString(t) : rowMapper.convert(t);
}
catch (JsonProcessingException e) {
logger.error("Error during processing of the line: ", e);
return null;
}
}
/**
* BigQuery uses <a href="https://github.com/ndjson/ndjson-spec">ndjson</a>.
* It is expected that to pass here JSON line generated by

15
spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryJsonItemWriterBuilder.java
View File

@@ -21,7 +21,7 @@ import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.WriteChannelConfiguration;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.core.convert.converter.Converter;
import org.springframework.batch.item.json.JsonObjectMarshaller;
import java.util.function.Consumer;
@@ -35,8 +35,7 @@ import java.util.function.Consumer;
*/
public class BigQueryJsonItemWriterBuilder<T> {
private Converter<T, String> rowMapper;
private JsonObjectMarshaller<T> marshaller;
private Consumer<Job> jobConsumer;
private DatasetInfo datasetInfo;
private WriteChannelConfiguration writeChannelConfig;
@@ -45,12 +44,12 @@ public class BigQueryJsonItemWriterBuilder<T> {
/**
* Converts your DTO into a {@link String}.
*
* @param rowMapper your mapping
* @param marshaller your mapper
* @return {@link BigQueryJsonItemWriter}
* @see BigQueryJsonItemWriter#setRowMapper(Converter)
* @see BigQueryJsonItemWriter#setMarshaller(JsonObjectMarshaller)
*/
public BigQueryJsonItemWriterBuilder<T> rowMapper(Converter<T, String> rowMapper) {
this.rowMapper = rowMapper;
public BigQueryJsonItemWriterBuilder<T> marshaller(JsonObjectMarshaller<T> marshaller) {
this.marshaller = marshaller;
return this;
}
@@ -110,7 +109,7 @@ public class BigQueryJsonItemWriterBuilder<T> {
public BigQueryJsonItemWriter<T> build() {
BigQueryJsonItemWriter<T> writer = new BigQueryJsonItemWriter<>();
writer.setRowMapper(this.rowMapper);
writer.setMarshaller(this.marshaller);
writer.setWriteChannelConfig(this.writeChannelConfig);
writer.setJobConsumer(this.jobConsumer);
writer.setBigQuery(this.bigQuery);

2
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/writer/BigQueryEmulatorJsonItemWriterTest.java
View File

@@ -31,6 +31,7 @@ import org.springframework.batch.extensions.bigquery.emulator.writer.base.BaseEm
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.item.json.JacksonJsonObjectMarshaller;
import java.util.stream.Stream;
@@ -52,6 +53,7 @@ class BigQueryEmulatorJsonItemWriterTest extends BaseEmulatorItemWriterTest {
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
.bigQuery(bigQuery)
.writeChannelConfig(channelConfig)
.marshaller(new JacksonJsonObjectMarshaller<>())
.build();
writer.afterPropertiesSet();

59
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/BigQueryJsonItemWriterTest.java
View File

@@ -16,8 +16,6 @@
package org.springframework.batch.extensions.bigquery.unit.writer;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.databind.ObjectWriter;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FormatOptions;
@@ -36,7 +34,9 @@ import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.core.convert.converter.Converter;
import org.springframework.batch.item.json.GsonJsonObjectMarshaller;
import org.springframework.batch.item.json.JacksonJsonObjectMarshaller;
import org.springframework.batch.item.json.JsonObjectMarshaller;
import java.lang.invoke.MethodHandles;
import java.util.List;
@@ -47,34 +47,15 @@ class BigQueryJsonItemWriterTest extends AbstractBigQueryTest {
private static final TableId TABLE_ID = TableId.of("1", "2");
@Test
void testDoInitializeProperties() throws IllegalAccessException, NoSuchFieldException {
TestWriter writer = new TestWriter();
List<PersonDto> items = TestConstants.CHUNK.getItems();
MethodHandles.Lookup handle = MethodHandles.privateLookupIn(BigQueryJsonItemWriter.class, MethodHandles.lookup());
// Exception
Assertions.assertThrows(IllegalStateException.class, () -> writer.testInitializeProperties(List.of()));
// No exception
writer.testInitializeProperties(items);
Assertions.assertEquals(
PersonDto.class.getSimpleName(),
((Class<PersonDto>) handle.findVarHandle(BigQueryJsonItemWriter.class, "itemClass", Class.class).get(writer)).getSimpleName()
);
ObjectWriter objectWriter = (ObjectWriter) handle.findVarHandle(BigQueryJsonItemWriter.class, "objectWriter", ObjectWriter.class).get(writer);
Assertions.assertInstanceOf(JsonFactory.class, objectWriter.getFactory());
}
@Test
void testSetRowMapper() throws IllegalAccessException, NoSuchFieldException {
void testSetMarshaller() throws IllegalAccessException, NoSuchFieldException {
BigQueryJsonItemWriter<PersonDto> reader = new BigQueryJsonItemWriter<>();
MethodHandles.Lookup handle = MethodHandles.privateLookupIn(BigQueryJsonItemWriter.class, MethodHandles.lookup());
Converter<PersonDto, String> expected = source -> null;
JsonObjectMarshaller<PersonDto> expected = new JacksonJsonObjectMarshaller<>();
reader.setRowMapper(expected);
reader.setMarshaller(expected);
Converter<PersonDto, String> actual = (Converter<PersonDto, String>) handle
.findVarHandle(BigQueryJsonItemWriter.class, "rowMapper", Converter.class)
JsonObjectMarshaller<PersonDto> actual = (JsonObjectMarshaller<PersonDto>) handle
.findVarHandle(BigQueryJsonItemWriter.class, "marshaller", JsonObjectMarshaller.class)
.get(reader);
Assertions.assertEquals(expected, actual);
@@ -83,14 +64,23 @@ class BigQueryJsonItemWriterTest extends AbstractBigQueryTest {
@Test
void testConvertObjectsToByteArrays() {
TestWriter writer = new TestWriter();
writer.setMarshaller(new JacksonJsonObjectMarshaller<>());
// Empty
Assertions.assertTrue(writer.testConvert(List.of()).isEmpty());
// Not empty
writer.setRowMapper(Record::toString);
writer.setMarshaller(Record::toString);
List<byte[]> actual = writer.testConvert(TestConstants.CHUNK.getItems());
List<byte[]> expected = TestConstants.CHUNK.getItems().stream().map(PersonDto::toString).map(s -> s.concat("\n")).map(String::getBytes).toList();
List<byte[]> expected = TestConstants.CHUNK
.getItems()
.stream()
.map(PersonDto::toString)
.map(s -> s.concat("\n"))
.map(String::getBytes)
.toList();
Assertions.assertEquals(expected.size(), actual.size());
for (int i = 0; i < actual.size(); i++) {
@@ -112,10 +102,15 @@ class BigQueryJsonItemWriterTest extends AbstractBigQueryTest {
BigQuery bigQuery = prepareMockedBigQuery();
Mockito.when(bigQuery.getTable(Mockito.any(TableId.class))).thenReturn(table);
// marshaller
IllegalArgumentException actual = Assertions.assertThrows(IllegalArgumentException.class, writer::testPerformFormatSpecificChecks);
Assertions.assertEquals("Marshaller is mandatory", actual.getMessage());
// schema
writer.setMarshaller(new JacksonJsonObjectMarshaller<>());
writer.setBigQuery(bigQuery);
writer.setWriteChannelConfig(WriteChannelConfiguration.of(TABLE_ID, FormatOptions.csv()));
IllegalArgumentException actual = Assertions.assertThrows(IllegalArgumentException.class, writer::testPerformFormatSpecificChecks);
actual = Assertions.assertThrows(IllegalArgumentException.class, writer::testPerformFormatSpecificChecks);
Assertions.assertEquals("Schema must be provided", actual.getMessage());
// schema equality
@@ -144,6 +139,7 @@ class BigQueryJsonItemWriterTest extends AbstractBigQueryTest {
TestWriter writer = new TestWriter();
writer.setBigQuery(bigQuery);
writer.setMarshaller(new GsonJsonObjectMarshaller<>());
writer.setWriteChannelConfig(WriteChannelConfiguration.newBuilder(TABLE_ID).setAutodetect(true).setFormatOptions(formatOptions).build());
IllegalArgumentException actual = Assertions.assertThrows(IllegalArgumentException.class, writer::testPerformFormatSpecificChecks);
@@ -164,9 +160,6 @@ class BigQueryJsonItemWriterTest extends AbstractBigQueryTest {
}
private static final class TestWriter extends BigQueryJsonItemWriter<PersonDto> {
public void testInitializeProperties(List<PersonDto> items) {
doInitializeProperties(items);
}
public List<byte[]> testConvert(List<PersonDto> items) {
return convertObjectsToByteArrays(items);

13
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java
View File

@@ -30,7 +30,8 @@ import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryT
import org.springframework.batch.extensions.bigquery.writer.BigQueryBaseItemWriter;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
import org.springframework.core.convert.converter.Converter;
import org.springframework.batch.item.json.JacksonJsonObjectMarshaller;
import org.springframework.batch.item.json.JsonObjectMarshaller;
import java.lang.invoke.MethodHandles;
import java.util.function.Consumer;
@@ -42,7 +43,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
MethodHandles.Lookup jsonWriterHandle = MethodHandles.privateLookupIn(BigQueryJsonItemWriter.class, MethodHandles.lookup());
MethodHandles.Lookup baseWriterHandle = MethodHandles.privateLookupIn(BigQueryBaseItemWriter.class, MethodHandles.lookup());
Converter<PersonDto, String> rowMapper = source -> "";
JsonObjectMarshaller<PersonDto> marshaller = new JacksonJsonObjectMarshaller<>();
DatasetInfo datasetInfo = DatasetInfo.newBuilder(TestConstants.DATASET).setLocation("europe-west-2").build();
Consumer<Job> jobConsumer = job -> {};
BigQuery mockedBigQuery = prepareMockedBigQuery();
@@ -53,7 +54,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
.build();
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
.rowMapper(rowMapper)
.marshaller(marshaller)
.writeChannelConfig(writeConfiguration)
.jobConsumer(jobConsumer)
.bigQuery(mockedBigQuery)
@@ -62,8 +63,8 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
Assertions.assertNotNull(writer);
Converter<PersonDto, String> actualRowMapper = (Converter<PersonDto, String>) jsonWriterHandle
.findVarHandle(BigQueryJsonItemWriter.class, "rowMapper", Converter.class)
JsonObjectMarshaller<PersonDto> actualMarshaller = (JsonObjectMarshaller<PersonDto>) jsonWriterHandle
.findVarHandle(BigQueryJsonItemWriter.class, "marshaller", JsonObjectMarshaller.class)
.get(writer);
WriteChannelConfiguration actualWriteChannelConfig = (WriteChannelConfiguration) jsonWriterHandle
@@ -82,7 +83,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
.findVarHandle(BigQueryJsonItemWriter.class, "datasetInfo", DatasetInfo.class)
.get(writer);
Assertions.assertEquals(rowMapper, actualRowMapper);
Assertions.assertEquals(marshaller, actualMarshaller);
Assertions.assertEquals(writeConfiguration, actualWriteChannelConfig);
Assertions.assertEquals(jobConsumer, actualJobConsumer);
Assertions.assertEquals(mockedBigQuery, actualBigQuery);