Spring/spring-batch-extensions
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bq] 0.2 introduce integration tests for writers

Browse Source
This commit is contained in:
Volodymyr
2022-12-22 13:38:16 +02:00
committed by GitHub
parent 79465739ce
commit 760cde444a
10 changed files with 360 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
spring-batch-bigquery/pom.xml
View File

@@ -51,6 +51,7 @@
<!-- Dependent on Spring Batch core -->
<java.version>17</java.version>
<logback.version>1.4.5</logback.version>
</properties>
<dependencies>
@@ -96,6 +97,25 @@
<version>4.9.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>2.0.6</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -117,6 +137,12 @@
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.22.2</version>
<configuration>
<includes>
<!-- Integration tests are omitted because they are designed to be run locally -->
<include>/unit</include>
</includes>
</configuration>
</plugin>
<!-- Generate javadoc and source jars -->

3
spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/package-info.java
View File

@@ -32,6 +32,9 @@
* Take into account that BigQuery has rate limits, and it is very easy to exceed those in concurrent environment.
* @see <a href="https://cloud.google.com/bigquery/quotas">BigQuery Quotas &amp; Limits</a>
*
* Also worth mentioning that you should ensure ordering of the fields in DTO that you are going to send to the BigQuery.
* In case of CSV/JSON and Jackson consider using {@link com.fasterxml.jackson.annotation.JsonPropertyOrder}.
*
* @author Volodymyr Perebykivskyi
* @since 0.1.0
* @see <a href="https://cloud.google.com/bigquery/">Google BigQuery</a>

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java Normal file
View File

@@ -0,0 +1,11 @@
/**
* In order to launch these tests you should provide a way how to authorize to Google BigQuery.
* A simple way is to create service account, store credentials as JSON file and provide environment variable.
* Example: GOOGLE_APPLICATION_CREDENTIALS=/home/dgray/Downloads/bq-key.json
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
*
* Test names should follow this pattern: test1, test2, testN.
* So later in BigQuery you will see generated table name: csv_test1, csv_test2, csv_testN.
* This way it will be easier to trace errors in BigQuery.
*/
package org.springframework.batch.extensions.bigquery.integration;

85
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright 2002-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.writer;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.JobId;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableResult;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest;
import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder;
import org.springframework.batch.item.Chunk;
import java.util.concurrent.atomic.AtomicReference;
@Tag("csv")
public class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
@Test
void test1(TestInfo testInfo) throws Exception {
AtomicReference<JobId> jobId = new AtomicReference<>();
BigQueryCsvItemWriter writer = new BigQueryCsvItemWriterBuilder<PersonDto>()
.bigQuery(bigQuery)
.writeChannelConfig(generateConfiguration(testInfo, FormatOptions.csv()))
.jobConsumer(j -> jobId.set(j.getJobId()))
.build();
writer.afterPropertiesSet();
Chunk<PersonDto> chunk = Chunk.of(new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26));
writer.write(chunk);
waitForJobToFinish(jobId.get());
Dataset dataset = bigQuery.getDataset(DATASET);
Table table = bigQuery.getTable(TableId.of(DATASET, getTableName(testInfo)));
TableId tableId = table.getTableId();
TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
Assertions.assertNotNull(dataset.getDatasetId());
Assertions.assertNotNull(tableId);
Assertions.assertEquals(chunk.size(), tableResult.getTotalRows());
tableResult
.getValues()
.forEach(field -> {
Assertions.assertTrue(
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(NumberUtils.INTEGER_ZERO).getStringValue().equals(name))
);
boolean ageCondition = chunk
.getItems()
.stream()
.map(PersonDto::age)
.map(Long::valueOf)
.anyMatch(age -> age.compareTo(field.get(NumberUtils.INTEGER_ONE).getLongValue()) == NumberUtils.INTEGER_ZERO);
Assertions.assertTrue(ageCondition);
});
}
}

85
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java Normal file
View File

@@ -0,0 +1,85 @@
/*
* Copyright 2002-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.writer;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.JobId;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableResult;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
import org.springframework.batch.item.Chunk;
import java.util.concurrent.atomic.AtomicReference;
@Tag("json")
public class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest {
@Test
void test1(TestInfo testInfo) throws Exception {
AtomicReference<JobId> jobId = new AtomicReference<>();
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
.bigQuery(bigQuery)
.writeChannelConfig(generateConfiguration(testInfo, FormatOptions.json()))
.jobConsumer(j -> jobId.set(j.getJobId()))
.build();
writer.afterPropertiesSet();
Chunk<PersonDto> chunk = Chunk.of(new PersonDto("Viktor", 57), new PersonDto("Nina", 57));
writer.write(chunk);
waitForJobToFinish(jobId.get());
Dataset dataset = bigQuery.getDataset(DATASET);
Table table = bigQuery.getTable(TableId.of(DATASET, getTableName(testInfo)));
TableId tableId = table.getTableId();
TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
Assertions.assertNotNull(dataset.getDatasetId());
Assertions.assertNotNull(tableId);
Assertions.assertEquals(chunk.size(), tableResult.getTotalRows());
tableResult
.getValues()
.forEach(field -> {
Assertions.assertTrue(
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(NumberUtils.INTEGER_ZERO).getStringValue().equals(name))
);
boolean ageCondition = chunk
.getItems()
.stream()
.map(PersonDto::age)
.map(Long::valueOf)
.anyMatch(age -> age.compareTo(field.get(NumberUtils.INTEGER_ONE).getLongValue()) == NumberUtils.INTEGER_ZERO);
Assertions.assertTrue(ageCondition);
});
}
}

103
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java Normal file
View File

@@ -0,0 +1,103 @@
/*
* Copyright 2002-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.writer.base;
import com.fasterxml.jackson.annotation.JsonPropertyOrder;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.Field;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.JobId;
import com.google.cloud.bigquery.JobStatus;
import com.google.cloud.bigquery.Schema;
import com.google.cloud.bigquery.StandardSQLTypeName;
import com.google.cloud.bigquery.StandardTableDefinition;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.bigquery.WriteChannelConfiguration;
import org.apache.commons.lang3.BooleanUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import java.lang.reflect.Method;
import java.util.Objects;
public abstract class BaseBigQueryItemWriterTest {
protected static final String DATASET = "spring_extensions";
protected final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
private static final String TABLE_PATTERN = "%s_%s";
@BeforeEach
void prepareTest(TestInfo testInfo) {
if (Objects.isNull(bigQuery.getDataset(DATASET))) {
bigQuery.create(DatasetInfo.of(DATASET));
}
if (Objects.isNull(bigQuery.getTable(DATASET, getTableName(testInfo)))) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
bigQuery.create(TableInfo.of(TableId.of(DATASET, getTableName(testInfo)), tableDefinition));
}
}
@AfterEach
void cleanupTest(TestInfo testInfo) {
bigQuery.delete(TableId.of(DATASET, getTableName(testInfo)));
}
protected String getTableName(TestInfo testInfo) {
return String.format(
TABLE_PATTERN,
testInfo.getTags().stream().findFirst().orElseThrow(),
testInfo.getTestMethod().map(Method::getName).orElseThrow()
);
}
protected WriteChannelConfiguration generateConfiguration(TestInfo testInfo, FormatOptions formatOptions) {
return WriteChannelConfiguration
.newBuilder(TableId.of(DATASET, getTableName(testInfo)))
.setSchema(PersonDto.getBigQuerySchema())
.setAutodetect(false)
.setFormatOptions(formatOptions)
.build();
}
protected void waitForJobToFinish(JobId jobId) {
JobStatus status = bigQuery.getJob(jobId).getStatus();
while (BooleanUtils.isFalse(JobStatus.State.DONE.equals(status.getState()))) {
status = bigQuery.getJob(jobId).getStatus();
}
}
@JsonPropertyOrder(value = {"name", "age"})
public record PersonDto(String name, Integer age) {
public static Schema getBigQuerySchema() {
Field nameField = Field.newBuilder("name", StandardSQLTypeName.STRING).build();
Field ageField = Field.newBuilder("age", StandardSQLTypeName.INT64).build();
return Schema.of(nameField, ageField);
}
}
}

24
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java Normal file
View File

@@ -0,0 +1,24 @@
package org.springframework.batch.extensions.bigquery.unit.base;
import com.google.cloud.bigquery.BigQuery;
import org.mockito.Mockito;
public abstract class AbstractBigQueryTest {
protected BigQuery prepareMockedBigQuery() {
BigQuery mockedBigQuery = Mockito.mock(BigQuery.class);
Mockito
.when(mockedBigQuery.getTable(Mockito.any()))
.thenReturn(null);
Mockito
.when(mockedBigQuery.getDataset(Mockito.anyString()))
.thenReturn(null);
return mockedBigQuery;
}
public record PersonDto(String name) {}
}

30
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryCsvItemWriterBuilderTests.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.writer.builder;
package org.springframework.batch.extensions.bigquery.unit.writer.builder;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.dataformat.csv.CsvMapper;
@@ -27,11 +27,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder;
class BigQueryCsvItemWriterBuilderTests {
class BigQueryCsvItemWriterBuilderTests extends AbstractBigQueryTest {
private static final String DATASET_NAME = "my_dataset";
@@ -94,28 +94,4 @@ class BigQueryCsvItemWriterBuilderTests {
}
}
private BigQuery prepareMockedBigQuery() {
BigQuery mockedBigQuery = Mockito.mock(BigQuery.class);
Mockito
.when(mockedBigQuery.getTable(Mockito.any()))
.thenReturn(null);
Mockito
.when(mockedBigQuery.getDataset(Mockito.anyString()))
.thenReturn(null);
return mockedBigQuery;
}
static class PersonDto {
private final String name;
public PersonDto(String name) {
this.name = name;
}
}
}

30
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryJsonItemWriterBuilderTests.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.writer.builder;
package org.springframework.batch.extensions.bigquery.unit.writer.builder;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -29,11 +29,11 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
class BigQueryJsonItemWriterBuilderTests {
class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
private static final String DATASET_NAME = "my_dataset";
@@ -95,28 +95,4 @@ class BigQueryJsonItemWriterBuilderTests {
}
}
private BigQuery prepareMockedBigQuery() {
BigQuery mockedBigQuery = Mockito.mock(BigQuery.class);
Mockito
.when(mockedBigQuery.getTable(Mockito.any()))
.thenReturn(null);
Mockito
.when(mockedBigQuery.getDataset(Mockito.anyString()))
.thenReturn(null);
return mockedBigQuery;
}
static class PersonDto {
private final String name;
public PersonDto(String name) {
this.name = name;
}
}
}

17
spring-batch-bigquery/src/test/resources/logback.xml Normal file
View File

@@ -0,0 +1,17 @@
<configuration>
<statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
<!-- Send debug messages to System.out -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- By default, encoders are assigned the type ch.qos.logback.classic.encoder.PatternLayoutEncoder -->
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %yellow(%-5level) %magenta(%logger{5}) : %msg%n</pattern>
</encoder>
</appender>
<root level="DEBUG">
<appender-ref ref="STDOUT"/>
</root>
</configuration>