diff --git a/spring-batch-bigquery/pom.xml b/spring-batch-bigquery/pom.xml index 64b31d3..93ae349 100644 --- a/spring-batch-bigquery/pom.xml +++ b/spring-batch-bigquery/pom.xml @@ -51,6 +51,7 @@ 17 + 1.4.5 @@ -96,6 +97,25 @@ 4.9.0 test + + ch.qos.logback + logback-core + ${logback.version} + test + + + ch.qos.logback + logback-classic + ${logback.version} + test + + + org.slf4j + slf4j-api + 2.0.6 + test + + @@ -117,6 +137,12 @@ org.apache.maven.plugins maven-surefire-plugin 2.22.2 + + + + /unit + + diff --git a/spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/package-info.java b/spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/package-info.java index c887e4e..e085743 100644 --- a/spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/package-info.java +++ b/spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/writer/package-info.java @@ -32,6 +32,9 @@ * Take into account that BigQuery has rate limits, and it is very easy to exceed those in concurrent environment. * @see BigQuery Quotas & Limits * + * Also worth mentioning that you should ensure ordering of the fields in DTO that you are going to send to the BigQuery. + * In case of CSV/JSON and Jackson consider using {@link com.fasterxml.jackson.annotation.JsonPropertyOrder}. + * * @author Volodymyr Perebykivskyi * @since 0.1.0 * @see Google BigQuery diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java new file mode 100644 index 0000000..63d60f4 --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java @@ -0,0 +1,11 @@ +/** + * In order to launch these tests you should provide a way how to authorize to Google BigQuery. + * A simple way is to create service account, store credentials as JSON file and provide environment variable. + * Example: GOOGLE_APPLICATION_CREDENTIALS=/home/dgray/Downloads/bq-key.json + * @see Authentication + * + * Test names should follow this pattern: test1, test2, testN. + * So later in BigQuery you will see generated table name: csv_test1, csv_test2, csv_testN. + * This way it will be easier to trace errors in BigQuery. + */ +package org.springframework.batch.extensions.bigquery.integration; \ No newline at end of file diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java new file mode 100644 index 0000000..9535d71 --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java @@ -0,0 +1,85 @@ +/* + * Copyright 2002-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.bigquery.integration.writer; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; +import org.apache.commons.lang3.math.NumberUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest; +import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter; +import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder; +import org.springframework.batch.item.Chunk; + +import java.util.concurrent.atomic.AtomicReference; + +@Tag("csv") +public class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest { + + @Test + void test1(TestInfo testInfo) throws Exception { + AtomicReference jobId = new AtomicReference<>(); + + BigQueryCsvItemWriter writer = new BigQueryCsvItemWriterBuilder() + .bigQuery(bigQuery) + .writeChannelConfig(generateConfiguration(testInfo, FormatOptions.csv())) + .jobConsumer(j -> jobId.set(j.getJobId())) + .build(); + + writer.afterPropertiesSet(); + + Chunk chunk = Chunk.of(new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26)); + writer.write(chunk); + + waitForJobToFinish(jobId.get()); + + Dataset dataset = bigQuery.getDataset(DATASET); + Table table = bigQuery.getTable(TableId.of(DATASET, getTableName(testInfo))); + TableId tableId = table.getTableId(); + TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L)); + + Assertions.assertNotNull(dataset.getDatasetId()); + Assertions.assertNotNull(tableId); + Assertions.assertEquals(chunk.size(), tableResult.getTotalRows()); + + tableResult + .getValues() + .forEach(field -> { + Assertions.assertTrue( + chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(NumberUtils.INTEGER_ZERO).getStringValue().equals(name)) + ); + + boolean ageCondition = chunk + .getItems() + .stream() + .map(PersonDto::age) + .map(Long::valueOf) + .anyMatch(age -> age.compareTo(field.get(NumberUtils.INTEGER_ONE).getLongValue()) == NumberUtils.INTEGER_ZERO); + + Assertions.assertTrue(ageCondition); + }); + } + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java new file mode 100644 index 0000000..5a1f2fd --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java @@ -0,0 +1,85 @@ +/* + * Copyright 2002-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.bigquery.integration.writer; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.Dataset; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.Table; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableResult; +import org.apache.commons.lang3.math.NumberUtils; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInfo; +import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest; +import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter; +import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder; +import org.springframework.batch.item.Chunk; + +import java.util.concurrent.atomic.AtomicReference; + +@Tag("json") +public class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest { + + @Test + void test1(TestInfo testInfo) throws Exception { + AtomicReference jobId = new AtomicReference<>(); + + BigQueryJsonItemWriter writer = new BigQueryJsonItemWriterBuilder() + .bigQuery(bigQuery) + .writeChannelConfig(generateConfiguration(testInfo, FormatOptions.json())) + .jobConsumer(j -> jobId.set(j.getJobId())) + .build(); + + writer.afterPropertiesSet(); + + Chunk chunk = Chunk.of(new PersonDto("Viktor", 57), new PersonDto("Nina", 57)); + writer.write(chunk); + + waitForJobToFinish(jobId.get()); + + Dataset dataset = bigQuery.getDataset(DATASET); + Table table = bigQuery.getTable(TableId.of(DATASET, getTableName(testInfo))); + TableId tableId = table.getTableId(); + TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L)); + + Assertions.assertNotNull(dataset.getDatasetId()); + Assertions.assertNotNull(tableId); + Assertions.assertEquals(chunk.size(), tableResult.getTotalRows()); + + tableResult + .getValues() + .forEach(field -> { + Assertions.assertTrue( + chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(NumberUtils.INTEGER_ZERO).getStringValue().equals(name)) + ); + + boolean ageCondition = chunk + .getItems() + .stream() + .map(PersonDto::age) + .map(Long::valueOf) + .anyMatch(age -> age.compareTo(field.get(NumberUtils.INTEGER_ONE).getLongValue()) == NumberUtils.INTEGER_ZERO); + + Assertions.assertTrue(ageCondition); + }); + } + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java new file mode 100644 index 0000000..816e619 --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java @@ -0,0 +1,103 @@ +/* + * Copyright 2002-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.bigquery.integration.writer.base; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.BigQueryOptions; +import com.google.cloud.bigquery.DatasetInfo; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.JobId; +import com.google.cloud.bigquery.JobStatus; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; +import com.google.cloud.bigquery.StandardTableDefinition; +import com.google.cloud.bigquery.TableDefinition; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.TableInfo; +import com.google.cloud.bigquery.WriteChannelConfiguration; +import org.apache.commons.lang3.BooleanUtils; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.TestInfo; + +import java.lang.reflect.Method; +import java.util.Objects; + +public abstract class BaseBigQueryItemWriterTest { + + protected static final String DATASET = "spring_extensions"; + + protected final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService(); + + private static final String TABLE_PATTERN = "%s_%s"; + + @BeforeEach + void prepareTest(TestInfo testInfo) { + if (Objects.isNull(bigQuery.getDataset(DATASET))) { + bigQuery.create(DatasetInfo.of(DATASET)); + } + + if (Objects.isNull(bigQuery.getTable(DATASET, getTableName(testInfo)))) { + TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema()); + bigQuery.create(TableInfo.of(TableId.of(DATASET, getTableName(testInfo)), tableDefinition)); + } + } + + @AfterEach + void cleanupTest(TestInfo testInfo) { + bigQuery.delete(TableId.of(DATASET, getTableName(testInfo))); + } + + protected String getTableName(TestInfo testInfo) { + return String.format( + TABLE_PATTERN, + testInfo.getTags().stream().findFirst().orElseThrow(), + testInfo.getTestMethod().map(Method::getName).orElseThrow() + ); + } + + protected WriteChannelConfiguration generateConfiguration(TestInfo testInfo, FormatOptions formatOptions) { + return WriteChannelConfiguration + .newBuilder(TableId.of(DATASET, getTableName(testInfo))) + .setSchema(PersonDto.getBigQuerySchema()) + .setAutodetect(false) + .setFormatOptions(formatOptions) + .build(); + } + + protected void waitForJobToFinish(JobId jobId) { + JobStatus status = bigQuery.getJob(jobId).getStatus(); + + while (BooleanUtils.isFalse(JobStatus.State.DONE.equals(status.getState()))) { + status = bigQuery.getJob(jobId).getStatus(); + } + } + + @JsonPropertyOrder(value = {"name", "age"}) + public record PersonDto(String name, Integer age) { + + public static Schema getBigQuerySchema() { + Field nameField = Field.newBuilder("name", StandardSQLTypeName.STRING).build(); + Field ageField = Field.newBuilder("age", StandardSQLTypeName.INT64).build(); + return Schema.of(nameField, ageField); + } + + } + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java new file mode 100644 index 0000000..41ebe16 --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java @@ -0,0 +1,24 @@ +package org.springframework.batch.extensions.bigquery.unit.base; + +import com.google.cloud.bigquery.BigQuery; +import org.mockito.Mockito; + +public abstract class AbstractBigQueryTest { + + protected BigQuery prepareMockedBigQuery() { + BigQuery mockedBigQuery = Mockito.mock(BigQuery.class); + + Mockito + .when(mockedBigQuery.getTable(Mockito.any())) + .thenReturn(null); + + Mockito + .when(mockedBigQuery.getDataset(Mockito.anyString())) + .thenReturn(null); + + return mockedBigQuery; + } + + public record PersonDto(String name) {} + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryCsvItemWriterBuilderTests.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java similarity index 84% rename from spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryCsvItemWriterBuilderTests.java rename to spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java index 534e5c4..8df9def 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryCsvItemWriterBuilderTests.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.batch.extensions.bigquery.writer.builder; +package org.springframework.batch.extensions.bigquery.unit.writer.builder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.dataformat.csv.CsvMapper; @@ -27,11 +27,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest; import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter; import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder; -class BigQueryCsvItemWriterBuilderTests { +class BigQueryCsvItemWriterBuilderTests extends AbstractBigQueryTest { private static final String DATASET_NAME = "my_dataset"; @@ -94,28 +94,4 @@ class BigQueryCsvItemWriterBuilderTests { } } - private BigQuery prepareMockedBigQuery() { - BigQuery mockedBigQuery = Mockito.mock(BigQuery.class); - - Mockito - .when(mockedBigQuery.getTable(Mockito.any())) - .thenReturn(null); - - Mockito - .when(mockedBigQuery.getDataset(Mockito.anyString())) - .thenReturn(null); - - return mockedBigQuery; - } - - - static class PersonDto { - - private final String name; - - public PersonDto(String name) { - this.name = name; - } - } - } diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryJsonItemWriterBuilderTests.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java similarity index 84% rename from spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryJsonItemWriterBuilderTests.java rename to spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java index a0fbbc4..412aeae 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/writer/builder/BigQueryJsonItemWriterBuilderTests.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.springframework.batch.extensions.bigquery.writer.builder; +package org.springframework.batch.extensions.bigquery.unit.writer.builder; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; @@ -29,11 +29,11 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.Mockito; +import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest; import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter; import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder; -class BigQueryJsonItemWriterBuilderTests { +class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest { private static final String DATASET_NAME = "my_dataset"; @@ -95,28 +95,4 @@ class BigQueryJsonItemWriterBuilderTests { } } - private BigQuery prepareMockedBigQuery() { - BigQuery mockedBigQuery = Mockito.mock(BigQuery.class); - - Mockito - .when(mockedBigQuery.getTable(Mockito.any())) - .thenReturn(null); - - Mockito - .when(mockedBigQuery.getDataset(Mockito.anyString())) - .thenReturn(null); - - return mockedBigQuery; - } - - - static class PersonDto { - - private final String name; - - public PersonDto(String name) { - this.name = name; - } - } - } diff --git a/spring-batch-bigquery/src/test/resources/logback.xml b/spring-batch-bigquery/src/test/resources/logback.xml new file mode 100644 index 0000000..08466f1 --- /dev/null +++ b/spring-batch-bigquery/src/test/resources/logback.xml @@ -0,0 +1,17 @@ + + + + + + + + + %d{HH:mm:ss.SSS} [%thread] %yellow(%-5level) %magenta(%logger{5}) : %msg%n + + + + + + + + \ No newline at end of file