diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java new file mode 100644 index 0000000..08251d5 --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java @@ -0,0 +1,91 @@ +/* + * Copyright 2002-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.bigquery.common; + +import com.google.cloud.bigquery.BigQuery; +import com.google.cloud.bigquery.FormatOptions; +import com.google.cloud.bigquery.Job; +import com.google.cloud.bigquery.TableId; +import com.google.cloud.bigquery.WriteChannelConfiguration; +import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter; +import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter; +import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder; +import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder; +import org.springframework.batch.item.Chunk; + +import java.util.concurrent.atomic.AtomicReference; + +public class BigQueryDataLoader { + + public static final Chunk CHUNK = Chunk.of( + new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26) + ); + + private final BigQuery bigQuery; + + public BigQueryDataLoader(BigQuery bigQuery) { + this.bigQuery = bigQuery; + } + + + public void loadCsvSample() throws Exception { + loadCsvSample(TestConstants.PERSONS_TABLE); + } + + public void loadCsvSample(String tableName) throws Exception { + AtomicReference job = new AtomicReference<>(); + + WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration + .newBuilder(TableId.of(TestConstants.DATASET, tableName)) + .setSchema(PersonDto.getBigQuerySchema()) + .setAutodetect(false) + .setFormatOptions(FormatOptions.csv()) + .build(); + + BigQueryCsvItemWriter writer = new BigQueryCsvItemWriterBuilder() + .bigQuery(bigQuery) + .writeChannelConfig(channelConfiguration) + .jobConsumer(job::set) + .build(); + + writer.afterPropertiesSet(); + writer.write(CHUNK); + job.get().waitFor(); + } + + public void loadJsonSample(String tableName) throws Exception { + AtomicReference job = new AtomicReference<>(); + + WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration + .newBuilder(TableId.of(TestConstants.DATASET, tableName)) + .setSchema(PersonDto.getBigQuerySchema()) + .setAutodetect(false) + .setFormatOptions(FormatOptions.json()) + .build(); + + BigQueryJsonItemWriter writer = new BigQueryJsonItemWriterBuilder() + .bigQuery(bigQuery) + .writeChannelConfig(channelConfiguration) + .jobConsumer(job::set) + .build(); + + writer.afterPropertiesSet(); + writer.write(CHUNK); + job.get().waitFor(); + } + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/PersonDto.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/PersonDto.java new file mode 100644 index 0000000..e4eabda --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/PersonDto.java @@ -0,0 +1,33 @@ +/* + * Copyright 2002-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.bigquery.common; + +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.google.cloud.bigquery.Field; +import com.google.cloud.bigquery.Schema; +import com.google.cloud.bigquery.StandardSQLTypeName; + +@JsonPropertyOrder(value = {"name", "age"}) +public record PersonDto(String name, Integer age) { + + public static Schema getBigQuerySchema() { + Field nameField = Field.newBuilder("name", StandardSQLTypeName.STRING).build(); + Field ageField = Field.newBuilder("age", StandardSQLTypeName.INT64).build(); + return Schema.of(nameField, ageField); + } + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/TestConstants.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/TestConstants.java new file mode 100644 index 0000000..1a9c001 --- /dev/null +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/TestConstants.java @@ -0,0 +1,26 @@ +/* + * Copyright 2002-2022 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.batch.extensions.bigquery.common; + +public class TestConstants { + + private TestConstants() {} + + public static final String DATASET = "spring_batch_extensions"; + public static final String PERSONS_TABLE = "persons"; + +} diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java index 9535d71..1b07f0d 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java @@ -18,8 +18,6 @@ package org.springframework.batch.extensions.bigquery.integration.writer; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.FormatOptions; -import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableResult; @@ -28,35 +26,23 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader; +import org.springframework.batch.extensions.bigquery.common.PersonDto; +import org.springframework.batch.extensions.bigquery.common.TestConstants; import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest; -import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter; -import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder; import org.springframework.batch.item.Chunk; -import java.util.concurrent.atomic.AtomicReference; - @Tag("csv") public class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest { @Test void test1(TestInfo testInfo) throws Exception { - AtomicReference jobId = new AtomicReference<>(); + String tableName = getTableName(testInfo); + new BigQueryDataLoader(bigQuery).loadCsvSample(tableName); + Chunk chunk = BigQueryDataLoader.CHUNK; - BigQueryCsvItemWriter writer = new BigQueryCsvItemWriterBuilder() - .bigQuery(bigQuery) - .writeChannelConfig(generateConfiguration(testInfo, FormatOptions.csv())) - .jobConsumer(j -> jobId.set(j.getJobId())) - .build(); - - writer.afterPropertiesSet(); - - Chunk chunk = Chunk.of(new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26)); - writer.write(chunk); - - waitForJobToFinish(jobId.get()); - - Dataset dataset = bigQuery.getDataset(DATASET); - Table table = bigQuery.getTable(TableId.of(DATASET, getTableName(testInfo))); + Dataset dataset = bigQuery.getDataset(TestConstants.DATASET); + Table table = bigQuery.getTable(TableId.of(TestConstants.DATASET, tableName)); TableId tableId = table.getTableId(); TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L)); diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java index 5a1f2fd..e48f67d 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java @@ -18,8 +18,6 @@ package org.springframework.batch.extensions.bigquery.integration.writer; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.Dataset; -import com.google.cloud.bigquery.FormatOptions; -import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.Table; import com.google.cloud.bigquery.TableId; import com.google.cloud.bigquery.TableResult; @@ -28,35 +26,23 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInfo; +import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader; +import org.springframework.batch.extensions.bigquery.common.PersonDto; +import org.springframework.batch.extensions.bigquery.common.TestConstants; import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest; -import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter; -import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder; import org.springframework.batch.item.Chunk; -import java.util.concurrent.atomic.AtomicReference; - @Tag("json") public class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest { @Test void test1(TestInfo testInfo) throws Exception { - AtomicReference jobId = new AtomicReference<>(); + String tableName = getTableName(testInfo); + new BigQueryDataLoader(bigQuery).loadJsonSample(tableName); + Chunk chunk = BigQueryDataLoader.CHUNK; - BigQueryJsonItemWriter writer = new BigQueryJsonItemWriterBuilder() - .bigQuery(bigQuery) - .writeChannelConfig(generateConfiguration(testInfo, FormatOptions.json())) - .jobConsumer(j -> jobId.set(j.getJobId())) - .build(); - - writer.afterPropertiesSet(); - - Chunk chunk = Chunk.of(new PersonDto("Viktor", 57), new PersonDto("Nina", 57)); - writer.write(chunk); - - waitForJobToFinish(jobId.get()); - - Dataset dataset = bigQuery.getDataset(DATASET); - Table table = bigQuery.getTable(TableId.of(DATASET, getTableName(testInfo))); + Dataset dataset = bigQuery.getDataset(TestConstants.DATASET); + Table table = bigQuery.getTable(TableId.of(TestConstants.DATASET, tableName)); TableId tableId = table.getTableId(); TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L)); diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java index 816e619..d3b99fb 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java @@ -16,16 +16,13 @@ package org.springframework.batch.extensions.bigquery.integration.writer.base; -import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.google.cloud.RetryOption; import com.google.cloud.bigquery.BigQuery; import com.google.cloud.bigquery.BigQueryOptions; import com.google.cloud.bigquery.DatasetInfo; -import com.google.cloud.bigquery.Field; import com.google.cloud.bigquery.FormatOptions; import com.google.cloud.bigquery.JobId; import com.google.cloud.bigquery.JobStatus; -import com.google.cloud.bigquery.Schema; -import com.google.cloud.bigquery.StandardSQLTypeName; import com.google.cloud.bigquery.StandardTableDefinition; import com.google.cloud.bigquery.TableDefinition; import com.google.cloud.bigquery.TableId; @@ -35,33 +32,33 @@ import org.apache.commons.lang3.BooleanUtils; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.TestInfo; +import org.springframework.batch.extensions.bigquery.common.PersonDto; +import org.springframework.batch.extensions.bigquery.common.TestConstants; import java.lang.reflect.Method; import java.util.Objects; public abstract class BaseBigQueryItemWriterTest { - protected static final String DATASET = "spring_extensions"; + private static final String TABLE_PATTERN = "%s_%s"; protected final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService(); - private static final String TABLE_PATTERN = "%s_%s"; - @BeforeEach void prepareTest(TestInfo testInfo) { - if (Objects.isNull(bigQuery.getDataset(DATASET))) { - bigQuery.create(DatasetInfo.of(DATASET)); + if (Objects.isNull(bigQuery.getDataset(TestConstants.DATASET))) { + bigQuery.create(DatasetInfo.of(TestConstants.DATASET)); } - if (Objects.isNull(bigQuery.getTable(DATASET, getTableName(testInfo)))) { + if (Objects.isNull(bigQuery.getTable(TestConstants.DATASET, getTableName(testInfo)))) { TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema()); - bigQuery.create(TableInfo.of(TableId.of(DATASET, getTableName(testInfo)), tableDefinition)); + bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, getTableName(testInfo)), tableDefinition)); } } @AfterEach void cleanupTest(TestInfo testInfo) { - bigQuery.delete(TableId.of(DATASET, getTableName(testInfo))); + bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo))); } protected String getTableName(TestInfo testInfo) { @@ -74,13 +71,14 @@ public abstract class BaseBigQueryItemWriterTest { protected WriteChannelConfiguration generateConfiguration(TestInfo testInfo, FormatOptions formatOptions) { return WriteChannelConfiguration - .newBuilder(TableId.of(DATASET, getTableName(testInfo))) + .newBuilder(TableId.of(TestConstants.DATASET, getTableName(testInfo))) .setSchema(PersonDto.getBigQuerySchema()) .setAutodetect(false) .setFormatOptions(formatOptions) .build(); } + /** TODO check {@link com.google.cloud.bigquery.Job#waitFor(RetryOption...)} */ protected void waitForJobToFinish(JobId jobId) { JobStatus status = bigQuery.getJob(jobId).getStatus(); @@ -89,15 +87,4 @@ public abstract class BaseBigQueryItemWriterTest { } } - @JsonPropertyOrder(value = {"name", "age"}) - public record PersonDto(String name, Integer age) { - - public static Schema getBigQuerySchema() { - Field nameField = Field.newBuilder("name", StandardSQLTypeName.STRING).build(); - Field ageField = Field.newBuilder("age", StandardSQLTypeName.INT64).build(); - return Schema.of(nameField, ageField); - } - - } - } diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java index 41ebe16..effc743 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java @@ -19,6 +19,4 @@ public abstract class AbstractBigQueryTest { return mockedBigQuery; } - public record PersonDto(String name) {} - } diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java index 8df9def..3486a19 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java @@ -27,13 +27,15 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.springframework.batch.extensions.bigquery.common.PersonDto; +import org.springframework.batch.extensions.bigquery.common.TestConstants; import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest; import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter; import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder; class BigQueryCsvItemWriterBuilderTests extends AbstractBigQueryTest { - private static final String DATASET_NAME = "my_dataset"; + private static final String TABLE = "persons_csv"; private final Log logger = LogFactory.getLog(getClass()); @@ -44,10 +46,10 @@ class BigQueryCsvItemWriterBuilderTests extends AbstractBigQueryTest { void testCsvWriterWithRowMapper() { BigQuery mockedBigQuery = prepareMockedBigQuery(); CsvMapper csvMapper = new CsvMapper(); - DatasetInfo datasetInfo = DatasetInfo.newBuilder(DATASET_NAME).setLocation("europe-west-2").build(); + DatasetInfo datasetInfo = DatasetInfo.newBuilder(TestConstants.DATASET).setLocation("europe-west-2").build(); WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration - .newBuilder(TableId.of(datasetInfo.getDatasetId().getDataset(), "csv_table")) + .newBuilder(TableId.of(datasetInfo.getDatasetId().getDataset(), TABLE)) .setAutodetect(true) .setFormatOptions(FormatOptions.csv()) .build(); @@ -70,7 +72,7 @@ class BigQueryCsvItemWriterBuilderTests extends AbstractBigQueryTest { BigQuery mockedBigQuery = prepareMockedBigQuery(); WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration - .newBuilder(TableId.of(DATASET_NAME, "csv_table")) + .newBuilder(TableId.of(TestConstants.DATASET, TABLE)) .setAutodetect(true) .setFormatOptions(FormatOptions.csv()) .build(); diff --git a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java index 412aeae..3755693 100644 --- a/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java +++ b/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java @@ -29,14 +29,14 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.springframework.batch.extensions.bigquery.common.PersonDto; +import org.springframework.batch.extensions.bigquery.common.TestConstants; import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest; import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter; import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder; class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest { - private static final String DATASET_NAME = "my_dataset"; - private final Log logger = LogFactory.getLog(getClass()); /** @@ -48,7 +48,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest { ObjectMapper objectMapper = new ObjectMapper(); WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration - .newBuilder(TableId.of(DATASET_NAME, "json_table")) + .newBuilder(TableId.of(TestConstants.DATASET, "persons_json")) .setFormatOptions(FormatOptions.json()) .setSchema(Schema.of( Field.newBuilder("name", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build() @@ -72,7 +72,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest { BigQuery mockedBigQuery = prepareMockedBigQuery(); WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration - .newBuilder(TableId.of(DATASET_NAME, "json_table")) + .newBuilder(TableId.of(TestConstants.DATASET, "persons_json")) .setAutodetect(true) .setFormatOptions(FormatOptions.json()) .build();