Spring/spring-batch-extensions
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bq] test BQ with Docker emulator

Browse Source
This commit is contained in:
Volodymyr
2024-08-18 21:13:09 +03:00
committed by GitHub
parent 6f3277a2a3
commit 42093bc813
21 changed files with 339 additions and 472 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
spring-batch-bigquery/pom.xml
View File

@@ -112,7 +112,12 @@
<version>2.0.16</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>junit-jupiter</artifactId>
<version>1.20.1</version>
<scope>test</scope>
</dependency>
</dependencies>
@@ -136,8 +141,9 @@
<version>3.3.1</version>
<configuration>
<includes>
<!-- Integration tests are omitted because they are designed to be run locally -->
<include>/unit</include>
<!-- Google cloud tests are omitted because they are designed to be run locally -->
<include>**/unit/**</include>
<include>**/emulator/**</include>
</includes>
</configuration>
</plugin>

23
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java
View File

@@ -22,9 +22,7 @@ import com.google.cloud.bigquery.Job;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.WriteChannelConfiguration;
import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
import org.springframework.batch.item.Chunk;
import java.util.Comparator;
@@ -69,25 +67,4 @@ public class BigQueryDataLoader {
job.get().waitFor();
}
public void loadJsonSample(String tableName) throws Exception {
AtomicReference<Job> job = new AtomicReference<>();
WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration
.newBuilder(TableId.of(TestConstants.DATASET, tableName))
.setSchema(PersonDto.getBigQuerySchema())
.setAutodetect(false)
.setFormatOptions(FormatOptions.json())
.build();
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
.bigQuery(bigQuery)
.writeChannelConfig(channelConfiguration)
.jobConsumer(job::set)
.build();
writer.afterPropertiesSet();
writer.write(CHUNK);
job.get().waitFor();
}
}

2
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/TestConstants.java
View File

@@ -26,6 +26,8 @@ public final class TestConstants {
public static final String DATASET = "spring_batch_extensions";
public static final String NAME = "name";
public static final String AGE = "age";
public static final String CSV = "csv";
public static final String JSON = "json";
public static final Converter<FieldValueList, PersonDto> PERSON_MAPPER = res -> new PersonDto(
res.get(NAME).getStringValue(), Long.valueOf(res.get(AGE).getLongValue()).intValue()

36
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/reader/BaseEmulatorItemReaderTest.java Normal file
View File

@@ -0,0 +1,36 @@
package org.springframework.batch.extensions.bigquery.emulator.reader;
import com.google.cloud.NoCredentials;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.MountableFile;
@Testcontainers
abstract class BaseEmulatorItemReaderTest {
private static final int PORT = 9050;
private static final String PROJECT = "batch-test";
@Container
private static final GenericContainer<?> CONTAINER = new GenericContainer<>("ghcr.io/goccy/bigquery-emulator")
.withExposedPorts(PORT)
.withCommand("--project=" + PROJECT, "--data-from-yaml=/test-data.yaml")
.withCopyFileToContainer(MountableFile.forClasspathResource("test-data.yaml"), "/test-data.yaml");
protected static BigQuery bigQuery;
@BeforeAll
static void init() {
bigQuery = BigQueryOptions
.newBuilder()
.setHost("http://%s:%d".formatted(CONTAINER.getHost(), CONTAINER.getMappedPort(PORT)))
.setProjectId(PROJECT)
.setCredentials(NoCredentials.getInstance())
.build()
.getService();
}
}

59
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/reader/BigQueryEmulatorItemReaderTest.java Normal file
View File

@@ -0,0 +1,59 @@
package org.springframework.batch.extensions.bigquery.emulator.reader;
import com.google.cloud.bigquery.*;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
class BigQueryEmulatorItemReaderTest extends BaseEmulatorItemReaderTest {
@Test
void testBatchReader() throws Exception {
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.csv p ORDER BY p.name LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, TestConstants.CSV))
.setPriority(QueryJobConfiguration.Priority.BATCH)
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.rowMapper(TestConstants.PERSON_MAPPER)
.jobConfiguration(jobConfiguration)
.build();
reader.afterPropertiesSet();
verifyResult(reader);
}
@Test
void testInteractiveReader() throws Exception {
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.csv p ORDER BY p.name LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, TestConstants.CSV))
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.rowMapper(TestConstants.PERSON_MAPPER)
.jobConfiguration(jobConfiguration)
.build();
reader.afterPropertiesSet();
verifyResult(reader);
}
private void verifyResult(BigQueryQueryItemReader<PersonDto> reader) throws Exception {
PersonDto actual1 = reader.read();
Assertions.assertEquals("Volodymyr", actual1.name());
Assertions.assertEquals(27, actual1.age());
PersonDto actual2 = reader.read();
Assertions.assertEquals("Oleksandra", actual2.name());
Assertions.assertEquals(26, actual2.age());
}
}

5
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/emulator/writer/JsonWriterTest.java Normal file
View File

@@ -0,0 +1,5 @@
package org.springframework.batch.extensions.bigquery.emulator.writer;
// TODO
public class JsonWriterTest {
}

20
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/base/BaseBigQueryIntegrationTest.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/base/BaseBigQueryGcloudIntegrationTest.java
View File

@@ -14,25 +14,11 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.base;
package org.springframework.batch.extensions.bigquery.gcloud.base;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import org.junit.jupiter.api.TestInfo;
import java.lang.reflect.Method;
public abstract class BaseBigQueryIntegrationTest {
private static final String TABLE_PATTERN = "%s_%s";
public final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
protected String getTableName(TestInfo testInfo) {
return String.format(
TABLE_PATTERN,
testInfo.getTags().iterator().next(),
testInfo.getTestMethod().map(Method::getName).orElseThrow()
);
}
public abstract class BaseBigQueryGcloudIntegrationTest {
protected static final BigQuery BIG_QUERY = BigQueryOptions.getDefaultInstance().getService();
}

2
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/package-info.java
View File

@@ -25,4 +25,4 @@
*
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
*/
package org.springframework.batch.extensions.bigquery.integration;
package org.springframework.batch.extensions.bigquery.gcloud;

68
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/reader/BigQueryGcloudItemReaderTest.java
View File

@@ -14,50 +14,76 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
package org.springframework.batch.extensions.bigquery.gcloud.reader;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import com.google.cloud.bigquery.*;
import org.junit.jupiter.api.*;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.gcloud.base.BaseBigQueryGcloudIntegrationTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("csv")
class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
class BigQueryGcloudItemReaderTest extends BaseBigQueryGcloudIntegrationTest {
@BeforeAll
static void init() throws Exception {
if (BIG_QUERY.getDataset(TestConstants.DATASET) == null) {
BIG_QUERY.create(DatasetInfo.of(TestConstants.DATASET));
}
if (BIG_QUERY.getTable(TestConstants.DATASET, TestConstants.CSV) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
BIG_QUERY.create(TableInfo.of(TableId.of(TestConstants.DATASET, TestConstants.CSV), tableDefinition));
}
new BigQueryDataLoader(BIG_QUERY).loadCsvSample(TestConstants.CSV);
}
@AfterAll
static void cleanupTest() {
BIG_QUERY.delete(TableId.of(TestConstants.DATASET, TestConstants.CSV));
}
@Test
void batchQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
void testBatchQuery() throws Exception {
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2".formatted(TestConstants.CSV))
.setDestinationTable(TableId.of(TestConstants.DATASET, TestConstants.CSV))
.setPriority(QueryJobConfiguration.Priority.BATCH)
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.bigQuery(BIG_QUERY)
.rowMapper(TestConstants.PERSON_MAPPER)
.jobConfiguration(jobConfiguration)
.build();
reader.afterPropertiesSet();
verifyResult(reader);
}
@Test
void testInteractiveQuery() throws Exception {
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(BIG_QUERY)
.query("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2".formatted(TestConstants.CSV))
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
verifyResult(reader);
}
private void verifyResult(BigQueryQueryItemReader<PersonDto> reader) throws Exception {
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto expectedFirstPerson = BigQueryDataLoader.CHUNK.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto expectedSecondPerson = BigQueryDataLoader.CHUNK.getItems().get(1);
PersonDto actualThirdPerson = reader.read();

36
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/writer/BaseBigQueryGcloudItemWriterTest.java
View File

@@ -14,49 +14,35 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.writer;
package org.springframework.batch.extensions.bigquery.gcloud.writer;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableResult;
import com.google.cloud.bigquery.*;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest;
import org.springframework.batch.item.Chunk;
import org.springframework.batch.extensions.bigquery.gcloud.base.BaseBigQueryGcloudIntegrationTest;
@Tag("csv")
class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
abstract class BaseBigQueryGcloudItemWriterTest extends BaseBigQueryGcloudIntegrationTest {
@Test
void test1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
Dataset dataset = bigQuery.getDataset(TestConstants.DATASET);
Table table = bigQuery.getTable(TableId.of(TestConstants.DATASET, tableName));
protected void verifyResults(String tableName) {
Dataset dataset = BIG_QUERY.getDataset(TestConstants.DATASET);
Table table = BIG_QUERY.getTable(TableId.of(TestConstants.DATASET, tableName));
TableId tableId = table.getTableId();
TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
TableResult tableResult = BIG_QUERY.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
Assertions.assertNotNull(dataset.getDatasetId());
Assertions.assertNotNull(tableId);
Assertions.assertEquals(chunk.size(), tableResult.getTotalRows());
Assertions.assertEquals(BigQueryDataLoader.CHUNK.size(), tableResult.getTotalRows());
tableResult
.getValues()
.forEach(field -> {
Assertions.assertTrue(
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
BigQueryDataLoader.CHUNK.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
);
boolean ageCondition = chunk
boolean ageCondition = BigQueryDataLoader.CHUNK
.getItems()
.stream()
.map(PersonDto::age)

72
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/writer/BigQueryGcloudCsvItemWriterTest.java Normal file
View File

@@ -0,0 +1,72 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.gcloud.writer;
import com.google.cloud.bigquery.*;
import org.junit.jupiter.api.*;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.writer.BigQueryCsvItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvItemWriterBuilder;
import java.util.concurrent.atomic.AtomicReference;
class BigQueryGcloudCsvItemWriterTest extends BaseBigQueryGcloudItemWriterTest {
@BeforeAll
static void prepareTest() {
if (BIG_QUERY.getDataset(TestConstants.DATASET) == null) {
BIG_QUERY.create(DatasetInfo.of(TestConstants.DATASET));
}
if (BIG_QUERY.getTable(TestConstants.DATASET, TestConstants.CSV) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
BIG_QUERY.create(TableInfo.of(TableId.of(TestConstants.DATASET, TestConstants.CSV), tableDefinition));
}
}
@AfterAll
static void cleanup() {
BIG_QUERY.delete(TableId.of(TestConstants.DATASET, TestConstants.CSV));
}
@Test
void testWriteCsv() throws Exception {
AtomicReference<Job> job = new AtomicReference<>();
WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration
.newBuilder(TableId.of(TestConstants.DATASET, TestConstants.CSV))
.setSchema(PersonDto.getBigQuerySchema())
.setAutodetect(false)
.setFormatOptions(FormatOptions.csv())
.build();
BigQueryCsvItemWriter<PersonDto> writer = new BigQueryCsvItemWriterBuilder<PersonDto>()
.bigQuery(BIG_QUERY)
.writeChannelConfig(channelConfiguration)
.jobConsumer(job::set)
.build();
writer.afterPropertiesSet();
writer.write(BigQueryDataLoader.CHUNK);
job.get().waitFor();
verifyResults(TestConstants.CSV);
}
}

74
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/gcloud/writer/BigQueryGcloudJsonItemWriterTest.java Normal file
View File

@@ -0,0 +1,74 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.gcloud.writer;
import com.google.cloud.bigquery.*;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.writer.BigQueryJsonItemWriter;
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
import java.util.concurrent.atomic.AtomicReference;
class BigQueryGcloudJsonItemWriterTest extends BaseBigQueryGcloudItemWriterTest {
@BeforeAll
static void prepareTest() {
if (BIG_QUERY.getDataset(TestConstants.DATASET) == null) {
BIG_QUERY.create(DatasetInfo.of(TestConstants.DATASET));
}
if (BIG_QUERY.getTable(TestConstants.DATASET, TestConstants.JSON) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
BIG_QUERY.create(TableInfo.of(TableId.of(TestConstants.DATASET, TestConstants.JSON), tableDefinition));
}
}
@AfterAll
static void cleanup() {
BIG_QUERY.delete(TableId.of(TestConstants.DATASET, TestConstants.JSON));
}
@Test
void testWriteJson() throws Exception {
AtomicReference<Job> job = new AtomicReference<>();
WriteChannelConfiguration channelConfiguration = WriteChannelConfiguration
.newBuilder(TableId.of(TestConstants.DATASET, TestConstants.JSON))
.setSchema(PersonDto.getBigQuerySchema())
.setAutodetect(false)
.setFormatOptions(FormatOptions.json())
.build();
BigQueryJsonItemWriter<PersonDto> writer = new BigQueryJsonItemWriterBuilder<PersonDto>()
.bigQuery(BIG_QUERY)
.writeChannelConfig(channelConfiguration)
.jobConsumer(job::set)
.build();
writer.afterPropertiesSet();
writer.write(BigQueryDataLoader.CHUNK);
job.get().waitFor();
verifyResults(TestConstants.JSON);
}
}

52
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/base/BaseCsvJsonInteractiveQueryItemReaderTest.java
View File

@@ -1,52 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.base;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.StandardTableDefinition;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
public abstract class BaseCsvJsonInteractiveQueryItemReaderTest extends BaseBigQueryIntegrationTest {
@BeforeEach
void prepareTest(TestInfo testInfo) {
if (bigQuery.getDataset(TestConstants.DATASET) == null) {
bigQuery.create(DatasetInfo.of(TestConstants.DATASET));
}
String tableName = getTableName(testInfo);
if (bigQuery.getTable(TestConstants.DATASET, tableName) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, tableName), tableDefinition));
}
}
@AfterEach
void cleanupTest(TestInfo testInfo) {
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
}
}

75
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryJsonItemReaderTest.java
View File

@@ -1,75 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("json")
class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void batchQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
.setPriority(QueryJobConfiguration.Priority.BATCH)
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.rowMapper(TestConstants.PERSON_MAPPER)
.jobConfiguration(jobConfiguration)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}

67
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryCsvItemReaderTest.java
View File

@@ -1,67 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("csv")
class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}

67
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryJsonItemReaderTest.java
View File

@@ -1,67 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("json")
class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}

70
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java
View File

@@ -1,70 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.writer;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableResult;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.writer.base.BaseBigQueryItemWriterTest;
import org.springframework.batch.item.Chunk;
@Tag("json")
class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest {
@Test
void test1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
Dataset dataset = bigQuery.getDataset(TestConstants.DATASET);
Table table = bigQuery.getTable(TableId.of(TestConstants.DATASET, tableName));
TableId tableId = table.getTableId();
TableResult tableResult = bigQuery.listTableData(tableId, BigQuery.TableDataListOption.pageSize(2L));
Assertions.assertNotNull(dataset.getDatasetId());
Assertions.assertNotNull(tableId);
Assertions.assertEquals(chunk.size(), tableResult.getTotalRows());
tableResult
.getValues()
.forEach(field -> {
Assertions.assertTrue(
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
);
boolean ageCondition = chunk
.getItems()
.stream()
.map(PersonDto::age)
.map(Long::valueOf)
.anyMatch(age -> age.compareTo(field.get(1).getLongValue()) == 0);
Assertions.assertTrue(ageCondition);
});
}
}

50
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java
View File

@@ -1,50 +0,0 @@
/*
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.writer.base;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.StandardTableDefinition;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
public abstract class BaseBigQueryItemWriterTest extends BaseBigQueryIntegrationTest {
@BeforeEach
void prepareTest(TestInfo testInfo) {
if (bigQuery.getDataset(TestConstants.DATASET) == null) {
bigQuery.create(DatasetInfo.of(TestConstants.DATASET));
}
if (bigQuery.getTable(TestConstants.DATASET, getTableName(testInfo)) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, getTableName(testInfo)), tableDefinition));
}
}
@AfterEach
void cleanupTest(TestInfo testInfo) {
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
}
}

1
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java
View File

@@ -27,6 +27,7 @@ import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemRea
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
// TODO verify all corner cases of afterPropertiesSet
class BigQueryBatchQueryItemReaderBuilderTests extends AbstractBigQueryTest {
@Test

2
spring-batch-bigquery/src/test/resources/logback.xml
View File

@@ -12,7 +12,7 @@
</encoder>
</appender>
<root level="DEBUG">
<root level="INFO">
<appender-ref ref="STDOUT"/>
</root>

18
spring-batch-bigquery/src/test/resources/test-data.yaml Normal file
View File

@@ -0,0 +1,18 @@
projects:
- id: batch-test
datasets:
- id: spring_batch_extensions
tables:
- id: csv
columns:
- name: name
type: STRING
mode: REQUIRED
- name: age
type: INTEGER
mode: REQUIRED
data:
- name: Volodymyr
age: 27
- name: Oleksandra
age: 26