[bq] 0.2 introduce BigQuery interactive reader

This commit is contained in:
Volodymyr
2023-01-17 16:19:56 +02:00
committed by GitHub
parent 038f973015
commit 9c02e11eac
24 changed files with 791 additions and 101 deletions

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -27,13 +27,20 @@ import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvI
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
import org.springframework.batch.item.Chunk;
import java.util.Comparator;
import java.util.List;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Stream;
public class BigQueryDataLoader {
public static final Chunk<PersonDto> CHUNK = Chunk.of(
new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26)
);
/** Order must be defined so later executed queries results could be predictable */
private static final List<PersonDto> PERSONS = Stream
.of(new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26))
.sorted(Comparator.comparing(PersonDto::name))
.toList();
public static final Chunk<PersonDto> CHUNK = new Chunk<>(PERSONS);
private final BigQuery bigQuery;
@@ -41,11 +48,6 @@ public class BigQueryDataLoader {
this.bigQuery = bigQuery;
}
public void loadCsvSample() throws Exception {
loadCsvSample(TestConstants.PERSONS_TABLE);
}
public void loadCsvSample(String tableName) throws Exception {
AtomicReference<Job> job = new AtomicReference<>();

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,11 +16,19 @@
package org.springframework.batch.extensions.bigquery.common;
public class TestConstants {
import com.google.cloud.bigquery.FieldValueList;
import org.springframework.core.convert.converter.Converter;
public final class TestConstants {
private TestConstants() {}
public static final String DATASET = "spring_batch_extensions";
public static final String PERSONS_TABLE = "persons";
public static final String NAME = "name";
public static final String AGE = "age";
public static final Converter<FieldValueList, PersonDto> PERSON_MAPPER = res -> new PersonDto(
res.get(NAME).getStringValue(), Long.valueOf(res.get(AGE).getLongValue()).intValue()
);
}

View File

@@ -0,0 +1,38 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.base;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import org.junit.jupiter.api.TestInfo;
import java.lang.reflect.Method;
public abstract class BaseBigQueryIntegrationTest {
private static final String TABLE_PATTERN = "%s_%s";
public final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
protected String getTableName(TestInfo testInfo) {
return String.format(
TABLE_PATTERN,
testInfo.getTags().stream().findFirst().orElseThrow(),
testInfo.getTestMethod().map(Method::getName).orElseThrow()
);
}
}

View File

@@ -1,11 +1,28 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* In order to launch these tests you should provide a way how to authorize to Google BigQuery.
* A simple way is to create service account, store credentials as JSON file and provide environment variable.
* Example: GOOGLE_APPLICATION_CREDENTIALS=/home/dgray/Downloads/bq-key.json
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
*
* <p>
* Test names should follow this pattern: test1, test2, testN.
* So later in BigQuery you will see generated table name: csv_test1, csv_test2, csv_testN.
* This way it will be easier to trace errors in BigQuery.
*
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
*/
package org.springframework.batch.extensions.bigquery.integration;

View File

@@ -0,0 +1,68 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("csv")
public class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNull(actualThirdPerson);
}
}

View File

@@ -0,0 +1,68 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("json")
public class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNull(actualThirdPerson);
}
}

View File

@@ -0,0 +1,54 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.base;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.StandardTableDefinition;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
import java.util.Objects;
public abstract class BaseCsvJsonInteractiveQueryItemReaderTest extends BaseBigQueryIntegrationTest {
@BeforeEach
void prepareTest(TestInfo testInfo) {
if (Objects.isNull(bigQuery.getDataset(TestConstants.DATASET))) {
bigQuery.create(DatasetInfo.of(TestConstants.DATASET));
}
String tableName = getTableName(testInfo);
if (Objects.isNull(bigQuery.getTable(TestConstants.DATASET, tableName))) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, tableName), tableDefinition));
}
}
@AfterEach
void cleanupTest(TestInfo testInfo) {
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,33 +16,21 @@
package org.springframework.batch.extensions.bigquery.integration.writer.base;
import com.google.cloud.RetryOption;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.BigQueryOptions;
import com.google.cloud.bigquery.DatasetInfo;
import com.google.cloud.bigquery.FormatOptions;
import com.google.cloud.bigquery.JobId;
import com.google.cloud.bigquery.JobStatus;
import com.google.cloud.bigquery.StandardTableDefinition;
import com.google.cloud.bigquery.TableDefinition;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableInfo;
import com.google.cloud.bigquery.WriteChannelConfiguration;
import org.apache.commons.lang3.BooleanUtils;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
import java.lang.reflect.Method;
import java.util.Objects;
public abstract class BaseBigQueryItemWriterTest {
private static final String TABLE_PATTERN = "%s_%s";
protected final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
public abstract class BaseBigQueryItemWriterTest extends BaseBigQueryIntegrationTest {
@BeforeEach
void prepareTest(TestInfo testInfo) {
@@ -61,30 +49,4 @@ public abstract class BaseBigQueryItemWriterTest {
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
}
protected String getTableName(TestInfo testInfo) {
return String.format(
TABLE_PATTERN,
testInfo.getTags().stream().findFirst().orElseThrow(),
testInfo.getTestMethod().map(Method::getName).orElseThrow()
);
}
protected WriteChannelConfiguration generateConfiguration(TestInfo testInfo, FormatOptions formatOptions) {
return WriteChannelConfiguration
.newBuilder(TableId.of(TestConstants.DATASET, getTableName(testInfo)))
.setSchema(PersonDto.getBigQuerySchema())
.setAutodetect(false)
.setFormatOptions(formatOptions)
.build();
}
/** TODO check {@link com.google.cloud.bigquery.Job#waitFor(RetryOption...)} */
protected void waitForJobToFinish(JobId jobId) {
JobStatus status = bigQuery.getJob(jobId).getStatus();
while (BooleanUtils.isFalse(JobStatus.State.DONE.equals(status.getState()))) {
status = bigQuery.getJob(jobId).getStatus();
}
}
}

View File

@@ -1,3 +1,19 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.unit.base;
import com.google.cloud.bigquery.BigQuery;

View File

@@ -0,0 +1,67 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.unit.reader.builder;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTest {
@Test
void testSimpleQueryItemReader() {
BigQuery mockedBigQuery = prepareMockedBigQuery();
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
.bigQuery(mockedBigQuery)
.query("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 1")
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
Assertions.assertNotNull(reader);
}
@Test
void testCustomQueryItemReader() {
BigQuery mockedBigQuery = prepareMockedBigQuery();
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
.build();
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
.bigQuery(mockedBigQuery)
.jobConfiguration(jobConfiguration)
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
Assertions.assertNotNull(reader);
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2022 the original author or authors.
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -37,6 +37,8 @@ import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJson
class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
private static final String TABLE = "persons_json";
private final Log logger = LogFactory.getLog(getClass());
/**
@@ -48,7 +50,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
ObjectMapper objectMapper = new ObjectMapper();
WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration
.newBuilder(TableId.of(TestConstants.DATASET, "persons_json"))
.newBuilder(TableId.of(TestConstants.DATASET, TABLE))
.setFormatOptions(FormatOptions.json())
.setSchema(Schema.of(
Field.newBuilder("name", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build()
@@ -59,7 +61,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
.bigQuery(mockedBigQuery)
.rowMapper(dto -> convertDtoToJsonByteArray(objectMapper, dto))
.writeChannelConfig(writeConfiguration)
.jobConsumer(job -> this.logger.debug("Job with id: " + job.getJobId() + " is created"))
.jobConsumer(job -> this.logger.debug("Job with id: {}" + job.getJobId() + " is created"))
.build();
writer.afterPropertiesSet();
@@ -72,7 +74,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
BigQuery mockedBigQuery = prepareMockedBigQuery();
WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration
.newBuilder(TableId.of(TestConstants.DATASET, "persons_json"))
.newBuilder(TableId.of(TestConstants.DATASET, TABLE))
.setAutodetect(true)
.setFormatOptions(FormatOptions.json())
.build();