[bq] 0.2 introduce BigQuery interactive reader
This commit is contained in:
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -27,13 +27,20 @@ import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryCsvI
|
||||
import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJsonItemWriterBuilder;
|
||||
import org.springframework.batch.item.Chunk;
|
||||
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
public class BigQueryDataLoader {
|
||||
|
||||
public static final Chunk<PersonDto> CHUNK = Chunk.of(
|
||||
new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26)
|
||||
);
|
||||
/** Order must be defined so later executed queries results could be predictable */
|
||||
private static final List<PersonDto> PERSONS = Stream
|
||||
.of(new PersonDto("Volodymyr", 27), new PersonDto("Oleksandra", 26))
|
||||
.sorted(Comparator.comparing(PersonDto::name))
|
||||
.toList();
|
||||
|
||||
public static final Chunk<PersonDto> CHUNK = new Chunk<>(PERSONS);
|
||||
|
||||
private final BigQuery bigQuery;
|
||||
|
||||
@@ -41,11 +48,6 @@ public class BigQueryDataLoader {
|
||||
this.bigQuery = bigQuery;
|
||||
}
|
||||
|
||||
|
||||
public void loadCsvSample() throws Exception {
|
||||
loadCsvSample(TestConstants.PERSONS_TABLE);
|
||||
}
|
||||
|
||||
public void loadCsvSample(String tableName) throws Exception {
|
||||
AtomicReference<Job> job = new AtomicReference<>();
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,11 +16,19 @@
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.common;
|
||||
|
||||
public class TestConstants {
|
||||
import com.google.cloud.bigquery.FieldValueList;
|
||||
import org.springframework.core.convert.converter.Converter;
|
||||
|
||||
public final class TestConstants {
|
||||
|
||||
private TestConstants() {}
|
||||
|
||||
public static final String DATASET = "spring_batch_extensions";
|
||||
public static final String PERSONS_TABLE = "persons";
|
||||
public static final String NAME = "name";
|
||||
public static final String AGE = "age";
|
||||
|
||||
public static final Converter<FieldValueList, PersonDto> PERSON_MAPPER = res -> new PersonDto(
|
||||
res.get(NAME).getStringValue(), Long.valueOf(res.get(AGE).getLongValue()).intValue()
|
||||
);
|
||||
|
||||
}
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.integration.base;
|
||||
|
||||
import com.google.cloud.bigquery.BigQuery;
|
||||
import com.google.cloud.bigquery.BigQueryOptions;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
|
||||
public abstract class BaseBigQueryIntegrationTest {
|
||||
|
||||
private static final String TABLE_PATTERN = "%s_%s";
|
||||
|
||||
public final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
|
||||
|
||||
protected String getTableName(TestInfo testInfo) {
|
||||
return String.format(
|
||||
TABLE_PATTERN,
|
||||
testInfo.getTags().stream().findFirst().orElseThrow(),
|
||||
testInfo.getTestMethod().map(Method::getName).orElseThrow()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,28 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/**
|
||||
* In order to launch these tests you should provide a way how to authorize to Google BigQuery.
|
||||
* A simple way is to create service account, store credentials as JSON file and provide environment variable.
|
||||
* Example: GOOGLE_APPLICATION_CREDENTIALS=/home/dgray/Downloads/bq-key.json
|
||||
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
|
||||
*
|
||||
* <p>
|
||||
* Test names should follow this pattern: test1, test2, testN.
|
||||
* So later in BigQuery you will see generated table name: csv_test1, csv_test2, csv_testN.
|
||||
* This way it will be easier to trace errors in BigQuery.
|
||||
*
|
||||
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
|
||||
*/
|
||||
package org.springframework.batch.extensions.bigquery.integration;
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.integration.reader;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
|
||||
import org.springframework.batch.extensions.bigquery.common.PersonDto;
|
||||
import org.springframework.batch.extensions.bigquery.common.TestConstants;
|
||||
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
|
||||
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
|
||||
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
|
||||
import org.springframework.batch.item.Chunk;
|
||||
|
||||
@Tag("csv")
|
||||
public class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
|
||||
|
||||
@Test
|
||||
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
|
||||
String tableName = getTableName(testInfo);
|
||||
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
|
||||
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
|
||||
|
||||
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
|
||||
.bigQuery(bigQuery)
|
||||
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
|
||||
.rowMapper(TestConstants.PERSON_MAPPER)
|
||||
.build();
|
||||
|
||||
reader.afterPropertiesSet();
|
||||
|
||||
PersonDto actualFirstPerson = reader.read();
|
||||
PersonDto expectedFirstPerson = chunk.getItems().get(0);
|
||||
|
||||
PersonDto actualSecondPerson = reader.read();
|
||||
PersonDto expectedSecondPerson = chunk.getItems().get(1);
|
||||
|
||||
PersonDto actualThirdPerson = reader.read();
|
||||
|
||||
Assertions.assertNotNull(actualFirstPerson);
|
||||
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
|
||||
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
|
||||
|
||||
Assertions.assertNotNull(actualSecondPerson);
|
||||
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
|
||||
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
|
||||
|
||||
Assertions.assertNull(actualThirdPerson);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.integration.reader;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Tag;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
|
||||
import org.springframework.batch.extensions.bigquery.common.PersonDto;
|
||||
import org.springframework.batch.extensions.bigquery.common.TestConstants;
|
||||
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
|
||||
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
|
||||
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
|
||||
import org.springframework.batch.item.Chunk;
|
||||
|
||||
@Tag("json")
|
||||
public class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
|
||||
|
||||
@Test
|
||||
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
|
||||
String tableName = getTableName(testInfo);
|
||||
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
|
||||
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
|
||||
|
||||
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
|
||||
.bigQuery(bigQuery)
|
||||
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
|
||||
.rowMapper(TestConstants.PERSON_MAPPER)
|
||||
.build();
|
||||
|
||||
reader.afterPropertiesSet();
|
||||
|
||||
PersonDto actualFirstPerson = reader.read();
|
||||
PersonDto expectedFirstPerson = chunk.getItems().get(0);
|
||||
|
||||
PersonDto actualSecondPerson = reader.read();
|
||||
PersonDto expectedSecondPerson = chunk.getItems().get(1);
|
||||
|
||||
PersonDto actualThirdPerson = reader.read();
|
||||
|
||||
Assertions.assertNotNull(actualFirstPerson);
|
||||
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
|
||||
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
|
||||
|
||||
Assertions.assertNotNull(actualSecondPerson);
|
||||
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
|
||||
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
|
||||
|
||||
Assertions.assertNull(actualThirdPerson);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.integration.reader.base;
|
||||
|
||||
import com.google.cloud.bigquery.DatasetInfo;
|
||||
import com.google.cloud.bigquery.StandardTableDefinition;
|
||||
import com.google.cloud.bigquery.TableDefinition;
|
||||
import com.google.cloud.bigquery.TableId;
|
||||
import com.google.cloud.bigquery.TableInfo;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.springframework.batch.extensions.bigquery.common.PersonDto;
|
||||
import org.springframework.batch.extensions.bigquery.common.TestConstants;
|
||||
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
|
||||
|
||||
import java.util.Objects;
|
||||
|
||||
public abstract class BaseCsvJsonInteractiveQueryItemReaderTest extends BaseBigQueryIntegrationTest {
|
||||
|
||||
@BeforeEach
|
||||
void prepareTest(TestInfo testInfo) {
|
||||
if (Objects.isNull(bigQuery.getDataset(TestConstants.DATASET))) {
|
||||
bigQuery.create(DatasetInfo.of(TestConstants.DATASET));
|
||||
}
|
||||
|
||||
String tableName = getTableName(testInfo);
|
||||
|
||||
if (Objects.isNull(bigQuery.getTable(TestConstants.DATASET, tableName))) {
|
||||
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
|
||||
bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, tableName), tableDefinition));
|
||||
}
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void cleanupTest(TestInfo testInfo) {
|
||||
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -16,33 +16,21 @@
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.integration.writer.base;
|
||||
|
||||
import com.google.cloud.RetryOption;
|
||||
import com.google.cloud.bigquery.BigQuery;
|
||||
import com.google.cloud.bigquery.BigQueryOptions;
|
||||
import com.google.cloud.bigquery.DatasetInfo;
|
||||
import com.google.cloud.bigquery.FormatOptions;
|
||||
import com.google.cloud.bigquery.JobId;
|
||||
import com.google.cloud.bigquery.JobStatus;
|
||||
import com.google.cloud.bigquery.StandardTableDefinition;
|
||||
import com.google.cloud.bigquery.TableDefinition;
|
||||
import com.google.cloud.bigquery.TableId;
|
||||
import com.google.cloud.bigquery.TableInfo;
|
||||
import com.google.cloud.bigquery.WriteChannelConfiguration;
|
||||
import org.apache.commons.lang3.BooleanUtils;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.TestInfo;
|
||||
import org.springframework.batch.extensions.bigquery.common.PersonDto;
|
||||
import org.springframework.batch.extensions.bigquery.common.TestConstants;
|
||||
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Objects;
|
||||
|
||||
public abstract class BaseBigQueryItemWriterTest {
|
||||
|
||||
private static final String TABLE_PATTERN = "%s_%s";
|
||||
|
||||
protected final BigQuery bigQuery = BigQueryOptions.getDefaultInstance().getService();
|
||||
public abstract class BaseBigQueryItemWriterTest extends BaseBigQueryIntegrationTest {
|
||||
|
||||
@BeforeEach
|
||||
void prepareTest(TestInfo testInfo) {
|
||||
@@ -61,30 +49,4 @@ public abstract class BaseBigQueryItemWriterTest {
|
||||
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
|
||||
}
|
||||
|
||||
protected String getTableName(TestInfo testInfo) {
|
||||
return String.format(
|
||||
TABLE_PATTERN,
|
||||
testInfo.getTags().stream().findFirst().orElseThrow(),
|
||||
testInfo.getTestMethod().map(Method::getName).orElseThrow()
|
||||
);
|
||||
}
|
||||
|
||||
protected WriteChannelConfiguration generateConfiguration(TestInfo testInfo, FormatOptions formatOptions) {
|
||||
return WriteChannelConfiguration
|
||||
.newBuilder(TableId.of(TestConstants.DATASET, getTableName(testInfo)))
|
||||
.setSchema(PersonDto.getBigQuerySchema())
|
||||
.setAutodetect(false)
|
||||
.setFormatOptions(formatOptions)
|
||||
.build();
|
||||
}
|
||||
|
||||
/** TODO check {@link com.google.cloud.bigquery.Job#waitFor(RetryOption...)} */
|
||||
protected void waitForJobToFinish(JobId jobId) {
|
||||
JobStatus status = bigQuery.getJob(jobId).getStatus();
|
||||
|
||||
while (BooleanUtils.isFalse(JobStatus.State.DONE.equals(status.getState()))) {
|
||||
status = bigQuery.getJob(jobId).getStatus();
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -1,3 +1,19 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.unit.base;
|
||||
|
||||
import com.google.cloud.bigquery.BigQuery;
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* https://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.batch.extensions.bigquery.unit.reader.builder;
|
||||
|
||||
import com.google.cloud.bigquery.BigQuery;
|
||||
import com.google.cloud.bigquery.QueryJobConfiguration;
|
||||
import com.google.cloud.bigquery.TableId;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.batch.extensions.bigquery.common.PersonDto;
|
||||
import org.springframework.batch.extensions.bigquery.common.TestConstants;
|
||||
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
|
||||
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
|
||||
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
|
||||
|
||||
class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTest {
|
||||
|
||||
@Test
|
||||
void testSimpleQueryItemReader() {
|
||||
BigQuery mockedBigQuery = prepareMockedBigQuery();
|
||||
|
||||
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
|
||||
.bigQuery(mockedBigQuery)
|
||||
.query("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 1")
|
||||
.rowMapper(TestConstants.PERSON_MAPPER)
|
||||
.build();
|
||||
|
||||
reader.afterPropertiesSet();
|
||||
|
||||
Assertions.assertNotNull(reader);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testCustomQueryItemReader() {
|
||||
BigQuery mockedBigQuery = prepareMockedBigQuery();
|
||||
|
||||
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
|
||||
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 2")
|
||||
.setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
|
||||
.build();
|
||||
|
||||
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
|
||||
.bigQuery(mockedBigQuery)
|
||||
.jobConfiguration(jobConfiguration)
|
||||
.rowMapper(TestConstants.PERSON_MAPPER)
|
||||
.build();
|
||||
|
||||
reader.afterPropertiesSet();
|
||||
|
||||
Assertions.assertNotNull(reader);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Copyright 2002-2022 the original author or authors.
|
||||
* Copyright 2002-2023 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -37,6 +37,8 @@ import org.springframework.batch.extensions.bigquery.writer.builder.BigQueryJson
|
||||
|
||||
class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
|
||||
|
||||
private static final String TABLE = "persons_json";
|
||||
|
||||
private final Log logger = LogFactory.getLog(getClass());
|
||||
|
||||
/**
|
||||
@@ -48,7 +50,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
|
||||
ObjectMapper objectMapper = new ObjectMapper();
|
||||
|
||||
WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration
|
||||
.newBuilder(TableId.of(TestConstants.DATASET, "persons_json"))
|
||||
.newBuilder(TableId.of(TestConstants.DATASET, TABLE))
|
||||
.setFormatOptions(FormatOptions.json())
|
||||
.setSchema(Schema.of(
|
||||
Field.newBuilder("name", StandardSQLTypeName.STRING).setMode(Field.Mode.REQUIRED).build()
|
||||
@@ -59,7 +61,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
|
||||
.bigQuery(mockedBigQuery)
|
||||
.rowMapper(dto -> convertDtoToJsonByteArray(objectMapper, dto))
|
||||
.writeChannelConfig(writeConfiguration)
|
||||
.jobConsumer(job -> this.logger.debug("Job with id: " + job.getJobId() + " is created"))
|
||||
.jobConsumer(job -> this.logger.debug("Job with id: {}" + job.getJobId() + " is created"))
|
||||
.build();
|
||||
|
||||
writer.afterPropertiesSet();
|
||||
@@ -72,7 +74,7 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
|
||||
BigQuery mockedBigQuery = prepareMockedBigQuery();
|
||||
|
||||
WriteChannelConfiguration writeConfiguration = WriteChannelConfiguration
|
||||
.newBuilder(TableId.of(TestConstants.DATASET, "persons_json"))
|
||||
.newBuilder(TableId.of(TestConstants.DATASET, TABLE))
|
||||
.setAutodetect(true)
|
||||
.setFormatOptions(FormatOptions.json())
|
||||
.build();
|
||||
|
||||
Reference in New Issue
Block a user