Spring/spring-batch-extensions
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bq] describe how to run batch query

Browse Source
This commit is contained in:
Volodymyr
2023-08-26 10:25:32 +03:00
committed by GitHub
parent 56f3fc6bbc
commit bc7fda30b6
8 changed files with 239 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java Normal file
View File

@@ -0,0 +1,76 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("csv")
public class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void batchQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
.setPriority(QueryJobConfiguration.Priority.BATCH)
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.rowMapper(TestConstants.PERSON_MAPPER)
.jobConfiguration(jobConfiguration)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNull(actualThirdPerson);
}
}

76
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryJsonItemReaderTest.java Normal file
View File

@@ -0,0 +1,76 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader.batch;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInfo;
import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("json")
public class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void batchQueryTest1(TestInfo testInfo) throws Exception {
String tableName = getTableName(testInfo);
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
.setPriority(QueryJobConfiguration.Priority.BATCH)
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.rowMapper(TestConstants.PERSON_MAPPER)
.jobConfiguration(jobConfiguration)
.build();
reader.afterPropertiesSet();
PersonDto actualFirstPerson = reader.read();
PersonDto expectedFirstPerson = chunk.getItems().get(0);
PersonDto actualSecondPerson = reader.read();
PersonDto expectedSecondPerson = chunk.getItems().get(1);
PersonDto actualThirdPerson = reader.read();
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertNull(actualThirdPerson);
}
}

8
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/BigQueryInteractiveQueryCsvItemReaderTest.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryCsvItemReaderTest.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader;
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
@@ -25,8 +25,8 @@ import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("csv")
@@ -38,7 +38,7 @@ public class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonIntera
new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
.rowMapper(TestConstants.PERSON_MAPPER)

8
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/BigQueryInteractiveQueryJsonItemReaderTest.java → spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryJsonItemReaderTest.java
View File

@@ -14,7 +14,7 @@
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.integration.reader;
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
@@ -25,8 +25,8 @@ import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.item.Chunk;
@Tag("json")
@@ -38,7 +38,7 @@ public class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInter
new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(bigQuery)
.query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
.rowMapper(TestConstants.PERSON_MAPPER)

53
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java Normal file
View File

@@ -0,0 +1,53 @@
/*
* Copyright 2002-2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.batch.extensions.bigquery.unit.reader.builder;
import com.google.cloud.bigquery.BigQuery;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
class BigQueryBatchQueryItemReaderBuilderTests extends AbstractBigQueryTest {
@Test
void testCustomQueryItemReader() {
BigQuery mockedBigQuery = prepareMockedBigQuery();
QueryJobConfiguration jobConfiguration = QueryJobConfiguration
.newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 2")
.setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
.setPriority(QueryJobConfiguration.Priority.BATCH)
.build();
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(mockedBigQuery)
.jobConfiguration(jobConfiguration)
.rowMapper(TestConstants.PERSON_MAPPER)
.build();
reader.afterPropertiesSet();
Assertions.assertNotNull(reader);
}
}

8
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryInteractiveQueryItemReaderBuilderTests.java
View File

@@ -24,8 +24,8 @@ import org.junit.jupiter.api.Test;
import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTest {
@@ -33,7 +33,7 @@ class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTes
void testSimpleQueryItemReader() {
BigQuery mockedBigQuery = prepareMockedBigQuery();
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(mockedBigQuery)
.query("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 1")
.rowMapper(TestConstants.PERSON_MAPPER)
@@ -53,7 +53,7 @@ class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTes
.setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
.build();
BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
.bigQuery(mockedBigQuery)
.jobConfiguration(jobConfiguration)
.rowMapper(TestConstants.PERSON_MAPPER)