[bq] describe how to run batch query

spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/reader/BigQueryQueryItemReader.java

@@ -43,9 +43,10 @@ import java.util.Objects;
  * @author Volodymyr Perebykivskyi
  * @since 0.2.0
  * @see <a href="https://cloud.google.com/bigquery/docs/running-queries#queries">Interactive queries</a>
+ * @see <a href="https://cloud.google.com/bigquery/docs/running-queries#batch">Batch queries</a>
  * @see <a href="https://cloud.google.com/bigquery/quotas#concurrent_rate_interactive_queries">Concurrency limits</a>
  */
-public class BigQueryInteractiveQueryItemReader<T> implements ItemReader<T>, InitializingBean {
+public class BigQueryQueryItemReader<T> implements ItemReader<T>, InitializingBean {
 
     private final Log logger = LogFactory.getLog(getClass());
 

spring-batch-bigquery/src/main/java/org/springframework/batch/extensions/bigquery/reader/builder/BigQueryQueryItemReaderBuilder.java

@@ -20,21 +20,22 @@ import com.google.cloud.bigquery.BigQuery;
 import com.google.cloud.bigquery.FieldValueList;
 import com.google.cloud.bigquery.QueryJobConfiguration;
 import org.apache.commons.lang3.StringUtils;
-import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
 import org.springframework.core.convert.converter.Converter;
 import org.springframework.util.Assert;
 
 import java.util.Objects;
 
 /**
- * A builder for {@link BigQueryInteractiveQueryItemReader}.
+ * A builder for {@link BigQueryQueryItemReader}.
  *
  * @param <T> your DTO type
  * @author Volodymyr Perebykivskyi
  * @since 0.2.0
  * @see <a href="https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryInteractiveQueryItemReaderBuilderTests.java">Examples</a>
+ * @see <a href="https://github.com/spring-projects/spring-batch-extensions/tree/main/spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java">Examples</a>
  */
-public class BigQueryInteractiveQueryItemReaderBuilder<T> {
+public class BigQueryQueryItemReaderBuilder<T> {
 
     private BigQuery bigQuery;
     private String query;
@@ -45,10 +46,10 @@ public class BigQueryInteractiveQueryItemReaderBuilder<T> {
      * BigQuery service, responsible for API calls.
      *
      * @param bigQuery BigQuery service
-     * @return {@link BigQueryInteractiveQueryItemReaderBuilder}
-     * @see BigQueryInteractiveQueryItemReader#setBigQuery(BigQuery)
+     * @return {@link BigQueryQueryItemReaderBuilder}
+     * @see BigQueryQueryItemReader#setBigQuery(BigQuery)
      */
-    public BigQueryInteractiveQueryItemReaderBuilder<T> bigQuery(BigQuery bigQuery) {
+    public BigQueryQueryItemReaderBuilder<T> bigQuery(BigQuery bigQuery) {
         this.bigQuery = bigQuery;
         return this;
     }
@@ -60,10 +61,10 @@ public class BigQueryInteractiveQueryItemReaderBuilder<T> {
      * because BigQuery charges you for the amount of data that is being processed.
      *
      * @param query your query to run
-     * @return {@link BigQueryInteractiveQueryItemReaderBuilder}
-     * @see BigQueryInteractiveQueryItemReader#setJobConfiguration(QueryJobConfiguration)
+     * @return {@link BigQueryQueryItemReaderBuilder}
+     * @see BigQueryQueryItemReader#setJobConfiguration(QueryJobConfiguration)
      */
-    public BigQueryInteractiveQueryItemReaderBuilder<T> query(String query) {
+    public BigQueryQueryItemReaderBuilder<T> query(String query) {
         this.query = query;
         return this;
     }
@@ -72,10 +73,10 @@ public class BigQueryInteractiveQueryItemReaderBuilder<T> {
      * Row mapper which transforms single BigQuery row into desired type.
      *
      * @param rowMapper your row mapper
-     * @return {@link BigQueryInteractiveQueryItemReaderBuilder}
-     * @see BigQueryInteractiveQueryItemReader#setRowMapper(Converter)
+     * @return {@link BigQueryQueryItemReaderBuilder}
+     * @see BigQueryQueryItemReader#setRowMapper(Converter)
      */
-    public BigQueryInteractiveQueryItemReaderBuilder<T> rowMapper(Converter<FieldValueList, T> rowMapper) {
+    public BigQueryQueryItemReaderBuilder<T> rowMapper(Converter<FieldValueList, T> rowMapper) {
         this.rowMapper = rowMapper;
         return this;
     }
@@ -84,21 +85,21 @@ public class BigQueryInteractiveQueryItemReaderBuilder<T> {
      * Specifies query to run, destination table, etc.
      *
      * @param jobConfiguration BigQuery job configuration
-     * @return {@link BigQueryInteractiveQueryItemReaderBuilder}
-     * @see BigQueryInteractiveQueryItemReader#setJobConfiguration(QueryJobConfiguration)
+     * @return {@link BigQueryQueryItemReaderBuilder}
+     * @see BigQueryQueryItemReader#setJobConfiguration(QueryJobConfiguration)
      */
-    public BigQueryInteractiveQueryItemReaderBuilder<T> jobConfiguration(QueryJobConfiguration jobConfiguration) {
+    public BigQueryQueryItemReaderBuilder<T> jobConfiguration(QueryJobConfiguration jobConfiguration) {
         this.jobConfiguration = jobConfiguration;
         return this;
     }
 
     /**
-     * Please do not forget about {@link BigQueryInteractiveQueryItemReader#afterPropertiesSet()}.
+     * Please do not forget about {@link BigQueryQueryItemReader#afterPropertiesSet()}.
      *
-     * @return {@link BigQueryInteractiveQueryItemReader}
+     * @return {@link BigQueryQueryItemReader}
      */
-    public BigQueryInteractiveQueryItemReader<T> build() {
-        BigQueryInteractiveQueryItemReader<T> reader = new BigQueryInteractiveQueryItemReader<>();
+    public BigQueryQueryItemReader<T> build() {
+        BigQueryQueryItemReader<T> reader = new BigQueryQueryItemReader<>();
 
         reader.setBigQuery(this.bigQuery);
         reader.setRowMapper(this.rowMapper);

spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2002-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.batch.extensions.bigquery.integration.reader.batch;
+
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.TableId;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
+import org.springframework.batch.extensions.bigquery.common.PersonDto;
+import org.springframework.batch.extensions.bigquery.common.TestConstants;
+import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
+import org.springframework.batch.item.Chunk;
+
+@Tag("csv")
+public class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
+
+    @Test
+    void batchQueryTest1(TestInfo testInfo) throws Exception {
+        String tableName = getTableName(testInfo);
+        new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
+        Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
+
+        QueryJobConfiguration jobConfiguration = QueryJobConfiguration
+                .newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
+                .setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
+                .setPriority(QueryJobConfiguration.Priority.BATCH)
+                .build();
+
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
+                .bigQuery(bigQuery)
+                .rowMapper(TestConstants.PERSON_MAPPER)
+                .jobConfiguration(jobConfiguration)
+                .build();
+
+        reader.afterPropertiesSet();
+
+        PersonDto actualFirstPerson = reader.read();
+        PersonDto expectedFirstPerson = chunk.getItems().get(0);
+
+        PersonDto actualSecondPerson = reader.read();
+        PersonDto expectedSecondPerson = chunk.getItems().get(1);
+
+        PersonDto actualThirdPerson = reader.read();
+
+        Assertions.assertNotNull(actualFirstPerson);
+        Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
+        Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
+
+        Assertions.assertNotNull(actualSecondPerson);
+        Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
+        Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
+
+        Assertions.assertNull(actualThirdPerson);
+    }
+
+}

spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryJsonItemReaderTest.java

@@ -0,0 +1,76 @@
+/*
+ * Copyright 2002-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.batch.extensions.bigquery.integration.reader.batch;
+
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.TableId;
+import org.apache.commons.lang3.math.NumberUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
+import org.springframework.batch.extensions.bigquery.common.PersonDto;
+import org.springframework.batch.extensions.bigquery.common.TestConstants;
+import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
+import org.springframework.batch.item.Chunk;
+
+@Tag("json")
+public class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
+
+    @Test
+    void batchQueryTest1(TestInfo testInfo) throws Exception {
+        String tableName = getTableName(testInfo);
+        new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
+        Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
+
+        QueryJobConfiguration jobConfiguration = QueryJobConfiguration
+                .newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2")
+                .setDestinationTable(TableId.of(TestConstants.DATASET, tableName))
+                .setPriority(QueryJobConfiguration.Priority.BATCH)
+                .build();
+
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
+                .bigQuery(bigQuery)
+                .rowMapper(TestConstants.PERSON_MAPPER)
+                .jobConfiguration(jobConfiguration)
+                .build();
+
+        reader.afterPropertiesSet();
+
+        PersonDto actualFirstPerson = reader.read();
+        PersonDto expectedFirstPerson = chunk.getItems().get(0);
+
+        PersonDto actualSecondPerson = reader.read();
+        PersonDto expectedSecondPerson = chunk.getItems().get(1);
+
+        PersonDto actualThirdPerson = reader.read();
+
+        Assertions.assertNotNull(actualFirstPerson);
+        Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
+        Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
+
+        Assertions.assertNotNull(actualSecondPerson);
+        Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
+        Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
+
+        Assertions.assertNull(actualThirdPerson);
+    }
+
+}

spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryCsvItemReaderTest.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.springframework.batch.extensions.bigquery.integration.reader;
+package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
 
 import org.apache.commons.lang3.math.NumberUtils;
 import org.junit.jupiter.api.Assertions;
@@ -25,8 +25,8 @@ import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
 import org.springframework.batch.extensions.bigquery.common.PersonDto;
 import org.springframework.batch.extensions.bigquery.common.TestConstants;
 import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
-import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
-import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
 import org.springframework.batch.item.Chunk;
 
 @Tag("csv")
@@ -38,7 +38,7 @@ public class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonIntera
         new BigQueryDataLoader(bigQuery).loadCsvSample(tableName);
         Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
 
-        BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
                 .bigQuery(bigQuery)
                 .query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
                 .rowMapper(TestConstants.PERSON_MAPPER)

spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryJsonItemReaderTest.java

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package org.springframework.batch.extensions.bigquery.integration.reader;
+package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
 
 import org.apache.commons.lang3.math.NumberUtils;
 import org.junit.jupiter.api.Assertions;
@@ -25,8 +25,8 @@ import org.springframework.batch.extensions.bigquery.common.BigQueryDataLoader;
 import org.springframework.batch.extensions.bigquery.common.PersonDto;
 import org.springframework.batch.extensions.bigquery.common.TestConstants;
 import org.springframework.batch.extensions.bigquery.integration.reader.base.BaseCsvJsonInteractiveQueryItemReaderTest;
-import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
-import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
 import org.springframework.batch.item.Chunk;
 
 @Tag("json")
@@ -38,7 +38,7 @@ public class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInter
         new BigQueryDataLoader(bigQuery).loadJsonSample(tableName);
         Chunk<PersonDto> chunk = BigQueryDataLoader.CHUNK;
 
-        BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
                 .bigQuery(bigQuery)
                 .query(String.format("SELECT p.name, p.age FROM spring_batch_extensions.%s p ORDER BY p.name LIMIT 2", tableName))
                 .rowMapper(TestConstants.PERSON_MAPPER)

spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2002-2023 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.batch.extensions.bigquery.unit.reader.builder;
+
+import com.google.cloud.bigquery.BigQuery;
+import com.google.cloud.bigquery.QueryJobConfiguration;
+import com.google.cloud.bigquery.TableId;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.springframework.batch.extensions.bigquery.common.PersonDto;
+import org.springframework.batch.extensions.bigquery.common.TestConstants;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
+import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
+
+class BigQueryBatchQueryItemReaderBuilderTests extends AbstractBigQueryTest {
+
+    @Test
+    void testCustomQueryItemReader() {
+        BigQuery mockedBigQuery = prepareMockedBigQuery();
+
+        QueryJobConfiguration jobConfiguration = QueryJobConfiguration
+                .newBuilder("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 2")
+                .setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
+                .setPriority(QueryJobConfiguration.Priority.BATCH)
+                .build();
+
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
+                .bigQuery(mockedBigQuery)
+                .jobConfiguration(jobConfiguration)
+                .rowMapper(TestConstants.PERSON_MAPPER)
+                .build();
+
+        reader.afterPropertiesSet();
+
+        Assertions.assertNotNull(reader);
+    }
+
+}

spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryInteractiveQueryItemReaderBuilderTests.java

@@ -24,8 +24,8 @@ import org.junit.jupiter.api.Test;
 import org.springframework.batch.extensions.bigquery.common.PersonDto;
 import org.springframework.batch.extensions.bigquery.common.TestConstants;
 import org.springframework.batch.extensions.bigquery.unit.base.AbstractBigQueryTest;
-import org.springframework.batch.extensions.bigquery.reader.BigQueryInteractiveQueryItemReader;
-import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryInteractiveQueryItemReaderBuilder;
+import org.springframework.batch.extensions.bigquery.reader.BigQueryQueryItemReader;
+import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQueryItemReaderBuilder;
 
 class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTest {
 
@@ -33,7 +33,7 @@ class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTes
     void testSimpleQueryItemReader() {
         BigQuery mockedBigQuery = prepareMockedBigQuery();
 
-        BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
                 .bigQuery(mockedBigQuery)
                 .query("SELECT p.name, p.age FROM spring_batch_extensions.persons p LIMIT 1")
                 .rowMapper(TestConstants.PERSON_MAPPER)
@@ -53,7 +53,7 @@ class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTes
                 .setDestinationTable(TableId.of(TestConstants.DATASET, "persons_duplicate"))
                 .build();
 
-        BigQueryInteractiveQueryItemReader<PersonDto> reader = new BigQueryInteractiveQueryItemReaderBuilder<PersonDto>()
+        BigQueryQueryItemReader<PersonDto> reader = new BigQueryQueryItemReaderBuilder<PersonDto>()
                 .bigQuery(mockedBigQuery)
                 .jobConfiguration(jobConfiguration)
                 .rowMapper(TestConstants.PERSON_MAPPER)