Spring/spring-batch-extensions
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[bq] refactoring & dependency upgrade

Browse Source
This commit is contained in:
Volodymyr
2024-08-18 18:36:21 +03:00
committed by GitHub
parent 50f93d3cab
commit 6f3277a2a3
29 changed files with 163 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/BigQueryDataLoader.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -90,4 +90,4 @@ public class BigQueryDataLoader {
job.get().waitFor();
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/PersonDto.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -30,4 +30,4 @@ public record PersonDto(String name, Integer age) {
return Schema.of(nameField, ageField);
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/common/TestConstants.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -31,4 +31,4 @@ public final class TestConstants {
res.get(NAME).getStringValue(), Long.valueOf(res.get(AGE).getLongValue()).intValue()
);
}
}

6
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/base/BaseBigQueryIntegrationTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -31,8 +31,8 @@ public abstract class BaseBigQueryIntegrationTest {
protected String getTableName(TestInfo testInfo) {
return String.format(
TABLE_PATTERN,
testInfo.getTags().stream().findFirst().orElseThrow(),
testInfo.getTags().iterator().next(),
testInfo.getTestMethod().map(Method::getName).orElseThrow()
);
}
}
}

10
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/package-info.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -15,13 +15,13 @@
*/
/**
* In order to launch these tests you should provide a way how to authorize to Google BigQuery.
* A simple way is to create service account, store credentials as JSON file and provide environment variable.
* Example: GOOGLE_APPLICATION_CREDENTIALS=/home/dgray/Downloads/bq-key.json
* To launch these tests, you should provide a way how to authorize to Google BigQuery.
* A simple way is to create a service account, store credentials as JSON file and provide environment variable.
* Example: GOOGLE_APPLICATION_CREDENTIALS=/home/user/Downloads/bq-key.json
* <p>
* Test names should follow this pattern: test1, test2, testN.
* So later in BigQuery you will see generated table name: csv_test1, csv_test2, csv_testN.
* This way it will be easier to trace errors in BigQuery.
* This way, it will be easier to trace errors in BigQuery.
*
* @see <a href="https://cloud.google.com/bigquery/docs/quickstarts/quickstart-client-libraries#before-you-begin">Authentication</a>
*/

10
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/base/BaseCsvJsonInteractiveQueryItemReaderTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,19 +28,17 @@ import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
import java.util.Objects;
public abstract class BaseCsvJsonInteractiveQueryItemReaderTest extends BaseBigQueryIntegrationTest {
@BeforeEach
void prepareTest(TestInfo testInfo) {
if (Objects.isNull(bigQuery.getDataset(TestConstants.DATASET))) {
if (bigQuery.getDataset(TestConstants.DATASET) == null) {
bigQuery.create(DatasetInfo.of(TestConstants.DATASET));
}
String tableName = getTableName(testInfo);
if (Objects.isNull(bigQuery.getTable(TestConstants.DATASET, tableName))) {
if (bigQuery.getTable(TestConstants.DATASET, tableName) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, tableName), tableDefinition));
}
@@ -51,4 +49,4 @@ public abstract class BaseCsvJsonInteractiveQueryItemReaderTest extends BaseBigQ
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
}
}
}

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryCsvItemReaderTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ package org.springframework.batch.extensions.bigquery.integration.reader.batch;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -32,7 +31,7 @@ import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQuer
import org.springframework.batch.item.Chunk;
@Tag("csv")
public class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void batchQueryTest1(TestInfo testInfo) throws Exception {
@@ -64,13 +63,13 @@ public class BigQueryBatchQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQ
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}
}

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/batch/BigQueryBatchQueryJsonItemReaderTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -18,7 +18,6 @@ package org.springframework.batch.extensions.bigquery.integration.reader.batch;
import com.google.cloud.bigquery.QueryJobConfiguration;
import com.google.cloud.bigquery.TableId;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -32,7 +31,7 @@ import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQuer
import org.springframework.batch.item.Chunk;
@Tag("json")
public class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void batchQueryTest1(TestInfo testInfo) throws Exception {
@@ -64,13 +63,13 @@ public class BigQueryBatchQueryJsonItemReaderTest extends BaseCsvJsonInteractive
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}
}

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryCsvItemReaderTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -30,7 +29,7 @@ import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQuer
import org.springframework.batch.item.Chunk;
@Tag("csv")
public class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
@@ -56,13 +55,13 @@ public class BigQueryInteractiveQueryCsvItemReaderTest extends BaseCsvJsonIntera
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}
}

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/reader/interactive/BigQueryInteractiveQueryJsonItemReaderTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,7 +16,6 @@
package org.springframework.batch.extensions.bigquery.integration.reader.interactive;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -30,7 +29,7 @@ import org.springframework.batch.extensions.bigquery.reader.builder.BigQueryQuer
import org.springframework.batch.item.Chunk;
@Tag("json")
public class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInteractiveQueryItemReaderTest {
@Test
void interactiveQueryTest1(TestInfo testInfo) throws Exception {
@@ -56,13 +55,13 @@ public class BigQueryInteractiveQueryJsonItemReaderTest extends BaseCsvJsonInter
Assertions.assertNotNull(actualFirstPerson);
Assertions.assertEquals(expectedFirstPerson.name(), actualFirstPerson.name());
Assertions.assertEquals(expectedFirstPerson.age().compareTo(actualFirstPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedFirstPerson.age().compareTo(actualFirstPerson.age()));
Assertions.assertNotNull(actualSecondPerson);
Assertions.assertEquals(expectedSecondPerson.name(), actualSecondPerson.name());
Assertions.assertEquals(expectedSecondPerson.age().compareTo(actualSecondPerson.age()), NumberUtils.INTEGER_ZERO);
Assertions.assertEquals(0, expectedSecondPerson.age().compareTo(actualSecondPerson.age()));
Assertions.assertNull(actualThirdPerson);
}
}
}

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryCsvItemWriterTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableResult;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -33,7 +32,7 @@ import org.springframework.batch.extensions.bigquery.integration.writer.base.Bas
import org.springframework.batch.item.Chunk;
@Tag("csv")
public class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
@Test
void test1(TestInfo testInfo) throws Exception {
@@ -54,7 +53,7 @@ public class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
.getValues()
.forEach(field -> {
Assertions.assertTrue(
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(NumberUtils.INTEGER_ZERO).getStringValue().equals(name))
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
);
boolean ageCondition = chunk
@@ -62,10 +61,10 @@ public class BigQueryCsvItemWriterTest extends BaseBigQueryItemWriterTest {
.stream()
.map(PersonDto::age)
.map(Long::valueOf)
.anyMatch(age -> age.compareTo(field.get(NumberUtils.INTEGER_ONE).getLongValue()) == NumberUtils.INTEGER_ZERO);
.anyMatch(age -> age.compareTo(field.get(1).getLongValue()) == 0);
Assertions.assertTrue(ageCondition);
});
}
}
}

11
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/BigQueryJsonItemWriterTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@ import com.google.cloud.bigquery.Dataset;
import com.google.cloud.bigquery.Table;
import com.google.cloud.bigquery.TableId;
import com.google.cloud.bigquery.TableResult;
import org.apache.commons.lang3.math.NumberUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
@@ -33,7 +32,7 @@ import org.springframework.batch.extensions.bigquery.integration.writer.base.Bas
import org.springframework.batch.item.Chunk;
@Tag("json")
public class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest {
class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest {
@Test
void test1(TestInfo testInfo) throws Exception {
@@ -54,7 +53,7 @@ public class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest {
.getValues()
.forEach(field -> {
Assertions.assertTrue(
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(NumberUtils.INTEGER_ZERO).getStringValue().equals(name))
chunk.getItems().stream().map(PersonDto::name).anyMatch(name -> field.get(0).getStringValue().equals(name))
);
boolean ageCondition = chunk
@@ -62,10 +61,10 @@ public class BigQueryJsonItemWriterTest extends BaseBigQueryItemWriterTest {
.stream()
.map(PersonDto::age)
.map(Long::valueOf)
.anyMatch(age -> age.compareTo(field.get(NumberUtils.INTEGER_ONE).getLongValue()) == NumberUtils.INTEGER_ZERO);
.anyMatch(age -> age.compareTo(field.get(1).getLongValue()) == 0);
Assertions.assertTrue(ageCondition);
});
}
}
}

10
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/integration/writer/base/BaseBigQueryItemWriterTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -28,17 +28,15 @@ import org.springframework.batch.extensions.bigquery.common.PersonDto;
import org.springframework.batch.extensions.bigquery.common.TestConstants;
import org.springframework.batch.extensions.bigquery.integration.base.BaseBigQueryIntegrationTest;
import java.util.Objects;
public abstract class BaseBigQueryItemWriterTest extends BaseBigQueryIntegrationTest {
@BeforeEach
void prepareTest(TestInfo testInfo) {
if (Objects.isNull(bigQuery.getDataset(TestConstants.DATASET))) {
if (bigQuery.getDataset(TestConstants.DATASET) == null) {
bigQuery.create(DatasetInfo.of(TestConstants.DATASET));
}
if (Objects.isNull(bigQuery.getTable(TestConstants.DATASET, getTableName(testInfo)))) {
if (bigQuery.getTable(TestConstants.DATASET, getTableName(testInfo)) == null) {
TableDefinition tableDefinition = StandardTableDefinition.of(PersonDto.getBigQuerySchema());
bigQuery.create(TableInfo.of(TableId.of(TestConstants.DATASET, getTableName(testInfo)), tableDefinition));
}
@@ -49,4 +47,4 @@ public abstract class BaseBigQueryItemWriterTest extends BaseBigQueryIntegration
bigQuery.delete(TableId.of(TestConstants.DATASET, getTableName(testInfo)));
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/base/AbstractBigQueryTest.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -35,4 +35,4 @@ public abstract class AbstractBigQueryTest {
return mockedBigQuery;
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryBatchQueryItemReaderBuilderTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -50,4 +50,4 @@ class BigQueryBatchQueryItemReaderBuilderTests extends AbstractBigQueryTest {
Assertions.assertNotNull(reader);
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/reader/builder/BigQueryInteractiveQueryItemReaderBuilderTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -64,4 +64,4 @@ class BigQueryInteractiveQueryItemReaderBuilderTests extends AbstractBigQueryTes
Assertions.assertNotNull(reader);
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryCsvItemWriterBuilderTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -96,4 +96,4 @@ class BigQueryCsvItemWriterBuilderTests extends AbstractBigQueryTest {
}
}
}
}

4
spring-batch-bigquery/src/test/java/org/springframework/batch/extensions/bigquery/unit/writer/builder/BigQueryJsonItemWriterBuilderTests.java
View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2023 the original author or authors.
* Copyright 2002-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -97,4 +97,4 @@ class BigQueryJsonItemWriterBuilderTests extends AbstractBigQueryTest {
}
}
}
}