Improved S3 Integration tests, adding shared network and using name for endpoint.

This commit is contained in:
Corneil du Plessis
2023-10-20 14:33:19 +02:00
parent 74c349e90d
commit c9eee64ac3
13 changed files with 150 additions and 82 deletions

View File

@@ -63,14 +63,14 @@ jobs:
shell: bash
env:
VERBOSE: ${{ github.debug && 'true' || '' }}
run: |
VERSION=${{ inputs.version }}
STREAM_APPS_VERSION: ${{ inputs.version }}
run: |
VERBOSE=${{ inputs.verbose }}
if [ "$VERBOSE" == "" ] || [ "$VERBOSE" == "false" ]; then
VERBOSE=${{ github.debug }}
fi
export VERBOSE
./run-ITs.sh $VERSION
./run-ITs.sh
- name: 'Upload: Error logs'
if: ${{ failure() }}
uses: actions/upload-artifact@v3

View File

@@ -20,15 +20,26 @@ import java.util.LinkedHashMap;
/**
* Fluent API wrapper for {@link java.util.LinkedHashMap}.
*
* @param <K> key type.
* @param <V> value type.
* @author David Turanski
* @author Corneil du Plessis
*/
public class FluentMap<K, V> extends LinkedHashMap<K, V> {
@SuppressWarnings("rawtypes")
public static FluentMap fluentMap() {
return new FluentMap();
}
public static <K, V> FluentMap<K, V> fluentMap(Class<K> kClass, Class<V> cClass) {
return new FluentMap<>();
}
public static FluentMap<String, String> fluentStringMap() {
return new FluentMap<String, String>();
}
public FluentMap<K, V> withEntry(K key, V value) {
put(key, value);
return this;

View File

@@ -21,6 +21,7 @@ import java.util.UUID;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.utility.DockerImageName;
import org.springframework.util.Assert;
@@ -32,6 +33,7 @@ import static org.springframework.cloud.stream.app.test.integration.AppLog.appLo
* Spring Cloud Stream applications. Currently this only supports apps with single I/O
* destinations. This configures standard input and output destination bindings.
* @author David Turanski
* @author Corneil du Plessis
*/
public abstract class StreamAppContainer extends GenericContainer<StreamAppContainer> {
@@ -58,7 +60,7 @@ public abstract class StreamAppContainer extends GenericContainer<StreamAppConta
Assert.notNull(messageBrokerContainer, "A Message broker container is required.");
Assert.isTrue(messageBrokerContainer.isRunning(), "Message broker container must be started first.");
this.messageBrokerContainer = messageBrokerContainer;
this.withNetwork(messageBrokerContainer.getNetwork()).dependsOn(this.messageBrokerContainer)
this.withNetwork(Network.SHARED).dependsOn(this.messageBrokerContainer)
.withOutputDestination(TestTopicListener.STREAM_APPLICATIONS_TEST_TOPIC)
.withInputDestination(TestTopicListener.STREAM_APPLICATIONS_TEST_TOPIC + "_IN_"
+ UUID.randomUUID().toString().substring(0, 8))

View File

@@ -32,10 +32,13 @@ import org.springframework.util.CollectionUtils;
/**
* The base class used for testing end-to-end Stream applications.
*
* @author David Turanski
* @author Corneil du Plessis
* @see org.springframework.cloud.stream.app.test.integration.kafka.KafkaStreamApps
* @see org.springframework.cloud.stream.app.test.integration.rabbitmq.RabbitMQStreamApps
*/
@SuppressWarnings("resource")
public abstract class StreamApps implements AutoCloseable, Startable {
protected Logger logger = LoggerFactory.getLogger(this.getClass());
@@ -81,6 +84,7 @@ public abstract class StreamApps implements AutoCloseable, Startable {
sourceContainer.stop();
}
@SuppressWarnings("unchecked")
private void logDebugInfo() {
logger.debug("Starting apps...");
logger.debug("Source container environment for {} :", sourceContainer().getImage().get());
@@ -97,6 +101,7 @@ public abstract class StreamApps implements AutoCloseable, Startable {
sinkContainer().getEnv().forEach((Consumer<String>) env -> logger.debug(env));
}
@SuppressWarnings({"rawtypes", "unchecked"})
public static abstract class Builder<S extends StreamApps> {
private final String streamName;
@@ -104,7 +109,7 @@ public abstract class StreamApps implements AutoCloseable, Startable {
private GenericContainer sink;
private List<GenericContainer> processors = new LinkedList<>();
private final List<GenericContainer> processors = new LinkedList<>();
protected final GenericContainer messageBrokerContainer;
@@ -146,31 +151,31 @@ public abstract class StreamApps implements AutoCloseable, Startable {
private GenericContainer setupSourceContainer() {
return source.withNetwork(messageBrokerContainer.getNetwork())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_OUTPUT_DESTINATION", sourceOutputDestination())
.withEnv(binderProperties())
.dependsOn(messageBrokerContainer);
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_OUTPUT_DESTINATION", sourceOutputDestination())
.withEnv(binderProperties())
.dependsOn(messageBrokerContainer);
}
private GenericContainer setupSinkContainer() {
return sink
.withNetwork(messageBrokerContainer.getNetwork())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_DESTINATION", sinkInputDestination())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_GROUP", streamName)
.withEnv(binderProperties())
.dependsOn(messageBrokerContainer);
.withNetwork(messageBrokerContainer.getNetwork())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_DESTINATION", sinkInputDestination())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_GROUP", streamName)
.withEnv(binderProperties())
.dependsOn(messageBrokerContainer);
}
private List<GenericContainer> setupProcessorContainers() {
IntStream.range(0, processors.size())
.forEach(i -> processors.get(i).withNetwork(messageBrokerContainer.getNetwork())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_DESTINATION",
i == 0 ? sourceOutputDestination() : "processor_ " + i)
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_OUTPUT_DESTINATION",
i == (processors.size() - 1) ? sinkInputDestination()
: "processor_" + (i + 1))
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_GROUP", streamName)
.withEnv(binderProperties())
.dependsOn(messageBrokerContainer));
.forEach(i -> processors.get(i).withNetwork(messageBrokerContainer.getNetwork())
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_DESTINATION",
i == 0 ? sourceOutputDestination() : "processor_ " + i)
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_OUTPUT_DESTINATION",
i == (processors.size() - 1) ? sinkInputDestination()
: "processor_" + (i + 1))
.withEnv("SPRING_CLOUD_STREAM_BINDINGS_INPUT_GROUP", streamName)
.withEnv(binderProperties())
.dependsOn(messageBrokerContainer));
return processors;
}
@@ -180,7 +185,7 @@ public abstract class StreamApps implements AutoCloseable, Startable {
private String sinkInputDestination() {
return (CollectionUtils.isEmpty(processors) || processors.size() <= 1) ? streamName
: "processor_" + (processors.size() - 1);
: "processor_" + (processors.size() - 1);
}
}
}

View File

@@ -27,10 +27,10 @@ import org.testcontainers.utility.DockerImageName;
*
* @author David Turanski
* @author Artem Bilan
* @author Corneil du Plessis
*/
public abstract class KafkaConfig {
final static Network network = Network.SHARED;
/**
* The KafkaContainer.
@@ -38,7 +38,7 @@ public abstract class KafkaConfig {
public final static KafkaContainer kafka = new KafkaContainer(
DockerImageName.parse("confluentinc/cp-kafka"))
.withExposedPorts(9092, 9093)
.withNetwork(network)
.withNetwork(Network.SHARED)
.withStartupTimeout(Duration.ofSeconds(120))
.withStartupAttempts(3);

View File

@@ -25,6 +25,7 @@ import org.testcontainers.utility.DockerImageName;
/**
* Initializes and starts a {@link RabbitMQContainer}.
* @author David Turanski
* @author Corneil du Plessis
*/
public abstract class RabbitMQConfig {
/**
@@ -32,11 +33,10 @@ public abstract class RabbitMQConfig {
*/
public static RabbitMQContainer rabbitmq;
final static Network network = Network.SHARED;
static {
rabbitmq = new RabbitMQContainer(DockerImageName.parse("rabbitmq:3.8-management"))
.withNetwork(network)
.withNetwork(Network.SHARED)
.withNetworkAliases("rabbitmq")
.withExposedPorts(5672, 15672)
.withStartupTimeout(Duration.ofSeconds(120))

View File

@@ -23,12 +23,13 @@ import org.testcontainers.containers.GenericContainer;
import org.springframework.cloud.stream.app.test.integration.StreamApps;
import static org.springframework.cloud.stream.app.test.integration.FluentMap.fluentMap;
import static org.springframework.cloud.stream.app.test.integration.FluentMap.fluentStringMap;
/**
* Configures an end to end Stream (source, processor(s), sink) using
* {@link org.springframework.cloud.stream.app.test.integration.rabbitmq.RabbitMQStreamAppContainer}s.
* @author David Turanski
* @author Corneil du Plessis
*/
public class RabbitMQStreamApps extends StreamApps {
@@ -50,9 +51,9 @@ public class RabbitMQStreamApps extends StreamApps {
protected Map<String, String> binderProperties() {
return fluentMap()
return fluentStringMap()
.withEntry("SPRING_RABBITMQ_HOST",
messageBrokerContainer.getNetworkAliases().get(0))
(String) messageBrokerContainer.getNetworkAliases().get(0))
.withEntry("SPRING_RABBITMQ_PORT", "5672");
}

View File

@@ -20,6 +20,7 @@ import java.util.Map;
import java.util.Optional;
import org.junit.jupiter.api.BeforeAll;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.localstack.LocalStackContainer;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;
@@ -39,15 +40,24 @@ import software.amazon.awssdk.services.s3.S3Client;
*
* @author Artem Bilan
* @author Chris Bono
* @author Corneil du Plessis
*/
@Testcontainers(disabledWithoutDocker = true)
public interface LocalstackContainerTest {
LocalStackContainer LOCAL_STACK_CONTAINER =
new LocalStackContainer(DockerImageName.parse("localstack/localstack:2.2.0"))
.withEnv(Optional.ofNullable(System.getenv("GH_TOKEN"))
.map(value -> Map.of("GITHUB_API_TOKEN", value))
.orElse(Map.of()));
new LocalStackContainer(DockerImageName.parse("localstack/localstack:2.3"))
.withNetwork(Network.SHARED)
.withServices(LocalStackContainer.Service.S3)
.withServices(LocalStackContainer.Service.EC2)
.withNetworkAliases("localstack-aws", "localstack")
.withEnv("PERSISTENCE", "1")
.withEnv("EAGER_SERVICE_LOADING", "1")
.withEnv("DEBUG", "1")
.withEnv("HOSTNAME_EXTERNAL", "localstack")
.withEnv(Optional.ofNullable(System.getenv("GH_TOKEN"))
.map(value -> Map.of("GITHUB_API_TOKEN", value))
.orElse(Map.of()));
@BeforeAll
static void startContainer() {
@@ -62,15 +72,15 @@ public interface LocalstackContainerTest {
static AwsCredentialsProvider credentialsProvider() {
return StaticCredentialsProvider.create(
AwsBasicCredentials.create(LOCAL_STACK_CONTAINER.getAccessKey(), LOCAL_STACK_CONTAINER.getSecretKey()));
AwsBasicCredentials.create(LOCAL_STACK_CONTAINER.getAccessKey(), LOCAL_STACK_CONTAINER.getSecretKey()));
}
private static <B extends AwsClientBuilder<B, T>, T> T applyAwsClientOptions(B clientBuilder) {
return clientBuilder
.region(Region.of(LOCAL_STACK_CONTAINER.getRegion()))
.credentialsProvider(credentialsProvider())
.endpointOverride(LOCAL_STACK_CONTAINER.getEndpoint())
.build();
.region(Region.of(LOCAL_STACK_CONTAINER.getRegion()))
.credentialsProvider(credentialsProvider())
.endpointOverride(LOCAL_STACK_CONTAINER.getEndpoint())
.build();
}
}

View File

@@ -17,9 +17,11 @@
package org.springframework.cloud.stream.app.integration.test.source.s3;
import java.util.Map;
import java.util.function.Predicate;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -34,10 +36,9 @@ import org.springframework.cloud.stream.app.test.integration.OutputMatcher;
import org.springframework.cloud.stream.app.test.integration.StreamAppContainer;
import org.springframework.cloud.stream.app.test.integration.junit.jupiter.BaseContainerExtension;
import static org.awaitility.Awaitility.await;
import static org.springframework.cloud.stream.app.integration.test.common.Configuration.DEFAULT_DURATION;
import static org.springframework.cloud.stream.app.test.integration.FluentMap.fluentMap;
import static org.springframework.cloud.stream.app.test.integration.FluentMap.fluentStringMap;
import static org.springframework.cloud.stream.app.test.integration.StreamAppContainerTestUtils.resourceAsFile;
@Tag("integration")
@@ -46,7 +47,7 @@ abstract class S3SourceTests implements LocalstackContainerTest {
private static final Logger logger = LoggerFactory.getLogger(S3SourceTests.class);
private static S3Client s3Client = LocalstackContainerTest.s3Client();
private static final S3Client s3Client = LocalstackContainerTest.s3Client();
private StreamAppContainer source;
@@ -55,46 +56,58 @@ abstract class S3SourceTests implements LocalstackContainerTest {
@BeforeEach
void configureSource() {
source = BaseContainerExtension.containerInstance()
.withEnv("SPRING_CLOUD_AWS_S3_ENDPOINT", LOCAL_STACK_CONTAINER.getEndpoint().toString())
.withEnv("SPRING_CLOUD_AWS_S3_PATH_STYLE_ACCESS_ENABLED", "true")
.withEnv("SPRING_CLOUD_AWS_CREDENTIALS_ACCESS_KEY", LOCAL_STACK_CONTAINER.getAccessKey())
.withEnv("SPRING_CLOUD_AWS_CREDENTIALS_SECRET_KEY", LOCAL_STACK_CONTAINER.getSecretKey())
.withEnv("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_INTEGRATION", "DEBUG")
.withEnv("SPRING_CLOUD_AWS_REGION_STATIC", LOCAL_STACK_CONTAINER.getRegion())
.log();
s3Client.createBucket(r -> r.bucket("bucket"));
// Use LocalStack container network
String endpoint = String.format("http://localstack:%d", LOCAL_STACK_CONTAINER.getExposedPorts().get(0));
String region = LOCAL_STACK_CONTAINER.getRegion();
logger.info("creating S3 source with region={}, endpoint={}, container={}", region, endpoint, LOCAL_STACK_CONTAINER.getEndpoint());
source = BaseContainerExtension.containerInstance()
.withNetwork(LOCAL_STACK_CONTAINER.getNetwork())
.withEnv("SPRING_CLOUD_CONFIG_ENABLED", "false")
.withEnv("SPRING_CLOUD_AWS_S3_ENDPOINT", endpoint)
.withEnv("SPRING_CLOUD_AWS_S3_PATH_STYLE_ACCESS_ENABLED", "true")
.withEnv("SPRING_CLOUD_AWS_CREDENTIALS_ACCESS_KEY", LOCAL_STACK_CONTAINER.getAccessKey())
.withEnv("SPRING_CLOUD_AWS_CREDENTIALS_SECRET_KEY", LOCAL_STACK_CONTAINER.getSecretKey())
.withEnv("SPRING_CLOUD_AWS_REGION_STATIC", region)
.withEnv("LOGGING_LEVEL_ORG_SPRINGFRAMEWORK_INTEGRATION", "DEBUG")
.log();
}
@SuppressWarnings("unchecked")
@Test
@Disabled
void testLines() {
startContainer(fluentMap()
.withEntry("FILE_CONSUMER_MODE", "lines"));
s3Client.putObject(r -> r.bucket("bucket").key("test"), resourceAsFile("minio/data").toPath());
startContainer(fluentStringMap().withEntry("FILE_CONSUMER_MODE", "lines"));
s3Client.putObject(r -> r.bucket("bucket").key("test"), resourceAsFile("s3/data").toPath());
await().atMost(DEFAULT_DURATION)
.until(outputMatcher.payloadMatches((String s) -> s.contains("Bart Simpson")));
.until(outputMatcher.payloadMatches((String s) -> s.contains("Bart Simpson")));
}
@Test
@Disabled
void testTaskLaunchRequest() {
startContainer(fluentMap()
.withEntry("SPRING_CLOUD_FUNCTION_DEFINITION", "s3Supplier|taskLaunchRequestFunction")
.withEntry("TASK_LAUNCH_REQUEST_ARG_EXPRESSIONS", "filename=payload")
.withEntry("TASK_LAUNCH_REQUEST_TASK_NAME", "myTask")
.withEntry("FILE_CONSUMER_MODE", "ref"));
s3Client.putObject(r -> r.bucket("bucket").key("test"), resourceAsFile("minio/data").toPath());
startContainer(fluentStringMap().withEntry("SPRING_CLOUD_FUNCTION_DEFINITION", "s3Supplier|taskLaunchRequestFunction")
.withEntry("TASK_LAUNCH_REQUEST_ARG_EXPRESSIONS", "filename=payload")
.withEntry("TASK_LAUNCH_REQUEST_TASK_NAME", "myTask")
.withEntry("FILE_CONSUMER_MODE", "ref"));
s3Client.putObject(r -> r.bucket("bucket").key("test"), resourceAsFile("s3/data").toPath());
Predicate<String> predicate = (String s) -> {
logger.info("payload:{}", s);
return s.equals("{\"args\":[\"filename=/tmp/s3-supplier/test\"],\"deploymentProps\":{},\"name\":\"myTask\"}");
};
await().atMost(DEFAULT_DURATION)
.until(outputMatcher.payloadMatches(s -> s.equals("{\"args\":[\"filename=/tmp/s3-supplier/test\"],\"deploymentProps\":{},\"name\":\"myTask\"}")));
.until(outputMatcher.payloadMatches(predicate));
}
@Test
void testListOnly() {
startContainer(fluentMap()
.withEntry("FILE_CONSUMER_MODE", "ref")
.withEntry("S3_SUPPLIER_LIST_ONLY", "true"));
s3Client.putObject(r -> r.bucket("bucket").key("test"), resourceAsFile("minio/data").toPath());
startContainer(fluentStringMap()
.withEntry("FILE_CONSUMER_MODE", "ref")
.withEntry("S3_SUPPLIER_LIST_ONLY", "true"));
s3Client.putObject(r -> r.bucket("bucket").key("test"), resourceAsFile("s3/data").toPath());
Predicate<String> predicate = (String s) -> s.contains("\"bucketName\":\"bucket\",\"key\":\"test\"");
await().atMost(DEFAULT_DURATION)
.until(outputMatcher.payloadMatches((String s) -> s.contains("\"bucketName\":\"bucket\",\"key\":\"test\"")));
.until(outputMatcher.payloadMatches(predicate));
}
private void startContainer(Map<String, String> environment) {

View File

@@ -1,17 +1,34 @@
<configuration>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
</filter>
<encoder>
<pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n</pattern>
</encoder>
</appender>
<appender name="FILE" class="ch.qos.logback.core.FileAppender">
<file>test.log</file>
<append>true</append>
<!-- set immediateFlush to false for much higher logging throughput -->
<immediateFlush>true</immediateFlush>
<!-- encoders are assigned the type
ch.qos.logback.classic.encoder.PatternLayoutEncoder by default -->
<encoder>
<pattern>%-4relative [%thread] %-5level %logger{35} -%kvp- %msg%n</pattern>
</encoder>
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>DEBUG</level>
</filter>
</appender>
<root level="info">
<appender-ref ref="STDOUT"/>
<appender-ref ref="FILE"/>
</root>
<logger name="org.testcontainers" level="INFO"/>
<logger name="org.testcontainers" level="DEBUG"/>
<logger name="com.github.dockerjava" level="WARN"/>
<!-- <logger name="org.springframework.amqp" level="DEBUG"/>-->
<!-- <logger name="org.springframework.retry" level="TRACE"/>-->
<logger name="http-request-processor" level="DEBUG"/>
</configuration>

View File

@@ -6,24 +6,24 @@ MVN_OPT=
if [ "$VERBOSE" == "true" ]; then
MVN_OPT="-X"
fi
VERSION=$1
if [ "$VERSION" == "" ]; then
VERSION=$($SCDIR/mvn-get-version.sh)
fi
if [ "$1" != "" ]; then
shift
if [ "$STREAM_APPS_VERSION" == "" ]; then
STREAM_APPS_VERSION=$($SCDIR/mvn-get-version.sh)
fi
CONTAINERS="s3-source sftp-source http-request-processor log-sink jdbc-source time-source http-source tcp-sink mongodb-sink"
BROKERS="rabbit kafka"
for container in $CONTAINERS; do
for broker in $BROKERS; do
echo "Pulling springcloudstream/${container}-${broker}:$VERSION"
docker pull "springcloudstream/${container}-${broker}:$VERSION"
docker tag "springcloudstream/${container}-${broker}:$VERSION" "springcloudstream/${container}-${broker}:latest"
echo "Pulling springcloudstream/${container}-${broker}:$STREAM_APPS_VERSION"
docker pull "springcloudstream/${container}-${broker}:$STREAM_APPS_VERSION"
docker tag "springcloudstream/${container}-${broker}:$STREAM_APPS_VERSION" "springcloudstream/${container}-${broker}:latest"
done
done
echo "Using version:$VERSION"
echo "Using STREAM_APPS_VERSION=$STREAM_APPS_VERSION"
$SCDIR/mvnw $MVN_OPT -pl :stream-applications-integration-tests -am install -DskipTests
$SCDIR/mvnw $MVN_OPT $@ -pl :stream-applications-integration-tests -Pintegration -Psnapshot test integration-test -Dspring.cloud.stream.applications.version=$VERSION
$SCDIR/mvnw $MVN_OPT $@ -pl :stream-applications-integration-tests -Pintegration -Psnapshot test integration-test -Dspring.cloud.stream.applications.version=$STREAM_APPS_VERSION
LOG_FILE=$SCDIR/stream-applications-integration-tests/test.log
if [ -f $LOG_FILE ]; then
cat $LOG_FILE
fi

View File

@@ -16,6 +16,15 @@
<properties>
<uniqueVersion>false</uniqueVersion>
</properties>
<dependencies>
<dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
<version>3.0.17</version>
<type>pom</type>
<scope>compile</scope>
</dependency>
</dependencies>
<build>
<resources>
<resource>