Spring/spring-data-cassandra
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DATACASS-699 - Introduce Testcontainers for integration tests.

Browse Source
This commit is contained in:
Mark Paluch
2020-01-14 17:23:07 +01:00
parent cfc3148dbd
commit 040a5938a3
10 changed files with 277 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
.travis.yml
View File

@@ -1,38 +1,21 @@
language: java
jdk:
- oraclejdk8
env:
global:
- CASSANDRA_VERSION=3.11.2
matrix:
- PROFILE=ci
- PROFILE=spring5-next
- CASSANDRA_VERSION=2.2.10
- CASSANDRA_VERSION=3.0.10
addons:
apt:
packages:
- oracle-java8-installer
cache:
directories:
- $HOME/.m2
- 'download'
sudo: required
sudo: false
before_install:
- sed -i.bak -e 's|https://nexus.codehaus.org/snapshots/|https://oss.sonatype.org/content/repositories/codehaus-snapshots/|g' ~/.m2/settings.xml
services:
- docker
install: if [ ! -z ${CASSANDRA_VERSION+x} ]; then ./setup-cassandra.sh; fi;
install: true
script:
- |
if [ ! -z ${CASSANDRA_DRIVER_VERSION+x} ]; then
mvn clean install -P${PROFILE},external-cassandra -Dcassandra-driver.version=${CASSANDRA_DRIVER_VERSION} -Dmaven.javadoc.skip=true
else
mvn clean install -P${PROFILE},external-cassandra -Dmaven.javadoc.skip=true
fi
script: ./mvnw clean install -Pci,testcontainers-cassandra,java11 -Dmaven.javadoc.skip=true -B

75
Jenkinsfile vendored
View File

@@ -12,10 +12,46 @@ pipeline {
}
stages {
stage("Docker images") {
parallel {
stage('Publish JDK 8 + Cassandra 3.11') {
when {
changeset "ci/openjdk8-cassandra-3.11/**"
}
agent { label 'data' }
options { timeout(time: 30, unit: 'MINUTES') }
steps {
script {
def image = docker.build("springci/spring-data-openjdk8-cassandra-3.11", "ci/openjdk8-cassandra-3.11/")
docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
image.push()
}
}
}
}
stage('Publish JDK 11 + Cassandra 3.11') {
when {
changeset "ci/openjdk11-8-cassandra-3.11/**"
}
agent { label 'data' }
options { timeout(time: 30, unit: 'MINUTES') }
steps {
script {
def image = docker.build("springci/spring-data-openjdk11-8-cassandra-3.11", "ci/openjdk11-8-cassandra-3.11/")
docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
image.push()
}
}
}
}
}
}
stage("Test") {
when {
anyOf {
branch 'master'
branch 'issue/DATACASS-699'
not { triggeredBy 'UpstreamCause' }
}
}
@@ -23,15 +59,39 @@ pipeline {
stage("test: baseline") {
agent {
docker {
image 'adoptopenjdk/openjdk8:latest'
image 'springci/spring-data-openjdk8-cassandra-3.11:latest'
label 'data'
args '-v $HOME:/tmp/jenkins-home'
}
}
options { timeout(time: 30, unit: 'MINUTES') }
steps {
sh 'rm -rf ?'
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw clean dependency:list verify -Dsort -U -B'
sh 'mkdir -p /tmp/jenkins-home'
sh 'JAVA_HOME=/opt/java/openjdk /opt/cassandra/bin/cassandra -R &'
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,external-cassandra clean dependency:list verify -Dsort -U -B'
}
}
}
}
stage("Test other configurations") {
when {
anyOf {
branch 'master'
not { triggeredBy 'UpstreamCause' }
}
}
parallel {
stage("test: baseline (jdk11)") {
agent {
docker {
image 'springci/spring-data-openjdk11-8-cassandra-3.11:latest'
label 'data'
}
}
options { timeout(time: 30, unit: 'MINUTES') }
steps {
sh 'mkdir -p /tmp/jenkins-home'
sh 'JAVA_HOME=/opt/java/openjdk8 /opt/cassandra/bin/cassandra -R &'
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,external-cassandra,java11 clean dependency:list verify -Dsort -U -B'
}
}
}
@@ -47,7 +107,6 @@ pipeline {
docker {
image 'adoptopenjdk/openjdk8:latest'
label 'data'
args '-v $HOME:/tmp/jenkins-home'
}
}
options { timeout(time: 20, unit: 'MINUTES') }
@@ -57,7 +116,7 @@ pipeline {
}
steps {
sh 'rm -rf ?'
sh 'mkdir -p /tmp/jenkins-home'
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,artifactory ' +
'-Dartifactory.server=https://repo.spring.io ' +
"-Dartifactory.username=${ARTIFACTORY_USR} " +
@@ -76,7 +135,6 @@ pipeline {
docker {
image 'adoptopenjdk/openjdk8:latest'
label 'data'
args '-v $HOME:/tmp/jenkins-home'
}
}
options { timeout(time: 20, unit: 'MINUTES') }
@@ -86,6 +144,7 @@ pipeline {
}
steps {
sh 'mkdir -p /tmp/jenkins-home'
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,distribute ' +
'-Dartifactory.server=https://repo.spring.io ' +
"-Dartifactory.username=${ARTIFACTORY_USR} " +

23
ci/openjdk11-8-cassandra-3.11/Dockerfile Normal file
View File

@@ -0,0 +1,23 @@
FROM adoptopenjdk/openjdk11:latest
RUN set -eux; \
CASSANDRA_URL='https://www-us.apache.org/dist/cassandra/3.11.5/apache-cassandra-3.11.5-bin.tar.gz'; \
curl -LfsSo /tmp/cassandra.tar.gz ${CASSANDRA_URL}; \
mkdir -p /opt/cassandra /opt/cassandra/data /opt/cassandra/logs; \
cd /opt/cassandra; \
tar -xf /tmp/cassandra.tar.gz --strip-components=1; \
rm -rf /tmp/cassandra.tar.gz; \
chmod -R a+rwx /opt/cassandra; \
useradd -d /home/jenkins-docker -m -u 1001 -U jenkins-docker;
RUN set -eux; \
BINARY_URL='https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u232-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u232b09.tar.gz'; \
curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \
mkdir -p /opt/java/openjdk8; \
cd /opt/java/openjdk8; \
tar -xf /tmp/openjdk.tar.gz --strip-components=1; \
rm -rf /tmp/openjdk.tar.gz;
ENV PATH="/opt/java/openjdk8/bin:$PATH"
ENV MAX_HEAP_SIZE=1500M
ENV HEAP_NEWSIZE=300M

14
ci/openjdk8-cassandra-3.11/Dockerfile Normal file
View File

@@ -0,0 +1,14 @@
FROM adoptopenjdk/openjdk8:latest
RUN set -eux; \
CASSANDRA_URL='https://www-us.apache.org/dist/cassandra/3.11.5/apache-cassandra-3.11.5-bin.tar.gz'; \
curl -LfsSo /tmp/cassandra.tar.gz ${CASSANDRA_URL}; \
mkdir -p /opt/cassandra /opt/cassandra/data /opt/cassandra/logs; \
cd /opt/cassandra; \
tar -xf /tmp/cassandra.tar.gz --strip-components=1; \
rm -rf /tmp/cassandra.tar.gz; \
chmod -R a+rwx /opt/cassandra; \
useradd -d /home/jenkins-docker -m -u 1001 -U jenkins-docker;
ENV MAX_HEAP_SIZE=1500M
ENV HEAP_NEWSIZE=300M

51
pom.xml
View File

@@ -1,5 +1,7 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
@@ -170,6 +172,14 @@
</exclusions>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers-bom</artifactId>
<version>${testcontainers}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>
@@ -237,7 +247,9 @@
<exclude>**/test/performance/**/*</exclude>
</excludes>
<systemPropertyVariables>
<java.util.logging.config.file>src/test/resources/logging.properties</java.util.logging.config.file>
<java.util.logging.config.file>
src/test/resources/logging.properties
</java.util.logging.config.file>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -259,7 +271,9 @@
<exclude>**/test/performance/**/*</exclude>
</excludes>
<systemPropertyVariables>
<java.util.logging.config.file>src/test/resources/logging.properties</java.util.logging.config.file>
<java.util.logging.config.file>
src/test/resources/logging.properties
</java.util.logging.config.file>
</systemPropertyVariables>
</configuration>
<executions>
@@ -283,12 +297,22 @@
</resource>
</resources>
<testResources>
<testResource>
<directory>src/test/resources</directory>
<filtering>false</filtering>
<includes>
<include>**/*.xml</include>
</includes>
</testResource>
<testResource>
<directory>src/test/resources</directory>
<filtering>true</filtering>
<includes>
<include>**/*</include>
</includes>
<excludes>
<exclude>**/*.xml</exclude>
</excludes>
</testResource>
</testResources>
</build>
@@ -322,10 +346,12 @@
<phase>generate-test-resources</phase>
<configuration>
<portNames>
<portName>build.cassandra.native_transport_port</portName>
<portName>build.cassandra.native_transport_port
</portName>
<portName>build.cassandra.rpc_port</portName>
<portName>build.cassandra.storage_port</portName>
<portName>build.cassandra.ssl_storage_port</portName>
<portName>build.cassandra.ssl_storage_port
</portName>
</portNames>
</configuration>
</execution>
@@ -339,12 +365,25 @@
<properties>
<build.cassandra.mode>external</build.cassandra.mode>
<build.cassandra.native_transport_port>9042</build.cassandra.native_transport_port>
<build.cassandra.native_transport_port>9042
</build.cassandra.native_transport_port>
<build.cassandra.rpc_port>9160</build.cassandra.rpc_port>
<build.cassandra.storage_port>7000</build.cassandra.storage_port>
<build.cassandra.ssl_storage_port>7001</build.cassandra.ssl_storage_port>
</properties>
</profile>
<profile>
<id>testcontainers-cassandra</id>
<properties>
<build.cassandra.mode>testcontainers</build.cassandra.mode>
<build.cassandra.native_transport_port>0
</build.cassandra.native_transport_port>
<build.cassandra.rpc_port>0</build.cassandra.rpc_port>
<build.cassandra.storage_port>0</build.cassandra.storage_port>
<build.cassandra.ssl_storage_port>0</build.cassandra.ssl_storage_port>
</properties>
</profile>
</profiles>
</project>

6
spring-data-cassandra/pom.xml
View File

@@ -161,6 +161,12 @@
<artifactId>cassandra-all</artifactId>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>cassandra</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.xerial.snappy</groupId>
<artifactId>snappy-java</artifactId>

2
spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/ReactiveCassandraBatchTemplate.java
View File

@@ -135,7 +135,7 @@ class ReactiveCassandraBatchTemplate implements ReactiveCassandraBatchOperations
.collectList() //
.flatMap(statements -> {
this.batch.addStatements(statements);
this.batch.addStatements((List<BatchableStatement<?>>) statements);
return this.operations.getReactiveCqlOperations().queryForResultSet(this.batch.build());

78
spring-data-cassandra/src/test/java/org/springframework/data/cassandra/support/CassandraConnectionProperties.java
View File

@@ -15,7 +15,12 @@
*/
package org.springframework.data.cassandra.support;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.lang.ref.WeakReference;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
import org.springframework.core.convert.converter.Converter;
@@ -30,6 +35,8 @@ import org.springframework.util.Assert;
@SuppressWarnings("serial")
public class CassandraConnectionProperties extends Properties {
private final static List<WeakReference<CassandraConnectionProperties>> instances = new ArrayList<>();
protected String resourceName;
/**
@@ -40,16 +47,43 @@ public class CassandraConnectionProperties extends Properties {
this("/config/cassandra-connection.properties");
}
public void update() {
try {
// Caution: Rewriting properties during initialization.
File file = new File(getClass().getResource(resourceName).toURI());
try (FileOutputStream fos = new FileOutputStream(file)) {
store(fos, "");
}
reload();
} catch (Exception e) {
e.printStackTrace();
throw new IllegalStateException(e);
}
}
public static void reload() {
for (WeakReference<CassandraConnectionProperties> ref : instances) {
CassandraConnectionProperties properties = ref.get();
if (properties != null) {
properties.loadProperties();
}
}
}
protected CassandraConnectionProperties(String resourceName) {
this.resourceName = resourceName;
loadProperties();
instances.add(new WeakReference<>(this));
}
private void loadProperties() {
loadProperties(this.resourceName);
putAll(System.getProperties());
}
private void loadProperties(String resourceName) {
@@ -68,11 +102,28 @@ public class CassandraConnectionProperties extends Properties {
if (in != null) {
try {
in.close();
} catch (Exception ignore) { }
} catch (Exception ignore) {}
}
}
}
@Override
public String getProperty(String key) {
String value = super.getProperty(key);
if (value == null) {
value = System.getProperty(key);
}
return value;
}
@Override
public synchronized Object setProperty(String key, String value) {
System.setProperty(key, value);
return super.setProperty(key, value);
}
/**
* @return the Cassandra hostname
*/
@@ -80,6 +131,10 @@ public class CassandraConnectionProperties extends Properties {
return getProperty("build.cassandra.host");
}
public void setCassandraHost(String host) {
setProperty("build.cassandra.host", host);
}
/**
* @return the Cassandra port (native).
*/
@@ -87,6 +142,10 @@ public class CassandraConnectionProperties extends Properties {
return getInt("build.cassandra.native_transport_port");
}
public void setCassandraPort(int port) {
setProperty("build.cassandra.native_transport_port", "" + port);
}
/**
* @return the Cassandra RPC port
*/
@@ -115,9 +174,12 @@ public class CassandraConnectionProperties extends Properties {
String cassandraType = getProperty("build.cassandra.mode");
return CassandraType.EXTERNAL.name().equalsIgnoreCase(cassandraType)
? CassandraType.EXTERNAL
: CassandraType.EMBEDDED;
if (CassandraType.TESTCONTAINERS.name().equalsIgnoreCase(cassandraType)) {
return CassandraType.TESTCONTAINERS;
}
return CassandraType.EXTERNAL.name().equalsIgnoreCase(cassandraType) ? CassandraType.EXTERNAL
: CassandraType.EMBEDDED;
}
/**
@@ -161,12 +223,12 @@ public class CassandraConnectionProperties extends Properties {
String message = "%1$s: cannot parse value [%2$s] of property [%3$s] as a [%4$s]";
throw new IllegalArgumentException(String.format(message, this.resourceName, propertyValue, propertyName,
type.getSimpleName()), cause);
throw new IllegalArgumentException(
String.format(message, this.resourceName, propertyValue, propertyName, type.getSimpleName()), cause);
}
}
public enum CassandraType {
EMBEDDED, EXTERNAL
EMBEDDED, EXTERNAL, TESTCONTAINERS;
}
}

49
spring-data-cassandra/src/test/java/org/springframework/data/cassandra/test/util/CassandraRule.java
View File

@@ -31,13 +31,15 @@ import org.springframework.util.Assert;
import org.springframework.util.SocketUtils;
import org.springframework.util.StringUtils;
import org.testcontainers.containers.CassandraContainer;
import com.datastax.oss.driver.api.core.CqlIdentifier;
import com.datastax.oss.driver.api.core.CqlSession;
import com.datastax.oss.driver.api.core.CqlSessionBuilder;
/**
* JUnit Rule used to provide a Cassandra context for integration tests. This rule can use/spin up either an embedded
* Cassandra instance or use an external instance. Typical usage:
* Cassandra instance, a TestContainer or use an external instance. Typical usage:
*
* <pre>
* public class MyIntegrationTest {
@@ -46,14 +48,20 @@ import com.datastax.oss.driver.api.core.CqlSessionBuilder;
* }
* </pre>
*
* Derives its configuration from {@code /config/cassandra-connection.properties} and {@link System#getenv(String)}
* {@code CASSANDRA_VERSION} to configure the Cassandra version via Testcontainers.
*
* @author Mark Paluch
* @author John Blum
* @since 1.5
* @see CassandraConnectionProperties
*/
public class CassandraRule extends ExternalResource {
private static ResourceHolder resourceHolder;
private static CassandraContainer<?> container;
private final long startupTimeout;
@SuppressWarnings("all") private final CassandraConnectionProperties properties = new CassandraConnectionProperties();
@@ -248,12 +256,17 @@ public class CassandraRule extends ExternalResource {
if (isStartNeeded()) {
configureRemoteJmxPort();
runEmbeddedCassandra();
if (isEmbedded()) {
runEmbeddedCassandra();
} else {
runTestcontainerCassandra();
}
}
}
private boolean isStartNeeded() {
return isParent() && isEmbedded();
return isParent() && (isEmbedded() || isTestcontainers());
}
private void configureRemoteJmxPort() {
@@ -270,6 +283,24 @@ public class CassandraRule extends ExternalResource {
}
}
private void runTestcontainerCassandra() {
if (container == null) {
String cassandra_version = System.getenv("CASSANDRA_VERSION");
if (StringUtils.hasText(cassandra_version)) {
container = new CassandraContainer<>("cassandra:" + cassandra_version);
} else {
container = new CassandraContainer<>();
}
container.start();
this.properties.setCassandraHost(container.getContainerIpAddress());
this.properties.setCassandraPort(container.getFirstMappedPort());
this.properties.update();
}
}
private synchronized void initializeConnection() {
if (isParent()) {
@@ -305,11 +336,19 @@ public class CassandraRule extends ExternalResource {
private String resolveHost() {
if (isTestcontainers()) {
return container.getContainerIpAddress();
}
return isEmbedded() ? EmbeddedCassandraServerHelper.getHost() : this.properties.getCassandraHost();
}
private int resolvePort() {
if (isTestcontainers()) {
return container.getFirstMappedPort();
}
return isEmbedded() ? EmbeddedCassandraServerHelper.getNativeTransportPort() : this.properties.getCassandraPort();
}
@@ -378,6 +417,10 @@ public class CassandraRule extends ExternalResource {
return CassandraConnectionProperties.CassandraType.EMBEDDED.equals(this.properties.getCassandraType());
}
private boolean isTestcontainers() {
return CassandraConnectionProperties.CassandraType.TESTCONTAINERS.equals(this.properties.getCassandraType());
}
private boolean isNotParent() {
return !isParent();
}

3
spring-data-cassandra/src/test/resources/config/cassandra-connection.properties
View File

@@ -1,9 +1,6 @@
# cassandra-connection.properties is needed twice because of enabled random port generation
# Generated ports are only valid for one module
build.cassandra.native_transport_port=@build.cassandra.native_transport_port@
build.cassandra.rpc_port=@build.cassandra.rpc_port@
build.cassandra.storage_port=@build.cassandra.storage_port@
build.cassandra.ssl_storage_port=@build.cassandra.ssl_storage_port@
build.cassandra.mode=@build.cassandra.mode@
build.cassandra.host=@build.cassandra.host@
build.cassandra.reuse-cluster=true