diff --git a/.travis.yml b/.travis.yml index 90a67a287..9bdf0471a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,38 +1,21 @@ language: java -jdk: - - oraclejdk8 - env: global: - CASSANDRA_VERSION=3.11.2 matrix: - - PROFILE=ci - - PROFILE=spring5-next - CASSANDRA_VERSION=2.2.10 - CASSANDRA_VERSION=3.0.10 -addons: - apt: - packages: - - oracle-java8-installer - cache: directories: - $HOME/.m2 - - 'download' -sudo: required +sudo: false -before_install: - - sed -i.bak -e 's|https://nexus.codehaus.org/snapshots/|https://oss.sonatype.org/content/repositories/codehaus-snapshots/|g' ~/.m2/settings.xml +services: + - docker -install: if [ ! -z ${CASSANDRA_VERSION+x} ]; then ./setup-cassandra.sh; fi; +install: true -script: - - | - if [ ! -z ${CASSANDRA_DRIVER_VERSION+x} ]; then - mvn clean install -P${PROFILE},external-cassandra -Dcassandra-driver.version=${CASSANDRA_DRIVER_VERSION} -Dmaven.javadoc.skip=true - else - mvn clean install -P${PROFILE},external-cassandra -Dmaven.javadoc.skip=true - fi +script: ./mvnw clean install -Pci,testcontainers-cassandra,java11 -Dmaven.javadoc.skip=true -B diff --git a/Jenkinsfile b/Jenkinsfile index c38309450..1f8672dc0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -12,10 +12,46 @@ pipeline { } stages { + stage("Docker images") { + parallel { + stage('Publish JDK 8 + Cassandra 3.11') { + when { + changeset "ci/openjdk8-cassandra-3.11/**" + } + agent { label 'data' } + options { timeout(time: 30, unit: 'MINUTES') } + + steps { + script { + def image = docker.build("springci/spring-data-openjdk8-cassandra-3.11", "ci/openjdk8-cassandra-3.11/") + docker.withRegistry('', 'hub.docker.com-springbuildmaster') { + image.push() + } + } + } + } + stage('Publish JDK 11 + Cassandra 3.11') { + when { + changeset "ci/openjdk11-8-cassandra-3.11/**" + } + agent { label 'data' } + options { timeout(time: 30, unit: 'MINUTES') } + + steps { + script { + def image = docker.build("springci/spring-data-openjdk11-8-cassandra-3.11", "ci/openjdk11-8-cassandra-3.11/") + docker.withRegistry('', 'hub.docker.com-springbuildmaster') { + image.push() + } + } + } + } + } + } stage("Test") { when { anyOf { - branch 'master' + branch 'issue/DATACASS-699' not { triggeredBy 'UpstreamCause' } } } @@ -23,15 +59,39 @@ pipeline { stage("test: baseline") { agent { docker { - image 'adoptopenjdk/openjdk8:latest' + image 'springci/spring-data-openjdk8-cassandra-3.11:latest' label 'data' - args '-v $HOME:/tmp/jenkins-home' } } options { timeout(time: 30, unit: 'MINUTES') } steps { - sh 'rm -rf ?' - sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw clean dependency:list verify -Dsort -U -B' + sh 'mkdir -p /tmp/jenkins-home' + sh 'JAVA_HOME=/opt/java/openjdk /opt/cassandra/bin/cassandra -R &' + sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,external-cassandra clean dependency:list verify -Dsort -U -B' + } + } + } + } + stage("Test other configurations") { + when { + anyOf { + branch 'master' + not { triggeredBy 'UpstreamCause' } + } + } + parallel { + stage("test: baseline (jdk11)") { + agent { + docker { + image 'springci/spring-data-openjdk11-8-cassandra-3.11:latest' + label 'data' + } + } + options { timeout(time: 30, unit: 'MINUTES') } + steps { + sh 'mkdir -p /tmp/jenkins-home' + sh 'JAVA_HOME=/opt/java/openjdk8 /opt/cassandra/bin/cassandra -R &' + sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,external-cassandra,java11 clean dependency:list verify -Dsort -U -B' } } } @@ -47,7 +107,6 @@ pipeline { docker { image 'adoptopenjdk/openjdk8:latest' label 'data' - args '-v $HOME:/tmp/jenkins-home' } } options { timeout(time: 20, unit: 'MINUTES') } @@ -57,7 +116,7 @@ pipeline { } steps { - sh 'rm -rf ?' + sh 'mkdir -p /tmp/jenkins-home' sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,artifactory ' + '-Dartifactory.server=https://repo.spring.io ' + "-Dartifactory.username=${ARTIFACTORY_USR} " + @@ -76,7 +135,6 @@ pipeline { docker { image 'adoptopenjdk/openjdk8:latest' label 'data' - args '-v $HOME:/tmp/jenkins-home' } } options { timeout(time: 20, unit: 'MINUTES') } @@ -86,6 +144,7 @@ pipeline { } steps { + sh 'mkdir -p /tmp/jenkins-home' sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/jenkins-home" ./mvnw -Pci,distribute ' + '-Dartifactory.server=https://repo.spring.io ' + "-Dartifactory.username=${ARTIFACTORY_USR} " + diff --git a/ci/openjdk11-8-cassandra-3.11/Dockerfile b/ci/openjdk11-8-cassandra-3.11/Dockerfile new file mode 100644 index 000000000..653764f8d --- /dev/null +++ b/ci/openjdk11-8-cassandra-3.11/Dockerfile @@ -0,0 +1,23 @@ +FROM adoptopenjdk/openjdk11:latest + +RUN set -eux; \ + CASSANDRA_URL='https://www-us.apache.org/dist/cassandra/3.11.5/apache-cassandra-3.11.5-bin.tar.gz'; \ + curl -LfsSo /tmp/cassandra.tar.gz ${CASSANDRA_URL}; \ + mkdir -p /opt/cassandra /opt/cassandra/data /opt/cassandra/logs; \ + cd /opt/cassandra; \ + tar -xf /tmp/cassandra.tar.gz --strip-components=1; \ + rm -rf /tmp/cassandra.tar.gz; \ + chmod -R a+rwx /opt/cassandra; \ + useradd -d /home/jenkins-docker -m -u 1001 -U jenkins-docker; + +RUN set -eux; \ + BINARY_URL='https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u232-b09/OpenJDK8U-jdk_x64_linux_hotspot_8u232b09.tar.gz'; \ + curl -LfsSo /tmp/openjdk.tar.gz ${BINARY_URL}; \ + mkdir -p /opt/java/openjdk8; \ + cd /opt/java/openjdk8; \ + tar -xf /tmp/openjdk.tar.gz --strip-components=1; \ + rm -rf /tmp/openjdk.tar.gz; + +ENV PATH="/opt/java/openjdk8/bin:$PATH" +ENV MAX_HEAP_SIZE=1500M +ENV HEAP_NEWSIZE=300M diff --git a/ci/openjdk8-cassandra-3.11/Dockerfile b/ci/openjdk8-cassandra-3.11/Dockerfile new file mode 100644 index 000000000..a410cea5d --- /dev/null +++ b/ci/openjdk8-cassandra-3.11/Dockerfile @@ -0,0 +1,14 @@ +FROM adoptopenjdk/openjdk8:latest + +RUN set -eux; \ + CASSANDRA_URL='https://www-us.apache.org/dist/cassandra/3.11.5/apache-cassandra-3.11.5-bin.tar.gz'; \ + curl -LfsSo /tmp/cassandra.tar.gz ${CASSANDRA_URL}; \ + mkdir -p /opt/cassandra /opt/cassandra/data /opt/cassandra/logs; \ + cd /opt/cassandra; \ + tar -xf /tmp/cassandra.tar.gz --strip-components=1; \ + rm -rf /tmp/cassandra.tar.gz; \ + chmod -R a+rwx /opt/cassandra; \ + useradd -d /home/jenkins-docker -m -u 1001 -U jenkins-docker; + +ENV MAX_HEAP_SIZE=1500M +ENV HEAP_NEWSIZE=300M diff --git a/pom.xml b/pom.xml index 6695cfe5c..1ad07adf1 100644 --- a/pom.xml +++ b/pom.xml @@ -1,5 +1,7 @@ - + 4.0.0 @@ -170,6 +172,14 @@ + + org.testcontainers + testcontainers-bom + ${testcontainers} + pom + import + + org.xerial.snappy snappy-java @@ -237,7 +247,9 @@ **/test/performance/**/* - src/test/resources/logging.properties + + src/test/resources/logging.properties + @@ -259,7 +271,9 @@ **/test/performance/**/* - src/test/resources/logging.properties + + src/test/resources/logging.properties + @@ -283,12 +297,22 @@ + + src/test/resources + false + + **/*.xml + + src/test/resources true **/* + + **/*.xml + @@ -322,10 +346,12 @@ generate-test-resources - build.cassandra.native_transport_port + build.cassandra.native_transport_port + build.cassandra.rpc_port build.cassandra.storage_port - build.cassandra.ssl_storage_port + build.cassandra.ssl_storage_port + @@ -339,12 +365,25 @@ external - 9042 + 9042 + 9160 7000 7001 + + testcontainers-cassandra + + + testcontainers + 0 + + 0 + 0 + 0 + + diff --git a/spring-data-cassandra/pom.xml b/spring-data-cassandra/pom.xml index 6cb3368dd..877d1a9ec 100644 --- a/spring-data-cassandra/pom.xml +++ b/spring-data-cassandra/pom.xml @@ -161,6 +161,12 @@ cassandra-all + + org.testcontainers + cassandra + test + + org.xerial.snappy snappy-java diff --git a/spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/ReactiveCassandraBatchTemplate.java b/spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/ReactiveCassandraBatchTemplate.java index 62384a875..2be9df9be 100644 --- a/spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/ReactiveCassandraBatchTemplate.java +++ b/spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/ReactiveCassandraBatchTemplate.java @@ -135,7 +135,7 @@ class ReactiveCassandraBatchTemplate implements ReactiveCassandraBatchOperations .collectList() // .flatMap(statements -> { - this.batch.addStatements(statements); + this.batch.addStatements((List>) statements); return this.operations.getReactiveCqlOperations().queryForResultSet(this.batch.build()); diff --git a/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/support/CassandraConnectionProperties.java b/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/support/CassandraConnectionProperties.java index ada2383d3..7fe6902b0 100644 --- a/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/support/CassandraConnectionProperties.java +++ b/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/support/CassandraConnectionProperties.java @@ -15,7 +15,12 @@ */ package org.springframework.data.cassandra.support; +import java.io.File; +import java.io.FileOutputStream; import java.io.InputStream; +import java.lang.ref.WeakReference; +import java.util.ArrayList; +import java.util.List; import java.util.Properties; import org.springframework.core.convert.converter.Converter; @@ -30,6 +35,8 @@ import org.springframework.util.Assert; @SuppressWarnings("serial") public class CassandraConnectionProperties extends Properties { + private final static List> instances = new ArrayList<>(); + protected String resourceName; /** @@ -40,16 +47,43 @@ public class CassandraConnectionProperties extends Properties { this("/config/cassandra-connection.properties"); } + public void update() { + try { + // Caution: Rewriting properties during initialization. + File file = new File(getClass().getResource(resourceName).toURI()); + + try (FileOutputStream fos = new FileOutputStream(file)) { + store(fos, ""); + } + + reload(); + } catch (Exception e) { + e.printStackTrace(); + throw new IllegalStateException(e); + } + } + + public static void reload() { + for (WeakReference ref : instances) { + + CassandraConnectionProperties properties = ref.get(); + if (properties != null) { + properties.loadProperties(); + } + } + } + protected CassandraConnectionProperties(String resourceName) { this.resourceName = resourceName; loadProperties(); + + instances.add(new WeakReference<>(this)); } private void loadProperties() { loadProperties(this.resourceName); - putAll(System.getProperties()); } private void loadProperties(String resourceName) { @@ -68,11 +102,28 @@ public class CassandraConnectionProperties extends Properties { if (in != null) { try { in.close(); - } catch (Exception ignore) { } + } catch (Exception ignore) {} } } } + @Override + public String getProperty(String key) { + + String value = super.getProperty(key); + if (value == null) { + value = System.getProperty(key); + } + + return value; + } + + @Override + public synchronized Object setProperty(String key, String value) { + System.setProperty(key, value); + return super.setProperty(key, value); + } + /** * @return the Cassandra hostname */ @@ -80,6 +131,10 @@ public class CassandraConnectionProperties extends Properties { return getProperty("build.cassandra.host"); } + public void setCassandraHost(String host) { + setProperty("build.cassandra.host", host); + } + /** * @return the Cassandra port (native). */ @@ -87,6 +142,10 @@ public class CassandraConnectionProperties extends Properties { return getInt("build.cassandra.native_transport_port"); } + public void setCassandraPort(int port) { + setProperty("build.cassandra.native_transport_port", "" + port); + } + /** * @return the Cassandra RPC port */ @@ -115,9 +174,12 @@ public class CassandraConnectionProperties extends Properties { String cassandraType = getProperty("build.cassandra.mode"); - return CassandraType.EXTERNAL.name().equalsIgnoreCase(cassandraType) - ? CassandraType.EXTERNAL - : CassandraType.EMBEDDED; + if (CassandraType.TESTCONTAINERS.name().equalsIgnoreCase(cassandraType)) { + return CassandraType.TESTCONTAINERS; + } + + return CassandraType.EXTERNAL.name().equalsIgnoreCase(cassandraType) ? CassandraType.EXTERNAL + : CassandraType.EMBEDDED; } /** @@ -161,12 +223,12 @@ public class CassandraConnectionProperties extends Properties { String message = "%1$s: cannot parse value [%2$s] of property [%3$s] as a [%4$s]"; - throw new IllegalArgumentException(String.format(message, this.resourceName, propertyValue, propertyName, - type.getSimpleName()), cause); + throw new IllegalArgumentException( + String.format(message, this.resourceName, propertyValue, propertyName, type.getSimpleName()), cause); } } public enum CassandraType { - EMBEDDED, EXTERNAL + EMBEDDED, EXTERNAL, TESTCONTAINERS; } } diff --git a/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/test/util/CassandraRule.java b/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/test/util/CassandraRule.java index fa102850c..3595ecfbe 100644 --- a/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/test/util/CassandraRule.java +++ b/spring-data-cassandra/src/test/java/org/springframework/data/cassandra/test/util/CassandraRule.java @@ -31,13 +31,15 @@ import org.springframework.util.Assert; import org.springframework.util.SocketUtils; import org.springframework.util.StringUtils; +import org.testcontainers.containers.CassandraContainer; + import com.datastax.oss.driver.api.core.CqlIdentifier; import com.datastax.oss.driver.api.core.CqlSession; import com.datastax.oss.driver.api.core.CqlSessionBuilder; /** * JUnit Rule used to provide a Cassandra context for integration tests. This rule can use/spin up either an embedded - * Cassandra instance or use an external instance. Typical usage: + * Cassandra instance, a TestContainer or use an external instance. Typical usage: * * 
  * public class MyIntegrationTest {
@@ -46,14 +48,20 @@ import com.datastax.oss.driver.api.core.CqlSessionBuilder;
  * }
  *
* + * Derives its configuration from {@code /config/cassandra-connection.properties} and {@link System#getenv(String)} + * {@code CASSANDRA_VERSION} to configure the Cassandra version via Testcontainers. + * * @author Mark Paluch * @author John Blum * @since 1.5 + * @see CassandraConnectionProperties */ public class CassandraRule extends ExternalResource { private static ResourceHolder resourceHolder; + private static CassandraContainer container; + private final long startupTimeout; @SuppressWarnings("all") private final CassandraConnectionProperties properties = new CassandraConnectionProperties(); @@ -248,12 +256,17 @@ public class CassandraRule extends ExternalResource { if (isStartNeeded()) { configureRemoteJmxPort(); - runEmbeddedCassandra(); + + if (isEmbedded()) { + runEmbeddedCassandra(); + } else { + runTestcontainerCassandra(); + } } } private boolean isStartNeeded() { - return isParent() && isEmbedded(); + return isParent() && (isEmbedded() || isTestcontainers()); } private void configureRemoteJmxPort() { @@ -270,6 +283,24 @@ public class CassandraRule extends ExternalResource { } } + private void runTestcontainerCassandra() { + + if (container == null) { + String cassandra_version = System.getenv("CASSANDRA_VERSION"); + if (StringUtils.hasText(cassandra_version)) { + container = new CassandraContainer<>("cassandra:" + cassandra_version); + } else { + container = new CassandraContainer<>(); + } + + container.start(); + + this.properties.setCassandraHost(container.getContainerIpAddress()); + this.properties.setCassandraPort(container.getFirstMappedPort()); + this.properties.update(); + } + } + private synchronized void initializeConnection() { if (isParent()) { @@ -305,11 +336,19 @@ public class CassandraRule extends ExternalResource { private String resolveHost() { + if (isTestcontainers()) { + return container.getContainerIpAddress(); + } + return isEmbedded() ? EmbeddedCassandraServerHelper.getHost() : this.properties.getCassandraHost(); } private int resolvePort() { + if (isTestcontainers()) { + return container.getFirstMappedPort(); + } + return isEmbedded() ? EmbeddedCassandraServerHelper.getNativeTransportPort() : this.properties.getCassandraPort(); } @@ -378,6 +417,10 @@ public class CassandraRule extends ExternalResource { return CassandraConnectionProperties.CassandraType.EMBEDDED.equals(this.properties.getCassandraType()); } + private boolean isTestcontainers() { + return CassandraConnectionProperties.CassandraType.TESTCONTAINERS.equals(this.properties.getCassandraType()); + } + private boolean isNotParent() { return !isParent(); } diff --git a/spring-data-cassandra/src/test/resources/config/cassandra-connection.properties b/spring-data-cassandra/src/test/resources/config/cassandra-connection.properties index c28cb2327..5cbe38a12 100644 --- a/spring-data-cassandra/src/test/resources/config/cassandra-connection.properties +++ b/spring-data-cassandra/src/test/resources/config/cassandra-connection.properties @@ -1,9 +1,6 @@ # cassandra-connection.properties is needed twice because of enabled random port generation # Generated ports are only valid for one module build.cassandra.native_transport_port=@build.cassandra.native_transport_port@ -build.cassandra.rpc_port=@build.cassandra.rpc_port@ -build.cassandra.storage_port=@build.cassandra.storage_port@ -build.cassandra.ssl_storage_port=@build.cassandra.ssl_storage_port@ build.cassandra.mode=@build.cassandra.mode@ build.cassandra.host=@build.cassandra.host@ build.cassandra.reuse-cluster=true