DATACASS-546 - Consider user-type dependencies during schema drop.

We now consider user-type dependencies while dropping user-defined types. User types can utilize other user-defined types and dropping a type that is in use causes the drop operation to fail. We're introspecting user types reported by Cassandra and construct a dependency graph that is used to determine the proper drop order.

spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/CassandraPersistentEntitySchemaCreator.java

@@ -157,7 +157,6 @@ public class CassandraPersistentEntitySchemaCreator {
 	private Map<CqlIdentifier, CassandraPersistentEntity<?>> getEntitiesByTableName(
 			Collection<? extends CassandraPersistentEntity<?>> entities) {
 
-		// TODO simplify by using Java 8 Streams API in 2.0.x
 		Map<CqlIdentifier, CassandraPersistentEntity<?>> byTableName = new HashMap<CqlIdentifier, CassandraPersistentEntity<?>>();
 
 		for (CassandraPersistentEntity<?> entity : entities) {

spring-data-cassandra/src/main/java/org/springframework/data/cassandra/core/CassandraPersistentEntitySchemaDropper.java

@@ -15,18 +15,26 @@
  */
 package org.springframework.data.cassandra.core;
 
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
 import java.util.Set;
 
 import org.springframework.cassandra.core.cql.CqlIdentifier;
 import org.springframework.data.cassandra.mapping.CassandraMappingContext;
 import org.springframework.data.cassandra.mapping.CassandraPersistentEntity;
 import org.springframework.util.Assert;
+import org.springframework.util.LinkedMultiValueMap;
+import org.springframework.util.MultiValueMap;
 
+import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.KeyspaceMetadata;
 import com.datastax.driver.core.TableMetadata;
+import com.datastax.driver.core.TupleType;
 import com.datastax.driver.core.UserType;
+import com.datastax.driver.core.UserType.Field;
 
 /**
  * Schema drop support for Cassandra based on {@link CassandraMappingContext} and {@link CassandraPersistentEntity}.
@@ -85,23 +93,216 @@ public class CassandraPersistentEntitySchemaDropper {
 	 */
 	public void dropUserTypes(boolean dropUnused) {
 
-		KeyspaceMetadata keyspaceMetadata = cassandraAdminOperations.getKeyspaceMetadata();
-
-		Collection<CassandraPersistentEntity<?>> userDefinedTypeEntities = mappingContext.getUserDefinedTypeEntities();
 		Set<CqlIdentifier> canRecreate = new HashSet<CqlIdentifier>();
 
-		for (CassandraPersistentEntity<?> userDefinedTypeEntity : userDefinedTypeEntities) {
+		for (CassandraPersistentEntity<?> entity : this.mappingContext.getUserDefinedTypeEntities()) {
-			canRecreate.add(userDefinedTypeEntity.getTableName());
+			canRecreate.add(entity.getTableName());
 		}
 
-		for (UserType userType : keyspaceMetadata.getUserTypes()) {
+		Collection<UserType> userTypes = this.cassandraAdminOperations.getKeyspaceMetadata().getUserTypes();
-			CqlIdentifier identifier = CqlIdentifier.cqlId(userType.getTypeName());
 
-			if (canRecreate.contains(identifier)) {
+		List<CqlIdentifier> identifiers = getUserTypesToDrop(userTypes);
-				cassandraAdminOperations.dropUserType(identifier);
+
-			} else if (dropUnused && !mappingContext.usesUserType(userType)) {
+		for (CqlIdentifier identifier : identifiers) {
-				cassandraAdminOperations.dropUserType(identifier);
+
+			if (canRecreate.contains(identifier) || (dropUnused && !mappingContext.usesUserType(identifier))) {
+				this.cassandraAdminOperations.dropUserType(identifier);
 			}
 		}
 	}
+
+	/**
+	 * Create {@link List} of {@link CqlIdentifier} with User-Defined type names to drop considering dependencies between
+	 * UDTs.
+	 *
+	 * @return {@link List} of {@link CqlIdentifier}.
+	 */
+	private List<CqlIdentifier> getUserTypesToDrop(Collection<UserType> knownUserTypes) {
+
+		List<CqlIdentifier> toDrop = new ArrayList<CqlIdentifier>();
+
+		UserTypeDependencyGraphBuilder builder = new UserTypeDependencyGraphBuilder();
+
+		for (UserType userType : knownUserTypes) {
+			builder.addUserType(userType);
+		}
+
+		UserTypeDependencyGraph dependencyGraph = builder.build();
+
+		final Set<CqlIdentifier> globalSeen = new LinkedHashSet<CqlIdentifier>();
+
+		for (UserType userType : knownUserTypes) {
+
+			CqlIdentifier typeName = CqlIdentifier.cqlId(userType.getTypeName());
+			toDrop.addAll(dependencyGraph.getDropOrder(typeName, new Predicate<CqlIdentifier>() {
+				@Override
+				public boolean test(CqlIdentifier type) {
+					return globalSeen.add(type);
+				}
+			}));
+		}
+
+		return toDrop;
+	}
+
+	/**
+	 * Builder for {@link UserTypeDependencyGraph}. Introspects {@link UserType} for dependencies to other user types to
+	 * build a dependency graph between user types.
+	 *
+	 * @author Mark Paluch
+	 * @since 2.0.7
+	 */
+	static class UserTypeDependencyGraphBuilder {
+
+		// Maps user types to other types they are referenced in.
+		private final MultiValueMap<CqlIdentifier, CqlIdentifier> dependencies = new LinkedMultiValueMap<CqlIdentifier, CqlIdentifier>();
+
+		/**
+		 * Add {@link UserType} to the builder and inspect its dependencies.
+		 *
+		 * @param userType must not be {@literal null.}
+		 */
+		void addUserType(UserType userType) {
+
+			final Set<CqlIdentifier> seen = new LinkedHashSet<CqlIdentifier>();
+
+			visitTypes(userType, new Predicate<CqlIdentifier>() {
+				@Override
+				public boolean test(CqlIdentifier type) {
+					return seen.add(type);
+				}
+			});
+		}
+
+		/**
+		 * Build the {@link UserTypeDependencyGraph}.
+		 *
+		 * @return the {@link UserTypeDependencyGraph}.
+		 */
+		UserTypeDependencyGraph build() {
+			return new UserTypeDependencyGraph(new LinkedMultiValueMap<CqlIdentifier, CqlIdentifier>(dependencies));
+		}
+
+		/**
+		 * Visit a {@link UserType} and its fields.
+		 *
+		 * @param userType
+		 * @param typeFilter
+		 */
+		private void visitTypes(UserType userType, final Predicate<CqlIdentifier> typeFilter) {
+
+			final CqlIdentifier typeName = CqlIdentifier.cqlId(userType.getTypeName());
+
+			if (!typeFilter.test(typeName)) {
+				return;
+			}
+
+			for (Field field : userType) {
+
+				if (field.getType() instanceof UserType) {
+
+					addDependency((UserType) field.getType(), typeName, typeFilter);
+
+					return;
+				}
+
+				doWithTypeArguments(field.getType(), new Consumer<DataType>() {
+					@Override
+					public void accept(DataType type) {
+						if (type instanceof UserType) {
+							addDependency((UserType) type, typeName, typeFilter);
+						}
+					}
+				});
+			}
+		}
+
+		private void addDependency(UserType userType, CqlIdentifier requiredBy, Predicate<CqlIdentifier> typeFilter) {
+
+			dependencies.add(CqlIdentifier.cqlId(userType.getTypeName()), requiredBy);
+
+			visitTypes(userType, typeFilter);
+		}
+
+		private static void doWithTypeArguments(DataType type, Consumer<DataType> callback) {
+
+			for (DataType nested : type.getTypeArguments()) {
+				callback.accept(nested);
+				doWithTypeArguments(nested, callback);
+			}
+
+			if (type instanceof TupleType) {
+
+				TupleType tupleType = (TupleType) type;
+
+				for (DataType nested : tupleType.getComponentTypes()) {
+					callback.accept(nested);
+					doWithTypeArguments(nested, callback);
+				}
+			}
+		}
+	}
+
+	/**
+	 * Dependency graph representing user type field dependencies to other user types.
+	 *
+	 * @author Mark Paluch
+	 * @since 1.5.12
+	 */
+	static class UserTypeDependencyGraph {
+
+		private final MultiValueMap<CqlIdentifier, CqlIdentifier> dependencies;
+
+		UserTypeDependencyGraph(MultiValueMap<CqlIdentifier, CqlIdentifier> dependencies) {
+			this.dependencies = dependencies;
+		}
+
+		/**
+		 * Returns the names of user types in the order they need to be dropped including type {@code typeName}.
+		 *
+		 * @param typeName
+		 * @param typeFilter
+		 * @return
+		 */
+		List<CqlIdentifier> getDropOrder(CqlIdentifier typeName, Predicate<CqlIdentifier> typeFilter) {
+
+			List<CqlIdentifier> toDrop = new ArrayList<CqlIdentifier>();
+
+			if (typeFilter.test(typeName)) {
+
+				List<CqlIdentifier> dependants = dependencies.get(typeName);
+				if (dependants != null) {
+
+					for (CqlIdentifier dependant : dependants) {
+						toDrop.addAll(getDropOrder(dependant, typeFilter));
+					}
+				}
+
+				toDrop.add(typeName);
+			}
+
+			return toDrop;
+		}
+	}
+
+	/**
+	 * Represents a predicate (boolean-valued function) of one argument.
+	 *
+	 * @param <T>
+	 */
+	interface Predicate<T> {
+
+		boolean test(T type);
+	}
+
+	/**
+	 * Represents an operation that accepts a single input argument and returns no result. Unlike most other functional
+	 * interfaces, {@code Consumer} is expected to operate via side-effects.
+	 *
+	 * @param <T>
+	 */
+	interface Consumer<T> {
+
+		void accept(T type);
+	}
 }

spring-data-cassandra/src/main/java/org/springframework/data/cassandra/mapping/BasicCassandraMappingContext.java

@@ -342,10 +342,15 @@ public class BasicCassandraMappingContext
 	 */
 	@Override
 	public boolean usesUserType(final UserType userType) {
+		return usesUserType(CqlIdentifier.cqlId(userType.getTypeName()));
+	}
 
-		CqlIdentifier identifier = CqlIdentifier.cqlId(userType.getTypeName());
+	/* (non-Javadoc)
-
+	 * @see org.springframework.data.cassandra.mapping.CassandraMappingContext#usesUserType(CqlIdentifier)
-		return (hasMappedUserType(identifier) || hasReferencedUserType(identifier));
+	 */
+	@Override
+	public boolean usesUserType(CqlIdentifier userType) {
+		return (hasMappedUserType(userType) || hasReferencedUserType(userType));
 	}
 
 	private boolean hasReferencedUserType(final CqlIdentifier identifier) {

spring-data-cassandra/src/main/java/org/springframework/data/cassandra/mapping/CassandraMappingContext.java

@@ -21,6 +21,7 @@ import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.TableMetadata;
 import com.datastax.driver.core.UserType;
 
+import org.springframework.cassandra.core.cql.CqlIdentifier;
 import org.springframework.cassandra.core.keyspace.CreateTableSpecification;
 import org.springframework.cassandra.core.keyspace.CreateUserTypeSpecification;
 import org.springframework.data.cassandra.convert.CustomConversions;
@@ -113,6 +114,15 @@ public interface CassandraMappingContext
 	 */
 	boolean usesUserType(UserType userType);
 
+	/**
+	 * Returns whether this mapping context has any entities using the given user type.
+	 *
+	 * @param userType must not be {@literal null}.
+	 * @return {@literal true} is this {@literal UserType} is used.
+	 * @since 1.5.12
+	 */
+	boolean usesUserType(CqlIdentifier userType);
+
 	/**
 	 * Returns the existing {@link CassandraPersistentEntity} for the given {@link Class}. If it is not yet known to this
 	 * {@link CassandraMappingContext}, an {@link IllegalArgumentException} is thrown.

spring-data-cassandra/src/test/java/org/springframework/data/cassandra/core/CassandraPersistentEntitySchemaCreatorUnitTests.java

@@ -17,9 +17,6 @@ package org.springframework.data.cassandra.core;
 
 import static org.mockito.Mockito.*;
 
-import java.util.List;
-import java.util.Set;
-
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -29,7 +26,6 @@ import org.mockito.Mockito;
 import org.mockito.runners.MockitoJUnitRunner;
 import org.springframework.cassandra.core.cql.CqlIdentifier;
 import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext;
-import org.springframework.data.cassandra.mapping.UserDefinedType;
 import org.springframework.data.cassandra.mapping.UserTypeResolver;
 
 import com.datastax.driver.core.UserType;
@@ -41,7 +37,7 @@ import com.datastax.driver.core.UserType;
  * @author Jens Schauder
  */
 @RunWith(MockitoJUnitRunner.class)
-public class CassandraPersistentEntitySchemaCreatorUnitTests {
+public class CassandraPersistentEntitySchemaCreatorUnitTests extends CassandraPersistentEntitySchemaTestSupport {
 
 	@Mock CassandraAdminOperations operations;
 
@@ -124,31 +120,4 @@ public class CassandraPersistentEntitySchemaCreatorUnitTests {
 			inOrder.verify(operations).execute(Mockito.contains("CREATE TYPE " + typename));
 		}
 	}
-
-	@UserDefinedType
-	static class UniverseType {
-		String name;
-	}
-
-	@UserDefinedType
-	static class MoonType {
-		UniverseType universeType;
-	}
-
-	@UserDefinedType
-	static class PlanetType {
-
-		Set<MoonType> moons;
-		UniverseType universeType;
-	}
-
-	@UserDefinedType
-	static class AstronautType {
-		String name;
-	}
-
-	@UserDefinedType
-	static class SpaceAgencyType {
-		List<AstronautType> astronauts;
-	}
 }

spring-data-cassandra/src/test/java/org/springframework/data/cassandra/core/CassandraPersistentEntitySchemaDropperUnitTests.java

@@ -17,8 +17,6 @@ package org.springframework.data.cassandra.core;
 
 import static org.mockito.Mockito.*;
 
-import lombok.Data;
-
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -26,18 +24,19 @@ import java.util.HashSet;
 import org.junit.Before;
 import org.junit.Test;
 import org.junit.runner.RunWith;
+import org.mockito.InOrder;
 import org.mockito.Mock;
+import org.mockito.Mockito;
 import org.mockito.runners.MockitoJUnitRunner;
 import org.springframework.cassandra.core.cql.CqlIdentifier;
-import org.springframework.data.annotation.Id;
 import org.springframework.data.cassandra.mapping.BasicCassandraMappingContext;
-import org.springframework.data.cassandra.mapping.Table;
-import org.springframework.data.cassandra.mapping.UserDefinedType;
 import org.springframework.data.cassandra.mapping.UserTypeResolver;
 
+import com.datastax.driver.core.DataType;
 import com.datastax.driver.core.KeyspaceMetadata;
 import com.datastax.driver.core.TableMetadata;
 import com.datastax.driver.core.UserType;
+import org.springframework.data.cassandra.repository.support.UserTypeBuilder;
 
 /**
  * Unit tests for {@link CassandraPersistentEntitySchemaDropper}.
@@ -46,13 +45,14 @@ import com.datastax.driver.core.UserType;
  */
 @SuppressWarnings("unchecked")
 @RunWith(MockitoJUnitRunner.class)
-public class CassandraPersistentEntitySchemaDropperUnitTests {
+public class CassandraPersistentEntitySchemaDropperUnitTests extends CassandraPersistentEntitySchemaTestSupport {
 
 	@Mock CassandraAdminOperations operations;
 	@Mock KeyspaceMetadata metadata;
-	@Mock UserType universetype;
+	UserType universetype = UserTypeBuilder.forName("universetype").withField("name", DataType.varchar()).build();
-	@Mock UserType moontype;
+	UserType moontype = UserTypeBuilder.forName("moontype").withField("universeType", universetype).build();
-	@Mock UserType planettype;
+	UserType planettype = UserTypeBuilder.forName("planettype").withField("moonType", DataType.set(moontype))
+			.withField("universeType", universetype).build();
 	@Mock TableMetadata person;
 	@Mock TableMetadata contact;
 
@@ -70,18 +70,12 @@ public class CassandraPersistentEntitySchemaDropperUnitTests {
 		});
 
 		when(operations.getKeyspaceMetadata()).thenReturn(metadata);
-		when(universetype.getTypeName()).thenReturn("universetype");
-		when(moontype.getTypeName()).thenReturn("moontype");
-		when(planettype.getTypeName()).thenReturn("planettype");
 		when(person.getName()).thenReturn("person");
 		when(contact.getName()).thenReturn("contact");
 	}
 
-	@Test // DATACASS-355
+	@Test // DATACASS-355, DATACASS-546
-	public void shouldDropTypes() throws Exception {
+	public void shouldDropTypesInOrderOfDependencies() throws Exception {
-
-		context.setInitialEntitySet(new HashSet<Class<?>>(Arrays.asList(MoonType.class, UniverseType.class)));
-		context.afterPropertiesSet();
 
 		when(metadata.getUserTypes()).thenReturn(Arrays.asList(universetype, moontype, planettype));
 
@@ -90,11 +84,7 @@ public class CassandraPersistentEntitySchemaDropperUnitTests {
 
 		schemaDropper.dropUserTypes(true);
 
-		verify(operations).dropUserType(CqlIdentifier.cqlId("universetype"));
+		verifyTypesGetDroppedInOrderFor("planettype", "moontype", "universetype");
-		verify(operations).dropUserType(CqlIdentifier.cqlId("moontype"));
-		verify(operations).dropUserType(CqlIdentifier.cqlId("planettype"));
-		verify(operations).getKeyspaceMetadata();
-		verifyNoMoreInteractions(operations);
 	}
 
 	@Test // DATACASS-355
@@ -153,18 +143,15 @@ public class CassandraPersistentEntitySchemaDropperUnitTests {
 		verifyNoMoreInteractions(operations);
 	}
 
-	@UserDefinedType
+	private void verifyTypesGetDroppedInOrderFor(String... typenames) {
-	@Data
-	static class UniverseType {}
 
-	@UserDefinedType
+		InOrder inOrder = Mockito.inOrder(operations);
-	static class MoonType {}
 
-	@UserDefinedType
+		for (String typename : typenames) {
-	static class PlanetType {}
+			inOrder.verify(operations).dropUserType(CqlIdentifier.cqlId(typename));
+		}
 
-	@Table
+		inOrder.verifyNoMoreInteractions();
-	static class Person {
-		@Id String id;
 	}
+
 }

spring-data-cassandra/src/test/java/org/springframework/data/cassandra/core/CassandraPersistentEntitySchemaTestSupport.java

@@ -0,0 +1,71 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.cassandra.core;
+
+import java.util.List;
+import java.util.Set;
+
+import org.springframework.data.annotation.Id;
+import org.springframework.data.cassandra.mapping.Indexed;
+import org.springframework.data.cassandra.mapping.Table;
+import org.springframework.data.cassandra.mapping.UserDefinedType;
+
+/**
+ * Support class for schema creation/drop tests.
+ *
+ * @author Mark Paluch
+ */
+public abstract class CassandraPersistentEntitySchemaTestSupport {
+
+	@UserDefinedType
+	static class UniverseType {
+		String name;
+	}
+
+	@UserDefinedType
+	static class MoonType {
+		UniverseType universeType;
+	}
+
+	@UserDefinedType
+	static class PlanetType {
+
+		Set<MoonType> moons;
+		UniverseType universeType;
+	}
+
+	@UserDefinedType
+	static class AstronautType {
+		String name;
+	}
+
+	@UserDefinedType
+	static class SpaceAgencyType {
+		List<AstronautType> astronauts;
+	}
+
+	@Table
+	static class IndexedEntity {
+
+		@Id String id;
+		@Indexed String firstName;
+	}
+
+	@Table
+	static class Person {
+		@Id String id;
+	}
+}

spring-data-cassandra/src/test/java/org/springframework/data/cassandra/repository/support/UserTypeBuilder.java

@@ -0,0 +1,83 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.cassandra.repository.support;
+
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.springframework.cassandra.core.cql.CqlIdentifier;
+
+import com.datastax.driver.core.CodecRegistry;
+import com.datastax.driver.core.DataType;
+import com.datastax.driver.core.ProtocolVersion;
+import com.datastax.driver.core.UserType;
+import com.datastax.driver.core.UserType.Field;
+
+/**
+ * @author Mark Paluch
+ */
+public class UserTypeBuilder {
+
+	private final CqlIdentifier typeName;
+	private List<Field> fields = new ArrayList<Field>();
+
+	private UserTypeBuilder(CqlIdentifier typeName) {
+		this.typeName = typeName;
+	}
+
+	public static UserTypeBuilder forName(String typeName) {
+		return forName(CqlIdentifier.cqlId(typeName));
+	}
+
+	public static UserTypeBuilder forName(CqlIdentifier typeName) {
+		return new UserTypeBuilder(typeName);
+	}
+
+	public UserTypeBuilder withField(String fieldName, DataType dataType) {
+		this.fields.add(createField(fieldName, dataType));
+		return this;
+	}
+
+	public UserType build() {
+		return createUserType(this.typeName.getUnquoted(), fields);
+	}
+
+	private Field createField(String fieldName, DataType dataType) {
+
+		try {
+			Constructor<Field> constructor = Field.class.getDeclaredConstructor(String.class, DataType.class);
+			constructor.setAccessible(true);
+			return constructor.newInstance(fieldName, dataType);
+		} catch (Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+
+	private UserType createUserType(String typeName, Collection<Field> fields) {
+
+		try {
+			Constructor<UserType> constructor = UserType.class.getDeclaredConstructor(String.class, String.class,
+					Collection.class, ProtocolVersion.class, CodecRegistry.class);
+			constructor.setAccessible(true);
+			return constructor.newInstance(typeName, typeName, fields, ProtocolVersion.NEWEST_SUPPORTED,
+					CodecRegistry.DEFAULT_INSTANCE);
+		} catch (Exception e) {
+			throw new IllegalStateException(e);
+		}
+	}
+}