Fix serialiation/deserialization for field level encryption. (#1622)

Closes #1621.
This commit is contained in:
Michael Reiche
2022-12-16 13:14:43 -08:00
committed by mikereiche
parent 761ed0d8a1
commit 0030214aa1
10 changed files with 85 additions and 68 deletions

View File

@@ -26,8 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import com.couchbase.client.java.encryption.annotation.Encrypted;
import com.fasterxml.jackson.annotation.JsonValue;
import org.springframework.beans.factory.config.BeanDefinition;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ClassPathScanningCandidateComponentProvider;
@@ -77,11 +75,13 @@ import com.couchbase.client.core.env.PasswordAuthenticator;
import com.couchbase.client.core.error.CouchbaseException;
import com.couchbase.client.java.Cluster;
import com.couchbase.client.java.codec.JacksonJsonSerializer;
import com.couchbase.client.java.encryption.annotation.Encrypted;
import com.couchbase.client.java.encryption.databind.jackson.EncryptionModule;
import com.couchbase.client.java.env.ClusterEnvironment;
import com.couchbase.client.java.json.JacksonTransformers;
import com.couchbase.client.java.json.JsonValueModule;
import com.couchbase.client.java.query.QueryScanConsistency;
import com.fasterxml.jackson.annotation.JsonValue;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -312,7 +312,7 @@ public abstract class AbstractCouchbaseConfiguration {
}
final public ObjectMapper getObjectMapper() {
if(objectMapper == null) {
if (objectMapper == null) {
synchronized (this) {
if (objectMapper == null) {
objectMapper = couchbaseObjectMapper();
@@ -403,7 +403,7 @@ public abstract class AbstractCouchbaseConfiguration {
*/
@Bean(name = BeanNames.COUCHBASE_CUSTOM_CONVERSIONS)
public CustomConversions customConversions() {
return customConversions(getCryptoManager());
return customConversions(getCryptoManager(), getObjectMapper());
}
/**
@@ -414,11 +414,12 @@ public abstract class AbstractCouchbaseConfiguration {
* @param cryptoManager
* @return must not be {@literal null}.
*/
public CustomConversions customConversions(CryptoManager cryptoManager) {
public CustomConversions customConversions(CryptoManager cryptoManager, ObjectMapper objectMapper) {
List<GenericConverter> newConverters = new ArrayList();
CustomConversions customConversions = CouchbaseCustomConversions.create(configurationAdapter -> {
SimplePropertyValueConversions valueConversions = new SimplePropertyValueConversions();
valueConversions.setConverterFactory(new CouchbasePropertyValueConverterFactory(cryptoManager, annotationToConverterMap()));
valueConversions.setConverterFactory(
new CouchbasePropertyValueConverterFactory(cryptoManager, annotationToConverterMap(), objectMapper));
valueConversions.setValueConverterRegistry(new PropertyValueConverterRegistrar().buildRegistry());
valueConversions.afterPropertiesSet(); // wraps the CouchbasePropertyValueConverterFactory with CachingPVCFactory
configurationAdapter.setPropertyValueConversions(valueConversions);
@@ -431,17 +432,18 @@ public abstract class AbstractCouchbaseConfiguration {
return customConversions;
}
Map<Class<? extends Annotation>,Class<?>> annotationToConverterMap(){
Map<Class<? extends Annotation>,Class<?>> map= new HashMap();
Map<Class<? extends Annotation>, Class<?>> annotationToConverterMap() {
Map<Class<? extends Annotation>, Class<?>> map = new HashMap();
map.put(Encrypted.class, CryptoConverter.class);
map.put(JsonValue.class, JsonValueConverter.class);
return map;
}
/**
* cryptoManager can be null, so it cannot be a bean and then used as an arg for bean methods
*/
private CryptoManager getCryptoManager() {
if(cryptoManager == null) {
if (cryptoManager == null) {
synchronized (this) {
if (cryptoManager == null) {
cryptoManager = cryptoManager();

View File

@@ -160,7 +160,8 @@ public abstract class AbstractCouchbaseConverter implements CouchbaseConverter,
Class<?> elementType = value.getClass();
if (elementType == null || conversions.isSimpleType(elementType)) {
// superseded by EnumCvtrs value = Enum.class.isAssignableFrom(value.getClass()) ? ((Enum<?>) value).name() : value;
// superseded by EnumCvtrs value = Enum.class.isAssignableFrom(value.getClass()) ? ((Enum<?>) value).name() :
// value;
} else if (value instanceof Collection || elementType.isArray()) {
TypeInformation<?> type = ClassTypeInformation.from(value.getClass());
value = ((MappingCouchbaseConverter) this).writeCollectionInternal(MappingCouchbaseConverter.asCollection(value),
@@ -168,7 +169,7 @@ public abstract class AbstractCouchbaseConverter implements CouchbaseConverter,
} else {
CouchbaseDocument embeddedDoc = new CouchbaseDocument();
TypeInformation<?> type = ClassTypeInformation.from(value.getClass());
((MappingCouchbaseConverter) this).writeInternalRoot(value, embeddedDoc, type, false, null);
((MappingCouchbaseConverter) this).writeInternalRoot(value, embeddedDoc, type, false, null, true);
value = embeddedDoc;
}
return value;

View File

@@ -31,7 +31,7 @@ import org.springframework.data.mapping.PersistentProperty;
import com.couchbase.client.core.encryption.CryptoManager;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonValue;
import org.springframework.util.Assert;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* Accept the Couchbase @Encrypted and @JsonValue annotations in addition to @ValueConverter annotation.<br>
@@ -48,14 +48,16 @@ import org.springframework.util.Assert;
*/
public class CouchbasePropertyValueConverterFactory implements PropertyValueConverterFactory {
final CryptoManager cryptoManager;
final Map<Class<? extends Annotation>, Class<?>> annotationToConverterMap;
private final CryptoManager cryptoManager;
private final ObjectMapper objectMapper;
private final Map<Class<? extends Annotation>, Class<?>> annotationToConverterMap;
static protected final Map<Class<?>, Optional<PropertyValueConverter<?, ?, ?>>> converterCacheForType = new ConcurrentHashMap<>();
public CouchbasePropertyValueConverterFactory(CryptoManager cryptoManager,
Map<Class<? extends Annotation>, Class<?>> annotationToConverterMap) {
Map<Class<? extends Annotation>, Class<?>> annotationToConverterMap, ObjectMapper objectMapper) {
this.cryptoManager = cryptoManager;
this.annotationToConverterMap = annotationToConverterMap;
this.objectMapper = objectMapper;
}
/**
@@ -155,7 +157,7 @@ public class CouchbasePropertyValueConverterFactory implements PropertyValueConv
// CryptoConverter takes a cryptoManager argument
if (CryptoConverter.class.isAssignableFrom(converterType)) {
return (PropertyValueConverter<DV, SV, P>) new CryptoConverter(cryptoManager);
return (PropertyValueConverter<DV, SV, P>) new CryptoConverter(cryptoManager, objectMapper);
} else if (property != null) { // try constructor that takes PersistentProperty
try {
Constructor<?> constructor = converterType.getConstructor(PersistentProperty.class);

View File

@@ -20,15 +20,12 @@ import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.springframework.core.convert.ConversionFailedException;
import org.springframework.core.convert.ConversionService;
import org.springframework.core.convert.ConverterNotFoundException;
import org.springframework.data.convert.CustomConversions;
import org.springframework.data.convert.PropertyValueConverter;
import org.springframework.data.convert.ValueConversionContext;
import org.springframework.data.couchbase.core.convert.translation.JacksonTranslationService;
import org.springframework.data.couchbase.core.convert.translation.TranslationService;
import org.springframework.data.couchbase.core.mapping.CouchbaseDocument;
import org.springframework.data.couchbase.core.mapping.CouchbasePersistentProperty;
import org.springframework.data.mapping.PersistentProperty;
@@ -41,20 +38,24 @@ import com.couchbase.client.java.encryption.annotation.Encrypted;
import com.couchbase.client.java.json.JsonArray;
import com.couchbase.client.java.json.JsonObject;
import com.couchbase.client.java.json.JsonValue;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* Encrypt/Decrypted properties annotated. This is registered in
* {@link org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration#customConversions(CryptoManager)}.
* {@link org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration#customConversions(CryptoManager, ObjectMapper)}.
*
* @author Michael Reiche
*/
public class CryptoConverter implements
PropertyValueConverter<Object, CouchbaseDocument, ValueConversionContext<? extends PersistentProperty<?>>> {
CryptoManager cryptoManager;
private final CryptoManager cryptoManager;
private final ObjectMapper objectMapper;
public CryptoConverter(CryptoManager cryptoManager) {
public CryptoConverter(CryptoManager cryptoManager, ObjectMapper objectMapper) {
this.cryptoManager = cryptoManager;
this.objectMapper = objectMapper;
}
@Override
@@ -81,22 +82,12 @@ public class CryptoConverter implements
CouchbasePersistentProperty property = context.getProperty();
CustomConversions cnvs = context.getConverter().getConversions();
ConversionService svc = context.getConverter().getConversionService();
Class<?> type = property.getType();
String decryptedString = new String(decrypted);
if ("null".equals(decryptedString)) {
return null;
}
// TODO - as-is, this never gets ran through ObjectMapper() -
// TODO - i.e. @JsonValue etc will not be processed by ObjectMapper
/* this what we would do if we could use a JsonParser with a beanPropertyTypeRef
final JsonParser plaintextParser = p.getCodec().getFactory().createParser(plaintext);
plaintextParser.setCodec(p.getCodec());
return plaintextParser.readValueAs(beanPropertyTypeRef);
*/
if (!cnvs.isSimpleType(type) && !type.isArray()) {
JsonObject jo = JsonObject.fromJson(decryptedString);
@@ -133,16 +124,20 @@ public class CryptoConverter implements
}
plainText = ja.toBytes();
} else if (cnvs.isSimpleType(sourceType)) { // simpleType
String plainString = value != null ? value.toString() : null; // TODO - this will ignore @JsonValue
String plainString = value != null ? value.toString() : null;
if ((sourceType == String.class || targetType == String.class) || sourceType == Character.class
|| sourceType == char.class || Enum.class.isAssignableFrom(sourceType)
|| Locale.class.isAssignableFrom(sourceType)) {
// TODO use jackson serializer here
plainString = "\"" + plainString.replaceAll("\"", "\\\"") + "\"";
|| sourceType == char.class || sourceType.isEnum() || Locale.class.isAssignableFrom(sourceType)) {
try {
plainString = objectMapper.writeValueAsString(plainString);// put quotes around strings
} catch (JsonProcessingException e) {
throw new RuntimeException(e);
}
}
plainText = plainString.getBytes(StandardCharsets.UTF_8);
} else { // an entity
plainText = JsonObject.fromJson(context.read(value).toString().getBytes(StandardCharsets.UTF_8)).toBytes();
CouchbaseDocument doc = new CouchbaseDocument();
context.getConverter().writeInternalRoot(value, doc, property.getTypeInformation(), false, property, false);
plainText = JsonObject.from(doc.export()).toBytes();
}
return plainText;
}

View File

@@ -443,7 +443,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
typeMapper.writeType(type, target);
}
writeInternalRoot(source, target, type, true, null);
writeInternalRoot(source, target, type, true, null, true);
if (target.getId() == null) {
throw new MappingException("An ID property is needed, but not found/could not be generated on this entity.");
}
@@ -459,7 +459,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
*/
@SuppressWarnings("unchecked")
public void writeInternalRoot(final Object source, CouchbaseDocument target, TypeInformation<?> typeHint,
boolean withId, CouchbasePersistentProperty property) {
boolean withId, CouchbasePersistentProperty property, boolean processValueConverter) {
if (source == null) {
return;
}
@@ -480,7 +480,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
}
CouchbasePersistentEntity<?> entity = mappingContext.getPersistentEntity(source.getClass());
writeInternalEntity(source, target, entity, withId, property);
writeInternalEntity(source, target, entity, withId, property, processValueConverter);
addCustomTypeKeyIfNecessary(typeHint, source, target);
}
@@ -517,7 +517,8 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
* @param withId one of the top-level properties is the id for the document
*/
protected void writeInternalEntity(final Object source, final CouchbaseDocument target,
final CouchbasePersistentEntity<?> entity, boolean withId, CouchbasePersistentProperty prop) {
final CouchbasePersistentEntity<?> entity, boolean withId, CouchbasePersistentProperty prop,
boolean processValueConverter) {
if (source == null) {
return;
}
@@ -566,7 +567,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
}
});
if (prop != null && conversions.hasValueConverter(prop)) { // whole entity is encrypted
if (prop != null && processValueConverter && conversions.hasValueConverter(prop)) { // whole entity is encrypted
Map<String, Object> propertyConverted = (Map<String, Object>) conversions.getPropertyValueConversions()
.getValueConverter(prop).write(source, new CouchbaseConversionContext(prop, this, accessor));
target.setContent(JsonObject.from(propertyConverted));
@@ -677,7 +678,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
CouchbasePersistentEntity<?> entity = isSubtype(prop.getType(), source.getClass())
? mappingContext.getRequiredPersistentEntity(source.getClass())
: mappingContext.getRequiredPersistentEntity(prop);
writeInternalEntity(source, propertyDoc, entity, false, prop);
writeInternalEntity(source, propertyDoc, entity, false, prop, true);
target.put(maybeMangle(prop), propertyDoc);
}
@@ -728,7 +729,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
prop.getTypeInformation(), prop, getPropertyAccessor(val)));
} else {
CouchbaseDocument embeddedDoc = new CouchbaseDocument();
writeInternalRoot(val, embeddedDoc, prop.getTypeInformation(), false, prop);
writeInternalRoot(val, embeddedDoc, prop.getTypeInformation(), false, prop, true);
target.put(simpleKey, embeddedDoc);
}
} else {
@@ -772,7 +773,7 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
} else {
CouchbaseDocument embeddedDoc = new CouchbaseDocument();
writeInternalRoot(element, embeddedDoc,
prop != null ? prop.getTypeInformation() : TypeInformation.of(elementType), false, prop);
prop != null ? prop.getTypeInformation() : TypeInformation.of(elementType), false, prop, true);
target.put(embeddedDoc);
}
@@ -873,9 +874,9 @@ public class MappingCouchbaseConverter extends AbstractCouchbaseConverter implem
* @param processValueConverter
* @return
*/
public Object getPotentiallyConvertedSimpleWrite(final CouchbasePersistentProperty value,
public Object getPotentiallyConvertedSimpleWrite(final CouchbasePersistentProperty property,
ConvertingPropertyAccessor<Object> accessor, boolean processValueConverter) {
return convertForWriteIfNeeded(value, accessor, processValueConverter); // can access annotations
return convertForWriteIfNeeded(property, accessor, processValueConverter); // can access annotations
}
/**

View File

@@ -27,7 +27,6 @@ import java.util.Collection;
import java.util.List;
import java.util.UUID;
import com.fasterxml.jackson.databind.ObjectWriter;
import org.springframework.core.convert.converter.Converter;
import org.springframework.data.convert.ReadingConverter;
import org.springframework.data.convert.WritingConverter;
@@ -200,7 +199,7 @@ public final class OtherConverters {
/**
* Writing converter for Enums. This is registered in
* {@link org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration#customConversions( CryptoManager)}.
* {@link org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration#customConversions( CryptoManager, ObjectMapper)}.
* The corresponding reading converters are in {@link IntegerToEnumConverterFactory} and
* {@link StringToEnumConverterFactory}
*/
@@ -220,12 +219,12 @@ public final class OtherConverters {
objectMapper.writeValue(generator, source);
String s = writer.toString();
if (s != null && s.startsWith("\"")) {
return objectMapper.readValue(s,String.class);
return objectMapper.readValue(s, String.class);
}
if ("true".equals(s) || "false".equals(s)) {
return objectMapper.readValue(s,Boolean.class);
return objectMapper.readValue(s, Boolean.class);
}
return objectMapper.readValue(s,Number.class);
return objectMapper.readValue(s, Number.class);
} catch (IOException e) {
throw new RuntimeException(e);
}

View File

@@ -15,7 +15,6 @@
*/
package org.springframework.data.couchbase.domain;
import com.couchbase.client.java.encryption.annotation.Encrypted;
import org.springframework.data.couchbase.core.mapping.Document;
@Document
@@ -26,6 +25,7 @@ public class Address extends ComparableEntity {
// for N1qlJoin
private String id;
private String parentId;
private ETurbulenceCategory turbulence;
public Address() {}
@@ -45,6 +45,13 @@ public class Address extends ComparableEntity {
this.city = city;
}
public ETurbulenceCategory getTurbulence() {
return turbulence;
}
public void setTurbulence(ETurbulenceCategory turbulence) {
this.turbulence = turbulence;
}
public String getParentId() {
return parentId;
}

View File

@@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonValue;
* @author Michael Reiche
*/
public enum ETurbulenceCategory {
T10("10%"), T20("20%"), T30("30%");
T10("\"10%"), T20("\"20%"), T30("\"30%");
private final String code;

View File

@@ -137,6 +137,7 @@ public class UserEncrypted extends AbstractUser implements Serializable {
@Encrypted public UUID encUUID;
@Encrypted public DateTime encDateTime;
@Encrypted public ETurbulenceCategory turbulence;
@Encrypted public Address encAddress = new Address();
public Date plainDate;
@@ -228,8 +229,8 @@ public class UserEncrypted extends AbstractUser implements Serializable {
encCharacter = 'a';
encCharacters = new Character[] { 'a', 'b' };
encString = "myString";
encStrings = new String[] { "myString" };
encString = "myS\"tring";
encStrings = new String[] { "mySt\"ring" };
encDate = NOW_Date;
encDates = new Date[] { NOW_Date };
@@ -248,13 +249,12 @@ public class UserEncrypted extends AbstractUser implements Serializable {
//clazz = String.class;
encDateTime = NOW_DateTime;
encAddress = new Address();
plainDate = NOW_Date;
plainDateTime = NOW_DateTime;
nicknames = List.of("Happy", "Sleepy");
turbulence = ETurbulenceCategory.T10;
}
}

View File

@@ -38,6 +38,7 @@ import org.springframework.data.couchbase.config.AbstractCouchbaseConfiguration;
import org.springframework.data.couchbase.core.CouchbaseTemplate;
import org.springframework.data.couchbase.domain.Address;
import org.springframework.data.couchbase.domain.AddressWithEncStreet;
import org.springframework.data.couchbase.domain.ETurbulenceCategory;
import org.springframework.data.couchbase.domain.TestEncrypted;
import org.springframework.data.couchbase.domain.UserEncrypted;
import org.springframework.data.couchbase.domain.UserEncryptedRepository;
@@ -97,6 +98,7 @@ public class CouchbaseRepositoryFieldLevelEncryptionIntegrationTests extends Clu
@Test
@IgnoreWhen(clusterTypes = ClusterType.MOCKED)
void saveAndFindByTestId() {
boolean cleanAfter = true;
TestEncrypted user = new TestEncrypted(UUID.randomUUID().toString());
user.initSimpleTypes();
couchbaseTemplate.save(user);
@@ -121,13 +123,20 @@ public class CouchbaseRepositoryFieldLevelEncryptionIntegrationTests extends Clu
writeSDKReadSpring.setId(user.getId());
assertEquals(user.toString(), writeSDKReadSpring.toString());
if (cleanAfter) {
try {
couchbaseTemplate.removeById(UserEncrypted.class).one(user.getId());
} catch (DataRetrievalFailureException iae) {
// ignore
}
}
}
@Test
@IgnoreWhen(clusterTypes = ClusterType.MOCKED)
void writeSpring_readSpring() {
boolean cleanAfter = false;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSpring_readSpring", "l", "hello");
boolean cleanAfter = true;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSpring_readSpring", "l", "hel\"lo");
AddressWithEncStreet address = new AddressWithEncStreet(); // plaintext address with encrypted street
address.setEncStreet("Olcott Street");
address.setCity("Santa Clara");
@@ -136,6 +145,7 @@ public class CouchbaseRepositoryFieldLevelEncryptionIntegrationTests extends Clu
Address encAddress = new Address(); // encrypted address with plaintext street.
encAddress.setStreet("Castro St");
encAddress.setCity("Mountain View");
encAddress.setTurbulence(ETurbulenceCategory.T10);
user.setEncAddress(encAddress);
user.initSimpleTypes();
@@ -159,8 +169,8 @@ public class CouchbaseRepositoryFieldLevelEncryptionIntegrationTests extends Clu
@Test
@IgnoreWhen(clusterTypes = ClusterType.MOCKED)
void writeSpring_readSDK() {
boolean cleanAfter = false;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSpring_readSDK", "l", "hello");
boolean cleanAfter = true;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSpring_readSDK", "l", "hel\"lo");
AddressWithEncStreet address = new AddressWithEncStreet(); // plaintext address with encrypted street
address.setEncStreet("Olcott Street");
address.setCity("Santa Clara");
@@ -195,8 +205,8 @@ public class CouchbaseRepositoryFieldLevelEncryptionIntegrationTests extends Clu
@Test
@IgnoreWhen(clusterTypes = ClusterType.MOCKED)
void writeSDK_readSpring() {
boolean cleanAfter = false;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSDK_readSpring", "l", "hello");
boolean cleanAfter = true;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSDK_readSpring", "l", "hel\"lo");
AddressWithEncStreet address = new AddressWithEncStreet(); // plaintext address with encrypted street
address.setEncStreet("Olcott Street");
address.setCity("Santa Clara");
@@ -232,8 +242,8 @@ public class CouchbaseRepositoryFieldLevelEncryptionIntegrationTests extends Clu
@Test
@IgnoreWhen(clusterTypes = ClusterType.MOCKED)
void writeSDK_readSDK() {
boolean cleanAfter = false;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSDK_readSDK", "l", "hello");
boolean cleanAfter = true;
UserEncrypted user = new UserEncrypted(UUID.randomUUID().toString(), "writeSDK_readSDK", "l", "hel\"lo");
AddressWithEncStreet address = new AddressWithEncStreet(); // plaintext address with encrypted street
address.setEncStreet("Olcott Street");
address.setCity("Santa Clara");