Spring/spring-data-redis
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

DATAREDIS-1091 - Add configuration to disable storage of shadow copies for entities using TimeToLive.

Browse Source 
Original pull request: #517.
This commit is contained in:
Christoph Strobl
2020-03-19 14:33:18 +01:00
committed by Mark Paluch
parent 2f0a5600dc
commit 546ebda2f2
5 changed files with 136 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/asciidoc/reference/redis-repositories.adoc
View File

@@ -591,6 +591,8 @@ NOTE: The keyspace notification message listener alters `notify-keyspace-events`
NOTE: Redis Pub/Sub messages are not persistent. If a key expires while the application is down, the expiry event is not processed, which may lead to secondary indexes containing references to the expired object.
NOTE: `@EnableKeyspaceEvents(shadowCopy = OFF)` disable storage of phantom copies and reduces data size within Redis. `RedisKeyExpiredEvent` will only contain the `id` of the expired key.
[[redis.repositories.references]]
== Persisting References
Marking properties with `@Reference` allows storing a simple key reference instead of copying values into the hash itself.

76
src/main/java/org/springframework/data/redis/core/RedisKeyValueAdapter.java
View File

@@ -117,6 +117,7 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
private EnableKeyspaceEvents enableKeyspaceEvents = EnableKeyspaceEvents.OFF;
private @Nullable String keyspaceNotificationsConfigParameter = null;
private ShadowCopy shadowCopy = ShadowCopy.DEFAULT;
/**
* Creates new {@link RedisKeyValueAdapter} with default {@link RedisMappingContext} and default
@@ -238,11 +239,13 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
connection.expire(objectKey, rdo.getTimeToLive());
// add phantom key so values can be restored
byte[] phantomKey = ByteUtils.concat(objectKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
connection.del(phantomKey);
connection.hMSet(phantomKey, rdo.getBucket().rawMap());
connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
if (keepShadowCopy()) { // add phantom key so values can be restored
byte[] phantomKey = ByteUtils.concat(objectKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
connection.del(phantomKey);
connection.hMSet(phantomKey, rdo.getBucket().rawMap());
connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
}
}
connection.sAdd(toBytes(rdo.getKeyspace()), key);
@@ -475,10 +478,12 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
connection.expire(redisKey, rdo.getTimeToLive());
// add phantom key so values can be restored
byte[] phantomKey = ByteUtils.concat(redisKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
connection.hMSet(phantomKey, rdo.getBucket().rawMap());
connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
if (keepShadowCopy()) { // add phantom key so values can be restored
byte[] phantomKey = ByteUtils.concat(redisKey, BinaryKeyspaceIdentifier.PHANTOM_SUFFIX);
connection.hMSet(phantomKey, rdo.getBucket().rawMap());
connection.expire(phantomKey, rdo.getTimeToLive() + PHANTOM_KEY_TTL);
}
} else {
@@ -661,6 +666,16 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
this.keyspaceNotificationsConfigParameter = keyspaceNotificationsConfigParameter;
}
/**
* Configure storage of phantom keys (shadow copies) of expiring entities.
*
* @param shadowCopy must not be {@literal null}.
* @since 2.3
*/
public void setShadowCopy(ShadowCopy shadowCopy) {
this.shadowCopy = shadowCopy;
}
/**
* @see org.springframework.beans.factory.InitializingBean#afterPropertiesSet()
* @since 1.8
@@ -773,7 +788,8 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
byte[] key = message.getBody();
byte[] phantomKey = ByteUtils.concat(key, converter.getConversionService().convert(KeyspaceIdentifier.PHANTOM_SUFFIX, byte[].class));
byte[] phantomKey = ByteUtils.concat(key,
converter.getConversionService().convert(KeyspaceIdentifier.PHANTOM_SUFFIX, byte[].class));
Map<byte[], byte[]> hash = ops.execute((RedisCallback<Map<byte[], byte[]>>) connection -> {
@@ -789,7 +805,8 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
Object value = converter.read(Object.class, new RedisData(hash));
String channel = !ObjectUtils.isEmpty(message.getChannel())
? converter.getConversionService().convert(message.getChannel(), String.class) : null;
? converter.getConversionService().convert(message.getChannel(), String.class)
: null;
RedisKeyExpiredEvent event = new RedisKeyExpiredEvent(channel, key, value);
@@ -813,6 +830,18 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
}
}
private boolean keepShadowCopy() {
switch (shadowCopy) {
case OFF:
return false;
case ON:
return true;
default:
return this.expirationListener.get() != null;
}
}
/**
* @author Christoph Strobl
* @since 1.8
@@ -835,6 +864,31 @@ public class RedisKeyValueAdapter extends AbstractKeyValueAdapter
OFF
}
/**
* Configuration flag controlling storage of phantom keys (shadow copies) of expiring entities to read them later when
* publishing {@link RedisKeyspaceEvent}.
*
* @author Christoph Strobl
* @since 2.3
*/
public enum ShadowCopy {
/**
* Store shadow copies of expiring entities depending on the {@link EnableKeyspaceEvents}.
*/
DEFAULT,
/**
* Store shadow copies of expiring entities.
*/
ON,
/**
* Do not store shadow copies.
*/
OFF
}
/**
* Container holding update information like fields to remove from the Redis Hash.
*

10
src/main/java/org/springframework/data/redis/repository/configuration/EnableRedisRepositories.java
View File

@@ -28,6 +28,7 @@ import org.springframework.context.annotation.Import;
import org.springframework.data.keyvalue.core.KeyValueOperations;
import org.springframework.data.keyvalue.repository.config.QueryCreatorType;
import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
import org.springframework.data.redis.core.RedisOperations;
import org.springframework.data.redis.core.convert.KeyspaceConfiguration;
import org.springframework.data.redis.core.index.IndexConfiguration;
@@ -166,6 +167,15 @@ public @interface EnableRedisRepositories {
*/
EnableKeyspaceEvents enableKeyspaceEvents() default EnableKeyspaceEvents.OFF;
/**
* Configuration flag controlling storage of phantom keys (shadow copies) of expiring entities to read them later when
* publishing {@link org.springframework.data.redis.core.RedisKeyspaceEvent keyspace events}.
*
* @return
* @since 2.3
*/
ShadowCopy shadowCopy() default ShadowCopy.DEFAULT;
/**
* Configure the {@literal notify-keyspace-events} property if not already set. <br />
* Use an empty {@link String} to keep (<b>not</b> alter) existing server configuration.

3
src/main/java/org/springframework/data/redis/repository/configuration/RedisRepositoryConfigurationExtension.java
View File

@@ -28,6 +28,7 @@ import org.springframework.data.keyvalue.repository.config.KeyValueRepositoryCon
import org.springframework.data.redis.core.RedisHash;
import org.springframework.data.redis.core.RedisKeyValueAdapter;
import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
import org.springframework.data.redis.core.RedisKeyValueTemplate;
import org.springframework.data.redis.core.convert.MappingConfiguration;
import org.springframework.data.redis.core.convert.MappingRedisConverter;
@@ -145,6 +146,8 @@ public class RedisRepositoryConfigurationExtension extends KeyValueRepositoryCon
configuration.getRequiredAttribute("enableKeyspaceEvents", EnableKeyspaceEvents.class)) //
.addPropertyValue("keyspaceNotificationsConfigParameter",
configuration.getAttribute("keyspaceNotificationsConfigParameter", String.class).orElse("")) //
.addPropertyValue("shadowCopy",
configuration.getRequiredAttribute("shadowCopy", ShadowCopy.class)) //
.getBeanDefinition();
}

58
src/test/java/org/springframework/data/redis/core/RedisKeyValueAdapterTests.java
View File

@@ -37,7 +37,6 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.data.annotation.Id;
import org.springframework.data.annotation.Reference;
@@ -50,6 +49,7 @@ import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory;
import org.springframework.data.redis.connection.lettuce.LettuceTestClientResources;
import org.springframework.data.redis.core.RedisKeyValueAdapter.EnableKeyspaceEvents;
import org.springframework.data.redis.core.RedisKeyValueAdapter.ShadowCopy;
import org.springframework.data.redis.core.convert.KeyspaceConfiguration;
import org.springframework.data.redis.core.convert.MappingConfiguration;
import org.springframework.data.redis.core.index.GeoIndexed;
@@ -341,7 +341,6 @@ public class RedisKeyValueAdapterTests {
assertThat(template.hasKey("persons:firstname:rand")).isFalse();
assertThat(template.hasKey("persons:1:idx")).isFalse();
assertThat(template.opsForSet().members("persons")).doesNotContain("1");
;
}
@Test // DATAREDIS-744
@@ -687,6 +686,56 @@ public class RedisKeyValueAdapterTests {
assertThat(updatedLocation.getY()).isCloseTo(18D, offset(0.005));
}
@Test // DATAREDIS-1091
public void phantomKeyNotInsertedOnPutWhenShadowCopyIsTurnedOff() {
RedisMappingContext mappingContext = new RedisMappingContext(
new MappingConfiguration(new IndexConfiguration(), new KeyspaceConfiguration()));
mappingContext.afterPropertiesSet();
RedisKeyValueAdapter kvAdapter = new RedisKeyValueAdapter(template, mappingContext);
kvAdapter.setShadowCopy(ShadowCopy.OFF);
ExpiringPerson rand = new ExpiringPerson();
rand.age = 24;
rand.ttl = 3000L;
kvAdapter.put("1", rand, "persons");
assertThat(template.hasKey("persons:1:phantom")).isFalse();
}
@Test // DATAREDIS-1091
public void phantomKeyInsertedOnPutWhenShadowCopyIsTurnedOn() {
RedisMappingContext mappingContext = new RedisMappingContext(
new MappingConfiguration(new IndexConfiguration(), new KeyspaceConfiguration()));
mappingContext.afterPropertiesSet();
RedisKeyValueAdapter kvAdapter = new RedisKeyValueAdapter(template, mappingContext);
kvAdapter.setShadowCopy(ShadowCopy.ON);
ExpiringPerson rand = new ExpiringPerson();
rand.age = 24;
rand.ttl = 3000L;
kvAdapter.put("1", rand, "persons");
assertThat(template.hasKey("persons:1:phantom")).isTrue();
}
@Test // DATAREDIS-1091
public void phantomKeyInsertedOnPutWhenShadowCopyIsInDefaultAndKeyspaceNotificationEnabled() {
ExpiringPerson rand = new ExpiringPerson();
rand.age = 24;
rand.ttl = 3000L;
adapter.put("1", rand, "persons");
assertThat(template.hasKey("persons:1:phantom")).isTrue();
}
/**
* Wait up to 5 seconds until {@code key} is no longer available in Redis.
*
@@ -776,6 +825,11 @@ public class RedisKeyValueAdapterTests {
}
static class ExpiringPerson extends Person {
@TimeToLive Long ttl;
}
@KeySpace("locations")
static class Location {