DATAREDIS-305 - Add support for HSCAN.

HSCAN is directly supported by jedis and can be emulated for lettuce via eval.

SRP and JRedis will throw UnsupportedOperationException.

We have modified the ScanCursor implementation to take Collection instead of List.

Currently naming for scan commands is not consistent for scan, sscan and hscan. There’s a clean up task for those commands when done with all scan related commands.

Original pull request: #78.
This commit is contained in:
Christoph Strobl
2014-06-02 10:39:48 +02:00
committed by Thomas Darimont
parent 1831ae9db7
commit ba88249188
16 changed files with 372 additions and 39 deletions

View File

@@ -21,6 +21,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Queue;
import java.util.Set;
@@ -2236,6 +2237,15 @@ public class DefaultStringRedisConnection implements StringRedisConnection {
return this.delegate.sScan(key, options);
}
/*
* (non-Javadoc)
* @see org.springframework.data.redis.connection.RedisHashCommands#hscan(byte[], org.springframework.data.redis.core.ScanOptions)
*/
@Override
public Cursor<Entry<byte[], byte[]>> hScan(byte[] key, ScanOptions options) {
return this.delegate.hScan(key, options);
}
/**
* Specifies if pipelined and tx results should be deserialized to Strings. If false, results of
* {@link #closePipeline()} and {@link #exec()} will be of the type returned by the underlying connection
@@ -2328,4 +2338,36 @@ public class DefaultStringRedisConnection implements StringRedisConnection {
}
/*
* (non-Javadoc)
* @see org.springframework.data.redis.connection.StringRedisConnection#hScan(java.lang.String, org.springframework.data.redis.core.ScanOptions)
*/
@Override
public Cursor<Entry<String, String>> hScan(String key, ScanOptions options) {
return new ConvertingCursor<Map.Entry<byte[], byte[]>, Map.Entry<String, String>>(this.delegate.hScan(
this.serialize(key), options), new Converter<Map.Entry<byte[], byte[]>, Map.Entry<String, String>>() {
@Override
public Entry<String, String> convert(final Entry<byte[], byte[]> source) {
return new Map.Entry<String, String>() {
@Override
public String getKey() {
return DefaultStringRedisConnection.this.serializer.deserialize(source.getKey());
}
@Override
public String getValue() {
return DefaultStringRedisConnection.this.serializer.deserialize(source.getValue());
}
@Override
public String setValue(String value) {
throw new UnsupportedOperationException("Cannot set value for entry in cursor");
}
};
}
});
}
}

View File

@@ -19,6 +19,9 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import org.springframework.data.redis.core.Cursor;
import org.springframework.data.redis.core.ScanOptions;
/**
* Hash-specific commands supported by Redis.
*
@@ -155,4 +158,14 @@ public interface RedisHashCommands {
* @return
*/
Map<byte[], byte[]> hGetAll(byte[] key);
/**
* Use a {@link Cursor} to iterate over entries in hash at {@code key}.
*
* @param key
* @param options
* @return
* @since 1.4
*/
Cursor<Map.Entry<byte[], byte[]>> hScan(byte[] key, ScanOptions options);
}

View File

@@ -327,4 +327,12 @@ public interface StringRedisConnection extends RedisConnection {
* @since 1.4
*/
Cursor<String> sScan(String key, ScanOptions options);
/**
* @param key
* @param options
* @return
* @since 1.4
*/
Cursor<Map.Entry<String, String>> hScan(String key, ScanOptions options);
}

View File

@@ -23,6 +23,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Queue;
import java.util.Set;
@@ -65,6 +66,7 @@ import redis.clients.jedis.Protocol.Command;
import redis.clients.jedis.Queable;
import redis.clients.jedis.Response;
import redis.clients.jedis.ScanParams;
import redis.clients.jedis.ScanResult;
import redis.clients.jedis.SortingParams;
import redis.clients.jedis.Transaction;
import redis.clients.jedis.ZParams;
@@ -2925,6 +2927,7 @@ public class JedisConnection implements RedisConnection {
* @param options
* @return
*/
@SuppressWarnings("resource")
public Cursor<byte[]> scan(long cursorId, ScanOptions options) {
return new ScanCursor<byte[]>(cursorId, options) {
@@ -2981,6 +2984,30 @@ public class JedisConnection implements RedisConnection {
}.open();
}
@Override
public Cursor<Entry<byte[], byte[]>> hScan(byte[] key, ScanOptions options) {
return hscan(key, 0, options);
}
public Cursor<Entry<byte[], byte[]>> hscan(byte[] key, long cursorId, ScanOptions options) {
return new KeyBoundCursor<Map.Entry<byte[], byte[]>>(key, cursorId, options) {
@Override
protected ScanIteration<Entry<byte[], byte[]>> doScan(byte[] key, long cursorId, ScanOptions options) {
if (isQueueing() || isPipelined()) {
throw new UnsupportedOperationException("'HSCAN' cannot be called in pipeline / transaction mode.");
}
ScanParams params = prepareScanParams(options);
ScanResult<Entry<byte[], byte[]>> result = jedis.hscan(key, JedisConverters.toBytes(cursorId), params);
return new ScanIteration<Map.Entry<byte[], byte[]>>(Long.valueOf(result.getStringCursor()), result.getResult());
}
}.open();
}
private ScanParams prepareScanParams(ScanOptions options) {
ScanParams sp = new ScanParams();
if (!options.equals(ScanOptions.NONE)) {

View File

@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
@@ -1256,4 +1257,13 @@ public class JredisConnection implements RedisConnection {
public Cursor<byte[]> sScan(byte[] key, ScanOptions options) {
throw new UnsupportedOperationException("'SSCAN' command is not uspported for jredis");
}
/*
* (non-Javadoc)
* @see org.springframework.data.redis.connection.RedisHashCommands#hscan(byte[], org.springframework.data.redis.core.ScanOptions)
*/
@Override
public Cursor<Entry<byte[], byte[]>> hScan(byte[] key, ScanOptions options) {
throw new UnsupportedOperationException("'HSCAN' command is not uspported for jredis");
}
}

View File

@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Queue;
import java.util.Set;
@@ -59,7 +60,6 @@ import org.springframework.data.redis.core.types.RedisClientInfo;
import org.springframework.util.Assert;
import org.springframework.util.ClassUtils;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import com.lambdaworks.redis.RedisAsyncConnection;
import com.lambdaworks.redis.RedisClient;
@@ -3063,7 +3063,7 @@ public class LettuceConnection implements RedisConnection {
throw new UnsupportedOperationException("'SCAN' cannot be called in pipeline / transaction mode.");
}
String params = " ," + cursorId + prepareScanParams(options);
String params = " ," + cursorId + options.toOptionString();
String script = "return redis.call('SCAN'" + params + ")";
List<?> result = eval(script.getBytes(), ReturnType.MULTI, 0);
@@ -3104,7 +3104,7 @@ public class LettuceConnection implements RedisConnection {
}
String params = " ,'" + LettuceConverters.bytesToString().convert(key) + "', " + cursorId
+ prepareScanParams(options);
+ options.toOptionString();
String script = "return redis.call('SSCAN'" + params + ")";
List<?> result = eval(script.getBytes(), ReturnType.MULTI, 0);
@@ -3115,18 +3115,44 @@ public class LettuceConnection implements RedisConnection {
}.open();
}
private String prepareScanParams(ScanOptions options) {
/* (non-Javadoc)
* @see org.springframework.data.redis.connection.RedisHashCommands#hScan(byte[], org.springframework.data.redis.core.ScanOptions)
*/
@Override
public Cursor<Entry<byte[], byte[]>> hScan(byte[] key, ScanOptions options) {
return hscan(key, 0, options);
}
String params = "";
if (!options.equals(ScanOptions.NONE)) {
if (options.getCount() != null) {
params += (", 'count', " + options.getCount());
/**
* @param key
* @param cursorId
* @param options
* @return
* @since 1.4
*/
public Cursor<Entry<byte[], byte[]>> hscan(byte[] key, long cursorId, ScanOptions options) {
return new KeyBoundCursor<Entry<byte[], byte[]>>(key, cursorId, options) {
@Override
protected ScanIteration<Entry<byte[], byte[]>> doScan(byte[] key, long cursorId, ScanOptions options) {
if (isQueueing() || isPipelined()) {
throw new UnsupportedOperationException("'HSCAN' cannot be called in pipeline / transaction mode.");
}
String params = " ,'" + LettuceConverters.bytesToString().convert(key) + "', " + cursorId
+ options.toOptionString();
String script = "return redis.call('HSCAN'" + params + ")";
List<?> result = eval(script.getBytes(), ReturnType.MULTI, 0);
String nextCursorId = LettuceConverters.bytesToString().convert((byte[]) result.get(0));
@SuppressWarnings("unchecked")
Map<byte[], byte[]> values = LettuceConverters.toMap((List<byte[]>) result.get(1));
return new ScanIteration<Entry<byte[], byte[]>>(Long.valueOf(nextCursorId), values.entrySet());
}
if (StringUtils.hasText(options.getPattern())) {
params += (", 'match' , '" + options.getPattern() + "'");
}
}
return params;
}.open();
}
/**

View File

@@ -19,8 +19,11 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.springframework.core.convert.converter.Converter;
@@ -36,6 +39,7 @@ import org.springframework.data.redis.connection.convert.LongToBooleanConverter;
import org.springframework.data.redis.connection.convert.StringToRedisClientInfoConverter;
import org.springframework.data.redis.core.types.RedisClientInfo;
import org.springframework.util.Assert;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
import com.lambdaworks.redis.KeyValue;
@@ -62,6 +66,7 @@ abstract public class LettuceConverters extends Converters {
private static final Converter<ScoredValue<byte[]>, Tuple> SCORED_VALUE_TO_TUPLE;
private static final Converter<Exception, DataAccessException> EXCEPTION_CONVERTER = new LettuceExceptionConverter();
private static final Converter<Long, Boolean> LONG_TO_BOOLEAN = new LongToBooleanConverter();
private static final Converter<List<byte[]>, Map<byte[], byte[]>> BYTES_LIST_TO_MAP;
private static final Converter<String[], List<RedisClientInfo>> STRING_TO_LIST_OF_CLIENT_INFO = new StringToRedisClientInfoConverter();
@@ -105,6 +110,26 @@ abstract public class LettuceConverters extends Converters {
return list;
}
};
BYTES_LIST_TO_MAP = new Converter<List<byte[]>, Map<byte[], byte[]>>() {
@Override
public Map<byte[], byte[]> convert(final List<byte[]> source) {
if (CollectionUtils.isEmpty(source)) {
Collections.emptyMap();
}
Map<byte[], byte[]> target = new LinkedHashMap<byte[], byte[]>();
Iterator<byte[]> kv = source.iterator();
while (kv.hasNext()) {
target.put(kv.next(), kv.hasNext() ? kv.next() : null);
}
return target;
}
};
SCORED_VALUES_TO_TUPLE_SET = new Converter<List<ScoredValue<byte[]>>, Set<Tuple>>() {
public Set<Tuple> convert(List<ScoredValue<byte[]>> source) {
if (source == null) {
@@ -238,6 +263,10 @@ abstract public class LettuceConverters extends Converters {
return (value ? 1 : 0);
}
public static Map<byte[], byte[]> toMap(List<byte[]> source) {
return BYTES_LIST_TO_MAP.convert(source);
}
public static SortArgs toSortArgs(SortParameters params) {
SortArgs args = new SortArgs();
if (params == null) {

View File

@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
import java.util.Queue;
import java.util.Set;
@@ -2351,6 +2352,15 @@ public class SrpConnection implements RedisConnection {
throw new UnsupportedOperationException("'SSCAN' command is not supported for Srp.");
}
/*
* (non-Javadoc)
* @see org.springframework.data.redis.connection.RedisHashCommands#hscan(byte[], org.springframework.data.redis.core.ScanOptions)
*/
@Override
public Cursor<Entry<byte[], byte[]>> hScan(byte[] key, ScanOptions options) {
throw new UnsupportedOperationException("'HSCAN' command is not supported for Srp.");
}
private List<Object> closeTransaction() {
List<Object> results = Collections.emptyList();
if (txTracker != null) {

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2011-2013 the original author or authors.
* Copyright 2011-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,6 +16,7 @@
package org.springframework.data.redis.core;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -24,6 +25,7 @@ import java.util.Set;
* Hash operations bound to a certain key.
*
* @author Costin Leau
* @author Christoph Strobl
*/
public interface BoundHashOperations<H, HK, HV> extends BoundKeyOperations<H> {
@@ -54,4 +56,11 @@ public interface BoundHashOperations<H, HK, HV> extends BoundKeyOperations<H> {
void delete(Object... keys);
Map<HK, HV> entries();
/**
* @param key
* @since 1.4
* @return
*/
Iterator<Map.Entry<HK, HV>> scan(H key, ScanOptions options);
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2011-2013 the original author or authors.
* Copyright 2011-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -16,8 +16,10 @@
package org.springframework.data.redis.core;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.springframework.data.redis.connection.DataType;
@@ -26,6 +28,7 @@ import org.springframework.data.redis.connection.DataType;
* Default implementation for {@link HashOperations}.
*
* @author Costin Leau
* @author Christoph Strobl
*/
class DefaultBoundHashOperations<H, HK, HV> extends DefaultBoundKeyOperations<H> implements
BoundHashOperations<H, HK, HV> {
@@ -102,4 +105,13 @@ class DefaultBoundHashOperations<H, HK, HV> extends DefaultBoundKeyOperations<H>
public DataType getType() {
return DataType.HASH;
}
/*
* (non-Javadoc)
* @see org.springframework.data.redis.core.BoundHashOperations#hscan(java.lang.Object)
*/
@Override
public Iterator<Entry<HK, HV>> scan(H key, ScanOptions options) {
return ops.scan(key, options);
}
}

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2011-2013 the original author or authors.
* Copyright 2011-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -17,17 +17,22 @@ package org.springframework.data.redis.core;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import org.springframework.core.convert.converter.Converter;
import org.springframework.dao.DataAccessException;
import org.springframework.data.redis.connection.RedisConnection;
/**
* Default implementation of {@link HashOperations}.
*
* @author Costin Leau
* @author Christoph Strobl
*/
class DefaultHashOperations<K, HK, HV> extends AbstractOperations<K, Object> implements HashOperations<K, HK, HV> {
@@ -222,4 +227,49 @@ class DefaultHashOperations<K, HK, HV> extends AbstractOperations<K, Object> imp
return deserializeHashMap(entries);
}
/*
* (non-Javadoc)
* @see org.springframework.data.redis.core.HashOperations#hscan(java.lang.Object, org.springframework.data.redis.core.ScanOptions)
*/
@Override
public Iterator<Entry<HK, HV>> scan(K key, final ScanOptions options) {
final byte[] rawKey = rawKey(key);
return execute(new RedisCallback<Cursor<Map.Entry<HK, HV>>>() {
@Override
public Cursor<Entry<HK, HV>> doInRedis(RedisConnection connection) throws DataAccessException {
return new ConvertingCursor<Map.Entry<byte[], byte[]>, Map.Entry<HK, HV>>(connection.hScan(rawKey, options),
new Converter<Map.Entry<byte[], byte[]>, Map.Entry<HK, HV>>() {
@Override
public Entry<HK, HV> convert(final Entry<byte[], byte[]> source) {
return new Map.Entry<HK, HV>() {
@Override
public HK getKey() {
return deserializeHashKey(source.getKey());
}
@Override
public HV getValue() {
return deserializeHashValue(source.getValue());
}
@Override
public HV setValue(HV value) {
throw new UnsupportedOperationException("Values cannot be set when scanning through entries.");
}
};
}
});
}
}, true);
}
}

View File

@@ -16,6 +16,7 @@
package org.springframework.data.redis.core;
import java.util.Collection;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -54,4 +55,12 @@ public interface HashOperations<H, HK, HV> {
Map<HK, HV> entries(H key);
RedisOperations<H, ?> getOperations();
/**
* @param key
* @since 1.4
* @param options
* @return
*/
Iterator<Map.Entry<HK, HV>> scan(H key, ScanOptions options);
}

View File

@@ -16,9 +16,9 @@
package org.springframework.data.redis.core;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
/**
* {@link ScanIteration} holds the values contained in Redis {@literal Multibulk reply} on exectuting {@literal SCAN}
@@ -30,13 +30,13 @@ import java.util.List;
public class ScanIteration<T> implements Iterable<T> {
private final long cursorId;
private final List<T> items;
private final Collection<T> items;
/**
* @param cursorId
* @param items
*/
public ScanIteration(long cursorId, List<T> items) {
public ScanIteration(long cursorId, Collection<T> items) {
this.cursorId = cursorId;
this.items = (items != null ? new ArrayList<T>(items) : Collections.<T> emptyList());
@@ -56,7 +56,7 @@ public class ScanIteration<T> implements Iterable<T> {
*
* @return
*/
public List<T> getItems() {
public Collection<T> getItems() {
return items;
}

View File

@@ -15,6 +15,8 @@
*/
package org.springframework.data.redis.core;
import org.springframework.util.StringUtils;
/**
* Options to be used for with {@literal SCAN} command.
*
@@ -29,15 +31,14 @@ public class ScanOptions {
private Long count;
private String pattern;
private ScanOptions() {
}
private ScanOptions() {}
/**
* Static factory method that returns a new {@link ScanOptionsBuilder}.
*
* @return
*/
public static ScanOptionsBuilder scanOptions(){
public static ScanOptionsBuilder scanOptions() {
return new ScanOptionsBuilder();
}
@@ -49,6 +50,24 @@ public class ScanOptions {
return pattern;
}
public String toOptionString() {
if (this.equals(ScanOptions.NONE)) {
return "";
}
String params = "";
if (this.count != null) {
params += (", 'count', " + count);
}
if (StringUtils.hasText(this.pattern)) {
params += (", 'match' , '" + this.pattern + "'");
}
return params;
}
/**
* @author Christoph Strobl
* @since 1.4