From 34d1759b4e8394774fd73ceabb4c18f3cba4508b Mon Sep 17 00:00:00 2001 From: Christoph Strobl Date: Mon, 26 May 2014 14:54:21 +0200 Subject: [PATCH] DATAREDIS-304 - Add support for SSCAN command. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSCAN command is natively supported by jedis and can be emulated for lettuce. JRedis and SRP will throw UnsupportedOperationException. sScan is available on RedisConneciton and RedisSetOperations returning basically a Cursor that allows iteration over the defined values for a given key. Currently the jedis driver does not directly expose the binary version of sscan which leads to invalid results when using a non String compatible converter along with the RedisTemplate. We’ll change this as soon as a newer version of jedis is available. Along the way loading behavior of ScanCursor has been fixed, preventing it from indicating next values available when there effectively are no more. Orignal pull request: #76. --- .../DefaultStringRedisConnection.java | 29 +++++ .../redis/connection/RedisSetCommands.java | 14 ++ .../connection/StringRedisConnection.java | 11 ++ .../connection/jedis/JedisConnection.java | 68 ++++++++-- .../connection/jredis/JredisConnection.java | 9 ++ .../connection/lettuce/LettuceConnection.java | 72 ++++++++-- .../redis/connection/srp/SrpConnection.java | 9 ++ .../data/redis/core/BoundSetOperations.java | 8 ++ .../data/redis/core/ConvertingCursor.java | 123 ++++++++++++++++++ .../redis/core/DefaultBoundSetOperations.java | 13 +- .../data/redis/core/DefaultSetOperations.java | 29 ++++- .../data/redis/core/KeyBoundCursor.java | 48 +++++++ .../data/redis/core/ScanCursor.java | 40 +++--- .../data/redis/core/SetOperations.java | 14 +- .../AbstractConnectionIntegrationTests.java | 29 +++++ .../core/AbstractOperationsTestParams.java | 29 +++-- .../redis/core/DefaultSetOperationsTests.java | 49 +++++-- .../data/redis/core/ScanCursorUnitTests.java | 18 +-- 18 files changed, 531 insertions(+), 81 deletions(-) create mode 100644 src/main/java/org/springframework/data/redis/core/ConvertingCursor.java create mode 100644 src/main/java/org/springframework/data/redis/core/KeyBoundCursor.java diff --git a/src/main/java/org/springframework/data/redis/connection/DefaultStringRedisConnection.java b/src/main/java/org/springframework/data/redis/connection/DefaultStringRedisConnection.java index 733e80a65..f9a86b09c 100644 --- a/src/main/java/org/springframework/data/redis/connection/DefaultStringRedisConnection.java +++ b/src/main/java/org/springframework/data/redis/connection/DefaultStringRedisConnection.java @@ -32,6 +32,7 @@ import org.springframework.data.redis.RedisSystemException; import org.springframework.data.redis.connection.convert.ListConverter; import org.springframework.data.redis.connection.convert.MapConverter; import org.springframework.data.redis.connection.convert.SetConverter; +import org.springframework.data.redis.core.ConvertingCursor; import org.springframework.data.redis.core.Cursor; import org.springframework.data.redis.core.ScanOptions; import org.springframework.data.redis.core.types.RedisClientInfo; @@ -2226,6 +2227,15 @@ public class DefaultStringRedisConnection implements StringRedisConnection { return this.delegate.scan(options); } + /* + * (non-Javadoc) + * @see org.springframework.data.redis.connection.RedisSetCommands#scan(byte[], org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(byte[] key, ScanOptions options) { + return this.delegate.sScan(key, options); + } + /** * Specifies if pipelined and tx results should be deserialized to Strings. If false, results of * {@link #closePipeline()} and {@link #exec()} will be of the type returned by the underlying connection @@ -2299,4 +2309,23 @@ public class DefaultStringRedisConnection implements StringRedisConnection { public String getClientName() { return this.delegate.getClientName(); } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.connection.StringRedisConnection#sScan(java.lang.String, org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(String key, ScanOptions options) { + + return new ConvertingCursor(this.delegate.sScan(this.serialize(key), options), + new Converter() { + + @Override + public String convert(byte[] source) { + return serializer.deserialize(source); + } + }); + + } + } diff --git a/src/main/java/org/springframework/data/redis/connection/RedisSetCommands.java b/src/main/java/org/springframework/data/redis/connection/RedisSetCommands.java index d196ce05e..c139d8297 100644 --- a/src/main/java/org/springframework/data/redis/connection/RedisSetCommands.java +++ b/src/main/java/org/springframework/data/redis/connection/RedisSetCommands.java @@ -18,6 +18,9 @@ package org.springframework.data.redis.connection; import java.util.List; import java.util.Set; +import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.ScanOptions; + /** * Set-specific commands supported by Redis. * @@ -166,4 +169,15 @@ public interface RedisSetCommands { * @return */ List sRandMember(byte[] key, long count); + + /** + * Use a {@link Cursor} to iterate over elements in set at {@code key}. + * + * @since 1.4 + * @see http://redis.io/commands/scan + * @param key + * @param options + * @return + */ + Cursor sScan(byte[] key, ScanOptions options); } diff --git a/src/main/java/org/springframework/data/redis/connection/StringRedisConnection.java b/src/main/java/org/springframework/data/redis/connection/StringRedisConnection.java index 9be9f35b3..b357a1a84 100644 --- a/src/main/java/org/springframework/data/redis/connection/StringRedisConnection.java +++ b/src/main/java/org/springframework/data/redis/connection/StringRedisConnection.java @@ -20,7 +20,9 @@ import java.util.List; import java.util.Map; import java.util.Set; +import org.springframework.data.redis.core.Cursor; import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.ScanOptions; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.types.RedisClientInfo; import org.springframework.data.redis.serializer.RedisSerializer; @@ -316,4 +318,13 @@ public interface StringRedisConnection extends RedisConnection { * @since 1.3 */ List getClientList(); + + /** + * @see RedisSetCommands#sScan(byte[], ScanOptions) + * @param key + * @param options + * @return + * @since 1.4 + */ + Cursor sScan(String key, ScanOptions options); } diff --git a/src/main/java/org/springframework/data/redis/connection/jedis/JedisConnection.java b/src/main/java/org/springframework/data/redis/connection/jedis/JedisConnection.java index ae3fc2a42..69919b266 100644 --- a/src/main/java/org/springframework/data/redis/connection/jedis/JedisConnection.java +++ b/src/main/java/org/springframework/data/redis/connection/jedis/JedisConnection.java @@ -43,6 +43,7 @@ import org.springframework.data.redis.connection.Subscription; import org.springframework.data.redis.connection.convert.Converters; import org.springframework.data.redis.connection.convert.TransactionResultConverter; import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.KeyBoundCursor; import org.springframework.data.redis.core.ScanCursor; import org.springframework.data.redis.core.ScanIteration; import org.springframework.data.redis.core.ScanOptions; @@ -2918,6 +2919,12 @@ public class JedisConnection implements RedisConnection { return scan(0, options != null ? options : ScanOptions.NONE); } + /** + * @since 1.4 + * @param cursorId + * @param options + * @return + */ public Cursor scan(long cursorId, ScanOptions options) { return new ScanCursor(cursorId, options) { @@ -2929,22 +2936,65 @@ public class JedisConnection implements RedisConnection { throw new UnsupportedOperationException("'SCAN' cannot be called in pipeline / transaction mode."); } - ScanParams sp = new ScanParams(); - if (!options.equals(ScanOptions.NONE)) { - if (options.getCount() != null) { - sp.count(options.getCount().intValue()); - } - if (StringUtils.hasText(options.getPattern())) { - sp.match(options.getPattern()); - } + ScanParams params = prepareScanParams(options); + redis.clients.jedis.ScanResult result = jedis.scan(Long.toString(cursorId), params); + return new ScanIteration(Long.valueOf(result.getStringCursor()), JedisConverters.stringListToByteList() + .convert(result.getResult())); + } + + }.open(); + + } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.connection.RedisSetCommands#sScan(byte[], org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(byte[] key, ScanOptions options) { + return sScan(key, 0, options); + } + + /** + * @since 1.4 + * @param key + * @param cursorId + * @param options + * @return + */ + public Cursor sScan(byte[] key, long cursorId, ScanOptions options) { + + return new KeyBoundCursor(key, cursorId, options) { + + @Override + protected ScanIteration doScan(byte[] key, long cursorId, ScanOptions options) { + + if (isQueueing() || isPipelined()) { + throw new UnsupportedOperationException("'SSCAN' cannot be called in pipeline / transaction mode."); } - redis.clients.jedis.ScanResult result = jedis.scan(Long.toString(cursorId), sp); + ScanParams params = prepareScanParams(options); + + // TODO: use binary version of jedis.sscan (in v.2.4.3) to avoid potentially invalid representations. + redis.clients.jedis.ScanResult result = jedis.sscan(JedisConverters.toString(key), + Long.toString(cursorId), params); return new ScanIteration(Long.valueOf(result.getStringCursor()), JedisConverters.stringListToByteList() .convert(result.getResult())); } }.open(); + } + private ScanParams prepareScanParams(ScanOptions options) { + ScanParams sp = new ScanParams(); + if (!options.equals(ScanOptions.NONE)) { + if (options.getCount() != null) { + sp.count(options.getCount().intValue()); + } + if (StringUtils.hasText(options.getPattern())) { + sp.match(options.getPattern()); + } + } + return sp; } /** diff --git a/src/main/java/org/springframework/data/redis/connection/jredis/JredisConnection.java b/src/main/java/org/springframework/data/redis/connection/jredis/JredisConnection.java index 1c797b4a7..ae0c7a4a1 100644 --- a/src/main/java/org/springframework/data/redis/connection/jredis/JredisConnection.java +++ b/src/main/java/org/springframework/data/redis/connection/jredis/JredisConnection.java @@ -1247,4 +1247,13 @@ public class JredisConnection implements RedisConnection { public Cursor scan(ScanOptions options) { throw new UnsupportedOperationException("'SCAN' command is not supported for jredis."); } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.connection.RedisSetCommands#sScan(byte[], org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(byte[] key, ScanOptions options) { + throw new UnsupportedOperationException("'SSCAN' command is not uspported for jredis"); + } } diff --git a/src/main/java/org/springframework/data/redis/connection/lettuce/LettuceConnection.java b/src/main/java/org/springframework/data/redis/connection/lettuce/LettuceConnection.java index 89c19b479..7ebb394ab 100644 --- a/src/main/java/org/springframework/data/redis/connection/lettuce/LettuceConnection.java +++ b/src/main/java/org/springframework/data/redis/connection/lettuce/LettuceConnection.java @@ -50,6 +50,7 @@ import org.springframework.data.redis.connection.Subscription; import org.springframework.data.redis.connection.convert.Converters; import org.springframework.data.redis.connection.convert.TransactionResultConverter; import org.springframework.data.redis.core.Cursor; +import org.springframework.data.redis.core.KeyBoundCursor; import org.springframework.data.redis.core.RedisCommand; import org.springframework.data.redis.core.ScanCursor; import org.springframework.data.redis.core.ScanIteration; @@ -3044,6 +3045,12 @@ public class LettuceConnection implements RedisConnection { return scan(0, options != null ? options : ScanOptions.NONE); } + /** + * @since 1.4 + * @param cursorId + * @param options + * @return + */ public Cursor scan(long cursorId, ScanOptions options) { return new ScanCursor(cursorId, options) { @@ -3056,16 +3063,7 @@ public class LettuceConnection implements RedisConnection { throw new UnsupportedOperationException("'SCAN' cannot be called in pipeline / transaction mode."); } - String params = " ," + cursorId; - if (!options.equals(ScanOptions.NONE)) { - if (options.getCount() != null) { - params += (", 'count', " + options.getCount()); - } - if (StringUtils.hasText(options.getPattern())) { - params += (", 'match' , '" + options.getPattern() + "'"); - } - } - + String params = " ," + cursorId + prepareScanParams(options); String script = "return redis.call('SCAN'" + params + ")"; List result = eval(script.getBytes(), ReturnType.MULTI, 0); @@ -3077,6 +3075,60 @@ public class LettuceConnection implements RedisConnection { } + /* + * (non-Javadoc) + * @see org.springframework.data.redis.connection.RedisSetCommands#sScan(byte[], org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(byte[] key, ScanOptions options) { + return sScan(key, 0, options); + } + + /** + * @since 1.4 + * @param key + * @param cursorId + * @param options + * @return + */ + public Cursor sScan(byte[] key, long cursorId, ScanOptions options) { + + return new KeyBoundCursor(key, cursorId, options) { + + @SuppressWarnings("unchecked") + @Override + protected ScanIteration doScan(byte[] key, long cursorId, ScanOptions options) { + + if (isQueueing() || isPipelined()) { + throw new UnsupportedOperationException("'SSCAN' cannot be called in pipeline / transaction mode."); + } + + String params = " ,'" + LettuceConverters.bytesToString().convert(key) + "', " + cursorId + + prepareScanParams(options); + String script = "return redis.call('SSCAN'" + params + ")"; + + List result = eval(script.getBytes(), ReturnType.MULTI, 0); + String nextCursorId = LettuceConverters.bytesToString().convert((byte[]) result.get(0)); + + return new ScanIteration(Long.valueOf(nextCursorId), ((ArrayList) result.get(1))); + } + }.open(); + } + + private String prepareScanParams(ScanOptions options) { + + String params = ""; + if (!options.equals(ScanOptions.NONE)) { + if (options.getCount() != null) { + params += (", 'count', " + options.getCount()); + } + if (StringUtils.hasText(options.getPattern())) { + params += (", 'match' , '" + options.getPattern() + "'"); + } + } + return params; + } + /** * Specifies if pipelined and transaction results should be converted to the expected data type. If false, results of * {@link #closePipeline()} and {@link #exec()} will be of the type returned by the Lettuce driver diff --git a/src/main/java/org/springframework/data/redis/connection/srp/SrpConnection.java b/src/main/java/org/springframework/data/redis/connection/srp/SrpConnection.java index 1babf2f3f..cf55276e5 100644 --- a/src/main/java/org/springframework/data/redis/connection/srp/SrpConnection.java +++ b/src/main/java/org/springframework/data/redis/connection/srp/SrpConnection.java @@ -2342,6 +2342,15 @@ public class SrpConnection implements RedisConnection { throw new UnsupportedOperationException("'SCAN' command is not supported for Srp."); } + /* + * (non-Javadoc) + * @see org.springframework.data.redis.connection.RedisSetCommands#sScan(byte[], org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(byte[] key, ScanOptions options) { + throw new UnsupportedOperationException("'SSCAN' command is not supported for Srp."); + } + private List closeTransaction() { List results = Collections.emptyList(); if (txTracker != null) { diff --git a/src/main/java/org/springframework/data/redis/core/BoundSetOperations.java b/src/main/java/org/springframework/data/redis/core/BoundSetOperations.java index 29ad8611b..14f4d44bf 100644 --- a/src/main/java/org/springframework/data/redis/core/BoundSetOperations.java +++ b/src/main/java/org/springframework/data/redis/core/BoundSetOperations.java @@ -17,6 +17,7 @@ package org.springframework.data.redis.core; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -72,4 +73,11 @@ public interface BoundSetOperations extends BoundKeyOperations { V pop(); Long size(); + + /** + * @param options + * @return + * @since 1.4 + */ + Iterator sScan(ScanOptions options); } diff --git a/src/main/java/org/springframework/data/redis/core/ConvertingCursor.java b/src/main/java/org/springframework/data/redis/core/ConvertingCursor.java new file mode 100644 index 000000000..a2a8994ba --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/ConvertingCursor.java @@ -0,0 +1,123 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.redis.core; + +import java.io.IOException; + +import org.springframework.core.convert.converter.Converter; +import org.springframework.util.Assert; + +/** + * {@link ConvertingCursor} wraps a given cursor and applies given {@link Converter} to items prior to returning them. + * This allows to easily perform required conversion whereas the underlying implementation may still work with its + * native types. + * + * @author Christoph Strobl + * @param + * @param + * @since 1.4 + */ +public class ConvertingCursor implements Cursor { + + private Cursor delegate; + private Converter converter; + + /** + * @param cursor Cursor must not be {@literal null}. + * @param converter Converter must not be {@literal null}. + */ + public ConvertingCursor(Cursor cursor, Converter converter) { + + Assert.notNull(cursor, "Cursor delegate must not be 'null'."); + Assert.notNull(cursor, "Converter must not be 'null'."); + this.delegate = cursor; + this.converter = converter; + } + + /* + * (non-Javadoc) + * @see java.util.Iterator#hasNext() + */ + @Override + public boolean hasNext() { + return delegate.hasNext(); + } + + /* + * (non-Javadoc) + * @see java.util.Iterator#next() + */ + @Override + public T next() { + return converter.convert(delegate.next()); + } + + /* + * (non-Javadoc) + * @see java.util.Iterator#remove() + */ + @Override + public void remove() { + delegate.remove(); + } + + /* + * (non-Javadoc) + * @see java.io.Closeable#close() + */ + @Override + public void close() throws IOException { + delegate.close(); + } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.core.Cursor#getCursorId() + */ + @Override + public long getCursorId() { + return delegate.getCursorId(); + } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.core.Cursor#isClosed() + */ + @Override + public boolean isClosed() { + return delegate.isClosed(); + } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.core.Cursor#open() + */ + @Override + public Cursor open() { + this.delegate = delegate.open(); + return this; + } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.core.Cursor#getPosition() + */ + @Override + public long getPosition() { + return delegate.getPosition(); + } + +} diff --git a/src/main/java/org/springframework/data/redis/core/DefaultBoundSetOperations.java b/src/main/java/org/springframework/data/redis/core/DefaultBoundSetOperations.java index c289d8e3b..f249e61cf 100644 --- a/src/main/java/org/springframework/data/redis/core/DefaultBoundSetOperations.java +++ b/src/main/java/org/springframework/data/redis/core/DefaultBoundSetOperations.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2013 the original author or authors. + * Copyright 2011-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package org.springframework.data.redis.core; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -26,6 +27,7 @@ import org.springframework.data.redis.connection.DataType; * Default implementation for {@link BoundSetOperations}. * * @author Costin Leau + * @author Christoph Strobl */ class DefaultBoundSetOperations extends DefaultBoundKeyOperations implements BoundSetOperations { @@ -137,4 +139,13 @@ class DefaultBoundSetOperations extends DefaultBoundKeyOperations imple public DataType getType() { return DataType.SET; } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.core.BoundSetOperations#sScan(org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Iterator sScan(ScanOptions options) { + return ops.sScan(getKey(), options); + } } diff --git a/src/main/java/org/springframework/data/redis/core/DefaultSetOperations.java b/src/main/java/org/springframework/data/redis/core/DefaultSetOperations.java index 03f9355da..18e5bb1f9 100644 --- a/src/main/java/org/springframework/data/redis/core/DefaultSetOperations.java +++ b/src/main/java/org/springframework/data/redis/core/DefaultSetOperations.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2013 the original author or authors. + * Copyright 2011-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,12 +21,15 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import org.springframework.core.convert.converter.Converter; +import org.springframework.dao.DataAccessException; import org.springframework.data.redis.connection.RedisConnection; /** * Default implementation of {@link SetOperations}. * * @author Costin Leau + * @author Christoph Strobl */ class DefaultSetOperations extends AbstractOperations implements SetOperations { @@ -244,4 +247,28 @@ class DefaultSetOperations extends AbstractOperations implements Set } }, true); } + + /* + * (non-Javadoc) + * @see org.springframework.data.redis.core.SetOperations#sScan(java.lang.Object, org.springframework.data.redis.core.ScanOptions) + */ + @Override + public Cursor sScan(K key, final ScanOptions options) { + + final byte[] rawKey = rawKey(key); + return execute(new RedisCallback>() { + + @Override + public Cursor doInRedis(RedisConnection connection) throws DataAccessException { + return new ConvertingCursor(connection.sScan(rawKey, options), new Converter() { + + @Override + public V convert(byte[] source) { + return deserializeValue(source); + } + }); + } + }, true); + + } } diff --git a/src/main/java/org/springframework/data/redis/core/KeyBoundCursor.java b/src/main/java/org/springframework/data/redis/core/KeyBoundCursor.java new file mode 100644 index 000000000..39f42bff2 --- /dev/null +++ b/src/main/java/org/springframework/data/redis/core/KeyBoundCursor.java @@ -0,0 +1,48 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.redis.core; + +/** + * @author Christoph Strobl + * @param + * @since 1.4 + */ +public abstract class KeyBoundCursor extends ScanCursor { + + private byte[] key; + + /** + * Crates new {@link ScanCursor} + * + * @param cursorId + * @param options Defaulted to {@link ScanOptions#NONE} if nulled. + */ + public KeyBoundCursor(byte[] key, long cursorId, ScanOptions options) { + super(cursorId, options != null ? options : ScanOptions.NONE); + this.key = key; + } + + protected ScanIteration doScan(long cursorId, ScanOptions options) { + return doScan(this.key, cursorId, options); + } + + protected abstract ScanIteration doScan(byte[] key, long cursorId, ScanOptions options); + + public byte[] getKey() { + return key; + } + +} diff --git a/src/main/java/org/springframework/data/redis/core/ScanCursor.java b/src/main/java/org/springframework/data/redis/core/ScanCursor.java index e26e021d3..bd088db81 100644 --- a/src/main/java/org/springframework/data/redis/core/ScanCursor.java +++ b/src/main/java/org/springframework/data/redis/core/ScanCursor.java @@ -76,7 +76,7 @@ public abstract class ScanCursor implements Cursor { this.scanOptions = options != null ? options : ScanOptions.NONE; this.cursorId = cursorId; - this.state = CursorState.CLOSED; + this.state = CursorState.READY; this.delegate = Collections.emptyIterator(); } @@ -101,11 +101,13 @@ public abstract class ScanCursor implements Cursor { */ public final ScanCursor open() { - if (isClosed()) { - doOpen(cursorId); - state = CursorState.OPEN; + if (!isReady()) { + throw new InvalidDataAccessApiUsageException("Cursor already " + state + ". Cannot (re)open it."); } + state = CursorState.OPEN; + doOpen(cursorId); + return this; } @@ -162,6 +164,10 @@ public abstract class ScanCursor implements Cursor { assertCursorIsOpen(); + if (!delegate.hasNext() && !CursorState.FINISHED.equals(state)) { + scan(cursorId); + } + if (delegate.hasNext()) { return true; } @@ -175,7 +181,7 @@ public abstract class ScanCursor implements Cursor { private void assertCursorIsOpen() { - if (isClosed()) { + if (isReady() || isClosed()) { throw new InvalidDataAccessApiUsageException("Cannot access closed cursor. Did you forget to call open()?"); } } @@ -189,11 +195,7 @@ public abstract class ScanCursor implements Cursor { assertCursorIsOpen(); - if (state != CursorState.FINISHED && !delegate.hasNext()) { - scan(cursorId); - } - - if (!delegate.hasNext()) { + if (!hasNext()) { throw new NoSuchElementException("No more elements available for cursor " + cursorId + "."); } @@ -231,7 +233,7 @@ public abstract class ScanCursor implements Cursor { try { doClose(); - }finally { + } finally { state = CursorState.CLOSED; } } @@ -239,9 +241,7 @@ public abstract class ScanCursor implements Cursor { /** * Customization hook for cleaning up resources on when calling {@link #close()}. */ - protected void doClose() { - resetDelegate(); - } + protected void doClose() {} /* * (non-Javadoc) @@ -252,6 +252,14 @@ public abstract class ScanCursor implements Cursor { return state == CursorState.CLOSED; } + protected final boolean isReady() { + return state == CursorState.READY; + } + + protected final boolean isOpen() { + return state == CursorState.OPEN; + } + /* * (non-Javadoc) * @see org.springframework.data.redis.core.Cursor#getPosition() @@ -264,7 +272,7 @@ public abstract class ScanCursor implements Cursor { /** * @author Thomas Darimont */ - enum CursorState{ - OPEN, FINISHED, CLOSED; + enum CursorState { + READY, OPEN, FINISHED, CLOSED; } } diff --git a/src/main/java/org/springframework/data/redis/core/SetOperations.java b/src/main/java/org/springframework/data/redis/core/SetOperations.java index 214440605..5825df3d0 100644 --- a/src/main/java/org/springframework/data/redis/core/SetOperations.java +++ b/src/main/java/org/springframework/data/redis/core/SetOperations.java @@ -1,5 +1,5 @@ /* - * Copyright 2011-2013 the original author or authors. + * Copyright 2011-2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,6 +17,7 @@ package org.springframework.data.redis.core; import java.util.Collection; +import java.util.Iterator; import java.util.List; import java.util.Set; @@ -24,6 +25,7 @@ import java.util.Set; * Redis set specific operations. * * @author Costin Leau + * @author Christoph Strobl */ public interface SetOperations { @@ -72,4 +74,14 @@ public interface SetOperations { Long size(K key); RedisOperations getOperations(); + + /** + * Iterate over elements in set at {@code key}. + * + * @param key + * @param options + * @return + * @since 1.4 + */ + Iterator sScan(K key, ScanOptions options); } diff --git a/src/test/java/org/springframework/data/redis/connection/AbstractConnectionIntegrationTests.java b/src/test/java/org/springframework/data/redis/connection/AbstractConnectionIntegrationTests.java index 40d0368bf..0117931f4 100644 --- a/src/test/java/org/springframework/data/redis/connection/AbstractConnectionIntegrationTests.java +++ b/src/test/java/org/springframework/data/redis/connection/AbstractConnectionIntegrationTests.java @@ -1950,6 +1950,35 @@ public abstract class AbstractConnectionIntegrationTests { assertThat(i, is(itemCount)); } + /** + * @see DATAREDIS-304 + */ + @Test + @IfProfileValue(name = "redisVersion", value = "2.8") + public void sScanShouldReadEntireValueRange() { + + if (!ConnectionUtils.isJedis(connectionFactory) && !ConnectionUtils.isLettuce(connectionFactory)) { + throw new AssumptionViolatedException("SCAN is only available for jedis and lettuce"); + } + + if (connection.isPipelined() || connection.isQueueing()) { + throw new AssumptionViolatedException("SCAN is only available in non pipeline | queue mode."); + } + + connection.sAdd("sscankey", "bar"); + connection.sAdd("sscankey", "foo-1", "foo-2", "foo-3", "foo-4", "foo-5", "foo-6"); + + Cursor cursor = connection.sScan("sscankey", scanOptions().count(2).match("fo*").build()); + + int i = 0; + while (cursor.hasNext()) { + assertThat(cursor.next(), not(containsString("bar"))); + i++; + } + + assertThat(i, is(6)); + } + protected void verifyResults(List expected) { assertEquals(expected, getResults()); } diff --git a/src/test/java/org/springframework/data/redis/core/AbstractOperationsTestParams.java b/src/test/java/org/springframework/data/redis/core/AbstractOperationsTestParams.java index 8cf95d743..a6369c4f9 100644 --- a/src/test/java/org/springframework/data/redis/core/AbstractOperationsTestParams.java +++ b/src/test/java/org/springframework/data/redis/core/AbstractOperationsTestParams.java @@ -26,7 +26,7 @@ import org.springframework.data.redis.PersonObjectFactory; import org.springframework.data.redis.RawObjectFactory; import org.springframework.data.redis.SettingsUtils; import org.springframework.data.redis.StringObjectFactory; -import org.springframework.data.redis.connection.srp.SrpConnectionFactory; +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; import org.springframework.data.redis.serializer.GenericToStringSerializer; import org.springframework.data.redis.serializer.Jackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.JacksonJsonRedisSerializer; @@ -39,6 +39,7 @@ import org.springframework.oxm.xstream.XStreamMarshaller; * * @author Jennifer Hickey * @author Thomas Darimont + * @author Christoph Strobl */ abstract public class AbstractOperationsTestParams { @@ -61,56 +62,56 @@ abstract public class AbstractOperationsTestParams { throw new RuntimeException("Cannot init XStream", ex); } - SrpConnectionFactory srConnFactory = new SrpConnectionFactory(); - srConnFactory.setPort(SettingsUtils.getPort()); - srConnFactory.setHostName(SettingsUtils.getHost()); - srConnFactory.afterPropertiesSet(); + JedisConnectionFactory jedisConnectionFactory = new JedisConnectionFactory(); + jedisConnectionFactory.setPort(SettingsUtils.getPort()); + jedisConnectionFactory.setHostName(SettingsUtils.getHost()); + jedisConnectionFactory.afterPropertiesSet(); RedisTemplate stringTemplate = new StringRedisTemplate(); - stringTemplate.setConnectionFactory(srConnFactory); + stringTemplate.setConnectionFactory(jedisConnectionFactory); stringTemplate.afterPropertiesSet(); RedisTemplate longTemplate = new RedisTemplate(); longTemplate.setKeySerializer(new StringRedisSerializer()); longTemplate.setValueSerializer(new GenericToStringSerializer(Long.class)); - longTemplate.setConnectionFactory(srConnFactory); + longTemplate.setConnectionFactory(jedisConnectionFactory); longTemplate.afterPropertiesSet(); RedisTemplate doubleTemplate = new RedisTemplate(); doubleTemplate.setKeySerializer(new StringRedisSerializer()); doubleTemplate.setValueSerializer(new GenericToStringSerializer(Double.class)); - doubleTemplate.setConnectionFactory(srConnFactory); + doubleTemplate.setConnectionFactory(jedisConnectionFactory); doubleTemplate.afterPropertiesSet(); RedisTemplate rawTemplate = new RedisTemplate(); rawTemplate.setEnableDefaultSerializer(false); - rawTemplate.setConnectionFactory(srConnFactory); + rawTemplate.setConnectionFactory(jedisConnectionFactory); rawTemplate.afterPropertiesSet(); RedisTemplate personTemplate = new RedisTemplate(); - personTemplate.setConnectionFactory(srConnFactory); + personTemplate.setConnectionFactory(jedisConnectionFactory); personTemplate.afterPropertiesSet(); OxmSerializer serializer = new OxmSerializer(xstream, xstream); RedisTemplate xstreamStringTemplate = new RedisTemplate(); - xstreamStringTemplate.setConnectionFactory(srConnFactory); + xstreamStringTemplate.setConnectionFactory(jedisConnectionFactory); xstreamStringTemplate.setDefaultSerializer(serializer); xstreamStringTemplate.afterPropertiesSet(); RedisTemplate xstreamPersonTemplate = new RedisTemplate(); - xstreamPersonTemplate.setConnectionFactory(srConnFactory); + xstreamPersonTemplate.setConnectionFactory(jedisConnectionFactory); xstreamPersonTemplate.setValueSerializer(serializer); xstreamPersonTemplate.afterPropertiesSet(); JacksonJsonRedisSerializer jacksonJsonSerializer = new JacksonJsonRedisSerializer(Person.class); RedisTemplate jsonPersonTemplate = new RedisTemplate(); - jsonPersonTemplate.setConnectionFactory(srConnFactory); + jsonPersonTemplate.setConnectionFactory(jedisConnectionFactory); jsonPersonTemplate.setValueSerializer(jacksonJsonSerializer); jsonPersonTemplate.afterPropertiesSet(); Jackson2JsonRedisSerializer jackson2JsonSerializer = new Jackson2JsonRedisSerializer(Person.class); RedisTemplate jackson2JsonPersonTemplate = new RedisTemplate(); - jackson2JsonPersonTemplate.setConnectionFactory(srConnFactory); + jackson2JsonPersonTemplate.setConnectionFactory(jedisConnectionFactory); jackson2JsonPersonTemplate.setValueSerializer(jackson2JsonSerializer); jackson2JsonPersonTemplate.afterPropertiesSet(); diff --git a/src/test/java/org/springframework/data/redis/core/DefaultSetOperationsTests.java b/src/test/java/org/springframework/data/redis/core/DefaultSetOperationsTests.java index 5ab5d1093..ee708a008 100644 --- a/src/test/java/org/springframework/data/redis/core/DefaultSetOperationsTests.java +++ b/src/test/java/org/springframework/data/redis/core/DefaultSetOperationsTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2013 the original author or authors. + * Copyright 2013 - 2014 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,25 +15,19 @@ */ package org.springframework.data.redis.core; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.hasItems; -import static org.hamcrest.CoreMatchers.either; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeTrue; -import static org.springframework.data.redis.matcher.RedisTestMatchers.isEqual; +import static org.hamcrest.CoreMatchers.*; +import static org.junit.Assert.*; +import static org.junit.Assume.*; +import static org.springframework.data.redis.matcher.RedisTestMatchers.*; import java.util.Collection; import java.util.Collections; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Set; import org.hamcrest.CoreMatchers; -import org.hamcrest.core.CombinableMatcher; -import org.hamcrest.core.CombinableMatcher.CombinableEitherMatcher; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -43,11 +37,13 @@ import org.junit.runners.Parameterized.Parameters; import org.springframework.data.redis.ObjectFactory; import org.springframework.data.redis.RedisTestProfileValueSource; import org.springframework.data.redis.connection.RedisConnection; +import org.springframework.data.redis.serializer.StringRedisSerializer; /** * Integration test of {@link DefaultSetOperations} * * @author Jennifer Hickey + * @author Christoph Strobl */ @RunWith(Parameterized.class) public class DefaultSetOperationsTests { @@ -118,8 +114,8 @@ public class DefaultSetOperationsTests { setOps.add(setKey, v2); List members = setOps.randomMembers(setKey, 2); assertEquals(2, members.size()); - - assertThat(members, CoreMatchers.>either(hasItem(v1)).or(hasItem(v2))); + + assertThat(members, CoreMatchers.> either(hasItem(v1)).or(hasItem(v2))); } @Test @@ -198,4 +194,29 @@ public class DefaultSetOperationsTests { assertEquals(Long.valueOf(2), setOps.remove(key, v1, v2, v4)); assertThat(setOps.members(key), isEqual(Collections.singleton(v3))); } + + /** + * @see DATAREDIS-304 + */ + @Test + @SuppressWarnings("unchecked") + public void testSSCanReadsValuesFully() { + + // TODO: remove this when upgrading to jedis v.2.4.3 as this guard to avoids key serialization + assumeThat(redisTemplate.getKeySerializer(), instanceOf(StringRedisSerializer.class)); + K key = keyFactory.instance(); + V v1 = valueFactory.instance(); + V v2 = valueFactory.instance(); + V v3 = valueFactory.instance(); + + setOps.add(key, v1, v2, v3); + Iterator it = setOps.sScan(key, ScanOptions.scanOptions().count(1).build()); + long count = 0; + while (it.hasNext()) { + assertThat(it.next(), anyOf(equalTo(v1), equalTo(v2), equalTo(v3))); + count++; + } + assertThat(count, is(setOps.size(key))); + + } } diff --git a/src/test/java/org/springframework/data/redis/core/ScanCursorUnitTests.java b/src/test/java/org/springframework/data/redis/core/ScanCursorUnitTests.java index d2a8d1295..ab20425c6 100644 --- a/src/test/java/org/springframework/data/redis/core/ScanCursorUnitTests.java +++ b/src/test/java/org/springframework/data/redis/core/ScanCursorUnitTests.java @@ -111,7 +111,7 @@ public class ScanCursorUnitTests { CapturingCursorDummy cursor = new CapturingCursorDummy(null); - assertThat(cursor.isClosed(), is(true)); + assertThat(cursor.isClosed(), is(false)); exception.expect(InvalidDataAccessApiUsageException.class); exception.expectMessage("closed cursor"); @@ -122,7 +122,7 @@ public class ScanCursorUnitTests { /** * @see DATAREDIS-290 */ - @Test + @Test(expected = InvalidDataAccessApiUsageException.class) public void repoeningCursorShouldHappenAtLastPosition() throws IOException { LinkedList> values = new LinkedList>(); @@ -131,11 +131,8 @@ public class ScanCursorUnitTests { values.add(createIteration(0, "redis")); Cursor cursor = initCursor(values).open(); - cursor.open(); - assertThat(cursor.next(), is("spring")); assertThat(cursor.getCursorId(), is(1L)); - assertThat(cursor.hasNext(), is(true)); // close the cursor cursor.close(); @@ -143,14 +140,6 @@ public class ScanCursorUnitTests { // reopen cursor at last position cursor.open(); - - assertThat(cursor.next(), is("data")); - assertThat(cursor.getCursorId(), is(2L)); - assertThat(cursor.hasNext(), is(true)); - - assertThat(cursor.next(), is("redis")); - assertThat(cursor.getCursorId(), is(0L)); - assertThat(cursor.hasNext(), is(false)); } /** @@ -163,9 +152,8 @@ public class ScanCursorUnitTests { values.add(createIteration(1, "spring")); values.add(createIteration(2, "data")); values.add(createIteration(0, "redis")); - Cursor cursor = initCursor(values).open(); + Cursor cursor = initCursor(values); - cursor.open(); assertThat(cursor.getPosition(), is(0L)); cursor.next();