GH-66: Add DynamoDbLockRegistry implementation (#93)

* GH-66: Add DynamoDbLockRegistry implementation Fixes https://github.com/spring-projects/spring-integration-aws/issues/66 * * Remove `Lifecycle` from `DynamoDbLockRegistry` in favor of a thread execution in the `afterPropertiesSet()` * Fix `lock()` interruptibility logic * * Remove `mavenLocal()` since the upstream PR is merged * decrease an amount of expectations in the Kinesis test * * Upgrade to SC-AWS-2.0.0.RC2 * * Add Docs for the `DynamoDbLockRegistry`
2018-06-08 16:49:23 -04:00
parent ffb4a6afe6
commit 5c262f1d3c
9 changed files with 1126 additions and 4 deletions
--- a/src/test/java/org/springframework/integration/aws/inbound/KinesisMessageDrivenChannelAdapterTests.java
+++ b/src/test/java/org/springframework/integration/aws/inbound/KinesisMessageDrivenChannelAdapterTests.java
@@ -207,7 +207,7 @@ public class KinesisMessageDrivenChannelAdapterTests {
 		assertThat(n).isLessThan(100);

 		// When resharding happens the describeStream() is performed again
-		verify(this.amazonKinesisForResharding, atLeast(2)).describeStream(any(DescribeStreamRequest.class));
+		verify(this.amazonKinesisForResharding, atLeast(1)).describeStream(any(DescribeStreamRequest.class));

 		this.reshardingChannelAdapter.stop();
 	}
--- a/src/test/java/org/springframework/integration/aws/leader/DynamoDbLockRegistryLeaderInitiatorTests.java
+++ b/src/test/java/org/springframework/integration/aws/leader/DynamoDbLockRegistryLeaderInitiatorTests.java
@@ -0,0 +1,249 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.aws.leader;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+
+import org.springframework.integration.aws.DynamoDbLocalRunning;
+import org.springframework.integration.aws.lock.DynamoDbLockRegistry;
+import org.springframework.integration.leader.Context;
+import org.springframework.integration.leader.DefaultCandidate;
+import org.springframework.integration.leader.event.LeaderEventPublisher;
+import org.springframework.integration.support.leader.LockRegistryLeaderInitiator;
+import org.springframework.scheduling.concurrent.CustomizableThreadFactory;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
+import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest;
+import com.amazonaws.waiters.FixedDelayStrategy;
+import com.amazonaws.waiters.MaxAttemptsRetryStrategy;
+import com.amazonaws.waiters.PollingStrategy;
+import com.amazonaws.waiters.Waiter;
+import com.amazonaws.waiters.WaiterParameters;
+
+/**
+ * @author Artem Bilan
+ *
+ * @since 2.0
+ */
+public class DynamoDbLockRegistryLeaderInitiatorTests {
+
+	@ClassRule
+	public static final DynamoDbLocalRunning DYNAMO_DB_RUNNING = DynamoDbLocalRunning.isRunning(4567);
+
+	private static AmazonDynamoDBAsync dynamoDB;
+
+	@BeforeClass
+	public static void init() {
+		dynamoDB = DYNAMO_DB_RUNNING.getDynamoDB();
+
+		try {
+			dynamoDB.deleteTableAsync(DynamoDbLockRegistry.DEFAULT_TABLE_NAME);
+
+			Waiter<DescribeTableRequest> waiter =
+					dynamoDB.waiters()
+							.tableNotExists();
+
+			waiter.run(new WaiterParameters<>(new DescribeTableRequest(DynamoDbLockRegistry.DEFAULT_TABLE_NAME))
+					.withPollingStrategy(new PollingStrategy(new MaxAttemptsRetryStrategy(25),
+							new FixedDelayStrategy(1))));
+		}
+		catch (Exception e) {
+
+		}
+	}
+
+	@AfterClass
+	public static void destroy() {
+		dynamoDB.deleteTable(DynamoDbLockRegistry.DEFAULT_TABLE_NAME);
+	}
+
+	@Test
+	public void testDistributedLeaderElection() throws Exception {
+		CountDownLatch granted = new CountDownLatch(1);
+		CountingPublisher countingPublisher = new CountingPublisher(granted);
+		List<DynamoDbLockRegistry> registries = new ArrayList<>();
+		List<LockRegistryLeaderInitiator> initiators = new ArrayList<>();
+		for (int i = 0; i < 2; i++) {
+			DynamoDbLockRegistry lockRepository = new DynamoDbLockRegistry(dynamoDB);
+			lockRepository.afterPropertiesSet();
+			registries.add(lockRepository);
+
+			LockRegistryLeaderInitiator initiator =
+					new LockRegistryLeaderInitiator(
+							lockRepository,
+							new DefaultCandidate("foo#" + i, "bar"));
+			initiator.setExecutorService(
+					Executors.newSingleThreadExecutor(new CustomizableThreadFactory("lock-leadership-" + i + "-")));
+			initiator.setLeaderEventPublisher(countingPublisher);
+			initiators.add(initiator);
+		}
+
+		for (LockRegistryLeaderInitiator initiator : initiators) {
+			initiator.start();
+		}
+
+		assertThat(granted.await(10, TimeUnit.SECONDS)).isTrue();
+
+		LockRegistryLeaderInitiator initiator1 = countingPublisher.initiator;
+
+		LockRegistryLeaderInitiator initiator2 = null;
+
+		for (LockRegistryLeaderInitiator initiator : initiators) {
+			if (initiator != initiator1) {
+				initiator2 = initiator;
+				break;
+			}
+		}
+
+		assertThat(initiator2).isNotNull();
+
+		assertThat(initiator1.getContext().isLeader()).isTrue();
+		assertThat(initiator2.getContext().isLeader()).isFalse();
+
+		final CountDownLatch granted1 = new CountDownLatch(1);
+		final CountDownLatch granted2 = new CountDownLatch(1);
+		CountDownLatch revoked1 = new CountDownLatch(1);
+		CountDownLatch revoked2 = new CountDownLatch(1);
+		CountDownLatch acquireLockFailed1 = new CountDownLatch(1);
+		CountDownLatch acquireLockFailed2 = new CountDownLatch(1);
+
+		initiator1.setLeaderEventPublisher(new CountingPublisher(granted1, revoked1, acquireLockFailed1));
+
+		initiator2.setLeaderEventPublisher(new CountingPublisher(granted2, revoked2, acquireLockFailed2));
+
+		// It's hard to see round-robin election, so let's make the yielding initiator to sleep long before restarting
+		initiator1.setBusyWaitMillis(1000);
+
+		initiator1.getContext().yield();
+
+		assertThat(revoked1.await(20, TimeUnit.SECONDS)).isTrue();
+		assertThat(granted2.await(20, TimeUnit.SECONDS)).isTrue();
+
+		assertThat(initiator2.getContext().isLeader()).isTrue();
+		assertThat(initiator1.getContext().isLeader()).isFalse();
+
+		initiator1.setBusyWaitMillis(LockRegistryLeaderInitiator.DEFAULT_BUSY_WAIT_TIME);
+		initiator2.setBusyWaitMillis(10000);
+
+		initiator2.getContext().yield();
+
+		assertThat(revoked2.await(20, TimeUnit.SECONDS)).isTrue();
+		assertThat(granted1.await(20, TimeUnit.SECONDS)).isTrue();
+
+		assertThat(initiator1.getContext().isLeader()).isTrue();
+		assertThat(initiator2.getContext().isLeader()).isFalse();
+
+		initiator2.stop();
+
+		CountDownLatch revoked11 = new CountDownLatch(1);
+		initiator1.setLeaderEventPublisher(new CountingPublisher(new CountDownLatch(1), revoked11,
+				new CountDownLatch(1)));
+
+		initiator1.getContext().yield();
+
+		assertThat(revoked11.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(initiator1.getContext().isLeader()).isFalse();
+
+		for (DynamoDbLockRegistry registry : registries) {
+			registry.destroy();
+		}
+	}
+
+	@Test
+	public void testLostConnection() throws Exception {
+		CountDownLatch granted = new CountDownLatch(1);
+		CountingPublisher countingPublisher = new CountingPublisher(granted);
+
+		DynamoDbLockRegistry lockRepository = new DynamoDbLockRegistry(dynamoDB);
+		lockRepository.afterPropertiesSet();
+
+		LockRegistryLeaderInitiator initiator = new LockRegistryLeaderInitiator(lockRepository);
+		initiator.setLeaderEventPublisher(countingPublisher);
+
+		initiator.start();
+
+		assertThat(granted.await(10, TimeUnit.SECONDS)).isTrue();
+
+		destroy();
+
+		assertThat(countingPublisher.revoked.await(10, TimeUnit.SECONDS)).isTrue();
+
+		granted = new CountDownLatch(1);
+		countingPublisher = new CountingPublisher(granted);
+		initiator.setLeaderEventPublisher(countingPublisher);
+
+		init();
+
+		lockRepository.afterPropertiesSet();
+
+		assertThat(granted.await(10, TimeUnit.SECONDS)).isTrue();
+
+		initiator.stop();
+
+		lockRepository.destroy();
+	}
+
+	private static class CountingPublisher implements LeaderEventPublisher {
+
+		private final CountDownLatch granted;
+
+		private final CountDownLatch revoked;
+
+		private final CountDownLatch acquireLockFailed;
+
+		private volatile LockRegistryLeaderInitiator initiator;
+
+		CountingPublisher(CountDownLatch granted, CountDownLatch revoked, CountDownLatch acquireLockFailed) {
+			this.granted = granted;
+			this.revoked = revoked;
+			this.acquireLockFailed = acquireLockFailed;
+		}
+
+		CountingPublisher(CountDownLatch granted) {
+			this(granted, new CountDownLatch(1), new CountDownLatch(1));
+		}
+
+		@Override
+		public void publishOnRevoked(Object source, Context context, String role) {
+			this.revoked.countDown();
+		}
+
+		@Override
+		public void publishOnFailedToAcquire(Object source, Context context, String role) {
+			this.acquireLockFailed.countDown();
+		}
+
+		@Override
+		public void publishOnGranted(Object source, Context context, String role) {
+			this.initiator = (LockRegistryLeaderInitiator) source;
+			this.granted.countDown();
+		}
+
+	}
+
+}
--- a/src/test/java/org/springframework/integration/aws/lock/DynamoDbLockRegistryTests.java
+++ b/src/test/java/org/springframework/integration/aws/lock/DynamoDbLockRegistryTests.java
@@ -0,0 +1,319 @@
+/*
+ * Copyright 2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.integration.aws.lock;
+
+import static org.assertj.core.api.Java6Assertions.assertThat;
+
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.Lock;
+
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.task.AsyncTaskExecutor;
+import org.springframework.core.task.SimpleAsyncTaskExecutor;
+import org.springframework.integration.aws.DynamoDbLocalRunning;
+import org.springframework.integration.test.util.TestUtils;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import com.amazonaws.services.dynamodbv2.AmazonDynamoDBAsync;
+import com.amazonaws.services.dynamodbv2.model.DescribeTableRequest;
+import com.amazonaws.waiters.FixedDelayStrategy;
+import com.amazonaws.waiters.MaxAttemptsRetryStrategy;
+import com.amazonaws.waiters.PollingStrategy;
+import com.amazonaws.waiters.Waiter;
+import com.amazonaws.waiters.WaiterParameters;
+
+/**
+ * @author Artem Bilan
+ *
+ * @since 2.0
+ */
+@RunWith(SpringRunner.class)
+@DirtiesContext
+public class DynamoDbLockRegistryTests {
+
+	@ClassRule
+	public static final DynamoDbLocalRunning DYNAMO_DB_RUNNING = DynamoDbLocalRunning.isRunning(4567);
+
+	private final AsyncTaskExecutor taskExecutor = new SimpleAsyncTaskExecutor();
+
+	@Autowired
+	private DynamoDbLockRegistry dynamoDbLockRegistry;
+
+	@BeforeClass
+	public static void setup() {
+		AmazonDynamoDBAsync dynamoDB = DYNAMO_DB_RUNNING.getDynamoDB();
+
+		try {
+			dynamoDB.deleteTableAsync(DynamoDbLockRegistry.DEFAULT_TABLE_NAME);
+
+			Waiter<DescribeTableRequest> waiter =
+					dynamoDB.waiters()
+							.tableNotExists();
+
+			waiter.run(new WaiterParameters<>(new DescribeTableRequest(DynamoDbLockRegistry.DEFAULT_TABLE_NAME))
+					.withPollingStrategy(new PollingStrategy(new MaxAttemptsRetryStrategy(25),
+							new FixedDelayStrategy(1))));
+		}
+		catch (Exception e) {
+
+		}
+	}
+
+	@Before
+	public void clear() {
+		this.dynamoDbLockRegistry.expireUnusedOlderThan(0);
+	}
+
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testLock() {
+		for (int i = 0; i < 10; i++) {
+			Lock lock = this.dynamoDbLockRegistry.obtain("foo");
+			lock.lock();
+			try {
+				assertThat(TestUtils.getPropertyValue(this.dynamoDbLockRegistry, "locks", Map.class)).hasSize(1);
+			}
+			finally {
+				lock.unlock();
+			}
+		}
+	}
+
+	@Test
+	@SuppressWarnings("unchecked")
+	public void testLockInterruptibly() throws Exception {
+		for (int i = 0; i < 10; i++) {
+			Lock lock = this.dynamoDbLockRegistry.obtain("foo");
+			lock.lockInterruptibly();
+			try {
+				assertThat(TestUtils.getPropertyValue(this.dynamoDbLockRegistry, "locks", Map.class)).hasSize(1);
+			}
+			finally {
+				lock.unlock();
+			}
+		}
+	}
+
+	@Test
+	public void testReentrantLock() {
+		for (int i = 0; i < 10; i++) {
+			Lock lock1 = this.dynamoDbLockRegistry.obtain("foo");
+			lock1.lock();
+			try {
+				Lock lock2 = this.dynamoDbLockRegistry.obtain("foo");
+				assertThat(lock1).isSameAs(lock2);
+				lock2.lock();
+				lock2.unlock();
+			}
+			finally {
+				lock1.unlock();
+			}
+		}
+	}
+
+	@Test
+	public void testReentrantLockInterruptibly() throws Exception {
+		for (int i = 0; i < 10; i++) {
+			Lock lock1 = this.dynamoDbLockRegistry.obtain("foo");
+			lock1.lockInterruptibly();
+			try {
+				Lock lock2 = this.dynamoDbLockRegistry.obtain("foo");
+				assertThat(lock1).isSameAs(lock2);
+				lock2.lockInterruptibly();
+				lock2.unlock();
+			}
+			finally {
+				lock1.unlock();
+			}
+		}
+	}
+
+	@Test
+	public void testTwoLocks() throws Exception {
+		for (int i = 0; i < 10; i++) {
+			Lock lock1 = this.dynamoDbLockRegistry.obtain("foo");
+			lock1.lockInterruptibly();
+			try {
+				Lock lock2 = this.dynamoDbLockRegistry.obtain("bar");
+				assertThat(lock1).isNotSameAs(lock2);
+				lock2.lockInterruptibly();
+				lock2.unlock();
+			}
+			finally {
+				lock1.unlock();
+			}
+		}
+	}
+
+	@Test
+	public void testTwoThreadsSecondFailsToGetLock() throws Exception {
+		final Lock lock1 = this.dynamoDbLockRegistry.obtain("foo");
+		lock1.lockInterruptibly();
+		final AtomicBoolean locked = new AtomicBoolean();
+		final CountDownLatch latch = new CountDownLatch(1);
+		Future<Object> result = this.taskExecutor.submit(() -> {
+			Lock lock2 = this.dynamoDbLockRegistry.obtain("foo");
+			locked.set(lock2.tryLock(200, TimeUnit.MILLISECONDS));
+			latch.countDown();
+			try {
+				lock2.unlock();
+			}
+			catch (Exception e) {
+				return e;
+			}
+			return null;
+		});
+		assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(locked.get()).isFalse();
+		lock1.unlock();
+		Object ise = result.get(10, TimeUnit.SECONDS);
+		assertThat(ise).isInstanceOf(IllegalMonitorStateException.class);
+		assertThat(((Exception) ise).getMessage()).contains("You do not own");
+	}
+
+	@Test
+	public void testTwoThreads() throws Exception {
+		final Lock lock1 = this.dynamoDbLockRegistry.obtain("foo");
+		final AtomicBoolean locked = new AtomicBoolean();
+		final CountDownLatch latch1 = new CountDownLatch(1);
+		final CountDownLatch latch2 = new CountDownLatch(1);
+		final CountDownLatch latch3 = new CountDownLatch(1);
+		lock1.lockInterruptibly();
+		this.taskExecutor.execute(() -> {
+			Lock lock2 = this.dynamoDbLockRegistry.obtain("foo");
+			try {
+				latch1.countDown();
+				lock2.lockInterruptibly();
+				latch2.await(10, TimeUnit.SECONDS);
+				locked.set(true);
+			}
+			catch (InterruptedException e) {
+				Thread.currentThread().interrupt();
+			}
+			finally {
+				lock2.unlock();
+				latch3.countDown();
+			}
+		});
+
+		assertThat(latch1.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(locked.get()).isFalse();
+
+		lock1.unlock();
+		latch2.countDown();
+
+		assertThat(latch3.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(locked.get()).isTrue();
+	}
+
+	@Test
+	public void testTwoThreadsDifferentRegistries() throws Exception {
+		final DynamoDbLockRegistry registry1 = new DynamoDbLockRegistry(DYNAMO_DB_RUNNING.getDynamoDB());
+		registry1.afterPropertiesSet();
+		final DynamoDbLockRegistry registry2 = new DynamoDbLockRegistry(DYNAMO_DB_RUNNING.getDynamoDB());
+		registry2.afterPropertiesSet();
+
+		final Lock lock1 = registry1.obtain("foo");
+		final AtomicBoolean locked = new AtomicBoolean();
+		final CountDownLatch latch1 = new CountDownLatch(1);
+		final CountDownLatch latch2 = new CountDownLatch(1);
+		final CountDownLatch latch3 = new CountDownLatch(1);
+		lock1.lockInterruptibly();
+		this.taskExecutor.execute(() -> {
+			Lock lock2 = registry2.obtain("foo");
+			try {
+				latch1.countDown();
+				lock2.lockInterruptibly();
+				latch2.await(10, TimeUnit.SECONDS);
+				locked.set(true);
+			}
+			catch (InterruptedException e) {
+				Thread.currentThread().interrupt();
+			}
+			finally {
+				lock2.unlock();
+				latch3.countDown();
+			}
+		});
+		assertThat(latch1.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(locked.get()).isFalse();
+
+		lock1.unlock();
+		latch2.countDown();
+
+		assertThat(latch3.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(locked.get()).isTrue();
+
+		registry1.destroy();
+		registry2.destroy();
+	}
+
+	@Test
+	public void testTwoThreadsWrongOneUnlocks() throws Exception {
+		final Lock lock = this.dynamoDbLockRegistry.obtain("foo");
+		lock.lockInterruptibly();
+		final AtomicBoolean locked = new AtomicBoolean();
+		final CountDownLatch latch = new CountDownLatch(1);
+		Future<Object> result =
+				this.taskExecutor.submit(() -> {
+					try {
+						lock.unlock();
+					}
+					catch (Exception e) {
+						latch.countDown();
+						return e;
+					}
+					return null;
+				});
+
+		assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThat(locked.get()).isFalse();
+
+		lock.unlock();
+		Object imse = result.get(10, TimeUnit.SECONDS);
+		assertThat(imse).isInstanceOf(IllegalMonitorStateException.class);
+		assertThat(((Exception) imse).getMessage()).contains("You do not own");
+	}
+
+
+	@Configuration
+	public static class ContextConfiguration {
+
+		@Bean
+		public DynamoDbLockRegistry dynamoDbLockRegistry() {
+			DynamoDbLockRegistry dynamoDbLockRegistry = new DynamoDbLockRegistry(DYNAMO_DB_RUNNING.getDynamoDB());
+			dynamoDbLockRegistry.setHeartbeatPeriod(1);
+			dynamoDbLockRegistry.setRefreshPeriod(10);
+			return dynamoDbLockRegistry;
+		}
+
+	}
+
+}