GH-3526: Fix Infinite Loop in FailoverCConnFactory

Resolves https://github.com/spring-projects/spring-integration/issues/3526

`FailoverClientConnectionFactory`

The logic to detect we had iterated over all factories and including
the one from which the previous connection was established was incorrect,
causing an infite loop until one of the factory connections was successful.

Change the logic to detect we have reset the iterator and the current failure
is from the same factory as the one from which the previous connection was
established.

**cherry-pick to 5.4.x, 5.3.x**

* Add diagnostics.

* Fix race in test.

* More race fixes and diagnostics.

* Remove diagnostics.

spring-integration-ip/src/test/java/org/springframework/integration/ip/tcp/connection/FailoverClientConnectionFactoryTests.java

@@ -28,8 +28,11 @@ import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.when;
 
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
 import java.io.UncheckedIOException;
+import java.net.ServerSocket;
 import java.net.Socket;
 import java.nio.channels.SocketChannel;
 import java.util.ArrayList;
@@ -41,7 +44,9 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
-import org.junit.Test;
+import javax.net.ServerSocketFactory;
+
+import org.junit.jupiter.api.Test;
 import org.mockito.InOrder;
 import org.mockito.Mockito;
 
@@ -62,6 +67,7 @@ import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageChannel;
 import org.springframework.messaging.SubscribableChannel;
 import org.springframework.messaging.support.GenericMessage;
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor;
 
 /**
  * @author Gary Russell
@@ -173,7 +179,7 @@ public class FailoverClientConnectionFactoryTests {
 		inOrder.verifyNoMoreInteractions();
 	}
 
-	@Test(expected = UncheckedIOException.class)
+	@Test
 	public void testFailoverAllDead() throws Exception {
 		AbstractClientConnectionFactory factory1 = mock(AbstractClientConnectionFactory.class);
 		AbstractClientConnectionFactory factory2 = mock(AbstractClientConnectionFactory.class);
@@ -193,10 +199,47 @@ public class FailoverClientConnectionFactoryTests {
 		FailoverClientConnectionFactory failoverFactory = new FailoverClientConnectionFactory(factories);
 		failoverFactory.start();
 		GenericMessage<String> message = new GenericMessage<String>("foo");
-		failoverFactory.getConnection().send(message);
+		assertThatExceptionOfType(UncheckedIOException.class).isThrownBy(() ->
+			failoverFactory.getConnection().send(message));
 		Mockito.verify(conn2).send(message);
 	}
 
+	@Test
+	void failoverAllDeadAfterSuccess() throws Exception {
+		ServerSocket ss1 = ServerSocketFactory.getDefault().createServerSocket(0);
+		ThreadPoolTaskExecutor exec = new ThreadPoolTaskExecutor();
+		exec.initialize();
+		exec.submit(() -> {
+			Socket accepted = ss1.accept();
+			BufferedReader br = new BufferedReader(new InputStreamReader(accepted.getInputStream()));
+			br.readLine();
+			accepted.getOutputStream().write("ok\r\n".getBytes());
+			accepted.close();
+			ss1.close();
+			return true;
+		});
+		TcpNetClientConnectionFactory cf1 = new TcpNetClientConnectionFactory("localhost", ss1.getLocalPort());
+		AbstractClientConnectionFactory cf2 = mock(AbstractClientConnectionFactory.class);
+		doThrow(new UncheckedIOException(new IOException("fail"))).when(cf2).getConnection();
+		CountDownLatch latch = new CountDownLatch(2);
+		cf1.setApplicationEventPublisher(event -> {
+			if (event instanceof TcpConnectionCloseEvent) {
+				latch.countDown();
+			}
+		});
+		cf2.setApplicationEventPublisher(event -> { });
+		FailoverClientConnectionFactory fccf = new FailoverClientConnectionFactory(List.of(cf1, cf2));
+		fccf.registerListener(msf -> {
+			latch.countDown();
+			return false;
+		});
+		fccf.start();
+		fccf.getConnection().send(new GenericMessage<>("test"));
+		assertThat(latch.await(10, TimeUnit.SECONDS)).isTrue();
+		assertThatExceptionOfType(UncheckedIOException.class).isThrownBy(() ->
+				fccf.getConnection().send(new GenericMessage<>("test")));
+	}
+
 	@Test
 	public void testFailoverAllDeadButOriginalOkAgain() throws Exception {
 		AbstractClientConnectionFactory factory1 = mock(AbstractClientConnectionFactory.class);
@@ -228,7 +271,7 @@ public class FailoverClientConnectionFactoryTests {
 		Mockito.verify(conn1, times(2)).send(message);
 	}
 
-	@Test(expected = UncheckedIOException.class)
+	@Test
 	public void testFailoverConnectNone() throws Exception {
 		AbstractClientConnectionFactory factory1 = mock(AbstractClientConnectionFactory.class);
 		AbstractClientConnectionFactory factory2 = mock(AbstractClientConnectionFactory.class);
@@ -242,7 +285,8 @@ public class FailoverClientConnectionFactoryTests {
 		FailoverClientConnectionFactory failoverFactory = new FailoverClientConnectionFactory(factories);
 		failoverFactory.start();
 		GenericMessage<String> message = new GenericMessage<String>("foo");
-		failoverFactory.getConnection().send(message);
+		assertThatExceptionOfType(UncheckedIOException.class).isThrownBy(() ->
+			failoverFactory.getConnection().send(message));
 	}
 
 	@Test