diff --git a/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonAutoConfiguration.java b/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonAutoConfiguration.java index a9729321..cc12ecd8 100644 --- a/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonAutoConfiguration.java +++ b/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonAutoConfiguration.java @@ -32,6 +32,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cloud.client.actuator.HasFeatures; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicyFactory; import org.springframework.cloud.client.loadbalancer.LoadBalancerAutoConfiguration; import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; import org.springframework.cloud.client.loadbalancer.RestTemplateCustomizer; @@ -78,6 +79,11 @@ public class RibbonAutoConfiguration { return new RibbonLoadBalancerClient(springClientFactory()); } + @Bean + public LoadBalancedRetryPolicyFactory loadBalancedRetryPolicyFactory(SpringClientFactory clientFactory) { + return new RibbonLoadBalancedRetryPolicyFactory(clientFactory); + } + @Bean @ConditionalOnMissingBean public PropertiesFactory propertiesFactory() { diff --git a/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancedRetryPolicyFactory.java b/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancedRetryPolicyFactory.java new file mode 100644 index 00000000..35facbe2 --- /dev/null +++ b/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancedRetryPolicyFactory.java @@ -0,0 +1,92 @@ +/* + * Copyright 2013-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.cloud.netflix.ribbon; + +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryContext; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicyFactory; +import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; +import org.springframework.http.HttpMethod; + +/** + * @author Ryan Baxter + */ +public class RibbonLoadBalancedRetryPolicyFactory implements LoadBalancedRetryPolicyFactory { + + private SpringClientFactory clientFactory; + + public RibbonLoadBalancedRetryPolicyFactory(SpringClientFactory clientFactory) { + this.clientFactory = clientFactory; + } + @Override + public LoadBalancedRetryPolicy create(final String serviceId, final LoadBalancerClient loadBalancerClient) { + final RibbonLoadBalancerContext lbContext = this.clientFactory + .getLoadBalancerContext(serviceId); + return new LoadBalancedRetryPolicy() { + private int sameServerCount = 0; + private int nextServerCount = 0; + private ServiceInstance lastServiceInstance = null; + public boolean canRetry(LoadBalancedRetryContext context) { + HttpMethod method = context.getRequest().getMethod(); + return HttpMethod.GET == method || lbContext.isOkToRetryOnAllOperations(); + } + + @Override + public boolean canRetrySameServer(LoadBalancedRetryContext context) { + return sameServerCount < lbContext.getRetryHandler().getMaxRetriesOnSameServer() && canRetry(context); + } + + @Override + public boolean canRetryNextServer(LoadBalancedRetryContext context) { + //this will be called after a failure occurs and we increment the counter + //so we check that the count is less than or equals to too make sure + //we try the next server the right number of times + return nextServerCount <= lbContext.getRetryHandler().getMaxRetriesOnNextServer() && canRetry(context); + } + + @Override + public void close(LoadBalancedRetryContext context) { + + } + + @Override + public void registerThrowable(LoadBalancedRetryContext context, Throwable throwable) { + //Check if we need to ask the load balancer for a new server. + //Do this before we increment the counters because the first call to this method + //is not a retry it is just an initial failure. + if(!canRetrySameServer(context) && canRetryNextServer(context)) { + context.setServiceInstance(loadBalancerClient.choose(serviceId)); + } + //This method is called regardless of whether we are retrying or making the first request. + //Since we do not count the initial request in the retry count we don't reset the counter + //until we actually equal the same server count limit. This will allow us to make the initial + //request plus the right number of retries. + if(sameServerCount >= lbContext.getRetryHandler().getMaxRetriesOnSameServer() && canRetry(context)) { + //reset same server since we are moving to a new server + sameServerCount = 0; + nextServerCount++; + if(!canRetryNextServer(context)) { + context.setExhaustedOnly(); + } + } else { + sameServerCount++; + } + + } + }; + } +} diff --git a/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancerClient.java b/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancerClient.java index 811a48bb..96ef9ae2 100644 --- a/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancerClient.java +++ b/spring-cloud-netflix-core/src/main/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancerClient.java @@ -21,11 +21,8 @@ import com.netflix.loadbalancer.ILoadBalancer; import com.netflix.loadbalancer.Server; import org.springframework.cloud.client.DefaultServiceInstance; import org.springframework.cloud.client.ServiceInstance; -import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryContext; -import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy; import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; import org.springframework.cloud.client.loadbalancer.LoadBalancerRequest; -import org.springframework.http.HttpMethod; import org.springframework.util.Assert; import org.springframework.util.ReflectionUtils; import org.springframework.web.util.UriComponentsBuilder; @@ -38,8 +35,9 @@ import java.util.Map; /** * @author Spencer Gibb * @author Dave Syer + * @author Ryan Baxter */ -public class RibbonLoadBalancerClient implements LoadBalancerClient { +public class RibbonLoadBalancerClient implements LoadBalancerClient{ private SpringClientFactory clientFactory; @@ -62,33 +60,6 @@ public class RibbonLoadBalancerClient implements LoadBalancerClient { return context.reconstructURIWithServer(server, uri); } - @Override - public LoadBalancedRetryPolicy getRetryPolicy(String serviceId) { - final RibbonLoadBalancerContext lbContext = this.clientFactory - .getLoadBalancerContext(serviceId); - return new LoadBalancedRetryPolicy() { - @Override - public boolean canRetry(LoadBalancedRetryContext context) { - HttpMethod method = context.getRequest().getMethod(); - if(HttpMethod.GET == method || lbContext.isOkToRetryOnAllOperations()) { - return true; - } else { - return false; - } - } - - @Override - public void close(LoadBalancedRetryContext context) { - - } - - @Override - public void registerThrowable(LoadBalancedRetryContext context, Throwable throwable) { - - } - }; - } - @Override public ServiceInstance choose(String serviceId) { Server server = getServer(serviceId); @@ -112,6 +83,7 @@ public class RibbonLoadBalancerClient implements LoadBalancerClient { return execute(serviceId, ribbonServer, request); } + @Override public T execute(String serviceId, ServiceInstance serviceInstance, LoadBalancerRequest request) throws IOException { Server server = null; if(serviceInstance instanceof RibbonServer) { diff --git a/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonInterceptorTests.java b/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonInterceptorTests.java index dbdf52d7..644a7c37 100644 --- a/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonInterceptorTests.java +++ b/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonInterceptorTests.java @@ -23,10 +23,11 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; import org.springframework.cloud.client.ServiceInstance; -import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicyFactory; import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; import org.springframework.cloud.client.loadbalancer.LoadBalancerInterceptor; import org.springframework.cloud.client.loadbalancer.LoadBalancerRequest; +import org.springframework.cloud.client.loadbalancer.LoadBalancerRetryProperties; import org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient.RibbonServer; import org.springframework.http.HttpRequest; import org.springframework.http.client.ClientHttpRequestExecution; @@ -36,6 +37,7 @@ import org.springframework.retry.support.RetryTemplate; import org.springframework.util.ReflectionUtils; import org.springframework.web.util.UriComponentsBuilder; +import java.io.IOException; import java.net.URI; import java.net.URL; @@ -67,7 +69,8 @@ public class RibbonInterceptorTests { @Test public void testIntercept() throws Exception { RibbonServer server = new RibbonServer("myservice", new Server("myhost", 8080)); - LoadBalancerInterceptor interceptor = new LoadBalancerInterceptor(new MyClient(server), new RetryTemplate()); + LoadBalancerInterceptor interceptor = new LoadBalancerInterceptor(new MyClient(server), new RetryTemplate(), + new LoadBalancerRetryProperties(), new LoadBalancedRetryPolicyFactory.NeverRetryFactory()); given(this.request.getURI()).willReturn(new URL("http://myservice").toURI()); given(this.execution.execute(isA(HttpRequest.class), isA(byte[].class))) .willReturn(this.response); @@ -106,17 +109,23 @@ public class RibbonInterceptorTests { return null; } + @Override + public T execute(String s, ServiceInstance serviceInstance, LoadBalancerRequest request) throws IOException { + try { + return request.apply(this.instance); + } + catch (Exception ex) { + ReflectionUtils.rethrowRuntimeException(ex); + } + return null; + } + @Override public URI reconstructURI(ServiceInstance instance, URI original) { return UriComponentsBuilder.fromUri(original).host(instance.getHost()) .port(instance.getPort()).build().toUri(); } - @Override - public LoadBalancedRetryPolicy getRetryPolicy(String serviceId) { - return null; - } - } } diff --git a/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancedRetryPolicyFactoryTest.java b/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancedRetryPolicyFactoryTest.java new file mode 100644 index 00000000..aecf19bc --- /dev/null +++ b/spring-cloud-netflix-core/src/test/java/org/springframework/cloud/netflix/ribbon/RibbonLoadBalancedRetryPolicyFactoryTest.java @@ -0,0 +1,214 @@ +package org.springframework.cloud.netflix.ribbon; + +import com.netflix.client.DefaultLoadBalancerRetryHandler; +import com.netflix.client.config.CommonClientConfigKey; +import com.netflix.client.config.IClientConfig; +import com.netflix.loadbalancer.BaseLoadBalancer; +import com.netflix.loadbalancer.LoadBalancerStats; +import com.netflix.loadbalancer.Server; +import com.netflix.loadbalancer.ServerStats; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mock; +import org.mockito.MockitoAnnotations; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryContext; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryPolicy; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpRequest; +import org.springframework.cloud.netflix.ribbon.RibbonLoadBalancerClient.RibbonServer; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.is; +import static org.mockito.BDDMockito.given; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyInt; +import static org.mockito.Matchers.anyObject; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.anyString; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +/** + * @author Ryan Baxter + */ +public class RibbonLoadBalancedRetryPolicyFactoryTest { + + @Mock + private SpringClientFactory clientFactory; + + @Mock + private BaseLoadBalancer loadBalancer; + + @Mock + private LoadBalancerStats loadBalancerStats; + + @Mock + private ServerStats serverStats; + + @Before + public void setUp() throws Exception { + MockitoAnnotations.initMocks(this); + given(this.clientFactory.getLoadBalancerContext(anyString())).willReturn( + new RibbonLoadBalancerContext(this.loadBalancer)); + given(this.clientFactory.getInstance(anyString(), eq(ServerIntrospector.class))) + .willReturn(new DefaultServerIntrospector() { + @Override + public Map getMetadata(Server server) { + return Collections.singletonMap("mykey", "myvalue"); + } + }); + + } + + @After + public void tearDown() throws Exception {} + + @Test + public void testGetRetryPolicyNoRetry() throws Exception { + int sameServer = 0; + int nextServer = 0; + boolean retryOnAllOps = false; + RibbonServer server = getRibbonServer(); + IClientConfig config = mock(IClientConfig.class); + doReturn(sameServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(sameServer).when(config).getPropertyAsInteger(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(nextServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(nextServer).when(config).getPropertyAsInteger(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(retryOnAllOps).when(config).get(eq(CommonClientConfigKey.OkToRetryOnAllOperations), anyBoolean()); + doReturn(retryOnAllOps).when(config).getPropertyAsBoolean(eq(CommonClientConfigKey.OkToRetryOnAllOperations), anyBoolean()); + doReturn(server.getServiceId()).when(config).getClientName(); + doReturn(config).when(clientFactory).getClientConfig(eq(server.getServiceId())); + clientFactory.getLoadBalancerContext(server.getServiceId()).setRetryHandler(new DefaultLoadBalancerRetryHandler(config)); + RibbonLoadBalancerClient client = getRibbonLoadBalancerClient(server); + RibbonLoadBalancedRetryPolicyFactory factory = new RibbonLoadBalancedRetryPolicyFactory(clientFactory); + LoadBalancedRetryPolicy policy = factory.create(server.getServiceId(), client); + HttpRequest request = mock(HttpRequest.class); + doReturn(HttpMethod.GET).when(request).getMethod(); + LoadBalancedRetryContext context = new LoadBalancedRetryContext(null, request); + assertThat(policy.canRetryNextServer(context), is(true)); + assertThat(policy.canRetrySameServer(context), is(false)); + } + + @Test + public void testGetRetryPolicyNotGet() throws Exception { + int sameServer = 3; + int nextServer = 3; + boolean retryOnAllOps = false; + RibbonServer server = getRibbonServer(); + IClientConfig config = mock(IClientConfig.class); + doReturn(sameServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(sameServer).when(config).getPropertyAsInteger(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(nextServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(nextServer).when(config).getPropertyAsInteger(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(retryOnAllOps).when(config).get(eq(CommonClientConfigKey.OkToRetryOnAllOperations), anyBoolean()); + doReturn(retryOnAllOps).when(config).getPropertyAsBoolean(eq(CommonClientConfigKey.OkToRetryOnAllOperations), anyBoolean()); + doReturn(server.getServiceId()).when(config).getClientName(); + doReturn(config).when(clientFactory).getClientConfig(eq(server.getServiceId())); + clientFactory.getLoadBalancerContext(server.getServiceId()).setRetryHandler(new DefaultLoadBalancerRetryHandler(config)); + RibbonLoadBalancerClient client = getRibbonLoadBalancerClient(server); + RibbonLoadBalancedRetryPolicyFactory factory = new RibbonLoadBalancedRetryPolicyFactory(clientFactory); + LoadBalancedRetryPolicy policy = factory.create(server.getServiceId(), client); + HttpRequest request = mock(HttpRequest.class); + doReturn(HttpMethod.POST).when(request).getMethod(); + LoadBalancedRetryContext context = new LoadBalancedRetryContext(null, request); + assertThat(policy.canRetryNextServer(context), is(false)); + assertThat(policy.canRetrySameServer(context), is(false)); + } + + @Test + public void testGetRetryPolicyRetryOnNonGet() throws Exception { + int sameServer = 3; + int nextServer = 3; + boolean retryOnAllOps = true; + RibbonServer server = getRibbonServer(); + IClientConfig config = mock(IClientConfig.class); + doReturn(sameServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(sameServer).when(config).getPropertyAsInteger(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(nextServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(nextServer).when(config).getPropertyAsInteger(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(retryOnAllOps).when(config).get(eq(CommonClientConfigKey.OkToRetryOnAllOperations), anyBoolean()); + doReturn(retryOnAllOps).when(config).getPropertyAsBoolean(eq(CommonClientConfigKey.OkToRetryOnAllOperations), anyBoolean()); + doReturn(server.getServiceId()).when(config).getClientName(); + doReturn(config).when(clientFactory).getClientConfig(eq(server.getServiceId())); + clientFactory.getLoadBalancerContext(server.getServiceId()).initWithNiwsConfig(config); + RibbonLoadBalancerClient client = getRibbonLoadBalancerClient(server); + RibbonLoadBalancedRetryPolicyFactory factory = new RibbonLoadBalancedRetryPolicyFactory(clientFactory); + LoadBalancedRetryPolicy policy = factory.create(server.getServiceId(), client); + HttpRequest request = mock(HttpRequest.class); + doReturn(HttpMethod.POST).when(request).getMethod(); + LoadBalancedRetryContext context = new LoadBalancedRetryContext(null, request); + assertThat(policy.canRetryNextServer(context), is(true)); + assertThat(policy.canRetrySameServer(context), is(true)); + } + + @Test + public void testGetRetryPolicyRetryCount() throws Exception { + int sameServer = 3; + int nextServer = 3; + RibbonServer server = getRibbonServer(); + IClientConfig config = mock(IClientConfig.class); + doReturn(sameServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetries), anyInt()); + doReturn(nextServer).when(config).get(eq(CommonClientConfigKey.MaxAutoRetriesNextServer), anyInt()); + doReturn(false).when(config).get(eq(CommonClientConfigKey.OkToRetryOnAllOperations), eq(false)); + doReturn(config).when(clientFactory).getClientConfig(eq(server.getServiceId())); + clientFactory.getLoadBalancerContext(server.getServiceId()).setRetryHandler(new DefaultLoadBalancerRetryHandler(config)); + RibbonLoadBalancerClient client = getRibbonLoadBalancerClient(server); + RibbonLoadBalancedRetryPolicyFactory factory = new RibbonLoadBalancedRetryPolicyFactory(clientFactory); + LoadBalancedRetryPolicy policy = factory.create(server.getServiceId(), client); + HttpRequest request = mock(HttpRequest.class); + doReturn(HttpMethod.GET).when(request).getMethod(); + LoadBalancedRetryContext context = spy(new LoadBalancedRetryContext(null, request)); + //Loop through as if we are retrying a request until we exhaust the number of retries + //outer loop is for next server retries + //inner loop is for same server retries + for(int i = 0; i < nextServer + 1; i++) { + //iterate once time beyond the same server retry limit to cause us to reset + //the same sever counter and increment the next server counter + for(int j = 0; j < sameServer + 1; j++) { + if(j < 3) { + assertThat(policy.canRetrySameServer(context), is(true)); + } else { + assertThat(policy.canRetrySameServer(context), is(false)); + } + policy.registerThrowable(context, new IOException()); + } + if(i < 3) { + assertThat(policy.canRetryNextServer(context), is(true)); + } else { + assertThat(policy.canRetryNextServer(context), is(false)); + } + } + assertThat(context.isExhaustedOnly(), is(true)); + verify(context, times(4)).setServiceInstance(any(ServiceInstance.class)); + } + + protected RibbonLoadBalancerClient getRibbonLoadBalancerClient( + RibbonServer ribbonServer) { + given(this.loadBalancer.getName()).willReturn(ribbonServer.getServiceId()); + given(this.loadBalancer.chooseServer(anyObject())).willReturn( + ribbonServer.getServer()); + given(this.loadBalancer.getLoadBalancerStats()) + .willReturn(this.loadBalancerStats); + given(this.loadBalancerStats.getSingleServerStat(ribbonServer.getServer())) + .willReturn(this.serverStats); + given(this.clientFactory.getLoadBalancer(this.loadBalancer.getName())) + .willReturn(this.loadBalancer); + return new RibbonLoadBalancerClient(this.clientFactory); + } + + protected RibbonServer getRibbonServer() { + return new RibbonServer("testService", new Server("myhost", 9080), false, + Collections.singletonMap("mykey", "myvalue")); + } + +} \ No newline at end of file