Add option to clone repositories on start

On startup the server will clone all git repositories if the property
"spring.cloud.config.server.git.cloneOnStart" is set to true. If the
property is not set or is set to false then the server will behave
as it does currently.

Fixes gh-149

docs/src/main/asciidoc/spring-cloud-config.adoc

@@ -135,6 +135,36 @@ In this example the server searches for config files in the top level
 and in the "foo/" sub-directory and also any sub-directory whose name
 begins with "bar".
 
+By default the server clones remote repositories when configuration 
+is first requested. The server can be configured to clone the repositories
+at startup. For example at the top level:
+
+----
+spring:
+  cloud:
+    config:
+      server:
+        git:
+          uri: https://git/common/config-repo.git
+          repos:
+            team-a:
+                pattern: team-a-*
+                cloneOnStart: true
+                uri: http://git/team-a/config-repo.git
+            team-b:
+                pattern: team-b-*
+                cloneOnStart: false
+                uri: http://git/team-b/config-repo.git
+            team-c:
+                pattern: team-c-*
+                uri: http://git/team-a/config-repo.git
+                
+----
+
+In this example the server clones team-a's config-repo on startup before it 
+accepts any requests. All other repositories will not be cloned until 
+configuration from the repository is requested.
+
 To use HTTP basic authentication on the remote repository add the
 "username" and "password" properties separately (not in the URL), 
 e.g.

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/JGitEnvironmentRepository.java

@@ -16,12 +16,12 @@
 
 package org.springframework.cloud.config.server;
 
+import static org.springframework.util.StringUtils.hasText;
+
 import java.io.File;
 import java.io.IOException;
 import java.util.List;
 
-import com.jcraft.jsch.Session;
-
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.eclipse.jgit.api.CheckoutCommand;
@@ -47,11 +47,11 @@ import org.springframework.core.io.UrlResource;
 import org.springframework.util.Assert;
 import org.springframework.util.StringUtils;
 
-import static org.springframework.util.StringUtils.hasText;
+import com.jcraft.jsch.Session;
 
 /**
  * An {@link EnvironmentRepository} backed by a single git repository.
- * 
+ *
  * @author Dave Syer
  * @author Roy Clarkson
  */
@@ -60,13 +60,35 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 	private static Log logger = LogFactory.getLog(JGitEnvironmentRepository.class);
 
 	private static final String DEFAULT_LABEL = "master";
+	private static final String FILE_URI_PREFIX = "file:";
 
 	private boolean initialized;
 
+	private boolean cloneOnStart = false;
+
+	private JGitEnvironmentRepository.JGitFactory gitFactory =
+			new JGitEnvironmentRepository.JGitFactory();
+
 	public JGitEnvironmentRepository(ConfigurableEnvironment environment) {
 		super(environment);
 	}
 
+	public boolean isCloneOnStart() {
+		return this.cloneOnStart;
+	}
+
+	public void setCloneOnStart(boolean cloneOnStart) {
+		this.cloneOnStart = cloneOnStart;
+	}
+
+	public JGitFactory getGitFactory() {
+		return this.gitFactory;
+	}
+
+	public void setGitFactory(JGitFactory gitFactory) {
+		this.gitFactory = gitFactory;
+	}
+
 	@Override
 	public String getDefaultLabel() {
 		return DEFAULT_LABEL;
@@ -105,6 +127,23 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 	public void afterPropertiesSet() throws Exception {
 		Assert.state(getUri() != null,
 				"You need to configure a uri for the git repository");
+		if (this.cloneOnStart) {
+			initClonedRepository();
+		}
+	}
+
+	/**
+	 * Clones the remote repository and then opens a connection to it.
+	 * @throws GitAPIException
+	 * @throws IOException
+	 */
+	private void initClonedRepository() throws GitAPIException, IOException {
+		if (!getUri().startsWith(FILE_URI_PREFIX)) {
+			deleteBaseDirIfExists();
+			cloneToBasedir();
+			openGitRepository();
+		}
+
 	}
 
 	private synchronized Environment loadEnvironment(Git git, String application,
@@ -161,7 +200,7 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 					+ ref
 					+ "), remote: "
 					+ git.getRepository().getConfig()
-							.getString("remote", "origin", "url"));
+					.getString("remote", "origin", "url"));
 		}
 	}
 
@@ -177,7 +216,7 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 	private Git copyRepository() throws IOException, GitAPIException {
 		deleteBaseDirIfExists();
 		Assert.state(getBasedir().mkdirs(), "Could not create basedir: " + getBasedir());
-		if (getUri().startsWith("file:")) {
+		if (getUri().startsWith(FILE_URI_PREFIX)) {
 			return copyFromLocalRepository();
 		}
 		else {
@@ -186,7 +225,7 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 	}
 
 	private Git openGitRepository() throws IOException {
-		Git git = Git.open(getWorkingDirectory());
+		Git git = this.gitFactory.getGitByOpen(getWorkingDirectory());
 		tryFetch(git);
 		return git;
 	}
@@ -198,13 +237,13 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 		File gitDir = new File(remote, ".git");
 		Assert.state(gitDir.exists(), "No .git at " + getUri());
 		Assert.state(gitDir.isDirectory(), "No .git directory at " + getUri());
-		git = Git.open(remote);
+		git = this.gitFactory.getGitByOpen(remote);
 		return git;
 	}
 
 	private Git cloneToBasedir() throws GitAPIException {
-		CloneCommand clone = Git.cloneRepository().setURI(getUri())
-				.setDirectory(getBasedir());
+		CloneCommand clone = this.gitFactory.getCloneCommandByCloneRepository()
+				.setURI(getUri()).setDirectory(getBasedir());
 		if (hasText(getUsername())) {
 			setCredentialsProvider(clone);
 		}
@@ -236,14 +275,14 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 	}
 
 	private void initialize() {
-		if (getUri().startsWith("file:") && !initialized) {
+		if (getUri().startsWith("file:") && !this.initialized) {
 			SshSessionFactory.setInstance(new JschConfigSessionFactory() {
 				@Override
 				protected void configure(Host hc, Session session) {
 					session.setConfig("StrictHostKeyChecking", "no");
 				}
 			});
-			initialized = true;
+			this.initialized = true;
 		}
 	}
 
@@ -254,8 +293,8 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 
 	private void trackBranch(Git git, CheckoutCommand checkout, String label) {
 		checkout.setCreateBranch(true).setName(label)
-				.setUpstreamMode(SetupUpstreamMode.TRACK)
-				.setStartPoint("origin/" + label);
+		.setUpstreamMode(SetupUpstreamMode.TRACK)
+		.setStartPoint("origin/" + label);
 	}
 
 	private boolean isBranch(Git git, String label) throws GitAPIException {
@@ -280,4 +319,20 @@ public class JGitEnvironmentRepository extends AbstractScmEnvironmentRepository
 		}
 		return false;
 	}
+
+	/**
+	 * Wraps the static method calls to {@link org.eclipse.jgit.api.Git} and
+	 * {@link org.eclipse.jgit.api.CloneCommand} allowing for easier unit
+	 * testing.
+	 */
+	static class JGitFactory {
+
+		public Git getGitByOpen(File file) throws IOException {
+			return Git.open(file);
+		}
+
+		public CloneCommand getCloneCommandByCloneRepository() {
+			return Git.cloneRepository();
+		}
+	}
 }

spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/JGitEnvironmentRepositoryIntegrationTests.java

@@ -16,6 +16,10 @@
 
 package org.springframework.cloud.config.server;
 
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
@@ -26,7 +30,6 @@ import org.eclipse.jgit.util.FileUtils;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
-
 import org.springframework.boot.autoconfigure.PropertyPlaceholderAutoConfiguration;
 import org.springframework.boot.builder.SpringApplicationBuilder;
 import org.springframework.cloud.config.environment.Environment;
@@ -36,9 +39,6 @@ import org.springframework.context.annotation.Import;
 import org.springframework.util.ResourceUtils;
 import org.springframework.util.StreamUtils;
 
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-
 /**
  * @author Dave Syer
  * @author Roy Clarkson
@@ -51,30 +51,31 @@ public class JGitEnvironmentRepositoryIntegrationTests {
 
 	@Before
 	public void init() throws Exception {
-		if (basedir.exists()) {
-			FileUtils.delete(basedir, FileUtils.RECURSIVE);
+		if (this.basedir.exists()) {
+			FileUtils.delete(this.basedir, FileUtils.RECURSIVE);
 		}
 		ConfigServerTestUtils.deleteLocalRepo("config-copy");
 	}
 
 	@After
 	public void close() {
-		if (context != null) {
-			context.close();
+		if (this.context != null) {
+			this.context.close();
 		}
 	}
 
 	@Test
 	public void vanilla() throws IOException {
 		String uri = ConfigServerTestUtils.prepareLocalRepo();
-		context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
 				.properties("spring.cloud.config.server.git.uri:" + uri).run();
-		EnvironmentRepository repository = context.getBean(EnvironmentRepository.class);
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
 		repository.findOne("bar", "staging", "master");
 		Environment environment = repository.findOne("bar", "staging", "master");
 		assertEquals(2, environment.getPropertySources().size());
 		assertEquals("bar", environment.getName());
-		assertArrayEquals(new String[] {"staging"}, environment.getProfiles());
+		assertArrayEquals(new String[] { "staging" }, environment.getProfiles());
 		assertEquals("master", environment.getLabel());
 	}
 
@@ -82,9 +83,10 @@ public class JGitEnvironmentRepositoryIntegrationTests {
 	public void pull() throws Exception {
 		ConfigServerTestUtils.prepareLocalRepo();
 		String uri = ConfigServerTestUtils.copyLocalRepo("config-copy");
-		context = new SpringApplicationBuilder(TestConfiguration.class).web(false).run(
-				"--spring.cloud.config.server.git.uri=" + uri);
-		EnvironmentRepository repository = context.getBean(EnvironmentRepository.class);
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+				.run("--spring.cloud.config.server.git.uri=" + uri);
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
 		repository.findOne("bar", "staging", "master");
 		Environment environment = repository.findOne("bar", "staging", "master");
 		assertEquals("bar", environment.getPropertySources().get(0).getSource()
@@ -103,11 +105,12 @@ public class JGitEnvironmentRepositoryIntegrationTests {
 	@Test
 	public void nested() throws IOException {
 		String uri = ConfigServerTestUtils.prepareLocalRepo("another-config-repo");
-		context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
-		// TODO: why didn't .properties() work for me?
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+				// TODO: why didn't .properties() work for me?
 				.run("--spring.cloud.config.server.git.uri=" + uri,
 						"--spring.cloud.config.server.git.searchPaths=sub");
-		EnvironmentRepository repository = context.getBean(EnvironmentRepository.class);
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
 		repository.findOne("bar", "staging", "master");
 		Environment environment = repository.findOne("bar", "staging", "master");
 		assertEquals(2, environment.getPropertySources().size());
@@ -116,23 +119,94 @@ public class JGitEnvironmentRepositoryIntegrationTests {
 	@Test
 	public void defaultLabel() throws Exception {
 		String uri = ConfigServerTestUtils.prepareLocalRepo();
-		context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
 				.properties("spring.cloud.config.server.git.uri:" + uri).run();
-		EnvironmentRepository repository = context.getBean(EnvironmentRepository.class);
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
 		assertEquals("master", repository.getDefaultLabel());
 	}
 
 	@Test(expected = NoSuchLabelException.class)
 	public void invalidLabel() throws IOException {
 		String uri = ConfigServerTestUtils.prepareLocalRepo();
-		context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
 				.properties("spring.cloud.config.server.git.uri:" + uri).run();
-		EnvironmentRepository repository = context.getBean(EnvironmentRepository.class);
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
+		repository.findOne("bar", "staging", "unknownlabel");
+	}
+
+	@Test
+	public void findOne_CloneOnStartTrue_FindOneSuccess() throws Exception {
+		ConfigServerTestUtils.prepareLocalRepo();
+		String uri = ConfigServerTestUtils.copyLocalRepo("config-copy");
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+				.run("--spring.cloud.config.server.git.uri=" + uri,
+						"--spring.cloud.config.server.git.cloneOnStart=true");
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
+		assertTrue(((JGitEnvironmentRepository) repository).isCloneOnStart());
+		Environment environment = repository.findOne("bar", "staging", "master");
+		assertEquals(2, environment.getPropertySources().size());
+		assertEquals("bar", environment.getName());
+		assertArrayEquals(new String[] { "staging" }, environment.getProfiles());
+		assertEquals("master", environment.getLabel());
+	}
+
+	@Test
+	public void findOne_FileAddedToRepo_FindOneSuccess() throws Exception {
+		ConfigServerTestUtils.prepareLocalRepo();
+		String uri = ConfigServerTestUtils.copyLocalRepo("config-copy");
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+				.run("--spring.cloud.config.server.git.uri=" + uri,
+						"--spring.cloud.config.server.git.cloneOnStart=true");
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
+		repository.findOne("bar", "staging", "master");
+		Environment environment = repository.findOne("bar", "staging", "master");
+		assertEquals("bar", environment.getPropertySources().get(0).getSource()
+				.get("foo"));
+		Git git = Git.open(ResourceUtils.getFile(uri).getAbsoluteFile());
+		git.checkout().setName("master").call();
+		StreamUtils.copy("foo: foo", Charset.defaultCharset(), new FileOutputStream(
+				ResourceUtils.getFile(uri + "/bar.properties")));
+		git.add().addFilepattern("bar.properties").call();
+		git.commit().setMessage("Updated for pull").call();
+		environment = repository.findOne("bar", "staging", "master");
+		assertEquals("foo", environment.getPropertySources().get(0).getSource()
+				.get("foo"));
+	}
+
+	@Test
+	public void findOne_NestedSearchPath_FindOneSuccess() throws IOException {
+		String uri = ConfigServerTestUtils.prepareLocalRepo("another-config-repo");
+		this.context = new SpringApplicationBuilder(TestConfiguration.class).web(false)
+				// TODO: why didn't .properties() work for me?
+				.run("--spring.cloud.config.server.git.uri=" + uri,
+						"--spring.cloud.config.server.git.searchPaths=sub",
+						"--spring.cloud.config.server.git.cloneOnStart=true");
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
+		repository.findOne("bar", "staging", "master");
+		Environment environment = repository.findOne("bar", "staging", "master");
+		assertEquals(2, environment.getPropertySources().size());
+	}
+
+	@Test(expected = NoSuchLabelException.class)
+	public void findOne_FindInvalidLabel_IllegalStateExceptionThrown() throws IOException {
+		String uri = ConfigServerTestUtils.prepareLocalRepo();
+		this.context = new SpringApplicationBuilder(TestConfiguration.class)
+		.web(false)
+		.properties("spring.cloud.config.server.git.uri:" + uri,
+				"--spring.cloud.config.server.git.cloneOnStart=true").run();
+		EnvironmentRepository repository = this.context
+				.getBean(EnvironmentRepository.class);
 		repository.findOne("bar", "staging", "unknownlabel");
 	}
 
 	@Configuration
-	@Import({ PropertyPlaceholderAutoConfiguration.class, EnvironmentRepositoryConfiguration.class })
+	@Import({ PropertyPlaceholderAutoConfiguration.class,
+		EnvironmentRepositoryConfiguration.class })
 	protected static class TestConfiguration {
 	}
 

spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/JGitEnvironmentRepositoryTests.java

@@ -17,10 +17,13 @@
 package org.springframework.cloud.config.server;
 
 import static org.junit.Assert.*;
+import static org.mockito.Mockito.*;
 
 import java.io.File;
 import java.io.IOException;
 
+import org.eclipse.jgit.api.CloneCommand;
+import org.eclipse.jgit.api.Git;
 import org.eclipse.jgit.util.FileUtils;
 import org.junit.Before;
 import org.junit.Test;
@@ -134,5 +137,79 @@ public class JGitEnvironmentRepositoryTests {
 		repository.setUri("git://localhost/foo/");
 		assertEquals("git://localhost/foo", repository.getUri());
 	}
+	
+	@Test
+	public void afterPropertiesSet_CloneOnStartTrue_CloneAndFetchCalled() 
+			throws Exception {
+		Git mockGit = mock(Git.class);
+		CloneCommand mockCloneCommand = mock(CloneCommand.class);
+		
+		when(mockCloneCommand.setURI(anyString())).thenReturn(mockCloneCommand);
+		when(mockCloneCommand.setDirectory(any(File.class))).thenReturn(mockCloneCommand);
 
+		JGitEnvironmentRepository envRepository = new JGitEnvironmentRepository(
+				environment);
+		envRepository.setGitFactory(new MockGitFactory(mockGit, mockCloneCommand));
+		envRepository.setUri("http://somegitserver/somegitrepo");
+		envRepository.setCloneOnStart(true);
+		envRepository.afterPropertiesSet();
+		verify(mockCloneCommand, times(1)).call();
+		verify(mockGit, times(1)).fetch();
+	}
+
+	@Test
+	public void afterPropertiesSet_CloneOnStartFalse_CloneAndFetchNotCalled() 
+			throws Exception {
+		Git mockGit = mock(Git.class);
+		CloneCommand mockCloneCommand = mock(CloneCommand.class);
+		
+		when(mockCloneCommand.setURI(anyString())).thenReturn(mockCloneCommand);
+		when(mockCloneCommand.setDirectory(any(File.class))).thenReturn(mockCloneCommand);
+
+		JGitEnvironmentRepository envRepository = new JGitEnvironmentRepository(
+				environment);
+		envRepository.setGitFactory(new MockGitFactory(mockGit, mockCloneCommand));
+		envRepository.setUri("http://somegitserver/somegitrepo");
+		envRepository.afterPropertiesSet();
+		verify(mockCloneCommand, times(0)).call();
+		verify(mockGit, times(0)).fetch();
+	}
+
+	@Test
+	public void afterPropertiesSet_CloneOnStartTrueWithFileURL_CloneAndFetchNotCalled() 
+			throws Exception {
+		Git mockGit = mock(Git.class);
+		CloneCommand mockCloneCommand = mock(CloneCommand.class);
+		
+		when(mockCloneCommand.setURI(anyString())).thenReturn(mockCloneCommand);
+		when(mockCloneCommand.setDirectory(any(File.class))).thenReturn(mockCloneCommand);
+
+		JGitEnvironmentRepository envRepository = new JGitEnvironmentRepository(
+				environment);
+		envRepository.setGitFactory(new MockGitFactory(mockGit, mockCloneCommand));
+		envRepository.setUri("file://somefilesystem/somegitrepo");
+		envRepository.setCloneOnStart(true);
+		envRepository.afterPropertiesSet();
+		verify(mockCloneCommand, times(0)).call();
+		verify(mockGit, times(0)).fetch();
+	}
+
+	class MockGitFactory extends JGitEnvironmentRepository.JGitFactory {
+		
+		private Git mockGit;
+		private CloneCommand mockCloneCommand;
+		
+		public MockGitFactory (Git mockGit, CloneCommand mockCloneCommand) {
+			this.mockGit = mockGit;
+			this.mockCloneCommand = mockCloneCommand;
+		}
+		
+		public Git getGitByOpen(File file) throws IOException {
+			return mockGit;
+		}
+
+		public CloneCommand getCloneCommandByCloneRepository() {
+			return mockCloneCommand;
+		}
+	}
 }