Spring/spring-boot
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Reduce memory consumption of fat/exploded jars

Browse Source 
Refactor `spring-boot-loader` to reduce the amount of memory required
to load fat & exploded jars. Jar files now no longer store a full list
of entry data records, but instead use an array of entry name hashes.

Since ClassLoaders often ask each JAR if they contain a particular
entry (and mostly they do not), the hash array provides a quick way to
deal with misses. Only when a hash does exist is data actually loaded
from the underlying file.

In addition to the JarFile changes, the Archive abstraction has also
been updated to reduce memory consumption.

See gh-4882
This commit is contained in:
Phillip Webb
2016-01-11 10:17:19 +00:00
parent 858a854ce1
commit e2368b909b
24 changed files with 1148 additions and 801 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/archive/ExplodedArchiveTests.java
View File

@@ -37,11 +37,9 @@ import org.junit.rules.TemporaryFolder;
import org.springframework.boot.loader.TestJarCreator;
import org.springframework.boot.loader.archive.Archive.Entry;
import org.springframework.boot.loader.util.AsciiBytes;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.greaterThan;
import static org.hamcrest.Matchers.nullValue;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertThat;
@@ -130,27 +128,6 @@ public class ExplodedArchiveTests {
equalTo("file:" + this.rootFolder.toURI().getPath() + "d/"));
}
@Test
public void getFilteredArchive() throws Exception {
Archive filteredArchive = this.archive
.getFilteredArchive(new Archive.EntryRenameFilter() {
@Override
public AsciiBytes apply(AsciiBytes entryName, Entry entry) {
if (entryName.toString().equals("1.dat")) {
return entryName;
}
return null;
}
});
Map<String, Entry> entries = getEntriesMap(filteredArchive);
assertThat(entries.size(), equalTo(1));
URLClassLoader classLoader = new URLClassLoader(
new URL[] { filteredArchive.getUrl() });
assertThat(classLoader.getResourceAsStream("1.dat").read(), equalTo(1));
assertThat(classLoader.getResourceAsStream("2.dat"), nullValue());
classLoader.close();
}
@Test
public void getNonRecursiveEntriesForRoot() throws Exception {
ExplodedArchive archive = new ExplodedArchive(new File("/"), false);
@@ -198,7 +175,7 @@ public class ExplodedArchiveTests {
private Map<String, Archive.Entry> getEntriesMap(Archive archive) {
Map<String, Archive.Entry> entries = new HashMap<String, Archive.Entry>();
for (Archive.Entry entry : archive.getEntries()) {
for (Archive.Entry entry : archive) {
entries.put(entry.getName().toString(), entry);
}
return entries;

19
spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/archive/JarFileArchiveTests.java
View File

@@ -28,7 +28,6 @@ import org.junit.rules.TemporaryFolder;
import org.springframework.boot.loader.TestJarCreator;
import org.springframework.boot.loader.archive.Archive.Entry;
import org.springframework.boot.loader.util.AsciiBytes;
import static org.hamcrest.Matchers.endsWith;
import static org.hamcrest.Matchers.equalTo;
@@ -126,25 +125,9 @@ public class JarFileArchiveTests {
assertThat(nested.getParent(), is(equalTo(anotherNested.getParent())));
}
@Test
public void getFilteredArchive() throws Exception {
Archive filteredArchive = this.archive
.getFilteredArchive(new Archive.EntryRenameFilter() {
@Override
public AsciiBytes apply(AsciiBytes entryName, Entry entry) {
if (entryName.toString().equals("1.dat")) {
return entryName;
}
return null;
}
});
Map<String, Entry> entries = getEntriesMap(filteredArchive);
assertThat(entries.size(), equalTo(1));
}
private Map<String, Archive.Entry> getEntriesMap(Archive archive) {
Map<String, Archive.Entry> entries = new HashMap<String, Archive.Entry>();
for (Archive.Entry entry : archive.getEntries()) {
for (Archive.Entry entry : archive) {
entries.put(entry.getName().toString(), entry);
}
return entries;

20
spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/AsciiBytesTests.java → spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/AsciiBytesTests.java
View File

@@ -14,14 +14,12 @@
* limitations under the License.
*/
package org.springframework.boot.loader;
package org.springframework.boot.loader.jar;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.springframework.boot.loader.util.AsciiBytes;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.not;
import static org.junit.Assert.assertThat;
@@ -138,8 +136,22 @@ public class AsciiBytesTests {
assertThat(bc, equalTo(bc));
assertThat(bc, equalTo(bc_substring));
assertThat(bc, equalTo(bc_string));
assertThat(bc.hashCode(), not(equalTo(abcd.hashCode())));
assertThat(bc, not(equalTo(abcd)));
}
@Test
public void hashCodeSameAsString() throws Exception {
String s = "abcABC123xyz!";
AsciiBytes a = new AsciiBytes(s);
assertThat(s.hashCode(), equalTo(a.hashCode()));
}
@Test
public void hashCodeSameAsStringWithSpecial() throws Exception {
String s = "special/\u00EB.dat";
AsciiBytes a = new AsciiBytes(s);
assertThat(s.hashCode(), equalTo(a.hashCode()));
}
}

122
spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/CentralDirectoryParserTests.java Normal file
View File

@@ -0,0 +1,122 @@
/*
* Copyright 2012-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.boot.loader.jar;
import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.InOrder;
import org.springframework.boot.loader.TestJarCreator;
import org.springframework.boot.loader.data.RandomAccessData;
import org.springframework.boot.loader.data.RandomAccessDataFile;
import static org.hamcrest.Matchers.equalTo;
import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
/**
* Tests for {@link CentralDirectoryParser}.
*
* @author Phillip Webb
*/
public class CentralDirectoryParserTests {
@Rule
public TemporaryFolder temporaryFolder = new TemporaryFolder();
private File jarFile;
private RandomAccessData jarData;
@Before
public void setup() throws Exception {
this.jarFile = this.temporaryFolder.newFile();
TestJarCreator.createTestJar(this.jarFile);
this.jarData = new RandomAccessDataFile(this.jarFile);
}
@Test
public void vistsInOrder() throws Exception {
CentralDirectoryVistor vistor = mock(CentralDirectoryVistor.class);
CentralDirectoryParser parser = new CentralDirectoryParser();
parser.addVistor(vistor);
parser.parse(this.jarData, false);
InOrder ordered = inOrder(vistor);
ordered.verify(vistor).visitStart(any(CentralDirectoryEndRecord.class),
any(RandomAccessData.class));
ordered.verify(vistor, atLeastOnce())
.visitFileHeader(any(CentralDirectoryFileHeader.class), anyInt());
ordered.verify(vistor).visitEnd();
}
@Test
public void vistRecords() throws Exception {
Collector collector = new Collector();
CentralDirectoryParser parser = new CentralDirectoryParser();
parser.addVistor(collector);
parser.parse(this.jarData, false);
Iterator<CentralDirectoryFileHeader> headers = collector.getHeaders().iterator();
assertThat(headers.next().getName().toString(), equalTo("META-INF/"));
assertThat(headers.next().getName().toString(), equalTo("META-INF/MANIFEST.MF"));
assertThat(headers.next().getName().toString(), equalTo("1.dat"));
assertThat(headers.next().getName().toString(), equalTo("2.dat"));
assertThat(headers.next().getName().toString(), equalTo("d/"));
assertThat(headers.next().getName().toString(), equalTo("d/9.dat"));
assertThat(headers.next().getName().toString(), equalTo("special/"));
assertThat(headers.next().getName().toString(), equalTo("special/\u00EB.dat"));
assertThat(headers.next().getName().toString(), equalTo("nested.jar"));
assertThat(headers.next().getName().toString(), equalTo("another-nested.jar"));
assertThat(headers.hasNext(), equalTo(false));
}
private static class Collector implements CentralDirectoryVistor {
private List<CentralDirectoryFileHeader> headers = new ArrayList<CentralDirectoryFileHeader>();
@Override
public void visitStart(CentralDirectoryEndRecord endRecord,
RandomAccessData centralDirectoryData) {
}
@Override
public void visitFileHeader(CentralDirectoryFileHeader fileHeader,
int dataOffset) {
this.headers.add(fileHeader);
}
@Override
public void visitEnd() {
}
public List<CentralDirectoryFileHeader> getHeaders() {
return this.headers;
}
}
}

22
spring-boot-tools/spring-boot-loader/src/test/java/org/springframework/boot/loader/jar/JarFileTests.java
View File

@@ -38,7 +38,6 @@ import org.junit.rules.TemporaryFolder;
import org.springframework.boot.loader.TestJarCreator;
import org.springframework.boot.loader.data.RandomAccessDataFile;
import org.springframework.boot.loader.util.AsciiBytes;
import org.springframework.util.FileCopyUtils;
import org.springframework.util.StreamUtils;
@@ -349,27 +348,6 @@ public class JarFileTests {
assertThat(inputStream.read(), equalTo(-1));
}
@Test
public void getFilteredJarFile() throws Exception {
JarFile filteredJarFile = this.jarFile.getFilteredJarFile(new JarEntryFilter() {
@Override
public AsciiBytes apply(AsciiBytes entryName, JarEntryData entry) {
if (entryName.toString().equals("1.dat")) {
return new AsciiBytes("x.dat");
}
return null;
}
});
Enumeration<java.util.jar.JarEntry> entries = filteredJarFile.entries();
assertThat(entries.nextElement().getName(), equalTo("x.dat"));
assertThat(entries.hasMoreElements(), equalTo(false));
InputStream inputStream = filteredJarFile
.getInputStream(filteredJarFile.getEntry("x.dat"));
assertThat(inputStream.read(), equalTo(1));
assertThat(inputStream.read(), equalTo(-1));
}
@Test
public void sensibleToString() throws Exception {
assertThat(this.jarFile.toString(), equalTo(this.rootJarFile.getPath()));