View Javadoc
1   /*
2    * Copyright (C) 2019, Matthias Sohn <matthias.sohn@sap.com> and others
3    *
4    * This program and the accompanying materials are made available under the
5    * terms of the Eclipse Distribution License v. 1.0 which is available at
6    * https://www.eclipse.org/org/documents/edl-v10.php.
7    *
8    * SPDX-License-Identifier: BSD-3-Clause
9    */
10  package org.eclipse.jgit.internal.storage.file;
11  
12  import static org.junit.Assert.assertEquals;
13  import static org.junit.Assert.assertFalse;
14  import static org.junit.Assert.assertNotNull;
15  import static org.junit.Assert.assertTrue;
16  import static org.junit.Assume.assumeFalse;
17  import static org.junit.Assume.assumeTrue;
18  
19  import java.io.File;
20  import java.io.IOException;
21  import java.io.OutputStream;
22  import java.io.Writer;
23  import java.nio.file.Files;
24  import java.nio.file.Path;
25  import java.nio.file.Paths;
26  import java.nio.file.StandardCopyOption;
27  import java.nio.file.StandardOpenOption;
28  import java.time.Instant;
29  import java.util.Collection;
30  import java.util.Iterator;
31  import java.util.Random;
32  import java.util.zip.Deflater;
33  
34  import org.eclipse.jgit.api.GarbageCollectCommand;
35  import org.eclipse.jgit.api.Git;
36  import org.eclipse.jgit.api.errors.AbortedByHookException;
37  import org.eclipse.jgit.api.errors.ConcurrentRefUpdateException;
38  import org.eclipse.jgit.api.errors.GitAPIException;
39  import org.eclipse.jgit.api.errors.NoFilepatternException;
40  import org.eclipse.jgit.api.errors.NoHeadException;
41  import org.eclipse.jgit.api.errors.NoMessageException;
42  import org.eclipse.jgit.api.errors.UnmergedPathsException;
43  import org.eclipse.jgit.api.errors.WrongRepositoryStateException;
44  import org.eclipse.jgit.junit.RepositoryTestCase;
45  import org.eclipse.jgit.lib.AnyObjectId;
46  import org.eclipse.jgit.lib.ConfigConstants;
47  import org.eclipse.jgit.lib.ObjectId;
48  import org.eclipse.jgit.storage.file.FileBasedConfig;
49  import org.eclipse.jgit.storage.pack.PackConfig;
50  import org.eclipse.jgit.util.FS;
51  import org.junit.Test;
52  
53  public class PackFileSnapshotTest extends RepositoryTestCase {
54  
55  	private static ObjectId unknownID = ObjectId
56  			.fromString("1234567890123456789012345678901234567890");
57  
58  	@Test
59  	public void testSamePackDifferentCompressionDetectChecksumChanged()
60  			throws Exception {
61  		Git git = Git.wrap(db);
62  		File f = writeTrashFile("file", "foobar ");
63  		for (int i = 0; i < 10; i++) {
64  			appendRandomLine(f);
65  			git.add().addFilepattern("file").call();
66  			git.commit().setMessage("message" + i).call();
67  		}
68  
69  		FileBasedConfig c = db.getConfig();
70  		c.setInt(ConfigConstants.CONFIG_GC_SECTION, null,
71  				ConfigConstants.CONFIG_KEY_AUTOPACKLIMIT, 1);
72  		c.save();
73  		Collection<Pack> packs = gc(Deflater.NO_COMPRESSION);
74  		assertEquals("expected 1 packfile after gc", 1, packs.size());
75  		Pack p1 = packs.iterator().next();
76  		PackFileSnapshot snapshot = p1.getFileSnapshot();
77  
78  		packs = gc(Deflater.BEST_COMPRESSION);
79  		assertEquals("expected 1 packfile after gc", 1, packs.size());
80  		Pack p2 = packs.iterator().next();
81  		File pf = p2.getPackFile();
82  
83  		// changing compression level with aggressive gc may change size,
84  		// fileKey (on *nix) and checksum. Hence FileSnapshot.isModified can
85  		// return true already based on size or fileKey.
86  		// So the only thing we can test here is that we ensure that checksum
87  		// also changed when we read it here in this test
88  		assertTrue("expected snapshot to detect modified pack",
89  				snapshot.isModified(pf));
90  		assertTrue("expected checksum changed", snapshot.isChecksumChanged(pf));
91  	}
92  
93  	private void appendRandomLine(File f, int length, Random r)
94  			throws IOException {
95  		try (Writer w = Files.newBufferedWriter(f.toPath(),
96  				StandardOpenOption.APPEND)) {
97  			appendRandomLine(w, length, r);
98  		}
99  	}
100 
101 	private void appendRandomLine(File f) throws IOException {
102 		appendRandomLine(f, 5, new Random());
103 	}
104 
105 	private void appendRandomLine(Writer w, int len, Random r)
106 			throws IOException {
107 		final int c1 = 32; // ' '
108 		int c2 = 126; // '~'
109 		for (int i = 0; i < len; i++) {
110 			w.append((char) (c1 + r.nextInt(1 + c2 - c1)));
111 		}
112 	}
113 
114 	private ObjectId createTestRepo(int testDataSeed, int testDataLength)
115 			throws IOException, GitAPIException, NoFilepatternException,
116 			NoHeadException, NoMessageException, UnmergedPathsException,
117 			ConcurrentRefUpdateException, WrongRepositoryStateException,
118 			AbortedByHookException {
119 		// Create a repo with two commits and one file. Each commit adds
120 		// testDataLength number of bytes. Data are random bytes. Since the
121 		// seed for the random number generator is specified we will get
122 		// the same set of bytes for every run and for every platform
123 		Random r = new Random(testDataSeed);
124 		Git git = Git.wrap(db);
125 		File f = writeTrashFile("file", "foobar ");
126 		appendRandomLine(f, testDataLength, r);
127 		git.add().addFilepattern("file").call();
128 		git.commit().setMessage("message1").call();
129 		appendRandomLine(f, testDataLength, r);
130 		git.add().addFilepattern("file").call();
131 		return git.commit().setMessage("message2").call().getId();
132 	}
133 
134 	// Try repacking so fast that you get two new packs which differ only in
135 	// content/chksum but have same name, size and lastmodified.
136 	// Since this is done with standard gc (which creates new tmp files and
137 	// renames them) the filekeys of the new packfiles differ helping jgit
138 	// to detect the fast modification
139 	@Test
140 	public void testDetectModificationAlthoughSameSizeAndModificationtime()
141 			throws Exception {
142 		int testDataSeed = 1;
143 		int testDataLength = 100;
144 		FileBasedConfig config = db.getConfig();
145 		// don't use mtime of the parent folder to detect pack file
146 		// modification.
147 		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
148 				ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, false);
149 		config.save();
150 
151 		createTestRepo(testDataSeed, testDataLength);
152 
153 		// repack to create initial packfile
154 		Pack p = repackAndCheck(5, null, null, null);
155 		Path packFilePath = p.getPackFile().toPath();
156 		AnyObjectId chk1 = p.getPackChecksum();
157 		String name = p.getPackName();
158 		Long length = Long.valueOf(p.getPackFile().length());
159 		FS fs = db.getFS();
160 		Instant m1 = fs.lastModifiedInstant(packFilePath);
161 
162 		// Wait for a filesystem timer tick to enhance probability the rest of
163 		// this test is done before the filesystem timer ticks again.
164 		fsTick(packFilePath.toFile());
165 
166 		// Repack to create packfile with same name, length. Lastmodified and
167 		// content and checksum are different since compression level differs
168 		AnyObjectId chk2 = repackAndCheck(6, name, length, chk1)
169 				.getPackChecksum();
170 		Instant m2 = fs.lastModifiedInstant(packFilePath);
171 		assumeFalse(m2.equals(m1));
172 
173 		// Repack to create packfile with same name, length. Lastmodified is
174 		// equal to the previous one because we are in the same filesystem timer
175 		// slot. Content and its checksum are different
176 		AnyObjectId chk3 = repackAndCheck(7, name, length, chk2)
177 				.getPackChecksum();
178 		Instant m3 = fs.lastModifiedInstant(packFilePath);
179 
180 		// ask for an unknown git object to force jgit to rescan the list of
181 		// available packs. If we would ask for a known objectid then JGit would
182 		// skip searching for new/modified packfiles
183 		db.getObjectDatabase().has(unknownID);
184 		assertEquals(chk3, getSinglePack(db.getObjectDatabase().getPacks())
185 				.getPackChecksum());
186 		assumeTrue(m3.equals(m2));
187 	}
188 
189 	// Try repacking so fast that we get two new packs which differ only in
190 	// content and checksum but have same name, size and lastmodified.
191 	// To avoid that JGit detects modification by checking the filekey create
192 	// two new packfiles upfront and create copies of them. Then modify the
193 	// packfiles in-place by opening them for write and then copying the
194 	// content.
195 	@Test
196 	public void testDetectModificationAlthoughSameSizeAndModificationtimeAndFileKey()
197 			throws Exception {
198 		int testDataSeed = 1;
199 		int testDataLength = 100;
200 		FileBasedConfig config = db.getConfig();
201 		config.setBoolean(ConfigConstants.CONFIG_CORE_SECTION, null,
202 				ConfigConstants.CONFIG_KEY_TRUSTFOLDERSTAT, false);
203 		config.save();
204 
205 		createTestRepo(testDataSeed, testDataLength);
206 
207 		// Repack to create initial packfile. Make a copy of it
208 		Pack p = repackAndCheck(5, null, null, null);
209 		Path packFilePath = p.getPackFile().toPath();
210 		Path fn = packFilePath.getFileName();
211 		assertNotNull(fn);
212 		String packFileName = fn.toString();
213 		Path packFileBasePath = packFilePath
214 				.resolveSibling(packFileName.replaceAll(".pack", ""));
215 		AnyObjectId chk1 = p.getPackChecksum();
216 		String name = p.getPackName();
217 		Long length = Long.valueOf(p.getPackFile().length());
218 		copyPack(packFileBasePath, "", ".copy1");
219 
220 		// Repack to create second packfile. Make a copy of it
221 		AnyObjectId chk2 = repackAndCheck(6, name, length, chk1)
222 				.getPackChecksum();
223 		copyPack(packFileBasePath, "", ".copy2");
224 
225 		// Repack to create third packfile
226 		AnyObjectId chk3 = repackAndCheck(7, name, length, chk2)
227 				.getPackChecksum();
228 		FS fs = db.getFS();
229 		Instant m3 = fs.lastModifiedInstant(packFilePath);
230 		db.getObjectDatabase().has(unknownID);
231 		assertEquals(chk3, getSinglePack(db.getObjectDatabase().getPacks())
232 				.getPackChecksum());
233 
234 		// Wait for a filesystem timer tick to enhance probability the rest of
235 		// this test is done before the filesystem timer ticks.
236 		fsTick(packFilePath.toFile());
237 
238 		// Copy copy2 to packfile data to force modification of packfile without
239 		// changing the packfile's filekey.
240 		copyPack(packFileBasePath, ".copy2", "");
241 		Instant m2 = fs.lastModifiedInstant(packFilePath);
242 		assumeFalse(m3.equals(m2));
243 
244 		db.getObjectDatabase().has(unknownID);
245 		assertEquals(chk2, getSinglePack(db.getObjectDatabase().getPacks())
246 				.getPackChecksum());
247 
248 		// Copy copy2 to packfile data to force modification of packfile without
249 		// changing the packfile's filekey.
250 		copyPack(packFileBasePath, ".copy1", "");
251 		Instant m1 = fs.lastModifiedInstant(packFilePath);
252 		assumeTrue(m2.equals(m1));
253 		db.getObjectDatabase().has(unknownID);
254 		assertEquals(chk1, getSinglePack(db.getObjectDatabase().getPacks())
255 				.getPackChecksum());
256 	}
257 
258 	// Copy file from src to dst but avoid creating a new File (with new
259 	// FileKey) if dst already exists
260 	private Path copyFile(Path src, Path dst) throws IOException {
261 		if (Files.exists(dst)) {
262 			dst.toFile().setWritable(true);
263 			try (OutputStream dstOut = Files.newOutputStream(dst)) {
264 				Files.copy(src, dstOut);
265 				return dst;
266 			}
267 		}
268 		return Files.copy(src, dst, StandardCopyOption.REPLACE_EXISTING);
269 	}
270 
271 	private Path copyPack(Path base, String srcSuffix, String dstSuffix)
272 			throws IOException {
273 		copyFile(Paths.get(base + ".idx" + srcSuffix),
274 				Paths.get(base + ".idx" + dstSuffix));
275 		copyFile(Paths.get(base + ".bitmap" + srcSuffix),
276 				Paths.get(base + ".bitmap" + dstSuffix));
277 		return copyFile(Paths.get(base + ".pack" + srcSuffix),
278 				Paths.get(base + ".pack" + dstSuffix));
279 	}
280 
281 	private Pack repackAndCheck(int compressionLevel, String oldName,
282 			Long oldLength, AnyObjectId oldChkSum) throws Exception {
283 		Pack p = getSinglePack(gc(compressionLevel));
284 		File pf = p.getPackFile();
285 		// The following two assumptions should not cause the test to fail. If
286 		// on a certain platform we get packfiles (containing the same git
287 		// objects) where the lengths differ or the checksums don't differ we
288 		// just skip this test. A reason for that could be that compression
289 		// works differently or random number generator works differently. Then
290 		// we have to search for more consistent test data or checkin these
291 		// packfiles as test resources
292 		assumeTrue(oldLength == null || pf.length() == oldLength.longValue());
293 		assumeTrue(oldChkSum == null || !p.getPackChecksum().equals(oldChkSum));
294 		assertTrue(oldName == null || p.getPackName().equals(oldName));
295 		return p;
296 	}
297 
298 	private Pack getSinglePack(Collection<Pack> packs) {
299 		Iterator<Pack> pIt = packs.iterator();
300 		Pack p = pIt.next();
301 		assertFalse(pIt.hasNext());
302 		return p;
303 	}
304 
305 	private Collection<Pack> gc(int compressionLevel) throws Exception {
306 		GC gc = new GC(db);
307 		PackConfig pc = new PackConfig(db.getConfig());
308 		pc.setCompressionLevel(compressionLevel);
309 
310 		pc.setSinglePack(true);
311 
312 		// --aggressive
313 		pc.setDeltaSearchWindowSize(
314 				GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_WINDOW);
315 		pc.setMaxDeltaDepth(GarbageCollectCommand.DEFAULT_GC_AGGRESSIVE_DEPTH);
316 		pc.setReuseObjects(false);
317 
318 		gc.setPackConfig(pc);
319 		gc.setExpireAgeMillis(0);
320 		gc.setPackExpireAgeMillis(0);
321 		return gc.gc().get();
322 	}
323 
324 }