View Javadoc
1   /*
2    * Copyright (C) 2009-2010, Google Inc.
3    * Copyright (C) 2009, Robin Rosenberg <robin.rosenberg@dewire.com> and others
4    *
5    * This program and the accompanying materials are made available under the
6    * terms of the Eclipse Distribution License v. 1.0 which is available at
7    * https://www.eclipse.org/org/documents/edl-v10.php.
8    *
9    * SPDX-License-Identifier: BSD-3-Clause
10   */
11  
12  package org.eclipse.jgit.internal.storage.file;
13  
14  import static org.junit.Assert.assertArrayEquals;
15  import static org.junit.Assert.assertEquals;
16  import static org.junit.Assert.assertFalse;
17  import static org.junit.Assert.assertNotNull;
18  import static org.junit.Assert.assertNotSame;
19  import static org.junit.Assert.fail;
20  
21  import java.io.BufferedOutputStream;
22  import java.io.File;
23  import java.io.FileOutputStream;
24  import java.io.IOException;
25  import java.io.OutputStream;
26  import java.time.Instant;
27  
28  import org.eclipse.jgit.errors.IncorrectObjectTypeException;
29  import org.eclipse.jgit.errors.MissingObjectException;
30  import org.eclipse.jgit.internal.storage.pack.PackExt;
31  import org.eclipse.jgit.internal.storage.pack.PackWriter;
32  import org.eclipse.jgit.junit.RepositoryTestCase;
33  import org.eclipse.jgit.lib.AnyObjectId;
34  import org.eclipse.jgit.lib.Constants;
35  import org.eclipse.jgit.lib.NullProgressMonitor;
36  import org.eclipse.jgit.lib.ObjectId;
37  import org.eclipse.jgit.lib.ObjectInserter;
38  import org.eclipse.jgit.lib.ObjectLoader;
39  import org.eclipse.jgit.lib.Repository;
40  import org.eclipse.jgit.revwalk.RevObject;
41  import org.eclipse.jgit.revwalk.RevWalk;
42  import org.eclipse.jgit.storage.file.WindowCacheConfig;
43  import org.eclipse.jgit.util.FS;
44  import org.eclipse.jgit.util.FileUtils;
45  import org.junit.After;
46  import org.junit.Before;
47  import org.junit.Test;
48  
49  public class ConcurrentRepackTest extends RepositoryTestCase {
50  	@Override
51  	@Before
52  	public void setUp() throws Exception {
53  		WindowCacheConfig windowCacheConfig = new WindowCacheConfig();
54  		windowCacheConfig.setPackedGitOpenFiles(1);
55  		windowCacheConfig.install();
56  		super.setUp();
57  	}
58  
59  	@Override
60  	@After
61  	public void tearDown() throws Exception {
62  		super.tearDown();
63  		new WindowCacheConfig().install();
64  	}
65  
66  	@Test
67  	public void testObjectInNewPack() throws IncorrectObjectTypeException,
68  			IOException {
69  		// Create a new object in a new pack, and test that it is present.
70  		//
71  		final Repository eden = createBareRepository();
72  		final RevObject o1 = writeBlob(eden, "o1");
73  		pack(eden, o1);
74  		assertEquals(o1.name(), parse(o1).name());
75  	}
76  
77  	@Test
78  	public void testObjectMovedToNewPack1()
79  			throws IncorrectObjectTypeException, IOException {
80  		// Create an object and pack it. Then remove that pack and put the
81  		// object into a different pack file, with some other object. We
82  		// still should be able to access the objects.
83  		//
84  		final Repository eden = createBareRepository();
85  		final RevObject o1 = writeBlob(eden, "o1");
86  		final File[] out1 = pack(eden, o1);
87  		assertEquals(o1.name(), parse(o1).name());
88  
89  		final RevObject o2 = writeBlob(eden, "o2");
90  		pack(eden, o2, o1);
91  
92  		// Force close, and then delete, the old pack.
93  		//
94  		whackCache();
95  		delete(out1);
96  
97  		// Now here is the interesting thing. Will git figure the new
98  		// object exists in the new pack, and not the old one.
99  		//
100 		assertEquals(o2.name(), parse(o2).name());
101 		assertEquals(o1.name(), parse(o1).name());
102 	}
103 
104 	@Test
105 	public void testObjectMovedWithinPack()
106 			throws IncorrectObjectTypeException, IOException {
107 		// Create an object and pack it.
108 		//
109 		final Repository eden = createBareRepository();
110 		final RevObject o1 = writeBlob(eden, "o1");
111 		final File[] out1 = pack(eden, o1);
112 		assertEquals(o1.name(), parse(o1).name());
113 
114 		// Force close the old pack.
115 		//
116 		whackCache();
117 
118 		// Now overwrite the old pack in place. This method of creating a
119 		// different pack under the same file name is partially broken. We
120 		// should also have a different file name because the list of objects
121 		// within the pack has been modified.
122 		//
123 		final RevObject o2 = writeBlob(eden, "o2");
124 		try (PackWriter pw = new PackWriter(eden)) {
125 			pw.addObject(o2);
126 			pw.addObject(o1);
127 			write(out1, pw);
128 		}
129 
130 		// Try the old name, then the new name. The old name should cause the
131 		// pack to reload when it opens and the index and pack mismatch.
132 		//
133 		assertEquals(o1.name(), parse(o1).name());
134 		assertEquals(o2.name(), parse(o2).name());
135 	}
136 
137 	@Test
138 	public void testObjectMovedToNewPack2()
139 			throws IncorrectObjectTypeException, IOException {
140 		// Create an object and pack it. Then remove that pack and put the
141 		// object into a different pack file, with some other object. We
142 		// still should be able to access the objects.
143 		//
144 		final Repository eden = createBareRepository();
145 		final RevObject o1 = writeBlob(eden, "o1");
146 		final File[] out1 = pack(eden, o1);
147 		assertEquals(o1.name(), parse(o1).name());
148 
149 		final ObjectLoader load1 = db.open(o1, Constants.OBJ_BLOB);
150 		assertNotNull(load1);
151 
152 		final RevObject o2 = writeBlob(eden, "o2");
153 		pack(eden, o2, o1);
154 
155 		// Force close, and then delete, the old pack.
156 		//
157 		whackCache();
158 		delete(out1);
159 
160 		// Now here is the interesting thing... can the loader we made
161 		// earlier still resolve the object, even though its underlying
162 		// pack is gone, but the object still exists.
163 		//
164 		final ObjectLoader load2 = db.open(o1, Constants.OBJ_BLOB);
165 		assertNotNull(load2);
166 		assertNotSame(load1, load2);
167 
168 		final byte[] data2 = load2.getCachedBytes();
169 		final byte[] data1 = load1.getCachedBytes();
170 		assertNotNull(data2);
171 		assertNotNull(data1);
172 		assertNotSame(data1, data2); // cache should be per-pack, not per object
173 		assertArrayEquals(data1, data2);
174 		assertEquals(load2.getType(), load1.getType());
175 	}
176 
177 	private static void whackCache() {
178 		final WindowCacheConfig config = new WindowCacheConfig();
179 		config.setPackedGitOpenFiles(1);
180 		config.install();
181 	}
182 
183 	private RevObject parse(AnyObjectId id)
184 			throws MissingObjectException, IOException {
185 		try (RevWalk rw = new RevWalk(db)) {
186 			return rw.parseAny(id);
187 		}
188 	}
189 
190 	private File[] pack(Repository src, RevObject... list)
191 			throws IOException {
192 		try (PackWriter pw = new PackWriter(src)) {
193 			for (RevObject o : list) {
194 				pw.addObject(o);
195 			}
196 
197 			PackFile packFile = new PackFile(
198 					db.getObjectDatabase().getPackDirectory(), pw.computeName(),
199 					PackExt.PACK);
200 			PackFile idxFile = packFile.create(PackExt.INDEX);
201 			final File[] files = new File[] { packFile, idxFile };
202 			write(files, pw);
203 			return files;
204 		}
205 	}
206 
207 	private static void write(File[] files, PackWriter pw)
208 			throws IOException {
209 		final Instant begin = FS.DETECTED
210 				.lastModifiedInstant(files[0].getParentFile());
211 		NullProgressMonitor m = NullProgressMonitor.INSTANCE;
212 
213 		try (OutputStream out = new BufferedOutputStream(
214 				new FileOutputStream(files[0]))) {
215 			pw.writePack(m, m, out);
216 		}
217 
218 		try (OutputStream out = new BufferedOutputStream(
219 				new FileOutputStream(files[1]))) {
220 			pw.writeIndex(out);
221 		}
222 
223 		touch(begin, files[0].getParentFile());
224 	}
225 
226 	private static void delete(File[] list) throws IOException {
227 		final Instant begin = FS.DETECTED
228 				.lastModifiedInstant(list[0].getParentFile());
229 		for (File f : list) {
230 			FileUtils.delete(f);
231 			assertFalse(f + " was removed", f.exists());
232 		}
233 		touch(begin, list[0].getParentFile());
234 	}
235 
236 	private static void touch(Instant begin, File dir) throws IOException {
237 		while (begin.compareTo(FS.DETECTED.lastModifiedInstant(dir)) >= 0) {
238 			try {
239 				Thread.sleep(25);
240 			} catch (InterruptedException ie) {
241 				//
242 			}
243 			FS.DETECTED.setLastModified(dir.toPath(), Instant.now());
244 		}
245 	}
246 
247 	private RevObject writeBlob(Repository repo, String data)
248 			throws IOException {
249 		final byte[] bytes = Constants.encode(data);
250 		final ObjectId id;
251 		try (ObjectInserter inserter = repo.newObjectInserter()) {
252 			id = inserter.insert(Constants.OBJ_BLOB, bytes);
253 			inserter.flush();
254 		}
255 		try {
256 			parse(id);
257 			fail("Object " + id.name() + " should not exist in test repository");
258 		} catch (MissingObjectException e) {
259 			// Ok
260 		}
261 		try (RevWalk revWalk = new RevWalk(repo)) {
262 			return revWalk.lookupBlob(id);
263 		}
264 	}
265 }