1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44 package org.eclipse.jgit.internal.storage.file;
45
46 import static java.util.Comparator.comparing;
47 import static java.util.stream.Collectors.toList;
48 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
49 import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT;
50 import static org.hamcrest.MatcherAssert.assertThat;
51 import static org.hamcrest.Matchers.greaterThan;
52 import static org.hamcrest.Matchers.lessThan;
53 import static org.junit.Assert.assertArrayEquals;
54 import static org.junit.Assert.assertEquals;
55 import static org.junit.Assert.assertNotEquals;
56 import static org.junit.Assert.fail;
57
58 import java.io.ByteArrayInputStream;
59 import java.io.File;
60 import java.io.IOException;
61 import java.nio.file.FileVisitResult;
62 import java.nio.file.Files;
63 import java.nio.file.Path;
64 import java.nio.file.SimpleFileVisitor;
65 import java.nio.file.attribute.BasicFileAttributes;
66 import java.util.ArrayList;
67 import java.util.Collection;
68 import java.util.List;
69 import java.util.Random;
70 import java.util.function.Predicate;
71 import java.util.regex.Matcher;
72 import java.util.regex.Pattern;
73
74 import org.eclipse.jgit.dircache.DirCache;
75 import org.eclipse.jgit.dircache.DirCacheBuilder;
76 import org.eclipse.jgit.dircache.DirCacheEntry;
77 import org.eclipse.jgit.errors.MissingObjectException;
78 import org.eclipse.jgit.junit.RepositoryTestCase;
79 import org.eclipse.jgit.lib.CommitBuilder;
80 import org.eclipse.jgit.lib.Constants;
81 import org.eclipse.jgit.lib.FileMode;
82 import org.eclipse.jgit.lib.ObjectId;
83 import org.eclipse.jgit.lib.ObjectLoader;
84 import org.eclipse.jgit.lib.ObjectReader;
85 import org.eclipse.jgit.lib.ObjectStream;
86 import org.eclipse.jgit.storage.file.WindowCacheConfig;
87 import org.eclipse.jgit.treewalk.CanonicalTreeParser;
88 import org.eclipse.jgit.util.IO;
89 import org.junit.After;
90 import org.junit.Before;
91 import org.junit.Test;
92
93 @SuppressWarnings("boxing")
94 public class PackInserterTest extends RepositoryTestCase {
95 private WindowCacheConfig origWindowCacheConfig;
96
97 private static final Random random = new Random(0);
98
99 @Before
100 public void setWindowCacheConfig() {
101 origWindowCacheConfig = new WindowCacheConfig();
102 origWindowCacheConfig.install();
103 }
104
105 @After
106 public void resetWindowCacheConfig() {
107 origWindowCacheConfig.install();
108 }
109
110 @Before
111 public void emptyAtSetUp() throws Exception {
112 assertEquals(0, listPacks().size());
113 assertNoObjects();
114 }
115
116 @Test
117 public void noFlush() throws Exception {
118 try (PackInserter ins = newInserter()) {
119 ins.insert(OBJ_BLOB, Constants.encode("foo contents"));
120
121 }
122 assertNoObjects();
123 }
124
125 @Test
126 public void flushEmptyPack() throws Exception {
127 try (PackInserter ins = newInserter()) {
128 ins.flush();
129 }
130 assertNoObjects();
131 }
132
133 @Test
134 public void singlePack() throws Exception {
135 ObjectId blobId;
136 byte[] blob = Constants.encode("foo contents");
137 ObjectId treeId;
138 ObjectId commitId;
139 byte[] commit;
140 try (PackInserter ins = newInserter()) {
141 blobId = ins.insert(OBJ_BLOB, blob);
142
143 DirCache dc = DirCache.newInCore();
144 DirCacheBuilder b = dc.builder();
145 DirCacheEntry dce = new DirCacheEntry("foo");
146 dce.setFileMode(FileMode.REGULAR_FILE);
147 dce.setObjectId(blobId);
148 b.add(dce);
149 b.finish();
150 treeId = dc.writeTree(ins);
151
152 CommitBuilder cb = new CommitBuilder();
153 cb.setTreeId(treeId);
154 cb.setAuthor(author);
155 cb.setCommitter(committer);
156 cb.setMessage("Commit message");
157 commit = cb.toByteArray();
158 commitId = ins.insert(cb);
159 ins.flush();
160 }
161
162 assertPacksOnly();
163 List<Pack> packs = listPacks();
164 assertEquals(1, packs.size());
165 assertEquals(3, packs.get(0).getObjectCount());
166
167 try (ObjectReader reader = db.newObjectReader()) {
168 assertBlob(reader, blobId, blob);
169
170 CanonicalTreeParser treeParser =
171 new CanonicalTreeParser(null, reader, treeId);
172 assertEquals("foo", treeParser.getEntryPathString());
173 assertEquals(blobId, treeParser.getEntryObjectId());
174
175 ObjectLoader commitLoader = reader.open(commitId);
176 assertEquals(OBJ_COMMIT, commitLoader.getType());
177 assertArrayEquals(commit, commitLoader.getBytes());
178 }
179 }
180
181 @Test
182 public void multiplePacks() throws Exception {
183 ObjectId blobId1;
184 ObjectId blobId2;
185 byte[] blob1 = Constants.encode("blob1");
186 byte[] blob2 = Constants.encode("blob2");
187
188 try (PackInserter ins = newInserter()) {
189 blobId1 = ins.insert(OBJ_BLOB, blob1);
190 ins.flush();
191 blobId2 = ins.insert(OBJ_BLOB, blob2);
192 ins.flush();
193 }
194
195 assertPacksOnly();
196 List<Pack> packs = listPacks();
197 assertEquals(2, packs.size());
198 assertEquals(1, packs.get(0).getObjectCount());
199 assertEquals(1, packs.get(1).getObjectCount());
200
201 try (ObjectReader reader = db.newObjectReader()) {
202 assertBlob(reader, blobId1, blob1);
203 assertBlob(reader, blobId2, blob2);
204 }
205 }
206
207 @Test
208 public void largeBlob() throws Exception {
209 ObjectId blobId;
210 byte[] blob = newLargeBlob();
211 try (PackInserter ins = newInserter()) {
212 assertThat(blob.length, greaterThan(ins.getBufferSize()));
213 blobId =
214 ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob));
215 ins.flush();
216 }
217
218 assertPacksOnly();
219 Collection<Pack> packs = listPacks();
220 assertEquals(1, packs.size());
221 Pack p = packs.iterator().next();
222 assertEquals(1, p.getObjectCount());
223
224 try (ObjectReader reader = db.newObjectReader()) {
225 assertBlob(reader, blobId, blob);
226 }
227 }
228
229 @Test
230 public void overwriteExistingPack() throws Exception {
231 ObjectId blobId;
232 byte[] blob = Constants.encode("foo contents");
233
234 try (PackInserter ins = newInserter()) {
235 blobId = ins.insert(OBJ_BLOB, blob);
236 ins.flush();
237 }
238
239 assertPacksOnly();
240 List<Pack> packs = listPacks();
241 assertEquals(1, packs.size());
242 Pack pack = packs.get(0);
243 assertEquals(1, pack.getObjectCount());
244
245 String inode = getInode(pack.getPackFile());
246
247 try (PackInserter ins = newInserter()) {
248 ins.checkExisting(false);
249 assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
250 ins.flush();
251 }
252
253 assertPacksOnly();
254 packs = listPacks();
255 assertEquals(1, packs.size());
256 pack = packs.get(0);
257 assertEquals(1, pack.getObjectCount());
258
259 if (inode != null) {
260
261
262 assertNotEquals(inode, getInode(pack.getPackFile()));
263 }
264 }
265
266 @Test
267 public void checkExisting() throws Exception {
268 ObjectId blobId;
269 byte[] blob = Constants.encode("foo contents");
270
271 try (PackInserter ins = newInserter()) {
272 blobId = ins.insert(OBJ_BLOB, blob);
273 ins.insert(OBJ_BLOB, Constants.encode("another blob"));
274 ins.flush();
275 }
276
277 assertPacksOnly();
278 assertEquals(1, listPacks().size());
279
280 try (PackInserter ins = newInserter()) {
281 assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
282 ins.flush();
283 }
284
285 assertPacksOnly();
286 assertEquals(1, listPacks().size());
287
288 try (PackInserter ins = newInserter()) {
289 ins.checkExisting(false);
290 assertEquals(blobId, ins.insert(OBJ_BLOB, blob));
291 ins.flush();
292 }
293
294 assertPacksOnly();
295 assertEquals(2, listPacks().size());
296
297 try (ObjectReader reader = db.newObjectReader()) {
298 assertBlob(reader, blobId, blob);
299 }
300 }
301
302 @Test
303 public void insertSmallInputStreamRespectsCheckExisting() throws Exception {
304 ObjectId blobId;
305 byte[] blob = Constants.encode("foo contents");
306 try (PackInserter ins = newInserter()) {
307 assertThat(blob.length, lessThan(ins.getBufferSize()));
308 blobId = ins.insert(OBJ_BLOB, blob);
309 ins.insert(OBJ_BLOB, Constants.encode("another blob"));
310 ins.flush();
311 }
312
313 assertPacksOnly();
314 assertEquals(1, listPacks().size());
315
316 try (PackInserter ins = newInserter()) {
317 assertEquals(blobId,
318 ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
319 ins.flush();
320 }
321
322 assertPacksOnly();
323 assertEquals(1, listPacks().size());
324 }
325
326 @Test
327 public void insertLargeInputStreamBypassesCheckExisting() throws Exception {
328 ObjectId blobId;
329 byte[] blob = newLargeBlob();
330
331 try (PackInserter ins = newInserter()) {
332 assertThat(blob.length, greaterThan(ins.getBufferSize()));
333 blobId = ins.insert(OBJ_BLOB, blob);
334 ins.insert(OBJ_BLOB, Constants.encode("another blob"));
335 ins.flush();
336 }
337
338 assertPacksOnly();
339 assertEquals(1, listPacks().size());
340
341 try (PackInserter ins = newInserter()) {
342 assertEquals(blobId,
343 ins.insert(OBJ_BLOB, blob.length, new ByteArrayInputStream(blob)));
344 ins.flush();
345 }
346
347 assertPacksOnly();
348 assertEquals(2, listPacks().size());
349 }
350
351 @Test
352 public void readBackSmallFiles() throws Exception {
353 ObjectId blobId1;
354 ObjectId blobId2;
355 ObjectId blobId3;
356 byte[] blob1 = Constants.encode("blob1");
357 byte[] blob2 = Constants.encode("blob2");
358 byte[] blob3 = Constants.encode("blob3");
359 try (PackInserter ins = newInserter()) {
360 assertThat(blob1.length, lessThan(ins.getBufferSize()));
361 blobId1 = ins.insert(OBJ_BLOB, blob1);
362
363 try (ObjectReader reader = ins.newReader()) {
364 assertBlob(reader, blobId1, blob1);
365 }
366
367
368 blobId2 = ins.insert(OBJ_BLOB, blob2);
369 ins.flush();
370
371 blobId3 = ins.insert(OBJ_BLOB, blob3);
372 }
373
374 assertPacksOnly();
375 List<Pack> packs = listPacks();
376 assertEquals(1, packs.size());
377 assertEquals(2, packs.get(0).getObjectCount());
378
379 try (ObjectReader reader = db.newObjectReader()) {
380 assertBlob(reader, blobId1, blob1);
381 assertBlob(reader, blobId2, blob2);
382
383 try {
384 reader.open(blobId3);
385 fail("Expected MissingObjectException");
386 } catch (MissingObjectException expected) {
387
388 }
389 }
390 }
391
392 @Test
393 public void readBackLargeFile() throws Exception {
394 ObjectId blobId;
395 byte[] blob = newLargeBlob();
396
397 WindowCacheConfig wcc = new WindowCacheConfig();
398 wcc.setStreamFileThreshold(1024);
399 wcc.install();
400 try (ObjectReader reader = db.newObjectReader()) {
401 assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
402 }
403
404 try (PackInserter ins = newInserter()) {
405 blobId = ins.insert(OBJ_BLOB, blob);
406
407 try (ObjectReader reader = ins.newReader()) {
408
409 assertThat(blob.length, greaterThan(reader.getStreamFileThreshold()));
410 assertBlob(reader, blobId, blob);
411 }
412 }
413
414 assertPacksOnly();
415
416
417 assertEquals(0, listPacks().size());
418
419 try (ObjectReader reader = db.newObjectReader()) {
420 try {
421 reader.open(blobId);
422 fail("Expected MissingObjectException");
423 } catch (MissingObjectException expected) {
424
425 }
426 }
427 }
428
429 @Test
430 public void readBackFallsBackToRepo() throws Exception {
431 ObjectId blobId;
432 byte[] blob = Constants.encode("foo contents");
433 try (PackInserter ins = newInserter()) {
434 assertThat(blob.length, lessThan(ins.getBufferSize()));
435 blobId = ins.insert(OBJ_BLOB, blob);
436 ins.flush();
437 }
438
439 try (PackInserter ins = newInserter();
440 ObjectReader reader = ins.newReader()) {
441 assertBlob(reader, blobId, blob);
442 }
443 }
444
445 @Test
446 public void readBackSmallObjectBeforeLargeObject() throws Exception {
447 WindowCacheConfig wcc = new WindowCacheConfig();
448 wcc.setStreamFileThreshold(1024);
449 wcc.install();
450
451 ObjectId blobId1;
452 ObjectId blobId2;
453 ObjectId largeId;
454 byte[] blob1 = Constants.encode("blob1");
455 byte[] blob2 = Constants.encode("blob2");
456 byte[] largeBlob = newLargeBlob();
457 try (PackInserter ins = newInserter()) {
458 assertThat(blob1.length, lessThan(ins.getBufferSize()));
459 assertThat(largeBlob.length, greaterThan(ins.getBufferSize()));
460
461 blobId1 = ins.insert(OBJ_BLOB, blob1);
462 largeId = ins.insert(OBJ_BLOB, largeBlob);
463
464 try (ObjectReader reader = ins.newReader()) {
465
466
467
468
469
470
471 assertBlob(reader, blobId1, blob1);
472 }
473
474 blobId2 = ins.insert(OBJ_BLOB, blob2);
475
476 try (ObjectReader reader = ins.newReader()) {
477 assertBlob(reader, blobId1, blob1);
478 assertBlob(reader, blobId2, blob2);
479 assertBlob(reader, largeId, largeBlob);
480 }
481
482 ins.flush();
483 }
484
485 try (ObjectReader reader = db.newObjectReader()) {
486 assertBlob(reader, blobId1, blob1);
487 assertBlob(reader, blobId2, blob2);
488 assertBlob(reader, largeId, largeBlob);
489 }
490 }
491
492 private List<Pack> listPacks() throws Exception {
493 List<Pack> fromOpenDb = listPacks(db);
494 List<Pack> reopened;
495 try (FileRepository db2 = new FileRepository(db.getDirectory())) {
496 reopened = listPacks(db2);
497 }
498 assertEquals(fromOpenDb.size(), reopened.size());
499 for (int i = 0 ; i < fromOpenDb.size(); i++) {
500 Pack a = fromOpenDb.get(i);
501 Pack b = reopened.get(i);
502 assertEquals(a.getPackName(), b.getPackName());
503 assertEquals(
504 a.getPackFile().getAbsolutePath(), b.getPackFile().getAbsolutePath());
505 assertEquals(a.getObjectCount(), b.getObjectCount());
506 a.getObjectCount();
507 }
508 return fromOpenDb;
509 }
510
511 private static List<Pack> listPacks(FileRepository db) throws Exception {
512 return db.getObjectDatabase().getPacks().stream()
513 .sorted(comparing(Pack::getPackName)).collect(toList());
514 }
515
516 private PackInserter newInserter() {
517 return db.getObjectDatabase().newPackInserter();
518 }
519
520 private static byte[] newLargeBlob() {
521 byte[] blob = new byte[10240];
522 random.nextBytes(blob);
523 return blob;
524 }
525
526 private static String getInode(File f) throws Exception {
527 BasicFileAttributes attrs = Files.readAttributes(
528 f.toPath(), BasicFileAttributes.class);
529 Object k = attrs.fileKey();
530 if (k == null) {
531 return null;
532 }
533 Pattern p = Pattern.compile("^\\(dev=[^,]*,ino=(\\d+)\\)$");
534 Matcher m = p.matcher(k.toString());
535 return m.matches() ? m.group(1) : null;
536 }
537
538 private static void assertBlob(ObjectReader reader, ObjectId id,
539 byte[] expected) throws Exception {
540 ObjectLoader loader = reader.open(id);
541 assertEquals(OBJ_BLOB, loader.getType());
542 assertEquals(expected.length, loader.getSize());
543 try (ObjectStream s = loader.openStream()) {
544 int n = (int) s.getSize();
545 byte[] actual = new byte[n];
546 assertEquals(n, IO.readFully(s, actual, 0));
547 assertArrayEquals(expected, actual);
548 }
549 }
550
551 private void assertPacksOnly() throws Exception {
552 new BadFileCollector(f -> !f.endsWith(".pack") && !f.endsWith(".idx"))
553 .assertNoBadFiles(db.getObjectDatabase().getDirectory());
554 }
555
556 private void assertNoObjects() throws Exception {
557 new BadFileCollector(f -> true)
558 .assertNoBadFiles(db.getObjectDatabase().getDirectory());
559 }
560
561 private static class BadFileCollector extends SimpleFileVisitor<Path> {
562 private final Predicate<String> badName;
563 private List<String> bad;
564
565 BadFileCollector(Predicate<String> badName) {
566 this.badName = badName;
567 }
568
569 void assertNoBadFiles(File f) throws IOException {
570 bad = new ArrayList<>();
571 Files.walkFileTree(f.toPath(), this);
572 if (!bad.isEmpty()) {
573 fail("unexpected files in object directory: " + bad);
574 }
575 }
576
577 @Override
578 public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
579 Path fileName = file.getFileName();
580 if (fileName != null) {
581 String name = fileName.toString();
582 if (!attrs.isDirectory() && badName.test(name)) {
583 bad.add(name);
584 }
585 }
586 return FileVisitResult.CONTINUE;
587 }
588 }
589 }