1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
14 import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES;
15 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
16 import static org.junit.Assert.assertEquals;
17 import static org.junit.Assert.assertFalse;
18 import static org.junit.Assert.assertNotNull;
19 import static org.junit.Assert.assertTrue;
20 import static org.junit.Assert.fail;
21 import static org.mockito.ArgumentMatchers.any;
22 import static org.mockito.Mockito.doNothing;
23 import static org.mockito.Mockito.times;
24 import static org.mockito.Mockito.verify;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.ByteArrayOutputStream;
28 import java.io.File;
29 import java.io.FileOutputStream;
30 import java.io.IOException;
31 import java.text.ParseException;
32 import java.time.Duration;
33 import java.util.ArrayList;
34 import java.util.Arrays;
35 import java.util.Collections;
36 import java.util.HashSet;
37 import java.util.List;
38 import java.util.Set;
39
40 import org.eclipse.jgit.api.Git;
41 import org.eclipse.jgit.errors.MissingObjectException;
42 import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
43 import org.eclipse.jgit.internal.storage.pack.PackExt;
44 import org.eclipse.jgit.internal.storage.pack.PackWriter;
45 import org.eclipse.jgit.junit.JGitTestUtil;
46 import org.eclipse.jgit.junit.TestRepository;
47 import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
48 import org.eclipse.jgit.lib.NullProgressMonitor;
49 import org.eclipse.jgit.lib.ObjectId;
50 import org.eclipse.jgit.lib.ObjectIdSet;
51 import org.eclipse.jgit.lib.ObjectInserter;
52 import org.eclipse.jgit.lib.Ref;
53 import org.eclipse.jgit.lib.Repository;
54 import org.eclipse.jgit.lib.Sets;
55 import org.eclipse.jgit.revwalk.DepthWalk;
56 import org.eclipse.jgit.revwalk.ObjectWalk;
57 import org.eclipse.jgit.revwalk.RevBlob;
58 import org.eclipse.jgit.revwalk.RevCommit;
59 import org.eclipse.jgit.revwalk.RevObject;
60 import org.eclipse.jgit.revwalk.RevWalk;
61 import org.eclipse.jgit.storage.pack.PackConfig;
62 import org.eclipse.jgit.storage.pack.PackStatistics;
63 import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase;
64 import org.eclipse.jgit.transport.PackParser;
65 import org.junit.After;
66 import org.junit.Before;
67 import org.junit.Test;
68 import org.mockito.Mockito;
69
70 public class PackWriterTest extends SampleDataRepositoryTestCase {
71
72 private static final List<RevObject> EMPTY_LIST_REVS = Collections
73 .<RevObject> emptyList();
74
75 private static final Set<ObjectIdSet> EMPTY_ID_SET = Collections
76 .<ObjectIdSet> emptySet();
77
78 private PackConfig config;
79
80 private PackWriter writer;
81
82 private ByteArrayOutputStream os;
83
84 private Pack pack;
85
86 private ObjectInserter inserter;
87
88 private FileRepository dst;
89
90 private RevBlob contentA;
91
92 private RevBlob contentB;
93
94 private RevBlob contentC;
95
96 private RevBlob contentD;
97
98 private RevBlob contentE;
99
100 private RevCommit c1;
101
102 private RevCommit c2;
103
104 private RevCommit c3;
105
106 private RevCommit c4;
107
108 private RevCommit c5;
109
110 @Override
111 @Before
112 public void setUp() throws Exception {
113 super.setUp();
114 os = new ByteArrayOutputStream();
115 config = new PackConfig(db);
116
117 dst = createBareRepository();
118 File alt = new File(dst.getObjectDatabase().getDirectory(), INFO_ALTERNATES);
119 alt.getParentFile().mkdirs();
120 write(alt, db.getObjectDatabase().getDirectory().getAbsolutePath() + "\n");
121 }
122
123 @Override
124 @After
125 public void tearDown() throws Exception {
126 if (writer != null) {
127 writer.close();
128 writer = null;
129 }
130 if (inserter != null) {
131 inserter.close();
132 inserter = null;
133 }
134 super.tearDown();
135 }
136
137
138
139
140
141
142 @Test
143 public void testContructor() throws IOException {
144 writer = new PackWriter(config, db.newObjectReader());
145 assertFalse(writer.isDeltaBaseAsOffset());
146 assertTrue(config.isReuseDeltas());
147 assertTrue(config.isReuseObjects());
148 assertEquals(0, writer.getObjectCount());
149 }
150
151
152
153
154 @Test
155 public void testModifySettings() {
156 config.setReuseDeltas(false);
157 config.setReuseObjects(false);
158 config.setDeltaBaseAsOffset(false);
159 assertFalse(config.isReuseDeltas());
160 assertFalse(config.isReuseObjects());
161 assertFalse(config.isDeltaBaseAsOffset());
162
163 writer = new PackWriter(config, db.newObjectReader());
164 writer.setDeltaBaseAsOffset(true);
165 assertTrue(writer.isDeltaBaseAsOffset());
166 assertFalse(config.isDeltaBaseAsOffset());
167 }
168
169
170
171
172
173
174
175 @Test
176 public void testWriteEmptyPack1() throws IOException {
177 createVerifyOpenPack(NONE, NONE, false, false);
178
179 assertEquals(0, writer.getObjectCount());
180 assertEquals(0, pack.getObjectCount());
181 assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
182 .computeName().name());
183 }
184
185
186
187
188
189
190
191 @Test
192 public void testWriteEmptyPack2() throws IOException {
193 createVerifyOpenPack(EMPTY_LIST_REVS);
194
195 assertEquals(0, writer.getObjectCount());
196 assertEquals(0, pack.getObjectCount());
197 }
198
199
200
201
202
203
204
205 @Test
206 public void testNotIgnoreNonExistingObjects() throws IOException {
207 final ObjectId nonExisting = ObjectId
208 .fromString("0000000000000000000000000000000000000001");
209 try {
210 createVerifyOpenPack(NONE, haves(nonExisting), false, false);
211 fail("Should have thrown MissingObjectException");
212 } catch (MissingObjectException x) {
213
214 }
215 }
216
217
218
219
220
221
222 @Test
223 public void testIgnoreNonExistingObjects() throws IOException {
224 final ObjectId nonExisting = ObjectId
225 .fromString("0000000000000000000000000000000000000001");
226 createVerifyOpenPack(NONE, haves(nonExisting), false, true);
227
228 }
229
230
231
232
233
234
235
236
237
238 @Test
239 public void testIgnoreNonExistingObjectsWithBitmaps() throws IOException,
240 ParseException {
241 final ObjectId nonExisting = ObjectId
242 .fromString("0000000000000000000000000000000000000001");
243 new GC(db).gc();
244 createVerifyOpenPack(NONE, haves(nonExisting), false, true, true);
245
246 }
247
248
249
250
251
252
253
254 @Test
255 public void testWritePack1() throws IOException {
256 config.setReuseDeltas(false);
257 writeVerifyPack1();
258 }
259
260
261
262
263
264
265
266 @Test
267 public void testWritePack1NoObjectReuse() throws IOException {
268 config.setReuseDeltas(false);
269 config.setReuseObjects(false);
270 writeVerifyPack1();
271 }
272
273
274
275
276
277
278
279 @Test
280 public void testWritePack2() throws IOException {
281 writeVerifyPack2(false);
282 }
283
284
285
286
287
288
289
290 @Test
291 public void testWritePack2DeltasReuseRefs() throws IOException {
292 writeVerifyPack2(true);
293 }
294
295
296
297
298
299
300
301 @Test
302 public void testWritePack2DeltasReuseOffsets() throws IOException {
303 config.setDeltaBaseAsOffset(true);
304 writeVerifyPack2(true);
305 }
306
307
308
309
310
311
312
313
314 @Test
315 public void testWritePack2DeltasCRC32Copy() throws IOException {
316 final File packDir = db.getObjectDatabase().getPackDirectory();
317 final PackFile crc32Pack = new PackFile(packDir,
318 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
319 final PackFile crc32Idx = new PackFile(packDir,
320 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
321 copyFile(JGitTestUtil.getTestResourceFile(
322 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
323 crc32Idx);
324 db.openPack(crc32Pack);
325
326 writeVerifyPack2(true);
327 }
328
329
330
331
332
333
334
335
336
337 @Test
338 public void testWritePack3() throws MissingObjectException, IOException {
339 config.setReuseDeltas(false);
340 final ObjectId forcedOrder[] = new ObjectId[] {
341 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
342 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
343 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
344 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
345 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
346 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
347 try (RevWalk parser = new RevWalk(db)) {
348 final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
349 for (int i = 0; i < forcedOrder.length; i++)
350 forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
351
352 createVerifyOpenPack(Arrays.asList(forcedOrderRevs));
353 }
354
355 assertEquals(forcedOrder.length, writer.getObjectCount());
356 verifyObjectsOrder(forcedOrder);
357 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
358 .computeName().name());
359 }
360
361
362
363
364
365
366
367
368 @Test
369 public void testWritePack4() throws IOException {
370 writeVerifyPack4(false);
371 }
372
373
374
375
376
377
378
379 @Test
380 public void testWritePack4ThinPack() throws IOException {
381 writeVerifyPack4(true);
382 }
383
384
385
386
387
388
389
390
391 @Test
392 public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
393 config.setReuseDeltas(false);
394 config.setDeltaCompress(false);
395 testWritePack2();
396 final long sizePack2NoDeltas = os.size();
397 tearDown();
398 setUp();
399 testWritePack2DeltasReuseRefs();
400 final long sizePack2DeltasRefs = os.size();
401
402 assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
403 }
404
405
406
407
408
409
410
411
412
413 @Test
414 public void testWritePack2SizeOffsetsVsRefs() throws Exception {
415 testWritePack2DeltasReuseRefs();
416 final long sizePack2DeltasRefs = os.size();
417 tearDown();
418 setUp();
419 testWritePack2DeltasReuseOffsets();
420 final long sizePack2DeltasOffsets = os.size();
421
422 assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
423 }
424
425
426
427
428
429
430
431
432 @Test
433 public void testWritePack4SizeThinVsNoThin() throws Exception {
434 testWritePack4();
435 final long sizePack4 = os.size();
436 tearDown();
437 setUp();
438 testWritePack4ThinPack();
439 final long sizePack4Thin = os.size();
440
441 assertTrue(sizePack4 > sizePack4Thin);
442 }
443
444 @Test
445 public void testDeltaStatistics() throws Exception {
446 config.setDeltaCompress(true);
447
448 FileRepository repo = createBareRepository();
449 ArrayList<RevObject> blobs = new ArrayList<>();
450 try (TestRepository<FileRepository> testRepo = new TestRepository<>(
451 repo)) {
452 blobs.add(testRepo.blob(genDeltableData(1000)));
453 blobs.add(testRepo.blob(genDeltableData(1005)));
454 try (PackWriter pw = new PackWriter(repo)) {
455 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
456 pw.preparePack(blobs.iterator());
457 pw.writePack(m, m, os);
458 PackStatistics stats = pw.getStatistics();
459 assertEquals(1, stats.getTotalDeltas());
460 assertTrue("Delta bytes not set.",
461 stats.byObjectType(OBJ_BLOB).getDeltaBytes() > 0);
462 }
463 }
464 }
465
466
467 private String genDeltableData(int length) {
468 assertTrue("Generated data must have a length > 0", length > 0);
469 char[] data = {'a', 'b', 'c', '\n'};
470 StringBuilder builder = new StringBuilder(length);
471 for (int i = 0; i < length; i++) {
472 builder.append(data[i % 4]);
473 }
474 return builder.toString();
475 }
476
477
478 @Test
479 public void testWriteIndex() throws Exception {
480 config.setIndexVersion(2);
481 writeVerifyPack4(false);
482
483 PackFile packFile = pack.getPackFile();
484 PackFile indexFile = packFile.create(PackExt.INDEX);
485
486
487 final PackIndex idx1 = PackIndex.open(indexFile);
488 assertTrue(idx1 instanceof PackIndexV2);
489 assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId
490 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
491
492
493 final File idx2File = new File(indexFile.getAbsolutePath() + ".2");
494 try (FileOutputStream is = new FileOutputStream(idx2File)) {
495 writer.writeIndex(is);
496 }
497 final PackIndex idx2 = PackIndex.open(idx2File);
498 assertTrue(idx2 instanceof PackIndexV2);
499 assertEquals(idx1.getObjectCount(), idx2.getObjectCount());
500 assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count());
501
502 for (int i = 0; i < idx1.getObjectCount(); i++) {
503 final ObjectId id = idx1.getObjectId(i);
504 assertEquals(id, idx2.getObjectId(i));
505 assertEquals(idx1.findOffset(id), idx2.findOffset(id));
506 assertEquals(idx1.findCRC32(id), idx2.findCRC32(id));
507 }
508 }
509
510 @Test
511 public void testExclude() throws Exception {
512
513 FileRepository repo = createBareRepository();
514
515 try (TestRepository<FileRepository> testRepo = new TestRepository<>(
516 repo)) {
517 BranchBuilder bb = testRepo.branch("refs/heads/master");
518 contentA = testRepo.blob("A");
519 c1 = bb.commit().add("f", contentA).create();
520 testRepo.getRevWalk().parseHeaders(c1);
521 PackIndex pf1 = writePack(repo, wants(c1), EMPTY_ID_SET);
522 assertContent(pf1, Arrays.asList(c1.getId(), c1.getTree().getId(),
523 contentA.getId()));
524 contentB = testRepo.blob("B");
525 c2 = bb.commit().add("f", contentB).create();
526 testRepo.getRevWalk().parseHeaders(c2);
527 PackIndex pf2 = writePack(repo, wants(c2),
528 Sets.of((ObjectIdSet) pf1));
529 assertContent(pf2, Arrays.asList(c2.getId(), c2.getTree().getId(),
530 contentB.getId()));
531 }
532 }
533
534 private static void assertContent(PackIndex pi, List<ObjectId> expected) {
535 assertEquals("Pack index has wrong size.", expected.size(),
536 pi.getObjectCount());
537 for (int i = 0; i < pi.getObjectCount(); i++)
538 assertTrue(
539 "Pack index didn't contain the expected id "
540 + pi.getObjectId(i),
541 expected.contains(pi.getObjectId(i)));
542 }
543
544 @Test
545 public void testShallowIsMinimalDepth1() throws Exception {
546 try (FileRepository repo = setupRepoForShallowFetch()) {
547 PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE);
548 assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(),
549 contentA.getId(), contentB.getId()));
550
551
552 idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2));
553 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
554 contentC.getId(), contentD.getId(), contentE.getId()));
555 }
556 }
557
558 @Test
559 public void testShallowIsMinimalDepth2() throws Exception {
560 try (FileRepository repo = setupRepoForShallowFetch()) {
561 PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE);
562 assertContent(idx,
563 Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(),
564 c2.getTree().getId(), contentA.getId(),
565 contentB.getId()));
566
567
568 idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2),
569 shallows(c1));
570 assertContent(idx,
571 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
572 c5.getTree().getId(), contentC.getId(),
573 contentD.getId(), contentE.getId()));
574 }
575 }
576
577 @Test
578 public void testShallowFetchShallowParentDepth1() throws Exception {
579 try (FileRepository repo = setupRepoForShallowFetch()) {
580 PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
581 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
582 contentA.getId(), contentB.getId(), contentC.getId(),
583 contentD.getId(), contentE.getId()));
584
585 idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5));
586 assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId()));
587 }
588 }
589
590 @Test
591 public void testShallowFetchShallowParentDepth2() throws Exception {
592 try (FileRepository repo = setupRepoForShallowFetch()) {
593 PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
594 assertContent(idx,
595 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
596 c5.getTree().getId(), contentA.getId(),
597 contentB.getId(), contentC.getId(),
598 contentD.getId(), contentE.getId()));
599
600 idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5),
601 shallows(c4));
602 assertContent(idx, Arrays.asList(c2.getId(), c3.getId(),
603 c2.getTree().getId(), c3.getTree().getId()));
604 }
605 }
606
607 @Test
608 public void testShallowFetchShallowAncestorDepth1() throws Exception {
609 try (FileRepository repo = setupRepoForShallowFetch()) {
610 PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
611 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
612 contentA.getId(), contentB.getId(), contentC.getId(),
613 contentD.getId(), contentE.getId()));
614
615 idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5));
616 assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId()));
617 }
618 }
619
620 @Test
621 public void testShallowFetchShallowAncestorDepth2() throws Exception {
622 try (FileRepository repo = setupRepoForShallowFetch()) {
623 PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
624 assertContent(idx,
625 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
626 c5.getTree().getId(), contentA.getId(),
627 contentB.getId(), contentC.getId(),
628 contentD.getId(), contentE.getId()));
629
630 idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5),
631 shallows(c4));
632 assertContent(idx, Arrays.asList(c1.getId(), c2.getId(),
633 c1.getTree().getId(), c2.getTree().getId()));
634 }
635 }
636
637 @Test
638 public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet()
639 throws Exception {
640 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
641 PackWriter mockedPackWriter = Mockito
642 .spy(new PackWriter(config, fileRepository.newObjectReader()));
643
644 doNothing().when(mockedPackWriter).select(any(), any());
645
646 try (FileOutputStream packOS = new FileOutputStream(
647 getPackFileToWrite(fileRepository, mockedPackWriter))) {
648 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
649 NullProgressMonitor.INSTANCE, packOS);
650 }
651
652 long numberOfPackFiles = new GC(fileRepository)
653 .getStatistics().numberOfPackFiles;
654 int expectedSelectCalls =
655
656 2 * (int) numberOfPackFiles +
657
658 1;
659 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
660 any());
661 }
662
663 @Test
664 public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck()
665 throws Exception {
666 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
667 PackConfig packConfig = new PackConfig();
668 packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1));
669 PackWriter mockedPackWriter = Mockito.spy(
670 new PackWriter(packConfig, fileRepository.newObjectReader()));
671
672 doNothing().when(mockedPackWriter).select(any(), any());
673
674 try (FileOutputStream packOS = new FileOutputStream(
675 getPackFileToWrite(fileRepository, mockedPackWriter))) {
676 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
677 NullProgressMonitor.INSTANCE, packOS);
678 }
679
680 long numberOfPackFiles = new GC(fileRepository)
681 .getStatistics().numberOfPackFiles;
682 int expectedSelectCalls =
683
684 2 * (int) numberOfPackFiles +
685
686 1;
687 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
688 any());
689 }
690
691 @Test
692 public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck()
693 throws Exception {
694 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
695 PackConfig packConfig = new PackConfig();
696 packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1));
697 PackWriter mockedPackWriter = Mockito.spy(
698 new PackWriter(packConfig, fileRepository.newObjectReader()));
699 mockedPackWriter.enableSearchForReuseTimeout();
700
701 doNothing().when(mockedPackWriter).select(any(), any());
702
703 try (FileOutputStream packOS = new FileOutputStream(
704 getPackFileToWrite(fileRepository, mockedPackWriter))) {
705 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
706 NullProgressMonitor.INSTANCE, packOS);
707 }
708
709 int expectedSelectCalls = 3;
710 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
711 any());
712 }
713
714
715
716
717
718
719
720
721
722
723
724
725
726 private FileRepository setUpRepoWithMultiplePackfiles() throws Exception {
727 FileRepository fileRepository = createWorkRepository();
728 try (Git git = new Git(fileRepository)) {
729
730 git.commit().setMessage("First commit").call();
731 GC gc = new GC(fileRepository);
732 gc.setPackExpireAgeMillis(Long.MAX_VALUE);
733 gc.setExpireAgeMillis(Long.MAX_VALUE);
734
735 gc.gc();
736
737 git.commit().setMessage("Second commit").call();
738
739 gc.gc();
740
741 git.commit().setMessage("Third commit").call();
742 }
743 return fileRepository;
744 }
745
746 private PackFile getPackFileToWrite(FileRepository fileRepository,
747 PackWriter mockedPackWriter) throws IOException {
748 File packdir = fileRepository.getObjectDatabase().getPackDirectory();
749 PackFile packFile = new PackFile(packdir,
750 mockedPackWriter.computeName(), PackExt.PACK);
751
752 Set<ObjectId> all = new HashSet<>();
753 for (Ref r : fileRepository.getRefDatabase().getRefs()) {
754 all.add(r.getObjectId());
755 }
756
757 mockedPackWriter.preparePack(NullProgressMonitor.INSTANCE, all,
758 PackWriter.NONE);
759 return packFile;
760 }
761
762 private FileRepository setupRepoForShallowFetch() throws Exception {
763 FileRepository repo = createBareRepository();
764
765
766 repo.incrementOpen();
767 try (TestRepository<Repository> r = new TestRepository<>(repo)) {
768 BranchBuilder bb = r.branch("refs/heads/master");
769 contentA = r.blob("A");
770 contentB = r.blob("B");
771 contentC = r.blob("C");
772 contentD = r.blob("D");
773 contentE = r.blob("E");
774 c1 = bb.commit().add("a", contentA).create();
775 c2 = bb.commit().add("b", contentB).create();
776 c3 = bb.commit().add("c", contentC).create();
777 c4 = bb.commit().add("d", contentD).create();
778 c5 = bb.commit().add("e", contentE).create();
779 r.getRevWalk().parseHeaders(c5);
780 return repo;
781 }
782 }
783
784 private static PackIndex writePack(FileRepository repo,
785 Set<? extends ObjectId> want, Set<ObjectIdSet> excludeObjects)
786 throws IOException {
787 try (RevWalk walk = new RevWalk(repo)) {
788 return writePack(repo, walk, 0, want, NONE, excludeObjects);
789 }
790 }
791
792 private static PackIndex writeShallowPack(FileRepository repo, int depth,
793 Set<? extends ObjectId> want, Set<? extends ObjectId> have,
794 Set<? extends ObjectId> shallow) throws IOException {
795
796
797 try (DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1)) {
798 walk.assumeShallow(shallow);
799 return writePack(repo, walk, depth, want, have, EMPTY_ID_SET);
800 }
801 }
802
803 private static PackIndex writePack(FileRepository repo, RevWalk walk,
804 int depth, Set<? extends ObjectId> want,
805 Set<? extends ObjectId> have, Set<ObjectIdSet> excludeObjects)
806 throws IOException {
807 try (PackWriter pw = new PackWriter(repo)) {
808 pw.setDeltaBaseAsOffset(true);
809 pw.setReuseDeltaCommits(false);
810 for (ObjectIdSet idx : excludeObjects) {
811 pw.excludeObjects(idx);
812 }
813 if (depth > 0) {
814 pw.setShallowPack(depth, null);
815 }
816
817 ObjectWalk ow = walk.toObjectWalkWithSameObjects();
818
819 pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE);
820 File packdir = repo.getObjectDatabase().getPackDirectory();
821 PackFile packFile = new PackFile(packdir, pw.computeName(),
822 PackExt.PACK);
823 try (FileOutputStream packOS = new FileOutputStream(packFile)) {
824 pw.writePack(NullProgressMonitor.INSTANCE,
825 NullProgressMonitor.INSTANCE, packOS);
826 }
827 PackFile idxFile = packFile.create(PackExt.INDEX);
828 try (FileOutputStream idxOS = new FileOutputStream(idxFile)) {
829 pw.writeIndex(idxOS);
830 }
831 return PackIndex.open(idxFile);
832 }
833 }
834
835
836
837
838 private void writeVerifyPack1() throws IOException {
839 final HashSet<ObjectId> interestings = new HashSet<>();
840 interestings.add(ObjectId
841 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
842 createVerifyOpenPack(interestings, NONE, false, false);
843
844 final ObjectId expectedOrder[] = new ObjectId[] {
845 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
846 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
847 ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
848 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
849 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
850 ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
851 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"),
852 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
853
854 assertEquals(expectedOrder.length, writer.getObjectCount());
855 verifyObjectsOrder(expectedOrder);
856 assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
857 .computeName().name());
858 }
859
860 private void writeVerifyPack2(boolean deltaReuse) throws IOException {
861 config.setReuseDeltas(deltaReuse);
862 final HashSet<ObjectId> interestings = new HashSet<>();
863 interestings.add(ObjectId
864 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
865 final HashSet<ObjectId> uninterestings = new HashSet<>();
866 uninterestings.add(ObjectId
867 .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
868 createVerifyOpenPack(interestings, uninterestings, false, false);
869
870 final ObjectId expectedOrder[] = new ObjectId[] {
871 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
872 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
873 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
874 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
875 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
876 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
877 if (!config.isReuseDeltas() && !config.isDeltaCompress()) {
878
879 swap(expectedOrder, 4, 5);
880 }
881 assertEquals(expectedOrder.length, writer.getObjectCount());
882 verifyObjectsOrder(expectedOrder);
883 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
884 .computeName().name());
885 }
886
887 private static void swap(ObjectId[] arr, int a, int b) {
888 ObjectId tmp = arr[a];
889 arr[a] = arr[b];
890 arr[b] = tmp;
891 }
892
893 private void writeVerifyPack4(final boolean thin) throws IOException {
894 final HashSet<ObjectId> interestings = new HashSet<>();
895 interestings.add(ObjectId
896 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
897 final HashSet<ObjectId> uninterestings = new HashSet<>();
898 uninterestings.add(ObjectId
899 .fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
900 createVerifyOpenPack(interestings, uninterestings, thin, false);
901
902 final ObjectId writtenObjects[] = new ObjectId[] {
903 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
904 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
905 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
906 assertEquals(writtenObjects.length, writer.getObjectCount());
907 ObjectId expectedObjects[];
908 if (thin) {
909 expectedObjects = new ObjectId[4];
910 System.arraycopy(writtenObjects, 0, expectedObjects, 0,
911 writtenObjects.length);
912 expectedObjects[3] = ObjectId
913 .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
914
915 } else {
916 expectedObjects = writtenObjects;
917 }
918 verifyObjectsOrder(expectedObjects);
919 assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
920 .computeName().name());
921 }
922
923 private void createVerifyOpenPack(final Set<ObjectId> interestings,
924 final Set<ObjectId> uninterestings, final boolean thin,
925 final boolean ignoreMissingUninteresting)
926 throws MissingObjectException, IOException {
927 createVerifyOpenPack(interestings, uninterestings, thin,
928 ignoreMissingUninteresting, false);
929 }
930
931 private void createVerifyOpenPack(final Set<ObjectId> interestings,
932 final Set<ObjectId> uninterestings, final boolean thin,
933 final boolean ignoreMissingUninteresting, boolean useBitmaps)
934 throws MissingObjectException, IOException {
935 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
936 writer = new PackWriter(config, db.newObjectReader());
937 writer.setUseBitmaps(useBitmaps);
938 writer.setThin(thin);
939 writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting);
940 writer.preparePack(m, interestings, uninterestings);
941 writer.writePack(m, m, os);
942 writer.close();
943 verifyOpenPack(thin);
944 }
945
946 private void createVerifyOpenPack(List<RevObject> objectSource)
947 throws MissingObjectException, IOException {
948 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
949 writer = new PackWriter(config, db.newObjectReader());
950 writer.preparePack(objectSource.iterator());
951 assertEquals(objectSource.size(), writer.getObjectCount());
952 writer.writePack(m, m, os);
953 writer.close();
954 verifyOpenPack(false);
955 }
956
957 private void verifyOpenPack(boolean thin) throws IOException {
958 final byte[] packData = os.toByteArray();
959
960 if (thin) {
961 PackParser p = index(packData);
962 try {
963 p.parse(NullProgressMonitor.INSTANCE);
964 fail("indexer should grumble about missing object");
965 } catch (IOException x) {
966
967 }
968 }
969
970 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(packData);
971 p.setKeepEmpty(true);
972 p.setAllowThin(thin);
973 p.setIndexVersion(2);
974 p.parse(NullProgressMonitor.INSTANCE);
975 pack = p.getPack();
976 assertNotNull("have PackFile after parsing", pack);
977 }
978
979 private PackParser index(byte[] packData) throws IOException {
980 if (inserter == null)
981 inserter = dst.newObjectInserter();
982 return inserter.newPackParser(new ByteArrayInputStream(packData));
983 }
984
985 private void verifyObjectsOrder(ObjectId objectsOrder[]) {
986 final List<PackIndex.MutableEntry> entries = new ArrayList<>();
987
988 for (MutableEntry me : pack) {
989 entries.add(me.cloneEntry());
990 }
991 Collections.sort(entries, (MutableEntry o1, MutableEntry o2) -> Long
992 .signum(o1.getOffset() - o2.getOffset()));
993
994 int i = 0;
995 for (MutableEntry me : entries) {
996 assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId());
997 }
998 }
999
1000 private static Set<ObjectId> haves(ObjectId... objects) {
1001 return Sets.of(objects);
1002 }
1003
1004 private static Set<ObjectId> wants(ObjectId... objects) {
1005 return Sets.of(objects);
1006 }
1007
1008 private static Set<ObjectId> shallows(ObjectId... objects) {
1009 return Sets.of(objects);
1010 }
1011 }