1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static org.eclipse.jgit.internal.storage.pack.PackWriter.NONE;
14 import static org.eclipse.jgit.lib.Constants.INFO_ALTERNATES;
15 import static org.eclipse.jgit.lib.Constants.OBJ_BLOB;
16 import static org.junit.Assert.assertEquals;
17 import static org.junit.Assert.assertFalse;
18 import static org.junit.Assert.assertNotNull;
19 import static org.junit.Assert.assertTrue;
20 import static org.junit.Assert.fail;
21 import static org.mockito.ArgumentMatchers.any;
22 import static org.mockito.Mockito.doNothing;
23 import static org.mockito.Mockito.times;
24 import static org.mockito.Mockito.verify;
25
26 import java.io.ByteArrayInputStream;
27 import java.io.ByteArrayOutputStream;
28 import java.io.File;
29 import java.io.FileOutputStream;
30 import java.io.IOException;
31 import java.time.Duration;
32 import java.util.ArrayList;
33 import java.util.Arrays;
34 import java.util.Collections;
35 import java.util.HashSet;
36 import java.util.List;
37 import java.util.Set;
38
39 import org.eclipse.jgit.api.Git;
40 import org.eclipse.jgit.errors.MissingObjectException;
41 import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
42 import org.eclipse.jgit.internal.storage.pack.PackExt;
43 import org.eclipse.jgit.internal.storage.pack.PackWriter;
44 import org.eclipse.jgit.junit.JGitTestUtil;
45 import org.eclipse.jgit.junit.TestRepository;
46 import org.eclipse.jgit.junit.TestRepository.BranchBuilder;
47 import org.eclipse.jgit.lib.NullProgressMonitor;
48 import org.eclipse.jgit.lib.ObjectId;
49 import org.eclipse.jgit.lib.ObjectIdSet;
50 import org.eclipse.jgit.lib.ObjectInserter;
51 import org.eclipse.jgit.lib.Ref;
52 import org.eclipse.jgit.lib.Repository;
53 import org.eclipse.jgit.lib.Sets;
54 import org.eclipse.jgit.revwalk.DepthWalk;
55 import org.eclipse.jgit.revwalk.ObjectWalk;
56 import org.eclipse.jgit.revwalk.RevBlob;
57 import org.eclipse.jgit.revwalk.RevCommit;
58 import org.eclipse.jgit.revwalk.RevObject;
59 import org.eclipse.jgit.revwalk.RevWalk;
60 import org.eclipse.jgit.storage.pack.PackConfig;
61 import org.eclipse.jgit.storage.pack.PackStatistics;
62 import org.eclipse.jgit.test.resources.SampleDataRepositoryTestCase;
63 import org.eclipse.jgit.transport.PackParser;
64 import org.junit.After;
65 import org.junit.Before;
66 import org.junit.Test;
67 import org.mockito.Mockito;
68
69 public class PackWriterTest extends SampleDataRepositoryTestCase {
70
71 private static final List<RevObject> EMPTY_LIST_REVS = Collections
72 .<RevObject> emptyList();
73
74 private static final Set<ObjectIdSet> EMPTY_ID_SET = Collections
75 .<ObjectIdSet> emptySet();
76
77 private PackConfig config;
78
79 private PackWriter writer;
80
81 private ByteArrayOutputStream os;
82
83 private Pack pack;
84
85 private ObjectInserter inserter;
86
87 private FileRepository dst;
88
89 private RevBlob contentA;
90
91 private RevBlob contentB;
92
93 private RevBlob contentC;
94
95 private RevBlob contentD;
96
97 private RevBlob contentE;
98
99 private RevCommit c1;
100
101 private RevCommit c2;
102
103 private RevCommit c3;
104
105 private RevCommit c4;
106
107 private RevCommit c5;
108
109 @Override
110 @Before
111 public void setUp() throws Exception {
112 super.setUp();
113 os = new ByteArrayOutputStream();
114 config = new PackConfig(db);
115
116 dst = createBareRepository();
117 File alt = new File(dst.getObjectDatabase().getDirectory(), INFO_ALTERNATES);
118 alt.getParentFile().mkdirs();
119 write(alt, db.getObjectDatabase().getDirectory().getAbsolutePath() + "\n");
120 }
121
122 @Override
123 @After
124 public void tearDown() throws Exception {
125 if (writer != null) {
126 writer.close();
127 writer = null;
128 }
129 if (inserter != null) {
130 inserter.close();
131 inserter = null;
132 }
133 super.tearDown();
134 }
135
136
137
138
139
140
141 @Test
142 public void testContructor() throws IOException {
143 writer = new PackWriter(config, db.newObjectReader());
144 assertFalse(writer.isDeltaBaseAsOffset());
145 assertTrue(config.isReuseDeltas());
146 assertTrue(config.isReuseObjects());
147 assertEquals(0, writer.getObjectCount());
148 }
149
150
151
152
153 @Test
154 public void testModifySettings() {
155 config.setReuseDeltas(false);
156 config.setReuseObjects(false);
157 config.setDeltaBaseAsOffset(false);
158 assertFalse(config.isReuseDeltas());
159 assertFalse(config.isReuseObjects());
160 assertFalse(config.isDeltaBaseAsOffset());
161
162 writer = new PackWriter(config, db.newObjectReader());
163 writer.setDeltaBaseAsOffset(true);
164 assertTrue(writer.isDeltaBaseAsOffset());
165 assertFalse(config.isDeltaBaseAsOffset());
166 }
167
168
169
170
171
172
173
174 @Test
175 public void testWriteEmptyPack1() throws IOException {
176 createVerifyOpenPack(NONE, NONE, false, false);
177
178 assertEquals(0, writer.getObjectCount());
179 assertEquals(0, pack.getObjectCount());
180 assertEquals("da39a3ee5e6b4b0d3255bfef95601890afd80709", writer
181 .computeName().name());
182 }
183
184
185
186
187
188
189
190 @Test
191 public void testWriteEmptyPack2() throws IOException {
192 createVerifyOpenPack(EMPTY_LIST_REVS);
193
194 assertEquals(0, writer.getObjectCount());
195 assertEquals(0, pack.getObjectCount());
196 }
197
198
199
200
201
202
203
204 @Test
205 public void testNotIgnoreNonExistingObjects() throws IOException {
206 final ObjectId nonExisting = ObjectId
207 .fromString("0000000000000000000000000000000000000001");
208 try {
209 createVerifyOpenPack(NONE, haves(nonExisting), false, false);
210 fail("Should have thrown MissingObjectException");
211 } catch (MissingObjectException x) {
212
213 }
214 }
215
216
217
218
219
220
221 @Test
222 public void testIgnoreNonExistingObjects() throws IOException {
223 final ObjectId nonExisting = ObjectId
224 .fromString("0000000000000000000000000000000000000001");
225 createVerifyOpenPack(NONE, haves(nonExisting), false, true);
226
227 }
228
229
230
231
232
233
234
235
236 @Test
237 public void testIgnoreNonExistingObjectsWithBitmaps() throws Exception {
238 final ObjectId nonExisting = ObjectId
239 .fromString("0000000000000000000000000000000000000001");
240 new GC(db).gc().get();
241 createVerifyOpenPack(NONE, haves(nonExisting), false, true, true);
242
243 }
244
245
246
247
248
249
250
251 @Test
252 public void testWritePack1() throws IOException {
253 config.setReuseDeltas(false);
254 writeVerifyPack1();
255 }
256
257
258
259
260
261
262
263 @Test
264 public void testWritePack1NoObjectReuse() throws IOException {
265 config.setReuseDeltas(false);
266 config.setReuseObjects(false);
267 writeVerifyPack1();
268 }
269
270
271
272
273
274
275
276 @Test
277 public void testWritePack2() throws IOException {
278 writeVerifyPack2(false);
279 }
280
281
282
283
284
285
286
287 @Test
288 public void testWritePack2DeltasReuseRefs() throws IOException {
289 writeVerifyPack2(true);
290 }
291
292
293
294
295
296
297
298 @Test
299 public void testWritePack2DeltasReuseOffsets() throws IOException {
300 config.setDeltaBaseAsOffset(true);
301 writeVerifyPack2(true);
302 }
303
304
305
306
307
308
309
310
311 @Test
312 public void testWritePack2DeltasCRC32Copy() throws IOException {
313 final File packDir = db.getObjectDatabase().getPackDirectory();
314 final PackFile crc32Pack = new PackFile(packDir,
315 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
316 final PackFile crc32Idx = new PackFile(packDir,
317 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idx");
318 copyFile(JGitTestUtil.getTestResourceFile(
319 "pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.idxV2"),
320 crc32Idx);
321 db.openPack(crc32Pack);
322
323 writeVerifyPack2(true);
324 }
325
326
327
328
329
330
331
332
333
334 @Test
335 public void testWritePack3() throws MissingObjectException, IOException {
336 config.setReuseDeltas(false);
337 final ObjectId forcedOrder[] = new ObjectId[] {
338 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
339 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
340 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
341 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
342 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
343 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
344 try (RevWalk parser = new RevWalk(db)) {
345 final RevObject forcedOrderRevs[] = new RevObject[forcedOrder.length];
346 for (int i = 0; i < forcedOrder.length; i++)
347 forcedOrderRevs[i] = parser.parseAny(forcedOrder[i]);
348
349 createVerifyOpenPack(Arrays.asList(forcedOrderRevs));
350 }
351
352 assertEquals(forcedOrder.length, writer.getObjectCount());
353 verifyObjectsOrder(forcedOrder);
354 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
355 .computeName().name());
356 }
357
358
359
360
361
362
363
364
365 @Test
366 public void testWritePack4() throws IOException {
367 writeVerifyPack4(false);
368 }
369
370
371
372
373
374
375
376 @Test
377 public void testWritePack4ThinPack() throws IOException {
378 writeVerifyPack4(true);
379 }
380
381
382
383
384
385
386
387
388 @Test
389 public void testWritePack2SizeDeltasVsNoDeltas() throws Exception {
390 config.setReuseDeltas(false);
391 config.setDeltaCompress(false);
392 testWritePack2();
393 final long sizePack2NoDeltas = os.size();
394 tearDown();
395 setUp();
396 testWritePack2DeltasReuseRefs();
397 final long sizePack2DeltasRefs = os.size();
398
399 assertTrue(sizePack2NoDeltas > sizePack2DeltasRefs);
400 }
401
402
403
404
405
406
407
408
409
410 @Test
411 public void testWritePack2SizeOffsetsVsRefs() throws Exception {
412 testWritePack2DeltasReuseRefs();
413 final long sizePack2DeltasRefs = os.size();
414 tearDown();
415 setUp();
416 testWritePack2DeltasReuseOffsets();
417 final long sizePack2DeltasOffsets = os.size();
418
419 assertTrue(sizePack2DeltasRefs > sizePack2DeltasOffsets);
420 }
421
422
423
424
425
426
427
428
429 @Test
430 public void testWritePack4SizeThinVsNoThin() throws Exception {
431 testWritePack4();
432 final long sizePack4 = os.size();
433 tearDown();
434 setUp();
435 testWritePack4ThinPack();
436 final long sizePack4Thin = os.size();
437
438 assertTrue(sizePack4 > sizePack4Thin);
439 }
440
441 @Test
442 public void testDeltaStatistics() throws Exception {
443 config.setDeltaCompress(true);
444
445 FileRepository repo = createBareRepository();
446 ArrayList<RevObject> blobs = new ArrayList<>();
447 try (TestRepository<FileRepository> testRepo = new TestRepository<>(
448 repo)) {
449 blobs.add(testRepo.blob(genDeltableData(1000)));
450 blobs.add(testRepo.blob(genDeltableData(1005)));
451 try (PackWriter pw = new PackWriter(repo)) {
452 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
453 pw.preparePack(blobs.iterator());
454 pw.writePack(m, m, os);
455 PackStatistics stats = pw.getStatistics();
456 assertEquals(1, stats.getTotalDeltas());
457 assertTrue("Delta bytes not set.",
458 stats.byObjectType(OBJ_BLOB).getDeltaBytes() > 0);
459 }
460 }
461 }
462
463
464 private String genDeltableData(int length) {
465 assertTrue("Generated data must have a length > 0", length > 0);
466 char[] data = {'a', 'b', 'c', '\n'};
467 StringBuilder builder = new StringBuilder(length);
468 for (int i = 0; i < length; i++) {
469 builder.append(data[i % 4]);
470 }
471 return builder.toString();
472 }
473
474
475 @Test
476 public void testWriteIndex() throws Exception {
477 config.setIndexVersion(2);
478 writeVerifyPack4(false);
479
480 PackFile packFile = pack.getPackFile();
481 PackFile indexFile = packFile.create(PackExt.INDEX);
482
483
484 final PackIndex idx1 = PackIndex.open(indexFile);
485 assertTrue(idx1 instanceof PackIndexV2);
486 assertEquals(0x4743F1E4L, idx1.findCRC32(ObjectId
487 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
488
489
490 final File idx2File = new File(indexFile.getAbsolutePath() + ".2");
491 try (FileOutputStream is = new FileOutputStream(idx2File)) {
492 writer.writeIndex(is);
493 }
494 final PackIndex idx2 = PackIndex.open(idx2File);
495 assertTrue(idx2 instanceof PackIndexV2);
496 assertEquals(idx1.getObjectCount(), idx2.getObjectCount());
497 assertEquals(idx1.getOffset64Count(), idx2.getOffset64Count());
498
499 for (int i = 0; i < idx1.getObjectCount(); i++) {
500 final ObjectId id = idx1.getObjectId(i);
501 assertEquals(id, idx2.getObjectId(i));
502 assertEquals(idx1.findOffset(id), idx2.findOffset(id));
503 assertEquals(idx1.findCRC32(id), idx2.findCRC32(id));
504 }
505 }
506
507 @Test
508 public void testExclude() throws Exception {
509
510 FileRepository repo = createBareRepository();
511
512 try (TestRepository<FileRepository> testRepo = new TestRepository<>(
513 repo)) {
514 BranchBuilder bb = testRepo.branch("refs/heads/master");
515 contentA = testRepo.blob("A");
516 c1 = bb.commit().add("f", contentA).create();
517 testRepo.getRevWalk().parseHeaders(c1);
518 PackIndex pf1 = writePack(repo, wants(c1), EMPTY_ID_SET);
519 assertContent(pf1, Arrays.asList(c1.getId(), c1.getTree().getId(),
520 contentA.getId()));
521 contentB = testRepo.blob("B");
522 c2 = bb.commit().add("f", contentB).create();
523 testRepo.getRevWalk().parseHeaders(c2);
524 PackIndex pf2 = writePack(repo, wants(c2),
525 Sets.of((ObjectIdSet) pf1));
526 assertContent(pf2, Arrays.asList(c2.getId(), c2.getTree().getId(),
527 contentB.getId()));
528 }
529 }
530
531 private static void assertContent(PackIndex pi, List<ObjectId> expected) {
532 assertEquals("Pack index has wrong size.", expected.size(),
533 pi.getObjectCount());
534 for (int i = 0; i < pi.getObjectCount(); i++)
535 assertTrue(
536 "Pack index didn't contain the expected id "
537 + pi.getObjectId(i),
538 expected.contains(pi.getObjectId(i)));
539 }
540
541 @Test
542 public void testShallowIsMinimalDepth1() throws Exception {
543 try (FileRepository repo = setupRepoForShallowFetch()) {
544 PackIndex idx = writeShallowPack(repo, 1, wants(c2), NONE, NONE);
545 assertContent(idx, Arrays.asList(c2.getId(), c2.getTree().getId(),
546 contentA.getId(), contentB.getId()));
547
548
549 idx = writeShallowPack(repo, 1, wants(c5), haves(c2), shallows(c2));
550 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
551 contentC.getId(), contentD.getId(), contentE.getId()));
552 }
553 }
554
555 @Test
556 public void testShallowIsMinimalDepth2() throws Exception {
557 try (FileRepository repo = setupRepoForShallowFetch()) {
558 PackIndex idx = writeShallowPack(repo, 2, wants(c2), NONE, NONE);
559 assertContent(idx,
560 Arrays.asList(c1.getId(), c2.getId(), c1.getTree().getId(),
561 c2.getTree().getId(), contentA.getId(),
562 contentB.getId()));
563
564
565 idx = writeShallowPack(repo, 2, wants(c5), haves(c1, c2),
566 shallows(c1));
567 assertContent(idx,
568 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
569 c5.getTree().getId(), contentC.getId(),
570 contentD.getId(), contentE.getId()));
571 }
572 }
573
574 @Test
575 public void testShallowFetchShallowParentDepth1() throws Exception {
576 try (FileRepository repo = setupRepoForShallowFetch()) {
577 PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
578 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
579 contentA.getId(), contentB.getId(), contentC.getId(),
580 contentD.getId(), contentE.getId()));
581
582 idx = writeShallowPack(repo, 1, wants(c4), haves(c5), shallows(c5));
583 assertContent(idx, Arrays.asList(c4.getId(), c4.getTree().getId()));
584 }
585 }
586
587 @Test
588 public void testShallowFetchShallowParentDepth2() throws Exception {
589 try (FileRepository repo = setupRepoForShallowFetch()) {
590 PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
591 assertContent(idx,
592 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
593 c5.getTree().getId(), contentA.getId(),
594 contentB.getId(), contentC.getId(),
595 contentD.getId(), contentE.getId()));
596
597 idx = writeShallowPack(repo, 2, wants(c3), haves(c4, c5),
598 shallows(c4));
599 assertContent(idx, Arrays.asList(c2.getId(), c3.getId(),
600 c2.getTree().getId(), c3.getTree().getId()));
601 }
602 }
603
604 @Test
605 public void testShallowFetchShallowAncestorDepth1() throws Exception {
606 try (FileRepository repo = setupRepoForShallowFetch()) {
607 PackIndex idx = writeShallowPack(repo, 1, wants(c5), NONE, NONE);
608 assertContent(idx, Arrays.asList(c5.getId(), c5.getTree().getId(),
609 contentA.getId(), contentB.getId(), contentC.getId(),
610 contentD.getId(), contentE.getId()));
611
612 idx = writeShallowPack(repo, 1, wants(c3), haves(c5), shallows(c5));
613 assertContent(idx, Arrays.asList(c3.getId(), c3.getTree().getId()));
614 }
615 }
616
617 @Test
618 public void testShallowFetchShallowAncestorDepth2() throws Exception {
619 try (FileRepository repo = setupRepoForShallowFetch()) {
620 PackIndex idx = writeShallowPack(repo, 2, wants(c5), NONE, NONE);
621 assertContent(idx,
622 Arrays.asList(c4.getId(), c5.getId(), c4.getTree().getId(),
623 c5.getTree().getId(), contentA.getId(),
624 contentB.getId(), contentC.getId(),
625 contentD.getId(), contentE.getId()));
626
627 idx = writeShallowPack(repo, 2, wants(c2), haves(c4, c5),
628 shallows(c4));
629 assertContent(idx, Arrays.asList(c1.getId(), c2.getId(),
630 c1.getTree().getId(), c2.getTree().getId()));
631 }
632 }
633
634 @Test
635 public void testTotalPackFilesScanWhenSearchForReuseTimeoutNotSet()
636 throws Exception {
637 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
638 PackWriter mockedPackWriter = Mockito
639 .spy(new PackWriter(config, fileRepository.newObjectReader()));
640
641 doNothing().when(mockedPackWriter).select(any(), any());
642
643 try (FileOutputStream packOS = new FileOutputStream(
644 getPackFileToWrite(fileRepository, mockedPackWriter))) {
645 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
646 NullProgressMonitor.INSTANCE, packOS);
647 }
648
649 long numberOfPackFiles = new GC(fileRepository)
650 .getStatistics().numberOfPackFiles;
651 int expectedSelectCalls =
652
653 2 * (int) numberOfPackFiles +
654
655 1;
656 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
657 any());
658 }
659
660 @Test
661 public void testTotalPackFilesScanWhenSkippingSearchForReuseTimeoutCheck()
662 throws Exception {
663 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
664 PackConfig packConfig = new PackConfig();
665 packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1));
666 PackWriter mockedPackWriter = Mockito.spy(
667 new PackWriter(packConfig, fileRepository.newObjectReader()));
668
669 doNothing().when(mockedPackWriter).select(any(), any());
670
671 try (FileOutputStream packOS = new FileOutputStream(
672 getPackFileToWrite(fileRepository, mockedPackWriter))) {
673 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
674 NullProgressMonitor.INSTANCE, packOS);
675 }
676
677 long numberOfPackFiles = new GC(fileRepository)
678 .getStatistics().numberOfPackFiles;
679 int expectedSelectCalls =
680
681 2 * (int) numberOfPackFiles +
682
683 1;
684 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
685 any());
686 }
687
688 @Test
689 public void testPartialPackFilesScanWhenDoingSearchForReuseTimeoutCheck()
690 throws Exception {
691 FileRepository fileRepository = setUpRepoWithMultiplePackfiles();
692 PackConfig packConfig = new PackConfig();
693 packConfig.setSearchForReuseTimeout(Duration.ofSeconds(-1));
694 PackWriter mockedPackWriter = Mockito.spy(
695 new PackWriter(packConfig, fileRepository.newObjectReader()));
696 mockedPackWriter.enableSearchForReuseTimeout();
697
698 doNothing().when(mockedPackWriter).select(any(), any());
699
700 try (FileOutputStream packOS = new FileOutputStream(
701 getPackFileToWrite(fileRepository, mockedPackWriter))) {
702 mockedPackWriter.writePack(NullProgressMonitor.INSTANCE,
703 NullProgressMonitor.INSTANCE, packOS);
704 }
705
706 int expectedSelectCalls = 3;
707 verify(mockedPackWriter, times(expectedSelectCalls)).select(any(),
708 any());
709 }
710
711
712
713
714
715
716
717
718
719
720
721
722
723 private FileRepository setUpRepoWithMultiplePackfiles() throws Exception {
724 FileRepository fileRepository = createWorkRepository();
725 try (Git git = new Git(fileRepository)) {
726
727 git.commit().setMessage("First commit").call();
728 GC gc = new GC(fileRepository);
729 gc.setPackExpireAgeMillis(Long.MAX_VALUE);
730 gc.setExpireAgeMillis(Long.MAX_VALUE);
731
732 gc.gc().get();
733
734 git.commit().setMessage("Second commit").call();
735
736 gc.gc().get();
737
738 git.commit().setMessage("Third commit").call();
739 }
740 return fileRepository;
741 }
742
743 private PackFile getPackFileToWrite(FileRepository fileRepository,
744 PackWriter mockedPackWriter) throws IOException {
745 File packdir = fileRepository.getObjectDatabase().getPackDirectory();
746 PackFile packFile = new PackFile(packdir,
747 mockedPackWriter.computeName(), PackExt.PACK);
748
749 Set<ObjectId> all = new HashSet<>();
750 for (Ref r : fileRepository.getRefDatabase().getRefs()) {
751 all.add(r.getObjectId());
752 }
753
754 mockedPackWriter.preparePack(NullProgressMonitor.INSTANCE, all,
755 PackWriter.NONE);
756 return packFile;
757 }
758
759 private FileRepository setupRepoForShallowFetch() throws Exception {
760 FileRepository repo = createBareRepository();
761
762
763 repo.incrementOpen();
764 try (TestRepository<Repository> r = new TestRepository<>(repo)) {
765 BranchBuilder bb = r.branch("refs/heads/master");
766 contentA = r.blob("A");
767 contentB = r.blob("B");
768 contentC = r.blob("C");
769 contentD = r.blob("D");
770 contentE = r.blob("E");
771 c1 = bb.commit().add("a", contentA).create();
772 c2 = bb.commit().add("b", contentB).create();
773 c3 = bb.commit().add("c", contentC).create();
774 c4 = bb.commit().add("d", contentD).create();
775 c5 = bb.commit().add("e", contentE).create();
776 r.getRevWalk().parseHeaders(c5);
777 return repo;
778 }
779 }
780
781 private static PackIndex writePack(FileRepository repo,
782 Set<? extends ObjectId> want, Set<ObjectIdSet> excludeObjects)
783 throws IOException {
784 try (RevWalk walk = new RevWalk(repo)) {
785 return writePack(repo, walk, 0, want, NONE, excludeObjects);
786 }
787 }
788
789 private static PackIndex writeShallowPack(FileRepository repo, int depth,
790 Set<? extends ObjectId> want, Set<? extends ObjectId> have,
791 Set<? extends ObjectId> shallow) throws IOException {
792
793
794 try (DepthWalk.RevWalk walk = new DepthWalk.RevWalk(repo, depth - 1)) {
795 walk.assumeShallow(shallow);
796 return writePack(repo, walk, depth, want, have, EMPTY_ID_SET);
797 }
798 }
799
800 private static PackIndex writePack(FileRepository repo, RevWalk walk,
801 int depth, Set<? extends ObjectId> want,
802 Set<? extends ObjectId> have, Set<ObjectIdSet> excludeObjects)
803 throws IOException {
804 try (PackWriter pw = new PackWriter(repo)) {
805 pw.setDeltaBaseAsOffset(true);
806 pw.setReuseDeltaCommits(false);
807 for (ObjectIdSet idx : excludeObjects) {
808 pw.excludeObjects(idx);
809 }
810 if (depth > 0) {
811 pw.setShallowPack(depth, null);
812 }
813
814 ObjectWalk ow = walk.toObjectWalkWithSameObjects();
815
816 pw.preparePack(NullProgressMonitor.INSTANCE, ow, want, have, NONE);
817 File packdir = repo.getObjectDatabase().getPackDirectory();
818 PackFile packFile = new PackFile(packdir, pw.computeName(),
819 PackExt.PACK);
820 try (FileOutputStream packOS = new FileOutputStream(packFile)) {
821 pw.writePack(NullProgressMonitor.INSTANCE,
822 NullProgressMonitor.INSTANCE, packOS);
823 }
824 PackFile idxFile = packFile.create(PackExt.INDEX);
825 try (FileOutputStream idxOS = new FileOutputStream(idxFile)) {
826 pw.writeIndex(idxOS);
827 }
828 return PackIndex.open(idxFile);
829 }
830 }
831
832
833
834
835 private void writeVerifyPack1() throws IOException {
836 final HashSet<ObjectId> interestings = new HashSet<>();
837 interestings.add(ObjectId
838 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
839 createVerifyOpenPack(interestings, NONE, false, false);
840
841 final ObjectId expectedOrder[] = new ObjectId[] {
842 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
843 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
844 ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"),
845 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
846 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
847 ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904"),
848 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3"),
849 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
850
851 assertEquals(expectedOrder.length, writer.getObjectCount());
852 verifyObjectsOrder(expectedOrder);
853 assertEquals("34be9032ac282b11fa9babdc2b2a93ca996c9c2f", writer
854 .computeName().name());
855 }
856
857 private void writeVerifyPack2(boolean deltaReuse) throws IOException {
858 config.setReuseDeltas(deltaReuse);
859 final HashSet<ObjectId> interestings = new HashSet<>();
860 interestings.add(ObjectId
861 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
862 final HashSet<ObjectId> uninterestings = new HashSet<>();
863 uninterestings.add(ObjectId
864 .fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab"));
865 createVerifyOpenPack(interestings, uninterestings, false, false);
866
867 final ObjectId expectedOrder[] = new ObjectId[] {
868 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
869 ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799"),
870 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
871 ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327"),
872 ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3") ,
873 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
874 if (!config.isReuseDeltas() && !config.isDeltaCompress()) {
875
876 swap(expectedOrder, 4, 5);
877 }
878 assertEquals(expectedOrder.length, writer.getObjectCount());
879 verifyObjectsOrder(expectedOrder);
880 assertEquals("ed3f96b8327c7c66b0f8f70056129f0769323d86", writer
881 .computeName().name());
882 }
883
884 private static void swap(ObjectId[] arr, int a, int b) {
885 ObjectId tmp = arr[a];
886 arr[a] = arr[b];
887 arr[b] = tmp;
888 }
889
890 private void writeVerifyPack4(final boolean thin) throws IOException {
891 final HashSet<ObjectId> interestings = new HashSet<>();
892 interestings.add(ObjectId
893 .fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"));
894 final HashSet<ObjectId> uninterestings = new HashSet<>();
895 uninterestings.add(ObjectId
896 .fromString("c59759f143fb1fe21c197981df75a7ee00290799"));
897 createVerifyOpenPack(interestings, uninterestings, thin, false);
898
899 final ObjectId writtenObjects[] = new ObjectId[] {
900 ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7"),
901 ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035"),
902 ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259") };
903 assertEquals(writtenObjects.length, writer.getObjectCount());
904 ObjectId expectedObjects[];
905 if (thin) {
906 expectedObjects = new ObjectId[4];
907 System.arraycopy(writtenObjects, 0, expectedObjects, 0,
908 writtenObjects.length);
909 expectedObjects[3] = ObjectId
910 .fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3");
911
912 } else {
913 expectedObjects = writtenObjects;
914 }
915 verifyObjectsOrder(expectedObjects);
916 assertEquals("cded4b74176b4456afa456768b2b5aafb41c44fc", writer
917 .computeName().name());
918 }
919
920 private void createVerifyOpenPack(final Set<ObjectId> interestings,
921 final Set<ObjectId> uninterestings, final boolean thin,
922 final boolean ignoreMissingUninteresting)
923 throws MissingObjectException, IOException {
924 createVerifyOpenPack(interestings, uninterestings, thin,
925 ignoreMissingUninteresting, false);
926 }
927
928 private void createVerifyOpenPack(final Set<ObjectId> interestings,
929 final Set<ObjectId> uninterestings, final boolean thin,
930 final boolean ignoreMissingUninteresting, boolean useBitmaps)
931 throws MissingObjectException, IOException {
932 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
933 writer = new PackWriter(config, db.newObjectReader());
934 writer.setUseBitmaps(useBitmaps);
935 writer.setThin(thin);
936 writer.setIgnoreMissingUninteresting(ignoreMissingUninteresting);
937 writer.preparePack(m, interestings, uninterestings);
938 writer.writePack(m, m, os);
939 writer.close();
940 verifyOpenPack(thin);
941 }
942
943 private void createVerifyOpenPack(List<RevObject> objectSource)
944 throws MissingObjectException, IOException {
945 NullProgressMonitor m = NullProgressMonitor.INSTANCE;
946 writer = new PackWriter(config, db.newObjectReader());
947 writer.preparePack(objectSource.iterator());
948 assertEquals(objectSource.size(), writer.getObjectCount());
949 writer.writePack(m, m, os);
950 writer.close();
951 verifyOpenPack(false);
952 }
953
954 private void verifyOpenPack(boolean thin) throws IOException {
955 final byte[] packData = os.toByteArray();
956
957 if (thin) {
958 PackParser p = index(packData);
959 try {
960 p.parse(NullProgressMonitor.INSTANCE);
961 fail("indexer should grumble about missing object");
962 } catch (IOException x) {
963
964 }
965 }
966
967 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(packData);
968 p.setKeepEmpty(true);
969 p.setAllowThin(thin);
970 p.setIndexVersion(2);
971 p.parse(NullProgressMonitor.INSTANCE);
972 pack = p.getPack();
973 assertNotNull("have PackFile after parsing", pack);
974 }
975
976 private PackParser index(byte[] packData) throws IOException {
977 if (inserter == null)
978 inserter = dst.newObjectInserter();
979 return inserter.newPackParser(new ByteArrayInputStream(packData));
980 }
981
982 private void verifyObjectsOrder(ObjectId objectsOrder[]) {
983 final List<PackIndex.MutableEntry> entries = new ArrayList<>();
984
985 for (MutableEntry me : pack) {
986 entries.add(me.cloneEntry());
987 }
988 Collections.sort(entries, (MutableEntry o1, MutableEntry o2) -> Long
989 .signum(o1.getOffset() - o2.getOffset()));
990
991 int i = 0;
992 for (MutableEntry me : entries) {
993 assertEquals(objectsOrder[i++].toObjectId(), me.toObjectId());
994 }
995 }
996
997 private static Set<ObjectId> haves(ObjectId... objects) {
998 return Sets.of(objects);
999 }
1000
1001 private static Set<ObjectId> wants(ObjectId... objects) {
1002 return Sets.of(objects);
1003 }
1004
1005 private static Set<ObjectId> shallows(ObjectId... objects) {
1006 return Sets.of(objects);
1007 }
1008 }