1
2
3
4
5
6
7
8
9
10
11
12
13
14 package org.eclipse.jgit.transport;
15
16 import static org.junit.Assert.assertEquals;
17 import static org.junit.Assert.assertTrue;
18 import static org.junit.Assert.fail;
19
20 import java.io.ByteArrayInputStream;
21 import java.io.File;
22 import java.io.FileInputStream;
23 import java.io.IOException;
24 import java.io.InputStream;
25 import java.security.MessageDigest;
26 import java.text.MessageFormat;
27 import java.util.zip.Deflater;
28
29 import org.eclipse.jgit.errors.TooLargeObjectInPackException;
30 import org.eclipse.jgit.internal.JGitText;
31 import org.eclipse.jgit.internal.storage.file.ObjectDirectoryPackParser;
32 import org.eclipse.jgit.internal.storage.file.Pack;
33 import org.eclipse.jgit.junit.JGitTestUtil;
34 import org.eclipse.jgit.junit.RepositoryTestCase;
35 import org.eclipse.jgit.junit.TestRepository;
36 import org.eclipse.jgit.lib.Constants;
37 import org.eclipse.jgit.lib.NullProgressMonitor;
38 import org.eclipse.jgit.lib.ObjectId;
39 import org.eclipse.jgit.lib.ObjectInserter;
40 import org.eclipse.jgit.lib.Repository;
41 import org.eclipse.jgit.revwalk.RevBlob;
42 import org.eclipse.jgit.util.NB;
43 import org.eclipse.jgit.util.TemporaryBuffer;
44 import org.eclipse.jgit.util.io.UnionInputStream;
45 import org.junit.After;
46 import org.junit.Test;
47
48
49
50
51
52
53
54 public class PackParserTest extends RepositoryTestCase {
55
56
57
58
59
60 @Test
61 public void test1() throws IOException {
62 File packFile = JGitTestUtil.getTestResourceFile("pack-34be9032ac282b11fa9babdc2b2a93ca996c9c2f.pack");
63 try (InputStream is = new FileInputStream(packFile)) {
64 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
65 p.parse(NullProgressMonitor.INSTANCE);
66 Pack pack = p.getPack();
67
68 assertTrue(pack.hasObject(ObjectId.fromString("4b825dc642cb6eb9a060e54bf8d69288fbee4904")));
69 assertTrue(pack.hasObject(ObjectId.fromString("540a36d136cf413e4b064c2b0e0a4db60f77feab")));
70 assertTrue(pack.hasObject(ObjectId.fromString("5b6e7c66c276e7610d4a73c70ec1a1f7c1003259")));
71 assertTrue(pack.hasObject(ObjectId.fromString("6ff87c4664981e4397625791c8ea3bbb5f2279a3")));
72 assertTrue(pack.hasObject(ObjectId.fromString("82c6b885ff600be425b4ea96dee75dca255b69e7")));
73 assertTrue(pack.hasObject(ObjectId.fromString("902d5476fa249b7abc9d84c611577a81381f0327")));
74 assertTrue(pack.hasObject(ObjectId.fromString("aabf2ffaec9b497f0950352b3e582d73035c2035")));
75 assertTrue(pack.hasObject(ObjectId.fromString("c59759f143fb1fe21c197981df75a7ee00290799")));
76 }
77 }
78
79
80
81
82
83
84
85 @Test
86 public void test2() throws IOException {
87 File packFile = JGitTestUtil.getTestResourceFile("pack-df2982f284bbabb6bdb59ee3fcc6eb0983e20371.pack");
88 try (InputStream is = new FileInputStream(packFile)) {
89 ObjectDirectoryPackParser p = (ObjectDirectoryPackParser) index(is);
90 p.parse(NullProgressMonitor.INSTANCE);
91 Pack pack = p.getPack();
92
93 assertTrue(pack.hasObject(ObjectId.fromString("02ba32d3649e510002c21651936b7077aa75ffa9")));
94 assertTrue(pack.hasObject(ObjectId.fromString("0966a434eb1a025db6b71485ab63a3bfbea520b6")));
95 assertTrue(pack.hasObject(ObjectId.fromString("09efc7e59a839528ac7bda9fa020dc9101278680")));
96 assertTrue(pack.hasObject(ObjectId.fromString("0a3d7772488b6b106fb62813c4d6d627918d9181")));
97 assertTrue(pack.hasObject(ObjectId.fromString("1004d0d7ac26fbf63050a234c9b88a46075719d3")));
98 assertTrue(pack.hasObject(ObjectId.fromString("10da5895682013006950e7da534b705252b03be6")));
99 assertTrue(pack.hasObject(ObjectId.fromString("1203b03dc816ccbb67773f28b3c19318654b0bc8")));
100 assertTrue(pack.hasObject(ObjectId.fromString("15fae9e651043de0fd1deef588aa3fbf5a7a41c6")));
101 assertTrue(pack.hasObject(ObjectId.fromString("16f9ec009e5568c435f473ba3a1df732d49ce8c3")));
102 assertTrue(pack.hasObject(ObjectId.fromString("1fd7d579fb6ae3fe942dc09c2c783443d04cf21e")));
103 assertTrue(pack.hasObject(ObjectId.fromString("20a8ade77639491ea0bd667bf95de8abf3a434c8")));
104 assertTrue(pack.hasObject(ObjectId.fromString("2675188fd86978d5bc4d7211698b2118ae3bf658")));
105
106 }
107 }
108
109 @Test
110 public void testTinyThinPack() throws Exception {
111 RevBlob a;
112 try (TestRepository d = new TestRepository<Repository>(db)) {
113 a = d.blob("a");
114 }
115
116 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
117
118 packHeader(pack, 1);
119
120 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
121 a.copyRawTo(pack);
122 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
123
124 digest(pack);
125
126 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
127 p.setAllowThin(true);
128 p.parse(NullProgressMonitor.INSTANCE);
129 }
130
131 @Test
132 public void testPackWithDuplicateBlob() throws Exception {
133 final byte[] data = Constants.encode("0123456789abcdefg");
134 try (TestRepository<Repository> d = new TestRepository<>(db)) {
135 assertTrue(db.getObjectDatabase().has(d.blob(data)));
136 }
137
138 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
139 packHeader(pack, 1);
140 pack.write((Constants.OBJ_BLOB) << 4 | 0x80 | 1);
141 pack.write(1);
142 deflate(pack, data);
143 digest(pack);
144
145 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
146 p.setAllowThin(false);
147 p.parse(NullProgressMonitor.INSTANCE);
148 }
149
150 @Test
151 public void testPackWithTrailingGarbage() throws Exception {
152 RevBlob a;
153 try (TestRepository d = new TestRepository<Repository>(db)) {
154 a = d.blob("a");
155 }
156
157 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
158 packHeader(pack, 1);
159 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
160 a.copyRawTo(pack);
161 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
162 digest(pack);
163
164 PackParser p = index(new UnionInputStream(
165 new ByteArrayInputStream(pack.toByteArray()),
166 new ByteArrayInputStream(new byte[] { 0x7e })));
167 p.setAllowThin(true);
168 p.setCheckEofAfterPackFooter(true);
169 try {
170 p.parse(NullProgressMonitor.INSTANCE);
171 fail("Pack with trailing garbage was accepted");
172 } catch (IOException err) {
173 assertEquals(
174 MessageFormat.format(JGitText.get().expectedEOFReceived, "\\x7e"),
175 err.getMessage());
176 }
177 }
178
179 @Test
180 public void testMaxObjectSizeFullBlob() throws Exception {
181 final byte[] data = Constants.encode("0123456789");
182 try (TestRepository d = new TestRepository<Repository>(db)) {
183 d.blob(data);
184 }
185
186 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
187
188 packHeader(pack, 1);
189 pack.write((Constants.OBJ_BLOB) << 4 | 10);
190 deflate(pack, data);
191 digest(pack);
192
193 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
194 p.setMaxObjectSizeLimit(11);
195 p.parse(NullProgressMonitor.INSTANCE);
196
197 p = index(new ByteArrayInputStream(pack.toByteArray()));
198 p.setMaxObjectSizeLimit(10);
199 p.parse(NullProgressMonitor.INSTANCE);
200
201 p = index(new ByteArrayInputStream(pack.toByteArray()));
202 p.setMaxObjectSizeLimit(9);
203 try {
204 p.parse(NullProgressMonitor.INSTANCE);
205 fail("PackParser should have failed");
206 } catch (TooLargeObjectInPackException e) {
207 assertTrue(e.getMessage().contains("10"));
208 assertTrue(e.getMessage().contains("9"));
209 }
210 }
211
212 @Test
213 public void testMaxObjectSizeDeltaBlock() throws Exception {
214 RevBlob a;
215 try (TestRepository d = new TestRepository<Repository>(db)) {
216 a = d.blob("a");
217 }
218
219 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
220
221 packHeader(pack, 1);
222 pack.write((Constants.OBJ_REF_DELTA) << 4 | 14);
223 a.copyRawTo(pack);
224 deflate(pack, new byte[] { 1, 11, 11, 'a', '0', '1', '2', '3', '4',
225 '5', '6', '7', '8', '9' });
226 digest(pack);
227
228 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
229 p.setAllowThin(true);
230 p.setMaxObjectSizeLimit(14);
231 p.parse(NullProgressMonitor.INSTANCE);
232
233 p = index(new ByteArrayInputStream(pack.toByteArray()));
234 p.setAllowThin(true);
235 p.setMaxObjectSizeLimit(13);
236 try {
237 p.parse(NullProgressMonitor.INSTANCE);
238 fail("PackParser should have failed");
239 } catch (TooLargeObjectInPackException e) {
240 assertTrue(e.getMessage().contains("13"));
241 assertTrue(e.getMessage().contains("14"));
242 }
243 }
244
245 @Test
246 public void testMaxObjectSizeDeltaResultSize() throws Exception {
247 RevBlob a;
248 try (TestRepository d = new TestRepository<Repository>(db)) {
249 a = d.blob("0123456789");
250 }
251
252 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
253
254 packHeader(pack, 1);
255 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
256 a.copyRawTo(pack);
257 deflate(pack, new byte[] { 10, 11, 1, 'a' });
258 digest(pack);
259
260 PackParser p = index(new ByteArrayInputStream(pack.toByteArray()));
261 p.setAllowThin(true);
262 p.setMaxObjectSizeLimit(11);
263 p.parse(NullProgressMonitor.INSTANCE);
264
265 p = index(new ByteArrayInputStream(pack.toByteArray()));
266 p.setAllowThin(true);
267 p.setMaxObjectSizeLimit(10);
268 try {
269 p.parse(NullProgressMonitor.INSTANCE);
270 fail("PackParser should have failed");
271 } catch (TooLargeObjectInPackException e) {
272 assertTrue(e.getMessage().contains("11"));
273 assertTrue(e.getMessage().contains("10"));
274 }
275 }
276
277 @Test
278 public void testNonMarkingInputStream() throws Exception {
279 RevBlob a;
280 try (TestRepository d = new TestRepository<Repository>(db)) {
281 a = d.blob("a");
282 }
283
284 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(1024);
285 packHeader(pack, 1);
286 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
287 a.copyRawTo(pack);
288 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
289 digest(pack);
290
291 InputStream in = new ByteArrayInputStream(pack.toByteArray()) {
292 @Override
293 public boolean markSupported() {
294 return false;
295 }
296
297 @Override
298 public void mark(int maxlength) {
299 fail("Mark should not be called");
300 }
301 };
302
303 PackParser p = index(in);
304 p.setAllowThin(true);
305 p.setCheckEofAfterPackFooter(false);
306 p.setExpectDataAfterPackFooter(true);
307
308 try {
309 p.parse(NullProgressMonitor.INSTANCE);
310 fail("PackParser should have failed");
311 } catch (IOException e) {
312 assertEquals(e.getMessage(),
313 JGitText.get().inputStreamMustSupportMark);
314 }
315 }
316
317 @Test
318 public void testDataAfterPackFooterSingleRead() throws Exception {
319 RevBlob a;
320 try (TestRepository d = new TestRepository<Repository>(db)) {
321 a = d.blob("a");
322 }
323
324 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32*1024);
325 packHeader(pack, 1);
326 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
327 a.copyRawTo(pack);
328 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
329 digest(pack);
330
331 byte packData[] = pack.toByteArray();
332 byte streamData[] = new byte[packData.length + 1];
333 System.arraycopy(packData, 0, streamData, 0, packData.length);
334 streamData[packData.length] = 0x7e;
335
336 InputStream in = new ByteArrayInputStream(streamData);
337 PackParser p = index(in);
338 p.setAllowThin(true);
339 p.setCheckEofAfterPackFooter(false);
340 p.setExpectDataAfterPackFooter(true);
341
342 p.parse(NullProgressMonitor.INSTANCE);
343
344 assertEquals(0x7e, in.read());
345 }
346
347 @Test
348 public void testDataAfterPackFooterSplitObjectRead() throws Exception {
349 final byte[] data = Constants.encode("0123456789");
350
351
352 int objects = 900;
353 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
354 packHeader(pack, objects);
355
356 for (int i = 0; i < objects; i++) {
357 pack.write((Constants.OBJ_BLOB) << 4 | 10);
358 deflate(pack, data);
359 }
360 digest(pack);
361
362 byte packData[] = pack.toByteArray();
363 byte streamData[] = new byte[packData.length + 1];
364 System.arraycopy(packData, 0, streamData, 0, packData.length);
365 streamData[packData.length] = 0x7e;
366 InputStream in = new ByteArrayInputStream(streamData);
367 PackParser p = index(in);
368 p.setAllowThin(true);
369 p.setCheckEofAfterPackFooter(false);
370 p.setExpectDataAfterPackFooter(true);
371
372 p.parse(NullProgressMonitor.INSTANCE);
373
374 assertEquals(0x7e, in.read());
375 }
376
377 @Test
378 public void testDataAfterPackFooterSplitHeaderRead() throws Exception {
379 final byte[] data = Constants.encode("a");
380 RevBlob b;
381 try (TestRepository d = new TestRepository<Repository>(db)) {
382 b = d.blob(data);
383 }
384
385 int objects = 248;
386 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(32 * 1024);
387 packHeader(pack, objects + 1);
388 int offset = 13;
389 StringBuilder sb = new StringBuilder();
390 for (int i = 0; i < offset; i++)
391 sb.append(i);
392 offset = sb.toString().length();
393 int lenByte = (Constants.OBJ_BLOB) << 4 | (offset & 0x0F);
394 offset >>= 4;
395 if (offset > 0)
396 lenByte |= 1 << 7;
397 pack.write(lenByte);
398 while (offset > 0) {
399 lenByte = offset & 0x7F;
400 offset >>= 6;
401 if (offset > 0)
402 lenByte |= 1 << 7;
403 pack.write(lenByte);
404 }
405 deflate(pack, Constants.encode(sb.toString()));
406
407 for (int i = 0; i < objects; i++) {
408
409
410 pack.write((Constants.OBJ_REF_DELTA) << 4 | 4);
411 b.copyRawTo(pack);
412 deflate(pack, new byte[] { 0x1, 0x1, 0x1, 'b' });
413 }
414 digest(pack);
415
416 byte packData[] = pack.toByteArray();
417 byte streamData[] = new byte[packData.length + 1];
418 System.arraycopy(packData, 0, streamData, 0, packData.length);
419 streamData[packData.length] = 0x7e;
420 InputStream in = new ByteArrayInputStream(streamData);
421 PackParser p = index(in);
422 p.setAllowThin(true);
423 p.setCheckEofAfterPackFooter(false);
424 p.setExpectDataAfterPackFooter(true);
425
426 p.parse(NullProgressMonitor.INSTANCE);
427
428 assertEquals(0x7e, in.read());
429 }
430
431 private static void packHeader(TemporaryBuffer.Heap tinyPack, int cnt)
432 throws IOException {
433 final byte[] hdr = new byte[8];
434 NB.encodeInt32(hdr, 0, 2);
435 NB.encodeInt32(hdr, 4, cnt);
436
437 tinyPack.write(Constants.PACK_SIGNATURE);
438 tinyPack.write(hdr, 0, 8);
439 }
440
441 private static void deflate(TemporaryBuffer.Heap tinyPack,
442 final byte[] content)
443 throws IOException {
444 final Deflater deflater = new Deflater();
445 final byte[] buf = new byte[128];
446 deflater.setInput(content, 0, content.length);
447 deflater.finish();
448 do {
449 final int n = deflater.deflate(buf, 0, buf.length);
450 if (n > 0)
451 tinyPack.write(buf, 0, n);
452 } while (!deflater.finished());
453 }
454
455 private static void digest(TemporaryBuffer.Heap buf) throws IOException {
456 MessageDigest md = Constants.newMessageDigest();
457 md.update(buf.toByteArray());
458 buf.write(md.digest());
459 }
460
461 private ObjectInserter inserter;
462
463 @After
464 public void release() {
465 if (inserter != null) {
466 inserter.close();
467 }
468 }
469
470 private PackParser index(InputStream in) throws IOException {
471 if (inserter == null)
472 inserter = db.newObjectInserter();
473 return inserter.newPackParser(in);
474 }
475 }