1
2
3
4
5
6
7
8
9
10
11 package org.eclipse.jgit.internal.storage.file;
12
13 import static org.junit.Assert.assertArrayEquals;
14 import static org.junit.Assert.assertEquals;
15 import static org.junit.Assert.assertFalse;
16 import static org.junit.Assert.assertNotNull;
17 import static org.junit.Assert.assertNull;
18 import static org.junit.Assert.assertTrue;
19 import static org.junit.Assert.fail;
20
21 import java.io.ByteArrayInputStream;
22 import java.io.ByteArrayOutputStream;
23 import java.io.File;
24 import java.io.FileOutputStream;
25 import java.io.IOException;
26 import java.security.MessageDigest;
27 import java.text.MessageFormat;
28 import java.util.ArrayList;
29 import java.util.Arrays;
30 import java.util.Collections;
31 import java.util.List;
32 import java.util.zip.Deflater;
33
34 import org.eclipse.jgit.errors.LargeObjectException;
35 import org.eclipse.jgit.internal.JGitText;
36 import org.eclipse.jgit.internal.storage.pack.DeltaEncoder;
37 import org.eclipse.jgit.internal.storage.pack.PackExt;
38 import org.eclipse.jgit.junit.JGitTestUtil;
39 import org.eclipse.jgit.junit.LocalDiskRepositoryTestCase;
40 import org.eclipse.jgit.junit.TestRepository;
41 import org.eclipse.jgit.junit.TestRng;
42 import org.eclipse.jgit.lib.Constants;
43 import org.eclipse.jgit.lib.NullProgressMonitor;
44 import org.eclipse.jgit.lib.ObjectId;
45 import org.eclipse.jgit.lib.ObjectInserter;
46 import org.eclipse.jgit.lib.ObjectLoader;
47 import org.eclipse.jgit.lib.ObjectStream;
48 import org.eclipse.jgit.lib.Repository;
49 import org.eclipse.jgit.revwalk.RevBlob;
50 import org.eclipse.jgit.storage.file.WindowCacheConfig;
51 import org.eclipse.jgit.transport.PackParser;
52 import org.eclipse.jgit.transport.PackedObjectInfo;
53 import org.eclipse.jgit.util.IO;
54 import org.eclipse.jgit.util.NB;
55 import org.eclipse.jgit.util.TemporaryBuffer;
56 import org.junit.After;
57 import org.junit.Before;
58 import org.junit.Test;
59
60 public class PackTest extends LocalDiskRepositoryTestCase {
61 private int streamThreshold = 16 * 1024;
62
63 private TestRng rng;
64
65 private FileRepository repo;
66
67 private TestRepository<Repository> tr;
68
69 private WindowCursor wc;
70
71 private TestRng getRng() {
72 if (rng == null)
73 rng = new TestRng(JGitTestUtil.getName());
74 return rng;
75 }
76
77 @Override
78 @Before
79 public void setUp() throws Exception {
80 super.setUp();
81
82 WindowCacheConfig cfg = new WindowCacheConfig();
83 cfg.setStreamFileThreshold(streamThreshold);
84 cfg.install();
85
86 repo = createBareRepository();
87 tr = new TestRepository<>(repo);
88 wc = (WindowCursor) repo.newObjectReader();
89 }
90
91 @Override
92 @After
93 public void tearDown() throws Exception {
94 if (wc != null)
95 wc.close();
96 new WindowCacheConfig().install();
97 super.tearDown();
98 }
99
100 @Test
101 public void testWhole_SmallObject() throws Exception {
102 final int type = Constants.OBJ_BLOB;
103 byte[] data = getRng().nextBytes(300);
104 RevBlob id = tr.blob(data);
105 tr.branch("master").commit().add("A", id).create();
106 tr.packAndPrune();
107 assertTrue("has blob", wc.has(id));
108
109 ObjectLoader ol = wc.open(id);
110 assertNotNull("created loader", ol);
111 assertEquals(type, ol.getType());
112 assertEquals(data.length, ol.getSize());
113 assertFalse("is not large", ol.isLarge());
114 assertTrue("same content", Arrays.equals(data, ol.getCachedBytes()));
115
116 try (ObjectStream in = ol.openStream()) {
117 assertNotNull("have stream", in);
118 assertEquals(type, in.getType());
119 assertEquals(data.length, in.getSize());
120 byte[] data2 = new byte[data.length];
121 IO.readFully(in, data2, 0, data.length);
122 assertTrue("same content", Arrays.equals(data2, data));
123 assertEquals("stream at EOF", -1, in.read());
124 }
125 }
126
127 @Test
128 public void testWhole_LargeObject() throws Exception {
129 final int type = Constants.OBJ_BLOB;
130 byte[] data = getRng().nextBytes(streamThreshold + 5);
131 RevBlob id = tr.blob(data);
132 tr.branch("master").commit().add("A", id).create();
133 tr.packAndPrune();
134 assertTrue("has blob", wc.has(id));
135
136 ObjectLoader ol = wc.open(id);
137 assertNotNull("created loader", ol);
138 assertEquals(type, ol.getType());
139 assertEquals(data.length, ol.getSize());
140 assertTrue("is large", ol.isLarge());
141 try {
142 ol.getCachedBytes();
143 fail("Should have thrown LargeObjectException");
144 } catch (LargeObjectException tooBig) {
145 assertEquals(MessageFormat.format(
146 JGitText.get().largeObjectException, id.name()), tooBig
147 .getMessage());
148 }
149
150 try (ObjectStream in = ol.openStream()) {
151 assertNotNull("have stream", in);
152 assertEquals(type, in.getType());
153 assertEquals(data.length, in.getSize());
154 byte[] data2 = new byte[data.length];
155 IO.readFully(in, data2, 0, data.length);
156 assertTrue("same content", Arrays.equals(data2, data));
157 assertEquals("stream at EOF", -1, in.read());
158 }
159 }
160
161 @Test
162 public void testDelta_SmallObjectChain() throws Exception {
163 try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) {
164 byte[] data0 = new byte[512];
165 Arrays.fill(data0, (byte) 0xf3);
166 ObjectId id0 = fmt.idFor(Constants.OBJ_BLOB, data0);
167
168 TemporaryBuffer.Heap pack = new TemporaryBuffer.Heap(64 * 1024);
169 packHeader(pack, 4);
170 objectHeader(pack, Constants.OBJ_BLOB, data0.length);
171 deflate(pack, data0);
172
173 byte[] data1 = clone(0x01, data0);
174 byte[] delta1 = delta(data0, data1);
175 ObjectId id1 = fmt.idFor(Constants.OBJ_BLOB, data1);
176 objectHeader(pack, Constants.OBJ_REF_DELTA, delta1.length);
177 id0.copyRawTo(pack);
178 deflate(pack, delta1);
179
180 byte[] data2 = clone(0x02, data1);
181 byte[] delta2 = delta(data1, data2);
182 ObjectId id2 = fmt.idFor(Constants.OBJ_BLOB, data2);
183 objectHeader(pack, Constants.OBJ_REF_DELTA, delta2.length);
184 id1.copyRawTo(pack);
185 deflate(pack, delta2);
186
187 byte[] data3 = clone(0x03, data2);
188 byte[] delta3 = delta(data2, data3);
189 ObjectId id3 = fmt.idFor(Constants.OBJ_BLOB, data3);
190 objectHeader(pack, Constants.OBJ_REF_DELTA, delta3.length);
191 id2.copyRawTo(pack);
192 deflate(pack, delta3);
193
194 digest(pack);
195 PackParser ip = index(pack.toByteArray());
196 ip.setAllowThin(true);
197 ip.parse(NullProgressMonitor.INSTANCE);
198
199 assertTrue("has blob", wc.has(id3));
200
201 ObjectLoader ol = wc.open(id3);
202 assertNotNull("created loader", ol);
203 assertEquals(Constants.OBJ_BLOB, ol.getType());
204 assertEquals(data3.length, ol.getSize());
205 assertFalse("is large", ol.isLarge());
206 assertNotNull(ol.getCachedBytes());
207 assertArrayEquals(data3, ol.getCachedBytes());
208
209 try (ObjectStream in = ol.openStream()) {
210 assertNotNull("have stream", in);
211 assertEquals(Constants.OBJ_BLOB, in.getType());
212 assertEquals(data3.length, in.getSize());
213 byte[] act = new byte[data3.length];
214 IO.readFully(in, act, 0, data3.length);
215 assertTrue("same content", Arrays.equals(act, data3));
216 assertEquals("stream at EOF", -1, in.read());
217 }
218 }
219 }
220
221 @Test
222 public void testDelta_FailsOver2GiB() throws Exception {
223 try (ObjectInserter.Formatter fmt = new ObjectInserter.Formatter()) {
224 byte[] base = new byte[] { 'a' };
225 ObjectId idA = fmt.idFor(Constants.OBJ_BLOB, base);
226 ObjectId idB = fmt.idFor(Constants.OBJ_BLOB, new byte[] { 'b' });
227
228 PackedObjectInfo a = new PackedObjectInfo(idA);
229 PackedObjectInfo b = new PackedObjectInfo(idB);
230
231 TemporaryBuffer.Heap packContents = new TemporaryBuffer.Heap(64 * 1024);
232 packHeader(packContents, 2);
233 a.setOffset(packContents.length());
234 objectHeader(packContents, Constants.OBJ_BLOB, base.length);
235 deflate(packContents, base);
236
237 ByteArrayOutputStream tmp = new ByteArrayOutputStream();
238 DeltaEncoder de = new DeltaEncoder(tmp, base.length, 3L << 30);
239 de.copy(0, 1);
240 byte[] delta = tmp.toByteArray();
241 b.setOffset(packContents.length());
242 objectHeader(packContents, Constants.OBJ_REF_DELTA, delta.length);
243 idA.copyRawTo(packContents);
244 deflate(packContents, delta);
245 byte[] footer = digest(packContents);
246
247 File dir = new File(repo.getObjectDatabase().getDirectory(),
248 "pack");
249 PackFile packName = new PackFile(dir, idA.name() + ".pack");
250 PackFile idxName = packName.create(PackExt.INDEX);
251
252 try (FileOutputStream f = new FileOutputStream(packName)) {
253 f.write(packContents.toByteArray());
254 }
255
256 try (FileOutputStream f = new FileOutputStream(idxName)) {
257 List<PackedObjectInfo> list = new ArrayList<>();
258 list.add(a);
259 list.add(b);
260 Collections.sort(list);
261 new PackIndexWriterV1(f).write(list, footer);
262 }
263
264 Pack pack = new Pack(packName, null);
265 try {
266 pack.get(wc, b);
267 fail("expected LargeObjectException.ExceedsByteArrayLimit");
268 } catch (LargeObjectException.ExceedsByteArrayLimit bad) {
269 assertNull(bad.getObjectId());
270 } finally {
271 pack.close();
272 }
273 }
274 }
275
276 @Test
277 public void testConfigurableStreamFileThreshold() throws Exception {
278 byte[] data = getRng().nextBytes(300);
279 RevBlob id = tr.blob(data);
280 tr.branch("master").commit().add("A", id).create();
281 tr.packAndPrune();
282 assertTrue("has blob", wc.has(id));
283
284 ObjectLoader ol = wc.open(id);
285 try (ObjectStream in = ol.openStream()) {
286 assertTrue(in instanceof ObjectStream.SmallStream);
287 assertEquals(300, in.available());
288 }
289
290 wc.setStreamFileThreshold(299);
291 ol = wc.open(id);
292 try (ObjectStream in = ol.openStream()) {
293 assertTrue(in instanceof ObjectStream.Filter);
294 assertEquals(1, in.available());
295 }
296 }
297
298 private static byte[] clone(int first, byte[] base) {
299 byte[] r = new byte[base.length];
300 System.arraycopy(base, 1, r, 1, r.length - 1);
301 r[0] = (byte) first;
302 return r;
303 }
304
305 private static byte[] delta(byte[] base, byte[] dest) throws IOException {
306 ByteArrayOutputStream tmp = new ByteArrayOutputStream();
307 DeltaEncoder de = new DeltaEncoder(tmp, base.length, dest.length);
308 de.insert(dest, 0, 1);
309 de.copy(1, base.length - 1);
310 return tmp.toByteArray();
311 }
312
313 private static void packHeader(TemporaryBuffer.Heap pack, int cnt)
314 throws IOException {
315 final byte[] hdr = new byte[8];
316 NB.encodeInt32(hdr, 0, 2);
317 NB.encodeInt32(hdr, 4, cnt);
318 pack.write(Constants.PACK_SIGNATURE);
319 pack.write(hdr, 0, 8);
320 }
321
322 private static void objectHeader(TemporaryBuffer.Heap pack, int type, int sz)
323 throws IOException {
324 byte[] buf = new byte[8];
325 int nextLength = sz >>> 4;
326 buf[0] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (type << 4) | (sz & 0x0F));
327 sz = nextLength;
328 int n = 1;
329 while (sz > 0) {
330 nextLength >>>= 7;
331 buf[n++] = (byte) ((nextLength > 0 ? 0x80 : 0x00) | (sz & 0x7F));
332 sz = nextLength;
333 }
334 pack.write(buf, 0, n);
335 }
336
337 private static void deflate(TemporaryBuffer.Heap pack, byte[] content)
338 throws IOException {
339 final Deflater deflater = new Deflater();
340 final byte[] buf = new byte[128];
341 deflater.setInput(content, 0, content.length);
342 deflater.finish();
343 do {
344 final int n = deflater.deflate(buf, 0, buf.length);
345 if (n > 0)
346 pack.write(buf, 0, n);
347 } while (!deflater.finished());
348 deflater.end();
349 }
350
351 private static byte[] digest(TemporaryBuffer.Heap buf)
352 throws IOException {
353 MessageDigest md = Constants.newMessageDigest();
354 md.update(buf.toByteArray());
355 byte[] footer = md.digest();
356 buf.write(footer);
357 return footer;
358 }
359
360 private ObjectInserter inserter;
361
362 @After
363 public void release() {
364 if (inserter != null) {
365 inserter.close();
366 }
367 }
368
369 private PackParser index(byte[] raw) throws IOException {
370 if (inserter == null)
371 inserter = repo.newObjectInserter();
372 return inserter.newPackParser(new ByteArrayInputStream(raw));
373 }
374 }