PackParser.java

  1. /*
  2.  * Copyright (C) 2008-2011, Google Inc.
  3.  * Copyright (C) 2007-2008, Robin Rosenberg <robin.rosenberg@dewire.com>
  4.  * Copyright (C) 2008, Shawn O. Pearce <spearce@spearce.org> and others
  5.  *
  6.  * This program and the accompanying materials are made available under the
  7.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  8.  * https://www.eclipse.org/org/documents/edl-v10.php.
  9.  *
  10.  * SPDX-License-Identifier: BSD-3-Clause
  11.  */

  13. package org.eclipse.jgit.transport;

  15. import java.io.EOFException;
  16. import java.io.IOException;
  17. import java.io.InputStream;
  18. import java.security.MessageDigest;
  19. import java.text.MessageFormat;
  20. import java.util.ArrayList;
  21. import java.util.Arrays;
  22. import java.util.Comparator;
  23. import java.util.List;
  24. import java.util.concurrent.TimeUnit;
  25. import java.util.zip.DataFormatException;
  26. import java.util.zip.Inflater;

  28. import org.eclipse.jgit.errors.CorruptObjectException;
  29. import org.eclipse.jgit.errors.MissingObjectException;
  30. import org.eclipse.jgit.errors.TooLargeObjectInPackException;
  31. import org.eclipse.jgit.internal.JGitText;
  32. import org.eclipse.jgit.internal.storage.pack.BinaryDelta;
  33. import org.eclipse.jgit.lib.AnyObjectId;
  34. import org.eclipse.jgit.lib.BatchingProgressMonitor;
  35. import org.eclipse.jgit.lib.BlobObjectChecker;
  36. import org.eclipse.jgit.lib.Constants;
  37. import org.eclipse.jgit.lib.InflaterCache;
  38. import org.eclipse.jgit.lib.MutableObjectId;
  39. import org.eclipse.jgit.lib.NullProgressMonitor;
  40. import org.eclipse.jgit.lib.ObjectChecker;
  41. import org.eclipse.jgit.lib.ObjectDatabase;
  42. import org.eclipse.jgit.lib.ObjectId;
  43. import org.eclipse.jgit.lib.ObjectIdOwnerMap;
  44. import org.eclipse.jgit.lib.ObjectIdSubclassMap;
  45. import org.eclipse.jgit.lib.ObjectLoader;
  46. import org.eclipse.jgit.lib.ObjectReader;
  47. import org.eclipse.jgit.lib.ObjectStream;
  48. import org.eclipse.jgit.lib.ProgressMonitor;
  49. import org.eclipse.jgit.util.BlockList;
  50. import org.eclipse.jgit.util.IO;
  51. import org.eclipse.jgit.util.LongMap;
  52. import org.eclipse.jgit.util.NB;
  53. import org.eclipse.jgit.util.sha1.SHA1;

  55. /**
  56.  * Parses a pack stream and imports it for an
  57.  * {@link org.eclipse.jgit.lib.ObjectInserter}.
  58.  * <p>
  59.  * Applications can acquire an instance of a parser from ObjectInserter's
  60.  * {@link org.eclipse.jgit.lib.ObjectInserter#newPackParser(InputStream)}
  61.  * method.
  62.  * <p>
  63.  * Implementations of {@link org.eclipse.jgit.lib.ObjectInserter} should
  64.  * subclass this type and provide their own logic for the various {@code on*()}
  65.  * event methods declared to be abstract.
  66.  */
  67. public abstract class PackParser {
  68.     /** Size of the internal stream buffer. */
  69.     private static final int BUFFER_SIZE = 8192;

  71.     /** Location data is being obtained from. */
  72.     public enum Source {
  73.         /** Data is read from the incoming stream. */
  74.         INPUT,

  76.         /** Data is read back from the database's buffers. */
  77.         DATABASE;
  78.     }

  80.     /** Object database used for loading existing objects. */
  81.     private final ObjectDatabase objectDatabase;

  83.     private InflaterStream inflater;

  85.     private byte[] tempBuffer;

  87.     private byte[] hdrBuf;

  89.     private final SHA1 objectHasher = SHA1.newInstance();
  90.     private final MutableObjectId tempObjectId;

  92.     private InputStream in;

  94.     byte[] buf;

  96.     /** Position in the input stream of {@code buf[0]}. */
  97.     private long bBase;

  99.     private int bOffset;

  101.     int bAvail;

  103.     private ObjectChecker objCheck;

  105.     private boolean allowThin;

  107.     private boolean checkObjectCollisions;

  109.     private boolean needBaseObjectIds;

  111.     private boolean checkEofAfterPackFooter;

  113.     private boolean expectDataAfterPackFooter;

  115.     private long expectedObjectCount;

  117.     private PackedObjectInfo[] entries;

  119.     /**
  120.      * Every object contained within the incoming pack.
  121.      * <p>
  122.      * This is a subset of {@link #entries}, as thin packs can add additional
  123.      * objects to {@code entries} by copying already existing objects from the
  124.      * repository onto the end of the thin pack to make it self-contained.
  125.      */
  126.     private ObjectIdSubclassMap<ObjectId> newObjectIds;

  128.     private int deltaCount;

  130.     private int entryCount;

  132.     private ObjectIdOwnerMap<DeltaChain> baseById;

  134.     /**
  135.      * Objects referenced by their name from deltas, that aren't in this pack.
  136.      * <p>
  137.      * This is the set of objects that were copied onto the end of this pack to
  138.      * make it complete. These objects were not transmitted by the remote peer,
  139.      * but instead were assumed to already exist in the local repository.
  140.      */
  141.     private ObjectIdSubclassMap<ObjectId> baseObjectIds;

  143.     private LongMap<UnresolvedDelta> baseByPos;

  145.     /** Objects need to be double-checked for collision after indexing. */
  146.     private BlockList<PackedObjectInfo> collisionCheckObjs;

  148.     private MessageDigest packDigest;

  150.     private ObjectReader readCurs;

  152.     /** Message to protect the pack data from garbage collection. */
  153.     private String lockMessage;

  155.     /** Git object size limit */
  156.     private long maxObjectSizeLimit;

  158.     private final ReceivedPackStatistics.Builder stats =
  159.             new ReceivedPackStatistics.Builder();

  161.     /**
  162.      * Initialize a pack parser.
  163.      *
  164.      * @param odb
  165.      *            database the parser will write its objects into.
  166.      * @param src
  167.      *            the stream the parser will read.
  168.      */
  169.     protected PackParser(ObjectDatabase odb, InputStream src) {
  170.         objectDatabase = odb.newCachedDatabase();
  171.         in = src;

  173.         inflater = new InflaterStream();
  174.         readCurs = objectDatabase.newReader();
  175.         buf = new byte[BUFFER_SIZE];
  176.         tempBuffer = new byte[BUFFER_SIZE];
  177.         hdrBuf = new byte[64];
  178.         tempObjectId = new MutableObjectId();
  179.         packDigest = Constants.newMessageDigest();
  180.         checkObjectCollisions = true;
  181.     }

  183.     /**
  184.      * Whether a thin pack (missing base objects) is permitted.
  185.      *
  186.      * @return {@code true} if a thin pack (missing base objects) is permitted.
  187.      */
  188.     public boolean isAllowThin() {
  189.         return allowThin;
  190.     }

  192.     /**
  193.      * Configure this index pack instance to allow a thin pack.
  194.      * <p>
  195.      * Thin packs are sometimes used during network transfers to allow a delta
  196.      * to be sent without a base object. Such packs are not permitted on disk.
  197.      *
  198.      * @param allow
  199.      *            true to enable a thin pack.
  200.      */
  201.     public void setAllowThin(boolean allow) {
  202.         allowThin = allow;
  203.     }

  205.     /**
  206.      * Whether received objects are verified to prevent collisions.
  207.      *
  208.      * @return if true received objects are verified to prevent collisions.
  209.      * @since 4.1
  210.      */
  211.     protected boolean isCheckObjectCollisions() {
  212.         return checkObjectCollisions;
  213.     }

  215.     /**
  216.      * Enable checking for collisions with existing objects.
  217.      * <p>
  218.      * By default PackParser looks for each received object in the repository.
  219.      * If the object already exists, the existing object is compared
  220.      * byte-for-byte with the newly received copy to ensure they are identical.
  221.      * The receive is aborted with an exception if any byte differs. This check
  222.      * is necessary to prevent an evil attacker from supplying a replacement
  223.      * object into this repository in the event that a discovery enabling SHA-1
  224.      * collisions is made.
  225.      * <p>
  226.      * This check may be very costly to perform, and some repositories may have
  227.      * other ways to segregate newly received object data. The check is enabled
  228.      * by default, but can be explicitly disabled if the implementation can
  229.      * provide the same guarantee, or is willing to accept the risks associated
  230.      * with bypassing the check.
  231.      *
  232.      * @param check
  233.      *            true to enable collision checking (strongly encouraged).
  234.      * @since 4.1
  235.      */
  236.     protected void setCheckObjectCollisions(boolean check) {
  237.         checkObjectCollisions = check;
  238.     }

  240.     /**
  241.      * Configure this index pack instance to keep track of new objects.
  242.      * <p>
  243.      * By default an index pack doesn't save the new objects that were created
  244.      * when it was instantiated. Setting this flag to {@code true} allows the
  245.      * caller to use {@link #getNewObjectIds()} to retrieve that list.
  246.      *
  247.      * @param b
  248.      *            {@code true} to enable keeping track of new objects.
  249.      */
  250.     public void setNeedNewObjectIds(boolean b) {
  251.         if (b)
  252.             newObjectIds = new ObjectIdSubclassMap<>();
  253.         else
  254.             newObjectIds = null;
  255.     }

  257.     private boolean needNewObjectIds() {
  258.         return newObjectIds != null;
  259.     }

  261.     /**
  262.      * Configure this index pack instance to keep track of the objects assumed
  263.      * for delta bases.
  264.      * <p>
  265.      * By default an index pack doesn't save the objects that were used as delta
  266.      * bases. Setting this flag to {@code true} will allow the caller to use
  267.      * {@link #getBaseObjectIds()} to retrieve that list.
  268.      *
  269.      * @param b
  270.      *            {@code true} to enable keeping track of delta bases.
  271.      */
  272.     public void setNeedBaseObjectIds(boolean b) {
  273.         this.needBaseObjectIds = b;
  274.     }

  276.     /**
  277.      * Whether the EOF should be read from the input after the footer.
  278.      *
  279.      * @return true if the EOF should be read from the input after the footer.
  280.      */
  281.     public boolean isCheckEofAfterPackFooter() {
  282.         return checkEofAfterPackFooter;
  283.     }

  285.     /**
  286.      * Ensure EOF is read from the input stream after the footer.
  287.      *
  288.      * @param b
  289.      *            true if the EOF should be read; false if it is not checked.
  290.      */
  291.     public void setCheckEofAfterPackFooter(boolean b) {
  292.         checkEofAfterPackFooter = b;
  293.     }

  295.     /**
  296.      * Whether there is data expected after the pack footer.
  297.      *
  298.      * @return true if there is data expected after the pack footer.
  299.      */
  300.     public boolean isExpectDataAfterPackFooter() {
  301.         return expectDataAfterPackFooter;
  302.     }

  304.     /**
  305.      * Set if there is additional data in InputStream after pack.
  306.      *
  307.      * @param e
  308.      *            true if there is additional data in InputStream after pack.
  309.      *            This requires the InputStream to support the mark and reset
  310.      *            functions.
  311.      */
  312.     public void setExpectDataAfterPackFooter(boolean e) {
  313.         expectDataAfterPackFooter = e;
  314.     }

  316.     /**
  317.      * Get the new objects that were sent by the user
  318.      *
  319.      * @return the new objects that were sent by the user
  320.      */
  321.     public ObjectIdSubclassMap<ObjectId> getNewObjectIds() {
  322.         if (newObjectIds != null)
  323.             return newObjectIds;
  324.         return new ObjectIdSubclassMap<>();
  325.     }

  327.     /**
  328.      * Get set of objects the incoming pack assumed for delta purposes
  329.      *
  330.      * @return set of objects the incoming pack assumed for delta purposes
  331.      */
  332.     public ObjectIdSubclassMap<ObjectId> getBaseObjectIds() {
  333.         if (baseObjectIds != null)
  334.             return baseObjectIds;
  335.         return new ObjectIdSubclassMap<>();
  336.     }

  338.     /**
  339.      * Configure the checker used to validate received objects.
  340.      * <p>
  341.      * Usually object checking isn't necessary, as Git implementations only
  342.      * create valid objects in pack files. However, additional checking may be
  343.      * useful if processing data from an untrusted source.
  344.      *
  345.      * @param oc
  346.      *            the checker instance; null to disable object checking.
  347.      */
  348.     public void setObjectChecker(ObjectChecker oc) {
  349.         objCheck = oc;
  350.     }

  352.     /**
  353.      * Configure the checker used to validate received objects.
  354.      * <p>
  355.      * Usually object checking isn't necessary, as Git implementations only
  356.      * create valid objects in pack files. However, additional checking may be
  357.      * useful if processing data from an untrusted source.
  358.      * <p>
  359.      * This is shorthand for:
  360.      *
  361.      * <pre>
  362.      * setObjectChecker(on ? new ObjectChecker() : null);
  363.      * </pre>
  364.      *
  365.      * @param on
  366.      *            true to enable the default checker; false to disable it.
  367.      */
  368.     public void setObjectChecking(boolean on) {
  369.         setObjectChecker(on ? new ObjectChecker() : null);
  370.     }

  372.     /**
  373.      * Get the message to record with the pack lock.
  374.      *
  375.      * @return the message to record with the pack lock.
  376.      */
  377.     public String getLockMessage() {
  378.         return lockMessage;
  379.     }

  381.     /**
  382.      * Set the lock message for the incoming pack data.
  383.      *
  384.      * @param msg
  385.      *            if not null, the message to associate with the incoming data
  386.      *            while it is locked to prevent garbage collection.
  387.      */
  388.     public void setLockMessage(String msg) {
  389.         lockMessage = msg;
  390.     }

  392.     /**
  393.      * Set the maximum allowed Git object size.
  394.      * <p>
  395.      * If an object is larger than the given size the pack-parsing will throw an
  396.      * exception aborting the parsing.
  397.      *
  398.      * @param limit
  399.      *            the Git object size limit. If zero then there is not limit.
  400.      */
  401.     public void setMaxObjectSizeLimit(long limit) {
  402.         maxObjectSizeLimit = limit;
  403.     }

  405.     /**
  406.      * Get the number of objects in the stream.
  407.      * <p>
  408.      * The object count is only available after {@link #parse(ProgressMonitor)}
  409.      * has returned. The count may have been increased if the stream was a thin
  410.      * pack, and missing bases objects were appending onto it by the subclass.
  411.      *
  412.      * @return number of objects parsed out of the stream.
  413.      */
  414.     public int getObjectCount() {
  415.         return entryCount;
  416.     }

  418.     /**
  419.      * Get the information about the requested object.
  420.      * <p>
  421.      * The object information is only available after
  422.      * {@link #parse(ProgressMonitor)} has returned.
  423.      *
  424.      * @param nth
  425.      *            index of the object in the stream. Must be between 0 and
  426.      *            {@link #getObjectCount()}-1.
  427.      * @return the object information.
  428.      */
  429.     public PackedObjectInfo getObject(int nth) {
  430.         return entries[nth];
  431.     }

  433.     /**
  434.      * Get all of the objects, sorted by their name.
  435.      * <p>
  436.      * The object information is only available after
  437.      * {@link #parse(ProgressMonitor)} has returned.
  438.      * <p>
  439.      * To maintain lower memory usage and good runtime performance, this method
  440.      * sorts the objects in-place and therefore impacts the ordering presented
  441.      * by {@link #getObject(int)}.
  442.      *
  443.      * @param cmp
  444.      *            comparison function, if null objects are stored by ObjectId.
  445.      * @return sorted list of objects in this pack stream.
  446.      */
  447.     public List<PackedObjectInfo> getSortedObjectList(
  448.             Comparator<PackedObjectInfo> cmp) {
  449.         Arrays.sort(entries, 0, entryCount, cmp);
  450.         List<PackedObjectInfo> list = Arrays.asList(entries);
  451.         if (entryCount < entries.length)
  452.             list = list.subList(0, entryCount);
  453.         return list;
  454.     }

  456.     /**
  457.      * Get the size of the newly created pack.
  458.      * <p>
  459.      * This will also include the pack index size if an index was created. This
  460.      * method should only be called after pack parsing is finished.
  461.      *
  462.      * @return the pack size (including the index size) or -1 if the size cannot
  463.      *         be determined
  464.      * @since 3.3
  465.      */
  466.     public long getPackSize() {
  467.         return -1;
  468.     }

  470.     /**
  471.      * Returns the statistics of the parsed pack.
  472.      * <p>
  473.      * This should only be called after pack parsing is finished.
  474.      *
  475.      * @return {@link org.eclipse.jgit.transport.ReceivedPackStatistics}
  476.      * @since 4.6
  477.      */
  478.     public ReceivedPackStatistics getReceivedPackStatistics() {
  479.         return stats.build();
  480.     }

  482.     /**
  483.      * Parse the pack stream.
  484.      *
  485.      * @param progress
  486.      *            callback to provide progress feedback during parsing. If null,
  487.      *            {@link org.eclipse.jgit.lib.NullProgressMonitor} will be used.
  488.      * @return the pack lock, if one was requested by setting
  489.      *         {@link #setLockMessage(String)}.
  490.      * @throws java.io.IOException
  491.      *             the stream is malformed, or contains corrupt objects.
  492.      * @since 6.0
  493.      */
  494.     public final PackLock parse(ProgressMonitor progress) throws IOException {
  495.         return parse(progress, progress);
  496.     }

  498.     /**
  499.      * Parse the pack stream.
  500.      *
  501.      * @param receiving
  502.      *            receives progress feedback during the initial receiving
  503.      *            objects phase. If null,
  504.      *            {@link org.eclipse.jgit.lib.NullProgressMonitor} will be used.
  505.      * @param resolving
  506.      *            receives progress feedback during the resolving objects phase.
  507.      * @return the pack lock, if one was requested by setting
  508.      *         {@link #setLockMessage(String)}.
  509.      * @throws java.io.IOException
  510.      *             the stream is malformed, or contains corrupt objects.
  511.      * @since 6.0
  512.      */
  513.     public PackLock parse(ProgressMonitor receiving, ProgressMonitor resolving)
  514.             throws IOException {
  515.         if (receiving == null)
  516.             receiving = NullProgressMonitor.INSTANCE;
  517.         if (resolving == null)
  518.             resolving = NullProgressMonitor.INSTANCE;

  520.         if (receiving == resolving)
  521.             receiving.start(2 /* tasks */);
  522.         try {
  523.             readPackHeader();

  525.             entries = new PackedObjectInfo[(int) expectedObjectCount];
  526.             baseById = new ObjectIdOwnerMap<>();
  527.             baseByPos = new LongMap<>();
  528.             collisionCheckObjs = new BlockList<>();

  530.             receiving.beginTask(JGitText.get().receivingObjects,
  531.                     (int) expectedObjectCount);
  532.             try {
  533.                 for (int done = 0; done < expectedObjectCount; done++) {
  534.                     indexOneObject();
  535.                     receiving.update(1);
  536.                     if (receiving.isCancelled())
  537.                         throw new IOException(JGitText.get().downloadCancelled);
  538.                 }
  539.                 readPackFooter();
  540.                 endInput();
  541.             } finally {
  542.                 receiving.endTask();
  543.             }

  545.             if (!collisionCheckObjs.isEmpty()) {
  546.                 checkObjectCollision();
  547.             }

  549.             if (deltaCount > 0) {
  550.                 processDeltas(resolving);
  551.             }

  553.             packDigest = null;
  554.             baseById = null;
  555.             baseByPos = null;
  556.         } finally {
  557.             try {
  558.                 if (readCurs != null)
  559.                     readCurs.close();
  560.             } finally {
  561.                 readCurs = null;
  562.             }

  564.             try {
  565.                 inflater.release();
  566.             } finally {
  567.                 inflater = null;
  568.             }
  569.         }
  570.         return null; // By default there is no locking.
  571.     }

  573.     private void processDeltas(ProgressMonitor resolving) throws IOException {
  574.         if (resolving instanceof BatchingProgressMonitor) {
  575.             ((BatchingProgressMonitor) resolving).setDelayStart(1000,
  576.                     TimeUnit.MILLISECONDS);
  577.         }
  578.         resolving.beginTask(JGitText.get().resolvingDeltas, deltaCount);
  579.         resolveDeltas(resolving);
  580.         if (entryCount < expectedObjectCount) {
  581.             if (!isAllowThin()) {
  582.                 throw new IOException(MessageFormat.format(
  583.                         JGitText.get().packHasUnresolvedDeltas,
  584.                         Long.valueOf(expectedObjectCount - entryCount)));
  585.             }

  587.             resolveDeltasWithExternalBases(resolving);

  589.             if (entryCount < expectedObjectCount) {
  590.                 throw new IOException(MessageFormat.format(
  591.                         JGitText.get().packHasUnresolvedDeltas,
  592.                         Long.valueOf(expectedObjectCount - entryCount)));
  593.             }
  594.         }
  595.         resolving.endTask();
  596.     }

  598.     private void resolveDeltas(ProgressMonitor progress)
  599.             throws IOException {
  600.         final int last = entryCount;
  601.         for (int i = 0; i < last; i++) {
  602.             resolveDeltas(entries[i], progress);
  603.             if (progress.isCancelled())
  604.                 throw new IOException(
  605.                         JGitText.get().downloadCancelledDuringIndexing);
  606.         }
  607.     }

  609.     private void resolveDeltas(final PackedObjectInfo oe,
  610.             ProgressMonitor progress) throws IOException {
  611.         UnresolvedDelta children = firstChildOf(oe);
  612.         if (children == null)
  613.             return;

  615.         DeltaVisit visit = new DeltaVisit();
  616.         visit.nextChild = children;

  618.         ObjectTypeAndSize info = openDatabase(oe, new ObjectTypeAndSize());
  619.         switch (info.type) {
  620.         case Constants.OBJ_COMMIT:
  621.         case Constants.OBJ_TREE:
  622.         case Constants.OBJ_BLOB:
  623.         case Constants.OBJ_TAG:
  624.             visit.data = inflateAndReturn(Source.DATABASE, info.size);
  625.             visit.id = oe;
  626.             break;
  627.         default:
  628.             throw new IOException(MessageFormat.format(
  629.                     JGitText.get().unknownObjectType,
  630.                     Integer.valueOf(info.type)));
  631.         }

  633.         if (!checkCRC(oe.getCRC())) {
  634.             throw new IOException(MessageFormat.format(
  635.                     JGitText.get().corruptionDetectedReReadingAt,
  636.                     Long.valueOf(oe.getOffset())));
  637.         }

  639.         resolveDeltas(visit.next(), info.type, info, progress);
  640.     }

  642.     private void resolveDeltas(DeltaVisit visit, final int type,
  643.             ObjectTypeAndSize info, ProgressMonitor progress)
  644.             throws IOException {
  645.         stats.addDeltaObject(type);
  646.         do {
  647.             progress.update(1);
  648.             info = openDatabase(visit.delta, info);
  649.             switch (info.type) {
  650.             case Constants.OBJ_OFS_DELTA:
  651.             case Constants.OBJ_REF_DELTA:
  652.                 break;

  654.             default:
  655.                 throw new IOException(MessageFormat.format(
  656.                         JGitText.get().unknownObjectType,
  657.                         Integer.valueOf(info.type)));
  658.             }

  660.             byte[] delta = inflateAndReturn(Source.DATABASE, info.size);
  661.             long finalSz = BinaryDelta.getResultSize(delta);
  662.             checkIfTooLarge(type, finalSz);

  664.             visit.data = BinaryDelta.apply(visit.parent.data, delta);
  665.             delta = null;

  667.             if (!checkCRC(visit.delta.crc))
  668.                 throw new IOException(MessageFormat.format(
  669.                         JGitText.get().corruptionDetectedReReadingAt,
  670.                         Long.valueOf(visit.delta.position)));

  672.             SHA1 objectDigest = objectHasher.reset();
  673.             objectDigest.update(Constants.encodedTypeString(type));
  674.             objectDigest.update((byte) ' ');
  675.             objectDigest.update(Constants.encodeASCII(visit.data.length));
  676.             objectDigest.update((byte) 0);
  677.             objectDigest.update(visit.data);
  678.             objectDigest.digest(tempObjectId);

  680.             verifySafeObject(tempObjectId, type, visit.data);
  681.             if (isCheckObjectCollisions() && readCurs.has(tempObjectId)) {
  682.                 checkObjectCollision(tempObjectId, type, visit.data,
  683.                         visit.delta.sizeBeforeInflating);
  684.             }

  686.             PackedObjectInfo oe;
  687.             oe = newInfo(tempObjectId, visit.delta, visit.parent.id);
  688.             oe.setFullSize(finalSz);
  689.             oe.setOffset(visit.delta.position);
  690.             oe.setType(type);
  691.             onInflatedObjectData(oe, type, visit.data);
  692.             addObjectAndTrack(oe);
  693.             visit.id = oe;

  695.             visit.nextChild = firstChildOf(oe);
  696.             visit = visit.next();
  697.         } while (visit != null);
  698.     }

  700.     private final void checkIfTooLarge(int typeCode, long size)
  701.             throws IOException {
  702.         if (0 < maxObjectSizeLimit && maxObjectSizeLimit < size) {
  703.             switch (typeCode) {
  704.             case Constants.OBJ_COMMIT:
  705.             case Constants.OBJ_TREE:
  706.             case Constants.OBJ_BLOB:
  707.             case Constants.OBJ_TAG:
  708.                 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);

  710.             case Constants.OBJ_OFS_DELTA:
  711.             case Constants.OBJ_REF_DELTA:
  712.                 throw new TooLargeObjectInPackException(size, maxObjectSizeLimit);

  714.             default:
  715.                 throw new IOException(MessageFormat.format(
  716.                         JGitText.get().unknownObjectType,
  717.                         Integer.valueOf(typeCode)));
  718.             }
  719.         }
  720.         if (size > Integer.MAX_VALUE - 8) {
  721.             throw new TooLargeObjectInPackException(size, Integer.MAX_VALUE - 8);
  722.         }
  723.     }

  725.     /**
  726.      * Read the header of the current object.
  727.      * <p>
  728.      * After the header has been parsed, this method automatically invokes
  729.      * {@link #onObjectHeader(Source, byte[], int, int)} to allow the
  730.      * implementation to update its internal checksums for the bytes read.
  731.      * <p>
  732.      * When this method returns the database will be positioned on the first
  733.      * byte of the deflated data stream.
  734.      *
  735.      * @param info
  736.      *            the info object to populate.
  737.      * @return {@code info}, after populating.
  738.      * @throws java.io.IOException
  739.      *             the size cannot be read.
  740.      */
  741.     protected ObjectTypeAndSize readObjectHeader(ObjectTypeAndSize info)
  742.             throws IOException {
  743.         int hdrPtr = 0;
  744.         int c = readFrom(Source.DATABASE);
  745.         hdrBuf[hdrPtr++] = (byte) c;

  747.         info.type = (c >> 4) & 7;
  748.         long sz = c & 15;
  749.         int shift = 4;
  750.         while ((c & 0x80) != 0) {
  751.             c = readFrom(Source.DATABASE);
  752.             hdrBuf[hdrPtr++] = (byte) c;
  753.             sz += ((long) (c & 0x7f)) << shift;
  754.             shift += 7;
  755.         }
  756.         info.size = sz;

  758.         switch (info.type) {
  759.         case Constants.OBJ_COMMIT:
  760.         case Constants.OBJ_TREE:
  761.         case Constants.OBJ_BLOB:
  762.         case Constants.OBJ_TAG:
  763.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  764.             break;

  766.         case Constants.OBJ_OFS_DELTA:
  767.             c = readFrom(Source.DATABASE);
  768.             hdrBuf[hdrPtr++] = (byte) c;
  769.             while ((c & 128) != 0) {
  770.                 c = readFrom(Source.DATABASE);
  771.                 hdrBuf[hdrPtr++] = (byte) c;
  772.             }
  773.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  774.             break;

  776.         case Constants.OBJ_REF_DELTA:
  777.             System.arraycopy(buf, fill(Source.DATABASE, 20), hdrBuf, hdrPtr, 20);
  778.             hdrPtr += 20;
  779.             use(20);
  780.             onObjectHeader(Source.DATABASE, hdrBuf, 0, hdrPtr);
  781.             break;

  783.         default:
  784.             throw new IOException(MessageFormat.format(
  785.                     JGitText.get().unknownObjectType,
  786.                     Integer.valueOf(info.type)));
  787.         }
  788.         return info;
  789.     }

  791.     private UnresolvedDelta removeBaseById(AnyObjectId id) {
  792.         final DeltaChain d = baseById.get(id);
  793.         return d != null ? d.remove() : null;
  794.     }

  796.     private static UnresolvedDelta reverse(UnresolvedDelta c) {
  797.         UnresolvedDelta tail = null;
  798.         while (c != null) {
  799.             final UnresolvedDelta n = c.next;
  800.             c.next = tail;
  801.             tail = c;
  802.             c = n;
  803.         }
  804.         return tail;
  805.     }

  807.     private UnresolvedDelta firstChildOf(PackedObjectInfo oe) {
  808.         UnresolvedDelta a = reverse(removeBaseById(oe));
  809.         UnresolvedDelta b = reverse(baseByPos.remove(oe.getOffset()));

  811.         if (a == null)
  812.             return b;
  813.         if (b == null)
  814.             return a;

  816.         UnresolvedDelta first = null;
  817.         UnresolvedDelta last = null;
  818.         while (a != null || b != null) {
  819.             UnresolvedDelta curr;
  820.             if (b == null || (a != null && a.position < b.position)) {
  821.                 curr = a;
  822.                 a = a.next;
  823.             } else {
  824.                 curr = b;
  825.                 b = b.next;
  826.             }
  827.             if (last != null)
  828.                 last.next = curr;
  829.             else
  830.                 first = curr;
  831.             last = curr;
  832.             curr.next = null;
  833.         }
  834.         return first;
  835.     }

  837.     private void resolveDeltasWithExternalBases(ProgressMonitor progress)
  838.             throws IOException {
  839.         growEntries(baseById.size());

  841.         if (needBaseObjectIds)
  842.             baseObjectIds = new ObjectIdSubclassMap<>();

  844.         final List<DeltaChain> missing = new ArrayList<>(64);
  845.         for (DeltaChain baseId : baseById) {
  846.             if (baseId.head == null)
  847.                 continue;

  849.             if (needBaseObjectIds)
  850.                 baseObjectIds.add(baseId);

  852.             final ObjectLoader ldr;
  853.             try {
  854.                 ldr = readCurs.open(baseId);
  855.             } catch (MissingObjectException notFound) {
  856.                 missing.add(baseId);
  857.                 continue;
  858.             }

  860.             final DeltaVisit visit = new DeltaVisit();
  861.             visit.data = ldr.getCachedBytes(Integer.MAX_VALUE);
  862.             visit.id = baseId;
  863.             final int typeCode = ldr.getType();
  864.             final PackedObjectInfo oe = newInfo(baseId, null, null);
  865.             oe.setType(typeCode);
  866.             oe.setFullSize(ldr.getSize());
  867.             if (onAppendBase(typeCode, visit.data, oe))
  868.                 entries[entryCount++] = oe;
  869.             visit.nextChild = firstChildOf(oe);
  870.             resolveDeltas(visit.next(), typeCode,
  871.                     new ObjectTypeAndSize(), progress);

  873.             if (progress.isCancelled())
  874.                 throw new IOException(
  875.                         JGitText.get().downloadCancelledDuringIndexing);
  876.         }

  878.         for (DeltaChain base : missing) {
  879.             if (base.head != null)
  880.                 throw new MissingObjectException(base, "delta base"); //$NON-NLS-1$
  881.         }

  883.         onEndThinPack();
  884.     }

  886.     private void growEntries(int extraObjects) {
  887.         final PackedObjectInfo[] ne;

  889.         ne = new PackedObjectInfo[(int) expectedObjectCount + extraObjects];
  890.         System.arraycopy(entries, 0, ne, 0, entryCount);
  891.         entries = ne;
  892.     }

  894.     private void readPackHeader() throws IOException {
  895.         if (expectDataAfterPackFooter) {
  896.             if (!in.markSupported())
  897.                 throw new IOException(
  898.                         JGitText.get().inputStreamMustSupportMark);
  899.             in.mark(buf.length);
  900.         }

  902.         final int hdrln = Constants.PACK_SIGNATURE.length + 4 + 4;
  903.         final int p = fill(Source.INPUT, hdrln);
  904.         for (int k = 0; k < Constants.PACK_SIGNATURE.length; k++)
  905.             if (buf[p + k] != Constants.PACK_SIGNATURE[k])
  906.                 throw new IOException(JGitText.get().notAPACKFile);

  908.         final long vers = NB.decodeUInt32(buf, p + 4);
  909.         if (vers != 2 && vers != 3)
  910.             throw new IOException(MessageFormat.format(
  911.                     JGitText.get().unsupportedPackVersion, Long.valueOf(vers)));
  912.         final long objectCount = NB.decodeUInt32(buf, p + 8);
  913.         use(hdrln);
  914.         setExpectedObjectCount(objectCount);
  915.         onPackHeader(objectCount);
  916.     }

  918.     private void readPackFooter() throws IOException {
  919.         sync();
  920.         final byte[] actHash = packDigest.digest();

  922.         final int c = fill(Source.INPUT, 20);
  923.         final byte[] srcHash = new byte[20];
  924.         System.arraycopy(buf, c, srcHash, 0, 20);
  925.         use(20);

  927.         if (bAvail != 0 && !expectDataAfterPackFooter)
  928.             throw new CorruptObjectException(MessageFormat.format(
  929.                     JGitText.get().expectedEOFReceived,
  930.                     "\\x" + Integer.toHexString(buf[bOffset] & 0xff))); //$NON-NLS-1$
  931.         if (isCheckEofAfterPackFooter()) {
  932.             int eof = in.read();
  933.             if (0 <= eof)
  934.                 throw new CorruptObjectException(MessageFormat.format(
  935.                         JGitText.get().expectedEOFReceived,
  936.                         "\\x" + Integer.toHexString(eof))); //$NON-NLS-1$
  937.         } else if (bAvail > 0 && expectDataAfterPackFooter) {
  938.             in.reset();
  939.             IO.skipFully(in, bOffset);
  940.         }

  942.         if (!Arrays.equals(actHash, srcHash))
  943.             throw new CorruptObjectException(
  944.                     JGitText.get().corruptObjectPackfileChecksumIncorrect);

  946.         onPackFooter(srcHash);
  947.     }

  949.     // Cleanup all resources associated with our input parsing.
  950.     private void endInput() {
  951.         stats.setNumBytesRead(streamPosition());
  952.         in = null;
  953.     }

  955.     // Read one entire object or delta from the input.
  956.     private void indexOneObject() throws IOException {
  957.         final long streamPosition = streamPosition();

  959.         int hdrPtr = 0;
  960.         int c = readFrom(Source.INPUT);
  961.         hdrBuf[hdrPtr++] = (byte) c;

  963.         final int typeCode = (c >> 4) & 7;
  964.         long sz = c & 15;
  965.         int shift = 4;
  966.         while ((c & 0x80) != 0) {
  967.             c = readFrom(Source.INPUT);
  968.             hdrBuf[hdrPtr++] = (byte) c;
  969.             sz += ((long) (c & 0x7f)) << shift;
  970.             shift += 7;
  971.         }

  973.         checkIfTooLarge(typeCode, sz);

  975.         switch (typeCode) {
  976.         case Constants.OBJ_COMMIT:
  977.         case Constants.OBJ_TREE:
  978.         case Constants.OBJ_BLOB:
  979.         case Constants.OBJ_TAG:
  980.             stats.addWholeObject(typeCode);
  981.             onBeginWholeObject(streamPosition, typeCode, sz);
  982.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  983.             whole(streamPosition, typeCode, sz);
  984.             break;

  986.         case Constants.OBJ_OFS_DELTA: {
  987.             stats.addOffsetDelta();
  988.             c = readFrom(Source.INPUT);
  989.             hdrBuf[hdrPtr++] = (byte) c;
  990.             long ofs = c & 127;
  991.             while ((c & 128) != 0) {
  992.                 ofs += 1;
  993.                 c = readFrom(Source.INPUT);
  994.                 hdrBuf[hdrPtr++] = (byte) c;
  995.                 ofs <<= 7;
  996.                 ofs += (c & 127);
  997.             }
  998.             final long base = streamPosition - ofs;
  999.             onBeginOfsDelta(streamPosition, base, sz);
  1000.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  1001.             inflateAndSkip(Source.INPUT, sz);
  1002.             UnresolvedDelta n = onEndDelta();
  1003.             n.position = streamPosition;
  1004.             n.next = baseByPos.put(base, n);
  1005.             n.sizeBeforeInflating = streamPosition() - streamPosition;
  1006.             deltaCount++;
  1007.             break;
  1008.         }

  1010.         case Constants.OBJ_REF_DELTA: {
  1011.             stats.addRefDelta();
  1012.             c = fill(Source.INPUT, 20);
  1013.             final ObjectId base = ObjectId.fromRaw(buf, c);
  1014.             System.arraycopy(buf, c, hdrBuf, hdrPtr, 20);
  1015.             hdrPtr += 20;
  1016.             use(20);
  1017.             DeltaChain r = baseById.get(base);
  1018.             if (r == null) {
  1019.                 r = new DeltaChain(base);
  1020.                 baseById.add(r);
  1021.             }
  1022.             onBeginRefDelta(streamPosition, base, sz);
  1023.             onObjectHeader(Source.INPUT, hdrBuf, 0, hdrPtr);
  1024.             inflateAndSkip(Source.INPUT, sz);
  1025.             UnresolvedDelta n = onEndDelta();
  1026.             n.position = streamPosition;
  1027.             n.sizeBeforeInflating = streamPosition() - streamPosition;
  1028.             r.add(n);
  1029.             deltaCount++;
  1030.             break;
  1031.         }

  1033.         default:
  1034.             throw new IOException(
  1035.                     MessageFormat.format(JGitText.get().unknownObjectType,
  1036.                             Integer.valueOf(typeCode)));
  1037.         }
  1038.     }

  1040.     private void whole(long pos, int type, long sz)
  1041.             throws IOException {
  1042.         SHA1 objectDigest = objectHasher.reset();
  1043.         objectDigest.update(Constants.encodedTypeString(type));
  1044.         objectDigest.update((byte) ' ');
  1045.         objectDigest.update(Constants.encodeASCII(sz));
  1046.         objectDigest.update((byte) 0);

  1048.         final byte[] data;
  1049.         if (type == Constants.OBJ_BLOB) {
  1050.             byte[] readBuffer = buffer();
  1051.             BlobObjectChecker checker = null;
  1052.             if (objCheck != null) {
  1053.                 checker = objCheck.newBlobObjectChecker();
  1054.             }
  1055.             if (checker == null) {
  1056.                 checker = BlobObjectChecker.NULL_CHECKER;
  1057.             }
  1058.             long cnt = 0;
  1059.             try (InputStream inf = inflate(Source.INPUT, sz)) {
  1060.                 while (cnt < sz) {
  1061.                     int r = inf.read(readBuffer);
  1062.                     if (r <= 0)
  1063.                         break;
  1064.                     objectDigest.update(readBuffer, 0, r);
  1065.                     checker.update(readBuffer, 0, r);
  1066.                     cnt += r;
  1067.                 }
  1068.             }
  1069.             objectDigest.digest(tempObjectId);
  1070.             checker.endBlob(tempObjectId);
  1071.             data = null;
  1072.         } else {
  1073.             data = inflateAndReturn(Source.INPUT, sz);
  1074.             objectDigest.update(data);
  1075.             objectDigest.digest(tempObjectId);
  1076.             verifySafeObject(tempObjectId, type, data);
  1077.         }

  1079.         long sizeBeforeInflating = streamPosition() - pos;
  1080.         PackedObjectInfo obj = newInfo(tempObjectId, null, null);
  1081.         obj.setOffset(pos);
  1082.         obj.setType(type);
  1083.         obj.setSize(sizeBeforeInflating);
  1084.         obj.setFullSize(sz);
  1085.         onEndWholeObject(obj);
  1086.         if (data != null)
  1087.             onInflatedObjectData(obj, type, data);
  1088.         addObjectAndTrack(obj);

  1090.         if (isCheckObjectCollisions()) {
  1091.             collisionCheckObjs.add(obj);
  1092.         }
  1093.     }

  1095.     /**
  1096.      * Verify the integrity of the object.
  1097.      *
  1098.      * @param id
  1099.      *            identity of the object to be checked.
  1100.      * @param type
  1101.      *            the type of the object.
  1102.      * @param data
  1103.      *            raw content of the object.
  1104.      * @throws org.eclipse.jgit.errors.CorruptObjectException
  1105.      * @since 4.9
  1106.      */
  1107.     protected void verifySafeObject(final AnyObjectId id, final int type,
  1108.             final byte[] data) throws CorruptObjectException {
  1109.         if (objCheck != null) {
  1110.             try {
  1111.                 objCheck.check(id, type, data);
  1112.             } catch (CorruptObjectException e) {
  1113.                 if (e.getErrorType() != null) {
  1114.                     throw e;
  1115.                 }
  1116.                 throw new CorruptObjectException(
  1117.                         MessageFormat.format(JGitText.get().invalidObject,
  1118.                                 Constants.typeString(type), id.name(),
  1119.                                 e.getMessage()),
  1120.                         e);
  1121.             }
  1122.         }
  1123.     }

  1125.     private void checkObjectCollision() throws IOException {
  1126.         for (PackedObjectInfo obj : collisionCheckObjs) {
  1127.             if (!readCurs.has(obj)) {
  1128.                 continue;
  1129.             }
  1130.             checkObjectCollision(obj);
  1131.         }
  1132.     }

  1134.     private void checkObjectCollision(PackedObjectInfo obj)
  1135.             throws IOException {
  1136.         ObjectTypeAndSize info = openDatabase(obj, new ObjectTypeAndSize());
  1137.         final byte[] readBuffer = buffer();
  1138.         final byte[] curBuffer = new byte[readBuffer.length];
  1139.         long sz = info.size;
  1140.         try (ObjectStream cur = readCurs.open(obj, info.type).openStream()) {
  1141.             if (cur.getSize() != sz) {
  1142.                 throw new IOException(MessageFormat.format(
  1143.                         JGitText.get().collisionOn, obj.name()));
  1144.             }
  1145.             try (InputStream pck = inflate(Source.DATABASE, sz)) {
  1146.                 while (0 < sz) {
  1147.                     int n = (int) Math.min(readBuffer.length, sz);
  1148.                     IO.readFully(cur, curBuffer, 0, n);
  1149.                     IO.readFully(pck, readBuffer, 0, n);
  1150.                     for (int i = 0; i < n; i++) {
  1151.                         if (curBuffer[i] != readBuffer[i]) {
  1152.                             throw new IOException(MessageFormat.format(
  1153.                                     JGitText.get().collisionOn, obj.name()));
  1154.                         }
  1155.                     }
  1156.                     sz -= n;
  1157.                 }
  1158.             }
  1159.             stats.incrementObjectsDuplicated();
  1160.             stats.incrementNumBytesDuplicated(obj.getSize());
  1161.         } catch (MissingObjectException notLocal) {
  1162.             // This is OK, we don't have a copy of the object locally
  1163.             // but the API throws when we try to read it as usually it's
  1164.             // an error to read something that doesn't exist.
  1165.         }
  1166.     }

  1168.     private void checkObjectCollision(AnyObjectId obj, int type, byte[] data,
  1169.             long sizeBeforeInflating) throws IOException {
  1170.         try {
  1171.             final ObjectLoader ldr = readCurs.open(obj, type);
  1172.             final byte[] existingData = ldr.getCachedBytes(data.length);
  1173.             if (!Arrays.equals(data, existingData)) {
  1174.                 throw new IOException(MessageFormat
  1175.                         .format(JGitText.get().collisionOn, obj.name()));
  1176.             }
  1177.             stats.incrementObjectsDuplicated();
  1178.             stats.incrementNumBytesDuplicated(sizeBeforeInflating);
  1179.         } catch (MissingObjectException notLocal) {
  1180.             // This is OK, we don't have a copy of the object locally
  1181.             // but the API throws when we try to read it as usually its
  1182.             // an error to read something that doesn't exist.
  1183.         }
  1184.     }

  1186.     /** @return current position of the input stream being parsed. */
  1187.     private long streamPosition() {
  1188.         return bBase + bOffset;
  1189.     }

  1191.     private ObjectTypeAndSize openDatabase(PackedObjectInfo obj,
  1192.             ObjectTypeAndSize info) throws IOException {
  1193.         bOffset = 0;
  1194.         bAvail = 0;
  1195.         return seekDatabase(obj, info);
  1196.     }

  1198.     private ObjectTypeAndSize openDatabase(UnresolvedDelta delta,
  1199.             ObjectTypeAndSize info) throws IOException {
  1200.         bOffset = 0;
  1201.         bAvail = 0;
  1202.         return seekDatabase(delta, info);
  1203.     }

  1205.     // Consume exactly one byte from the buffer and return it.
  1206.     private int readFrom(Source src) throws IOException {
  1207.         if (bAvail == 0)
  1208.             fill(src, 1);
  1209.         bAvail--;
  1210.         return buf[bOffset++] & 0xff;
  1211.     }

  1213.     // Consume cnt bytes from the buffer.
  1214.     void use(int cnt) {
  1215.         bOffset += cnt;
  1216.         bAvail -= cnt;
  1217.     }

  1219.     // Ensure at least need bytes are available in {@link #buf}.
  1220.     int fill(Source src, int need) throws IOException {
  1221.         while (bAvail < need) {
  1222.             int next = bOffset + bAvail;
  1223.             int free = buf.length - next;
  1224.             if (free + bAvail < need) {
  1225.                 switch (src) {
  1226.                 case INPUT:
  1227.                     sync();
  1228.                     break;
  1229.                 case DATABASE:
  1230.                     if (bAvail > 0)
  1231.                         System.arraycopy(buf, bOffset, buf, 0, bAvail);
  1232.                     bOffset = 0;
  1233.                     break;
  1234.                 }
  1235.                 next = bAvail;
  1236.                 free = buf.length - next;
  1237.             }
  1238.             switch (src) {
  1239.             case INPUT:
  1240.                 next = in.read(buf, next, free);
  1241.                 break;
  1242.             case DATABASE:
  1243.                 next = readDatabase(buf, next, free);
  1244.                 break;
  1245.             }
  1246.             if (next <= 0)
  1247.                 throw new EOFException(
  1248.                         JGitText.get().packfileIsTruncatedNoParam);
  1249.             bAvail += next;
  1250.         }
  1251.         return bOffset;
  1252.     }

  1254.     // Store consumed bytes in {@link #buf} up to {@link #bOffset}.
  1255.     private void sync() throws IOException {
  1256.         packDigest.update(buf, 0, bOffset);
  1257.         onStoreStream(buf, 0, bOffset);
  1258.         if (expectDataAfterPackFooter) {
  1259.             if (bAvail > 0) {
  1260.                 in.reset();
  1261.                 IO.skipFully(in, bOffset);
  1262.                 bAvail = 0;
  1263.             }
  1264.             in.mark(buf.length);
  1265.         } else if (bAvail > 0)
  1266.             System.arraycopy(buf, bOffset, buf, 0, bAvail);
  1267.         bBase += bOffset;
  1268.         bOffset = 0;
  1269.     }

  1271.     /**
  1272.      * Get a temporary byte array for use by the caller.
  1273.      *
  1274.      * @return a temporary byte array for use by the caller.
  1275.      */
  1276.     protected byte[] buffer() {
  1277.         return tempBuffer;
  1278.     }

  1280.     /**
  1281.      * Construct a PackedObjectInfo instance for this parser.
  1282.      *
  1283.      * @param id
  1284.      *            identity of the object to be tracked.
  1285.      * @param delta
  1286.      *            if the object was previously an unresolved delta, this is the
  1287.      *            delta object that was tracking it. Otherwise null.
  1288.      * @param deltaBase
  1289.      *            if the object was previously an unresolved delta, this is the
  1290.      *            ObjectId of the base of the delta. The base may be outside of
  1291.      *            the pack stream if the stream was a thin-pack.
  1292.      * @return info object containing this object's data.
  1293.      */
  1294.     protected PackedObjectInfo newInfo(AnyObjectId id, UnresolvedDelta delta,
  1295.             ObjectId deltaBase) {
  1296.         PackedObjectInfo oe = new PackedObjectInfo(id);
  1297.         if (delta != null)
  1298.             oe.setCRC(delta.crc);
  1299.         return oe;
  1300.     }

  1302.     /**
  1303.      * Set the expected number of objects in the pack stream.
  1304.      * <p>
  1305.      * The object count in the pack header is not always correct for some Dfs
  1306.      * pack files. e.g. INSERT pack always assume 1 object in the header since
  1307.      * the actual object count is unknown when the pack is written.
  1308.      * <p>
  1309.      * If external implementation wants to overwrite the expectedObjectCount,
  1310.      * they should call this method during {@link #onPackHeader(long)}.
  1311.      *
  1312.      * @param expectedObjectCount a long.
  1313.      * @since 4.9
  1314.      */
  1315.     protected void setExpectedObjectCount(long expectedObjectCount) {
  1316.         this.expectedObjectCount = expectedObjectCount;
  1317.     }

  1319.     /**
  1320.      * Store bytes received from the raw stream.
  1321.      * <p>
  1322.      * This method is invoked during {@link #parse(ProgressMonitor)} as data is
  1323.      * consumed from the incoming stream. Implementors may use this event to
  1324.      * archive the raw incoming stream to the destination repository in large
  1325.      * chunks, without paying attention to object boundaries.
  1326.      * <p>
  1327.      * The only component of the pack not supplied to this method is the last 20
  1328.      * bytes of the pack that comprise the trailing SHA-1 checksum. Those are
  1329.      * passed to {@link #onPackFooter(byte[])}.
  1330.      *
  1331.      * @param raw
  1332.      *            buffer to copy data out of.
  1333.      * @param pos
  1334.      *            first offset within the buffer that is valid.
  1335.      * @param len
  1336.      *            number of bytes in the buffer that are valid.
  1337.      * @throws java.io.IOException
  1338.      *             the stream cannot be archived.
  1339.      */
  1340.     protected abstract void onStoreStream(byte[] raw, int pos, int len)
  1341.             throws IOException;

  1343.     /**
  1344.      * Store (and/or checksum) an object header.
  1345.      * <p>
  1346.      * Invoked after any of the {@code onBegin()} events. The entire header is
  1347.      * supplied in a single invocation, before any object data is supplied.
  1348.      *
  1349.      * @param src
  1350.      *            where the data came from
  1351.      * @param raw
  1352.      *            buffer to read data from.
  1353.      * @param pos
  1354.      *            first offset within buffer that is valid.
  1355.      * @param len
  1356.      *            number of bytes in buffer that are valid.
  1357.      * @throws java.io.IOException
  1358.      *             the stream cannot be archived.
  1359.      */
  1360.     protected abstract void onObjectHeader(Source src, byte[] raw, int pos,
  1361.             int len) throws IOException;

  1363.     /**
  1364.      * Store (and/or checksum) a portion of an object's data.
  1365.      * <p>
  1366.      * This method may be invoked multiple times per object, depending on the
  1367.      * size of the object, the size of the parser's internal read buffer, and
  1368.      * the alignment of the object relative to the read buffer.
  1369.      * <p>
  1370.      * Invoked after {@link #onObjectHeader(Source, byte[], int, int)}.
  1371.      *
  1372.      * @param src
  1373.      *            where the data came from
  1374.      * @param raw
  1375.      *            buffer to read data from.
  1376.      * @param pos
  1377.      *            first offset within buffer that is valid.
  1378.      * @param len
  1379.      *            number of bytes in buffer that are valid.
  1380.      * @throws java.io.IOException
  1381.      *             the stream cannot be archived.
  1382.      */
  1383.     protected abstract void onObjectData(Source src, byte[] raw, int pos,
  1384.             int len) throws IOException;

  1386.     /**
  1387.      * Invoked for commits, trees, tags, and small blobs.
  1388.      *
  1389.      * @param obj
  1390.      *            the object info, populated.
  1391.      * @param typeCode
  1392.      *            the type of the object.
  1393.      * @param data
  1394.      *            inflated data for the object.
  1395.      * @throws java.io.IOException
  1396.      *             the object cannot be archived.
  1397.      */
  1398.     protected abstract void onInflatedObjectData(PackedObjectInfo obj,
  1399.             int typeCode, byte[] data) throws IOException;

  1401.     /**
  1402.      * Provide the implementation with the original stream's pack header.
  1403.      *
  1404.      * @param objCnt
  1405.      *            number of objects expected in the stream.
  1406.      * @throws java.io.IOException
  1407.      *             the implementation refuses to work with this many objects.
  1408.      */
  1409.     protected abstract void onPackHeader(long objCnt) throws IOException;

  1411.     /**
  1412.      * Provide the implementation with the original stream's pack footer.
  1413.      *
  1414.      * @param hash
  1415.      *            the trailing 20 bytes of the pack, this is a SHA-1 checksum of
  1416.      *            all of the pack data.
  1417.      * @throws java.io.IOException
  1418.      *             the stream cannot be archived.
  1419.      */
  1420.     protected abstract void onPackFooter(byte[] hash) throws IOException;

  1422.     /**
  1423.      * Provide the implementation with a base that was outside of the pack.
  1424.      * <p>
  1425.      * This event only occurs on a thin pack for base objects that were outside
  1426.      * of the pack and came from the local repository. Usually an implementation
  1427.      * uses this event to compress the base and append it onto the end of the
  1428.      * pack, so the pack stays self-contained.
  1429.      *
  1430.      * @param typeCode
  1431.      *            type of the base object.
  1432.      * @param data
  1433.      *            complete content of the base object.
  1434.      * @param info
  1435.      *            packed object information for this base. Implementors must
  1436.      *            populate the CRC and offset members if returning true.
  1437.      * @return true if the {@code info} should be included in the object list
  1438.      *         returned by {@link #getSortedObjectList(Comparator)}, false if it
  1439.      *         should not be included.
  1440.      * @throws java.io.IOException
  1441.      *             the base could not be included into the pack.
  1442.      */
  1443.     protected abstract boolean onAppendBase(int typeCode, byte[] data,
  1444.             PackedObjectInfo info) throws IOException;

  1446.     /**
  1447.      * Event indicating a thin pack has been completely processed.
  1448.      * <p>
  1449.      * This event is invoked only if a thin pack has delta references to objects
  1450.      * external from the pack. The event is called after all of those deltas
  1451.      * have been resolved.
  1452.      *
  1453.      * @throws java.io.IOException
  1454.      *             the pack cannot be archived.
  1455.      */
  1456.     protected abstract void onEndThinPack() throws IOException;

  1458.     /**
  1459.      * Reposition the database to re-read a previously stored object.
  1460.      * <p>
  1461.      * If the database is computing CRC-32 checksums for object data, it should
  1462.      * reset its internal CRC instance during this method call.
  1463.      *
  1464.      * @param obj
  1465.      *            the object position to begin reading from. This is from
  1466.      *            {@link #newInfo(AnyObjectId, UnresolvedDelta, ObjectId)}.
  1467.      * @param info
  1468.      *            object to populate with type and size.
  1469.      * @return the {@code info} object.
  1470.      * @throws java.io.IOException
  1471.      *             the database cannot reposition to this location.
  1472.      */
  1473.     protected abstract ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  1474.             ObjectTypeAndSize info) throws IOException;

  1476.     /**
  1477.      * Reposition the database to re-read a previously stored object.
  1478.      * <p>
  1479.      * If the database is computing CRC-32 checksums for object data, it should
  1480.      * reset its internal CRC instance during this method call.
  1481.      *
  1482.      * @param delta
  1483.      *            the object position to begin reading from. This is an instance
  1484.      *            previously returned by {@link #onEndDelta()}.
  1485.      * @param info
  1486.      *            object to populate with type and size.
  1487.      * @return the {@code info} object.
  1488.      * @throws java.io.IOException
  1489.      *             the database cannot reposition to this location.
  1490.      */
  1491.     protected abstract ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  1492.             ObjectTypeAndSize info) throws IOException;

  1494.     /**
  1495.      * Read from the database's current position into the buffer.
  1496.      *
  1497.      * @param dst
  1498.      *            the buffer to copy read data into.
  1499.      * @param pos
  1500.      *            position within {@code dst} to start copying data into.
  1501.      * @param cnt
  1502.      *            ideal target number of bytes to read. Actual read length may
  1503.      *            be shorter.
  1504.      * @return number of bytes stored.
  1505.      * @throws java.io.IOException
  1506.      *             the database cannot be accessed.
  1507.      */
  1508.     protected abstract int readDatabase(byte[] dst, int pos, int cnt)
  1509.             throws IOException;

  1511.     /**
  1512.      * Check the current CRC matches the expected value.
  1513.      * <p>
  1514.      * This method is invoked when an object is read back in from the database
  1515.      * and its data is used during delta resolution. The CRC is validated after
  1516.      * the object has been fully read, allowing the parser to verify there was
  1517.      * no silent data corruption.
  1518.      * <p>
  1519.      * Implementations are free to ignore this check by always returning true if
  1520.      * they are performing other data integrity validations at a lower level.
  1521.      *
  1522.      * @param oldCRC
  1523.      *            the prior CRC that was recorded during the first scan of the
  1524.      *            object from the pack stream.
  1525.      * @return true if the CRC matches; false if it does not.
  1526.      */
  1527.     protected abstract boolean checkCRC(int oldCRC);

  1529.     /**
  1530.      * Event notifying the start of an object stored whole (not as a delta).
  1531.      *
  1532.      * @param streamPosition
  1533.      *            position of this object in the incoming stream.
  1534.      * @param type
  1535.      *            type of the object; one of
  1536.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_COMMIT},
  1537.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_TREE},
  1538.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_BLOB}, or
  1539.      *            {@link org.eclipse.jgit.lib.Constants#OBJ_TAG}.
  1540.      * @param inflatedSize
  1541.      *            size of the object when fully inflated. The size stored within
  1542.      *            the pack may be larger or smaller, and is not yet known.
  1543.      * @throws java.io.IOException
  1544.      *             the object cannot be recorded.
  1545.      */
  1546.     protected abstract void onBeginWholeObject(long streamPosition, int type,
  1547.             long inflatedSize) throws IOException;

  1549.     /**
  1550.      * Event notifying the current object.
  1551.      *
  1552.      *@param info
  1553.      *            object information.
  1554.      * @throws java.io.IOException
  1555.      *             the object cannot be recorded.
  1556.      */
  1557.     protected abstract void onEndWholeObject(PackedObjectInfo info)
  1558.             throws IOException;

  1560.     /**
  1561.      * Event notifying start of a delta referencing its base by offset.
  1562.      *
  1563.      * @param deltaStreamPosition
  1564.      *            position of this object in the incoming stream.
  1565.      * @param baseStreamPosition
  1566.      *            position of the base object in the incoming stream. The base
  1567.      *            must be before the delta, therefore {@code baseStreamPosition
  1568.      *            &lt; deltaStreamPosition}. This is <b>not</b> the position
  1569.      *            returned by a prior end object event.
  1570.      * @param inflatedSize
  1571.      *            size of the delta when fully inflated. The size stored within
  1572.      *            the pack may be larger or smaller, and is not yet known.
  1573.      * @throws java.io.IOException
  1574.      *             the object cannot be recorded.
  1575.      */
  1576.     protected abstract void onBeginOfsDelta(long deltaStreamPosition,
  1577.             long baseStreamPosition, long inflatedSize) throws IOException;

  1579.     /**
  1580.      * Event notifying start of a delta referencing its base by ObjectId.
  1581.      *
  1582.      * @param deltaStreamPosition
  1583.      *            position of this object in the incoming stream.
  1584.      * @param baseId
  1585.      *            name of the base object. This object may be later in the
  1586.      *            stream, or might not appear at all in the stream (in the case
  1587.      *            of a thin-pack).
  1588.      * @param inflatedSize
  1589.      *            size of the delta when fully inflated. The size stored within
  1590.      *            the pack may be larger or smaller, and is not yet known.
  1591.      * @throws java.io.IOException
  1592.      *             the object cannot be recorded.
  1593.      */
  1594.     protected abstract void onBeginRefDelta(long deltaStreamPosition,
  1595.             AnyObjectId baseId, long inflatedSize) throws IOException;

  1597.     /**
  1598.      * Event notifying the current object.
  1599.      *
  1600.      *@return object information that must be populated with at least the
  1601.      *         offset.
  1602.      * @throws java.io.IOException
  1603.      *             the object cannot be recorded.
  1604.      */
  1605.     protected UnresolvedDelta onEndDelta() throws IOException {
  1606.         return new UnresolvedDelta();
  1607.     }

  1609.     /** Type and size information about an object in the database buffer. */
  1610.     public static class ObjectTypeAndSize {
  1611.         /** The type of the object. */
  1612.         public int type;

  1614.         /** The inflated size of the object. */
  1615.         public long size;
  1616.     }

  1618.     private void inflateAndSkip(Source src, long inflatedSize)
  1619.             throws IOException {
  1620.         try (InputStream inf = inflate(src, inflatedSize)) {
  1621.             IO.skipFully(inf, inflatedSize);
  1622.         }
  1623.     }

  1625.     private byte[] inflateAndReturn(Source src, long inflatedSize)
  1626.             throws IOException {
  1627.         final byte[] dst = new byte[(int) inflatedSize];
  1628.         try (InputStream inf = inflate(src, inflatedSize)) {
  1629.             IO.readFully(inf, dst, 0, dst.length);
  1630.         }
  1631.         return dst;
  1632.     }

  1634.     private InputStream inflate(Source src, long inflatedSize)
  1635.             throws IOException {
  1636.         inflater.open(src, inflatedSize);
  1637.         return inflater;
  1638.     }

  1640.     private static class DeltaChain extends ObjectIdOwnerMap.Entry {
  1641.         UnresolvedDelta head;

  1643.         DeltaChain(AnyObjectId id) {
  1644.             super(id);
  1645.         }

  1647.         UnresolvedDelta remove() {
  1648.             final UnresolvedDelta r = head;
  1649.             if (r != null)
  1650.                 head = null;
  1651.             return r;
  1652.         }

  1654.         void add(UnresolvedDelta d) {
  1655.             d.next = head;
  1656.             head = d;
  1657.         }
  1658.     }

  1660.     /** Information about an unresolved delta in this pack stream. */
  1661.     public static class UnresolvedDelta {
  1662.         long position;

  1664.         int crc;

  1666.         UnresolvedDelta next;

  1668.         long sizeBeforeInflating;

  1670.         /** @return offset within the input stream. */
  1671.         public long getOffset() {
  1672.             return position;
  1673.         }

  1675.         /** @return the CRC-32 checksum of the stored delta data. */
  1676.         public int getCRC() {
  1677.             return crc;
  1678.         }

  1680.         /**
  1681.          * @param crc32
  1682.          *            the CRC-32 checksum of the stored delta data.
  1683.          */
  1684.         public void setCRC(int crc32) {
  1685.             crc = crc32;
  1686.         }
  1687.     }

  1689.     private static class DeltaVisit {
  1690.         final UnresolvedDelta delta;

  1692.         ObjectId id;

  1694.         byte[] data;

  1696.         DeltaVisit parent;

  1698.         UnresolvedDelta nextChild;

  1700.         DeltaVisit() {
  1701.             this.delta = null; // At the root of the stack we have a base.
  1702.         }

  1704.         DeltaVisit(DeltaVisit parent) {
  1705.             this.parent = parent;
  1706.             this.delta = parent.nextChild;
  1707.             parent.nextChild = delta.next;
  1708.         }

  1710.         DeltaVisit next() {
  1711.             // If our parent has no more children, discard it.
  1712.             if (parent != null && parent.nextChild == null) {
  1713.                 parent.data = null;
  1714.                 parent = parent.parent;
  1715.             }

  1717.             if (nextChild != null)
  1718.                 return new DeltaVisit(this);

  1720.             // If we have no child ourselves, our parent must (if it exists),
  1721.             // due to the discard rule above. With no parent, we are done.
  1722.             if (parent != null)
  1723.                 return new DeltaVisit(parent);
  1724.             return null;
  1725.         }
  1726.     }

  1728.     private void addObjectAndTrack(PackedObjectInfo oe) {
  1729.         entries[entryCount++] = oe;
  1730.         if (needNewObjectIds())
  1731.             newObjectIds.add(oe);
  1732.     }

  1734.     private class InflaterStream extends InputStream {
  1735.         private final Inflater inf;

  1737.         private final byte[] skipBuffer;

  1739.         private Source src;

  1741.         private long expectedSize;

  1743.         private long actualSize;

  1745.         private int p;

  1747.         InflaterStream() {
  1748.             inf = InflaterCache.get();
  1749.             skipBuffer = new byte[512];
  1750.         }

  1752.         void release() {
  1753.             inf.reset();
  1754.             InflaterCache.release(inf);
  1755.         }

  1757.         void open(Source source, long inflatedSize) throws IOException {
  1758.             src = source;
  1759.             expectedSize = inflatedSize;
  1760.             actualSize = 0;

  1762.             p = fill(src, 1);
  1763.             inf.setInput(buf, p, bAvail);
  1764.         }

  1766.         @Override
  1767.         public long skip(long toSkip) throws IOException {
  1768.             long n = 0;
  1769.             while (n < toSkip) {
  1770.                 final int cnt = (int) Math.min(skipBuffer.length, toSkip - n);
  1771.                 final int r = read(skipBuffer, 0, cnt);
  1772.                 if (r <= 0)
  1773.                     break;
  1774.                 n += r;
  1775.             }
  1776.             return n;
  1777.         }

  1779.         @Override
  1780.         public int read() throws IOException {
  1781.             int n = read(skipBuffer, 0, 1);
  1782.             return n == 1 ? skipBuffer[0] & 0xff : -1;
  1783.         }

  1785.         @Override
  1786.         public int read(byte[] dst, int pos, int cnt) throws IOException {
  1787.             try {
  1788.                 int n = 0;
  1789.                 while (n < cnt) {
  1790.                     int r = inf.inflate(dst, pos + n, cnt - n);
  1791.                     n += r;
  1792.                     if (inf.finished())
  1793.                         break;
  1794.                     if (inf.needsInput()) {
  1795.                         onObjectData(src, buf, p, bAvail);
  1796.                         use(bAvail);

  1798.                         p = fill(src, 1);
  1799.                         inf.setInput(buf, p, bAvail);
  1800.                     } else if (r == 0) {
  1801.                         throw new CorruptObjectException(MessageFormat.format(
  1802.                                 JGitText.get().packfileCorruptionDetected,
  1803.                                 JGitText.get().unknownZlibError));
  1804.                     }
  1805.                 }
  1806.                 actualSize += n;
  1807.                 return 0 < n ? n : -1;
  1808.             } catch (DataFormatException dfe) {
  1809.                 throw new CorruptObjectException(MessageFormat.format(JGitText
  1810.                         .get().packfileCorruptionDetected, dfe.getMessage()));
  1811.             }
  1812.         }

  1814.         @Override
  1815.         public void close() throws IOException {
  1816.             // We need to read here to enter the loop above and pump the
  1817.             // trailing checksum into the Inflater. It should return -1 as the
  1818.             // caller was supposed to consume all content.
  1819.             //
  1820.             if (read(skipBuffer) != -1 || actualSize != expectedSize) {
  1821.                 throw new CorruptObjectException(MessageFormat.format(JGitText
  1822.                         .get().packfileCorruptionDetected,
  1823.                         JGitText.get().wrongDecompressedLength));
  1824.             }

  1826.             int used = bAvail - inf.getRemaining();
  1827.             if (0 < used) {
  1828.                 onObjectData(src, buf, p, used);
  1829.                 use(used);
  1830.             }

  1832.             inf.reset();
  1833.         }
  1834.     }
  1835. }
Created with JaCoCo 0.8.7.202105040129