FsckPackParser.java

  1. /*
  2.  * Copyright (C) 2017, Google Inc. and others
  3.  *
  4.  * This program and the accompanying materials are made available under the
  5.  * terms of the Eclipse Distribution License v. 1.0 which is available at
  6.  * https://www.eclipse.org/org/documents/edl-v10.php.
  7.  *
  8.  * SPDX-License-Identifier: BSD-3-Clause
  9.  */

  11. package org.eclipse.jgit.internal.fsck;

  13. import java.io.IOException;
  14. import java.nio.ByteBuffer;
  15. import java.nio.channels.Channels;
  16. import java.text.MessageFormat;
  17. import java.util.Arrays;
  18. import java.util.HashSet;
  19. import java.util.Set;
  20. import java.util.zip.CRC32;

  22. import org.eclipse.jgit.errors.CorruptObjectException;
  23. import org.eclipse.jgit.errors.CorruptPackIndexException;
  24. import org.eclipse.jgit.errors.CorruptPackIndexException.ErrorType;
  25. import org.eclipse.jgit.errors.MissingObjectException;
  26. import org.eclipse.jgit.internal.JGitText;
  27. import org.eclipse.jgit.internal.fsck.FsckError.CorruptObject;
  28. import org.eclipse.jgit.internal.storage.dfs.ReadableChannel;
  29. import org.eclipse.jgit.internal.storage.file.PackIndex;
  30. import org.eclipse.jgit.internal.storage.file.PackIndex.MutableEntry;
  31. import org.eclipse.jgit.lib.AnyObjectId;
  32. import org.eclipse.jgit.lib.ObjectDatabase;
  33. import org.eclipse.jgit.lib.ObjectIdOwnerMap;
  34. import org.eclipse.jgit.transport.PackParser;
  35. import org.eclipse.jgit.transport.PackedObjectInfo;

  37. /**
  38.  * A read-only pack parser for object validity checking.
  39.  */
  40. public class FsckPackParser extends PackParser {
  41.     private final CRC32 crc;

  43.     private final ReadableChannel channel;

  45.     private final Set<CorruptObject> corruptObjects = new HashSet<>();

  47.     private long expectedObjectCount = -1L;

  49.     private long offset;

  51.     private int blockSize;

  53.     /**
  54.      * Constructor for FsckPackParser
  55.      *
  56.      * @param db
  57.      *            the object database which stores repository's data.
  58.      * @param channel
  59.      *            readable channel of the pack file.
  60.      */
  61.     public FsckPackParser(ObjectDatabase db, ReadableChannel channel) {
  62.         super(db, Channels.newInputStream(channel));
  63.         this.channel = channel;
  64.         setCheckObjectCollisions(false);
  65.         this.crc = new CRC32();
  66.         this.blockSize = channel.blockSize() > 0 ? channel.blockSize() : 65536;
  67.     }

  69.     /** {@inheritDoc} */
  70.     @Override
  71.     protected void onPackHeader(long objCnt) throws IOException {
  72.         if (expectedObjectCount >= 0) {
  73.             // Some DFS pack files don't contain the correct object count, e.g.
  74.             // INSERT/RECEIVE packs don't always contain the correct object
  75.             // count in their headers. Overwrite the expected object count
  76.             // after parsing the pack header.
  77.             setExpectedObjectCount(expectedObjectCount);
  78.         }
  79.     }

  81.     /** {@inheritDoc} */
  82.     @Override
  83.     protected void onBeginWholeObject(long streamPosition, int type,
  84.             long inflatedSize) throws IOException {
  85.         crc.reset();
  86.     }

  88.     /** {@inheritDoc} */
  89.     @Override
  90.     protected void onObjectHeader(Source src, byte[] raw, int pos, int len)
  91.             throws IOException {
  92.         crc.update(raw, pos, len);
  93.     }

  95.     /** {@inheritDoc} */
  96.     @Override
  97.     protected void onObjectData(Source src, byte[] raw, int pos, int len)
  98.             throws IOException {
  99.         crc.update(raw, pos, len);
  100.     }

  102.     /** {@inheritDoc} */
  103.     @Override
  104.     protected void onEndWholeObject(PackedObjectInfo info) throws IOException {
  105.         info.setCRC((int) crc.getValue());
  106.     }

  108.     /** {@inheritDoc} */
  109.     @Override
  110.     protected void onBeginOfsDelta(long deltaStreamPosition,
  111.             long baseStreamPosition, long inflatedSize) throws IOException {
  112.         crc.reset();
  113.     }

  115.     /** {@inheritDoc} */
  116.     @Override
  117.     protected void onBeginRefDelta(long deltaStreamPosition, AnyObjectId baseId,
  118.             long inflatedSize) throws IOException {
  119.         crc.reset();
  120.     }

  122.     /** {@inheritDoc} */
  123.     @Override
  124.     protected UnresolvedDelta onEndDelta() throws IOException {
  125.         UnresolvedDelta delta = new UnresolvedDelta();
  126.         delta.setCRC((int) crc.getValue());
  127.         return delta;
  128.     }

  130.     /** {@inheritDoc} */
  131.     @Override
  132.     protected void onInflatedObjectData(PackedObjectInfo obj, int typeCode,
  133.             byte[] data) throws IOException {
  134.         // FsckPackParser ignores this event.
  135.     }

  137.     /** {@inheritDoc} */
  138.     @Override
  139.     protected void verifySafeObject(final AnyObjectId id, final int type,
  140.             final byte[] data) {
  141.         try {
  142.             super.verifySafeObject(id, type, data);
  143.         } catch (CorruptObjectException e) {
  144.             corruptObjects.add(
  145.                     new CorruptObject(id.toObjectId(), type, e.getErrorType()));
  146.         }
  147.     }

  149.     /** {@inheritDoc} */
  150.     @Override
  151.     protected void onPackFooter(byte[] hash) throws IOException {
  152.         // Do nothing.
  153.     }

  155.     /** {@inheritDoc} */
  156.     @Override
  157.     protected boolean onAppendBase(int typeCode, byte[] data,
  158.             PackedObjectInfo info) throws IOException {
  159.         // Do nothing.
  160.         return false;
  161.     }

  163.     /** {@inheritDoc} */
  164.     @Override
  165.     protected void onEndThinPack() throws IOException {
  166.         // Do nothing.
  167.     }

  169.     /** {@inheritDoc} */
  170.     @Override
  171.     protected ObjectTypeAndSize seekDatabase(PackedObjectInfo obj,
  172.             ObjectTypeAndSize info) throws IOException {
  173.         crc.reset();
  174.         offset = obj.getOffset();
  175.         return readObjectHeader(info);
  176.     }

  178.     /** {@inheritDoc} */
  179.     @Override
  180.     protected ObjectTypeAndSize seekDatabase(UnresolvedDelta delta,
  181.             ObjectTypeAndSize info) throws IOException {
  182.         crc.reset();
  183.         offset = delta.getOffset();
  184.         return readObjectHeader(info);
  185.     }

  187.     /** {@inheritDoc} */
  188.     @Override
  189.     protected int readDatabase(byte[] dst, int pos, int cnt)
  190.             throws IOException {
  191.         // read from input instead of database.
  192.         int n = read(offset, dst, pos, cnt);
  193.         if (n > 0) {
  194.             offset += n;
  195.         }
  196.         return n;
  197.     }

  199.     int read(long channelPosition, byte[] dst, int pos, int cnt)
  200.             throws IOException {
  201.         long block = channelPosition / blockSize;
  202.         byte[] bytes = readFromChannel(block);
  203.         if (bytes == null) {
  204.             return -1;
  205.         }
  206.         int offs = (int) (channelPosition - block * blockSize);
  207.         int bytesToCopy = Math.min(cnt, bytes.length - offs);
  208.         if (bytesToCopy < 1) {
  209.             return -1;
  210.         }
  211.         System.arraycopy(bytes, offs, dst, pos, bytesToCopy);
  212.         return bytesToCopy;
  213.     }

  215.     private byte[] readFromChannel(long block) throws IOException {
  216.         channel.position(block * blockSize);
  217.         ByteBuffer buf = ByteBuffer.allocate(blockSize);
  218.         int totalBytesRead = 0;
  219.         while (totalBytesRead < blockSize) {
  220.             int bytesRead = channel.read(buf);
  221.             if (bytesRead == -1) {
  222.                 if (totalBytesRead == 0) {
  223.                     return null;
  224.                 }
  225.                 return Arrays.copyOf(buf.array(), totalBytesRead);
  226.             }
  227.             totalBytesRead += bytesRead;
  228.         }
  229.         return buf.array();
  230.     }

  232.     /** {@inheritDoc} */
  233.     @Override
  234.     protected boolean checkCRC(int oldCRC) {
  235.         return oldCRC == (int) crc.getValue();
  236.     }

  238.     /** {@inheritDoc} */
  239.     @Override
  240.     protected void onStoreStream(byte[] raw, int pos, int len)
  241.             throws IOException {
  242.         // Do nothing.
  243.     }

  245.     /**
  246.      * Get corrupt objects reported by
  247.      * {@link org.eclipse.jgit.lib.ObjectChecker}
  248.      *
  249.      * @return corrupt objects that are reported by
  250.      *         {@link org.eclipse.jgit.lib.ObjectChecker}.
  251.      */
  252.     public Set<CorruptObject> getCorruptObjects() {
  253.         return corruptObjects;
  254.     }

  256.     /**
  257.      * Verify the existing index file with all objects from the pack.
  258.      *
  259.      * @param idx
  260.      *            index file associate with the pack
  261.      * @throws org.eclipse.jgit.errors.CorruptPackIndexException
  262.      *             when the index file is corrupt.
  263.      */
  264.     public void verifyIndex(PackIndex idx)
  265.             throws CorruptPackIndexException {
  266.         ObjectIdOwnerMap<ObjFromPack> inPack = new ObjectIdOwnerMap<>();
  267.         for (int i = 0; i < getObjectCount(); i++) {
  268.             PackedObjectInfo entry = getObject(i);
  269.             inPack.add(new ObjFromPack(entry));

  271.             long offs = idx.findOffset(entry);
  272.             if (offs == -1) {
  273.                 throw new CorruptPackIndexException(
  274.                         MessageFormat.format(JGitText.get().missingObject,
  275.                                 Integer.valueOf(entry.getType()),
  276.                                 entry.getName()),
  277.                         ErrorType.MISSING_OBJ);
  278.             } else if (offs != entry.getOffset()) {
  279.                 throw new CorruptPackIndexException(MessageFormat
  280.                         .format(JGitText.get().mismatchOffset, entry.getName()),
  281.                         ErrorType.MISMATCH_OFFSET);
  282.             }

  284.             try {
  285.                 if (idx.hasCRC32Support()
  286.                         && (int) idx.findCRC32(entry) != entry.getCRC()) {
  287.                     throw new CorruptPackIndexException(
  288.                             MessageFormat.format(JGitText.get().mismatchCRC,
  289.                                     entry.getName()),
  290.                             ErrorType.MISMATCH_CRC);
  291.                 }
  292.             } catch (MissingObjectException e) {
  293.                 CorruptPackIndexException cpe = new CorruptPackIndexException(
  294.                         MessageFormat.format(JGitText.get().missingCRC,
  295.                                 entry.getName()),
  296.                         ErrorType.MISSING_CRC);
  297.                 cpe.initCause(e);
  298.                 throw cpe;
  299.             }
  300.         }

  302.         for (MutableEntry entry : idx) {
  303.             if (!inPack.contains(entry.toObjectId())) {
  304.                 throw new CorruptPackIndexException(MessageFormat.format(
  305.                         JGitText.get().unknownObjectInIndex, entry.name()),
  306.                         ErrorType.UNKNOWN_OBJ);
  307.             }
  308.         }
  309.     }

  311.     /**
  312.      * Set the object count for overwriting the expected object count from pack
  313.      * header.
  314.      *
  315.      * @param objectCount
  316.      *            the actual expected object count.
  317.      */
  318.     public void overwriteObjectCount(long objectCount) {
  319.         this.expectedObjectCount = objectCount;
  320.     }

  322.     static class ObjFromPack extends ObjectIdOwnerMap.Entry {
  323.         ObjFromPack(AnyObjectId id) {
  324.             super(id);
  325.         }
  326.     }
  327. }
Created with JaCoCo 0.8.7.202105040129