BasePackConnection.java
/*
* Copyright (C) 2008, 2010 Google Inc.
* Copyright (C) 2008, Marek Zawirski <marek.zawirski@gmail.com>
* Copyright (C) 2008, Robin Rosenberg <robin.rosenberg@dewire.com>
* Copyright (C) 2008, 2020 Shawn O. Pearce <spearce@spearce.org> and others
*
* This program and the accompanying materials are made available under the
* terms of the Eclipse Distribution License v. 1.0 which is available at
* https://www.eclipse.org/org/documents/edl-v10.php.
*
* SPDX-License-Identifier: BSD-3-Clause
*/
package org.eclipse.jgit.transport;
import static org.eclipse.jgit.transport.GitProtocolConstants.COMMAND_LS_REFS;
import static org.eclipse.jgit.transport.GitProtocolConstants.OPTION_AGENT;
import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_PEELED;
import static org.eclipse.jgit.transport.GitProtocolConstants.REF_ATTR_SYMREF_TARGET;
import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_1;
import static org.eclipse.jgit.transport.GitProtocolConstants.VERSION_2;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.text.MessageFormat;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import org.eclipse.jgit.annotations.NonNull;
import org.eclipse.jgit.errors.InvalidObjectIdException;
import org.eclipse.jgit.errors.NoRemoteRepositoryException;
import org.eclipse.jgit.errors.PackProtocolException;
import org.eclipse.jgit.errors.RemoteRepositoryException;
import org.eclipse.jgit.errors.TransportException;
import org.eclipse.jgit.internal.JGitText;
import org.eclipse.jgit.lib.Constants;
import org.eclipse.jgit.lib.ObjectId;
import org.eclipse.jgit.lib.ObjectIdRef;
import org.eclipse.jgit.lib.Ref;
import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.lib.SymbolicRef;
import org.eclipse.jgit.util.StringUtils;
import org.eclipse.jgit.util.io.InterruptTimer;
import org.eclipse.jgit.util.io.TimeoutInputStream;
import org.eclipse.jgit.util.io.TimeoutOutputStream;
/**
* Base helper class for pack-based operations implementations. Provides partial
* implementation of pack-protocol - refs advertising and capabilities support,
* and some other helper methods.
*
* @see BasePackFetchConnection
* @see BasePackPushConnection
*/
abstract class BasePackConnection extends BaseConnection {
protected static final String CAPABILITY_SYMREF_PREFIX = "symref="; //$NON-NLS-1$
/** The repository this transport fetches into, or pushes out of. */
protected final Repository local;
/** Remote repository location. */
protected final URIish uri;
/** A transport connected to {@link #uri}. */
protected final Transport transport;
/** Low-level input stream, if a timeout was configured. */
protected TimeoutInputStream timeoutIn;
/** Low-level output stream, if a timeout was configured. */
protected TimeoutOutputStream timeoutOut;
/** Timer to manage {@link #timeoutIn} and {@link #timeoutOut}. */
private InterruptTimer myTimer;
/** Input stream reading from the remote. */
protected InputStream in;
/** Output stream sending to the remote. */
protected OutputStream out;
/** Packet line decoder around {@link #in}. */
protected PacketLineIn pckIn;
/** Packet line encoder around {@link #out}. */
protected PacketLineOut pckOut;
/** Send {@link PacketLineOut#end()} before closing {@link #out}? */
protected boolean outNeedsEnd;
/** True if this is a stateless RPC connection. */
protected boolean statelessRPC;
/** Capability tokens advertised by the remote side. */
private final Map<String, String> remoteCapabilities = new HashMap<>();
/** Extra objects the remote has, but which aren't offered as refs. */
protected final Set<ObjectId> additionalHaves = new HashSet<>();
private TransferConfig.ProtocolVersion protocol = TransferConfig.ProtocolVersion.V0;
BasePackConnection(PackTransport packTransport) {
transport = (Transport) packTransport;
local = transport.local;
uri = transport.uri;
}
TransferConfig.ProtocolVersion getProtocolVersion() {
return protocol;
}
void setProtocolVersion(@NonNull TransferConfig.ProtocolVersion protocol) {
this.protocol = protocol;
}
/**
* Configure this connection with the directional pipes.
*
* @param myIn
* input stream to receive data from the peer. Caller must ensure
* the input is buffered, otherwise read performance may suffer.
* @param myOut
* output stream to transmit data to the peer. Caller must ensure
* the output is buffered, otherwise write performance may
* suffer.
*/
protected final void init(InputStream myIn, OutputStream myOut) {
final int timeout = transport.getTimeout();
if (timeout > 0) {
final Thread caller = Thread.currentThread();
if (myTimer == null) {
myTimer = new InterruptTimer(caller.getName() + "-Timer"); //$NON-NLS-1$
}
timeoutIn = new TimeoutInputStream(myIn, myTimer);
timeoutOut = new TimeoutOutputStream(myOut, myTimer);
timeoutIn.setTimeout(timeout * 1000);
timeoutOut.setTimeout(timeout * 1000);
myIn = timeoutIn;
myOut = timeoutOut;
}
in = myIn;
out = myOut;
pckIn = new PacketLineIn(in);
pckOut = new PacketLineOut(out);
outNeedsEnd = true;
}
/**
* Reads the advertised references through the initialized stream.
* <p>
* Subclass implementations may call this method only after setting up the
* input and output streams with {@link #init(InputStream, OutputStream)}.
* <p>
* If any errors occur, this connection is automatically closed by invoking
* {@link #close()} and the exception is wrapped (if necessary) and thrown
* as a {@link org.eclipse.jgit.errors.TransportException}.
*
* @return {@code true} if the refs were read; {@code false} otherwise
* indicating that {@link #lsRefs} must be called
*
* @throws org.eclipse.jgit.errors.TransportException
* the reference list could not be scanned.
*/
protected boolean readAdvertisedRefs() throws TransportException {
try {
return readAdvertisedRefsImpl();
} catch (TransportException err) {
close();
throw err;
} catch (IOException | RuntimeException err) {
close();
throw new TransportException(err.getMessage(), err);
}
}
private String readLine() throws IOException {
String line = pckIn.readString();
if (PacketLineIn.isEnd(line)) {
return null;
}
if (line.startsWith("ERR ")) { //$NON-NLS-1$
// This is a customized remote service error.
// Users should be informed about it.
throw new RemoteRepositoryException(uri, line.substring(4));
}
return line;
}
private boolean readAdvertisedRefsImpl() throws IOException {
final Map<String, Ref> avail = new LinkedHashMap<>();
final Map<String, String> symRefs = new LinkedHashMap<>();
for (boolean first = true;; first = false) {
String line;
if (first) {
boolean isV1 = false;
try {
line = readLine();
} catch (EOFException e) {
TransportException noRepo = noRepository();
noRepo.initCause(e);
throw noRepo;
}
if (line != null && VERSION_1.equals(line)) {
// Same as V0, except for this extra line. We shouldn't get
// it since we never request V1.
setProtocolVersion(TransferConfig.ProtocolVersion.V0);
isV1 = true;
line = readLine();
}
if (line == null) {
break;
}
final int nul = line.indexOf('\0');
if (nul >= 0) {
// Protocol V0: The first line (if any) may contain
// "hidden" capability values after a NUL byte.
for (String capability : line.substring(nul + 1)
.split(" ")) { //$NON-NLS-1$
if (capability.startsWith(CAPABILITY_SYMREF_PREFIX)) {
String[] parts = capability
.substring(
CAPABILITY_SYMREF_PREFIX.length())
.split(":", 2); //$NON-NLS-1$
if (parts.length == 2) {
symRefs.put(parts[0], parts[1]);
}
} else {
addCapability(capability);
}
}
line = line.substring(0, nul);
setProtocolVersion(TransferConfig.ProtocolVersion.V0);
} else if (!isV1 && VERSION_2.equals(line)) {
// Protocol V2: remaining lines are capabilities as
// key=value pairs
setProtocolVersion(TransferConfig.ProtocolVersion.V2);
readCapabilitiesV2();
// Break out here so that stateless RPC transports get a
// chance to set up the output stream.
return false;
} else {
setProtocolVersion(TransferConfig.ProtocolVersion.V0);
}
} else {
line = readLine();
if (line == null) {
break;
}
}
// Expecting to get a line in the form "sha1 refname"
if (line.length() < 41 || line.charAt(40) != ' ') {
throw invalidRefAdvertisementLine(line);
}
String name = line.substring(41, line.length());
if (first && name.equals("capabilities^{}")) { //$NON-NLS-1$
// special line from git-receive-pack (protocol V0) to show
// capabilities when there are no refs to advertise
continue;
}
final ObjectId id = toId(line, line.substring(0, 40));
if (name.equals(".have")) { //$NON-NLS-1$
additionalHaves.add(id);
} else {
processLineV1(name, id, avail);
}
}
updateWithSymRefs(avail, symRefs);
available(avail);
return true;
}
/**
* Issue a protocol V2 ls-refs command and read its response.
*
* @param refSpecs
* to produce ref prefixes from if the server supports git
* protocol V2
* @param additionalPatterns
* to use for ref prefixes if the server supports git protocol V2
* @throws TransportException
* if the command could not be run or its output not be read
*/
protected void lsRefs(Collection<RefSpec> refSpecs,
String... additionalPatterns) throws TransportException {
try {
lsRefsImpl(refSpecs, additionalPatterns);
} catch (TransportException err) {
close();
throw err;
} catch (IOException | RuntimeException err) {
close();
throw new TransportException(err.getMessage(), err);
}
}
private void lsRefsImpl(Collection<RefSpec> refSpecs,
String... additionalPatterns) throws IOException {
pckOut.writeString("command=" + COMMAND_LS_REFS); //$NON-NLS-1$
// Add the user-agent
String agent = UserAgent.get();
if (agent != null && isCapableOf(OPTION_AGENT)) {
pckOut.writeString(OPTION_AGENT + '=' + agent);
}
pckOut.writeDelim();
pckOut.writeString("peel"); //$NON-NLS-1$
pckOut.writeString("symrefs"); //$NON-NLS-1$
for (String refPrefix : getRefPrefixes(refSpecs, additionalPatterns)) {
pckOut.writeString("ref-prefix " + refPrefix); //$NON-NLS-1$
}
pckOut.end();
final Map<String, Ref> avail = new LinkedHashMap<>();
final Map<String, String> symRefs = new LinkedHashMap<>();
for (;;) {
String line = readLine();
if (line == null) {
break;
}
// Expecting to get a line in the form "sha1 refname"
if (line.length() < 41 || line.charAt(40) != ' ') {
throw invalidRefAdvertisementLine(line);
}
String name = line.substring(41, line.length());
final ObjectId id = toId(line, line.substring(0, 40));
if (name.equals(".have")) { //$NON-NLS-1$
additionalHaves.add(id);
} else {
processLineV2(line, id, name, avail, symRefs);
}
}
updateWithSymRefs(avail, symRefs);
available(avail);
}
private Collection<String> getRefPrefixes(Collection<RefSpec> refSpecs,
String... additionalPatterns) {
if (refSpecs.isEmpty() && (additionalPatterns == null
|| additionalPatterns.length == 0)) {
return Collections.emptyList();
}
Set<String> patterns = new HashSet<>();
if (additionalPatterns != null) {
Arrays.stream(additionalPatterns).filter(Objects::nonNull)
.forEach(patterns::add);
}
for (RefSpec spec : refSpecs) {
// TODO: for now we only do protocol V2 for fetch. For push
// RefSpecs, the logic would need to be different. (At the
// minimum, take spec.getDestination().)
String src = spec.getSource();
if (ObjectId.isId(src)) {
continue;
}
if (spec.isWildcard()) {
patterns.add(src.substring(0, src.indexOf('*')));
} else {
patterns.add(src);
patterns.add(Constants.R_REFS + src);
patterns.add(Constants.R_HEADS + src);
patterns.add(Constants.R_TAGS + src);
}
}
return patterns;
}
private void readCapabilitiesV2() throws IOException {
// In git protocol V2, capabilities are different. If it's a key-value
// pair, the key may be a command name, and the value a space-separated
// list of capabilities for that command. We still store it in the same
// map as for protocol v0/v1. Protocol v2 code has to account for this.
for (;;) {
String line = readLine();
if (line == null) {
break;
}
addCapability(line);
}
}
private void addCapability(String capability) {
String parts[] = capability.split("=", 2); //$NON-NLS-1$
if (parts.length == 2) {
remoteCapabilities.put(parts[0], parts[1]);
}
remoteCapabilities.put(capability, null);
}
private ObjectId toId(String line, String value)
throws PackProtocolException {
try {
return ObjectId.fromString(value);
} catch (InvalidObjectIdException e) {
PackProtocolException ppe = invalidRefAdvertisementLine(line);
ppe.initCause(e);
throw ppe;
}
}
private void processLineV1(String name, ObjectId id, Map<String, Ref> avail)
throws IOException {
if (name.endsWith("^{}")) { //$NON-NLS-1$
name = name.substring(0, name.length() - 3);
final Ref prior = avail.get(name);
if (prior == null) {
throw new PackProtocolException(uri, MessageFormat.format(
JGitText.get().advertisementCameBefore, name, name));
}
if (prior.getPeeledObjectId() != null) {
throw duplicateAdvertisement(name + "^{}"); //$NON-NLS-1$
}
avail.put(name, new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name,
prior.getObjectId(), id));
} else {
final Ref prior = avail.put(name, new ObjectIdRef.PeeledNonTag(
Ref.Storage.NETWORK, name, id));
if (prior != null) {
throw duplicateAdvertisement(name);
}
}
}
private void processLineV2(String line, ObjectId id, String rest,
Map<String, Ref> avail, Map<String, String> symRefs)
throws IOException {
String[] parts = rest.split(" "); //$NON-NLS-1$
String name = parts[0];
// Two attributes possible, symref-target or peeled
String symRefTarget = null;
String peeled = null;
for (int i = 1; i < parts.length; i++) {
if (parts[i].startsWith(REF_ATTR_SYMREF_TARGET)) {
if (symRefTarget != null) {
throw new PackProtocolException(uri, MessageFormat.format(
JGitText.get().duplicateRefAttribute, line));
}
symRefTarget = parts[i]
.substring(REF_ATTR_SYMREF_TARGET.length());
} else if (parts[i].startsWith(REF_ATTR_PEELED)) {
if (peeled != null) {
throw new PackProtocolException(uri, MessageFormat.format(
JGitText.get().duplicateRefAttribute, line));
}
peeled = parts[i].substring(REF_ATTR_PEELED.length());
}
if (peeled != null && symRefTarget != null) {
break;
}
}
Ref idRef;
if (peeled != null) {
idRef = new ObjectIdRef.PeeledTag(Ref.Storage.NETWORK, name, id,
toId(line, peeled));
} else {
idRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK, name, id);
}
Ref prior = avail.put(name, idRef);
if (prior != null) {
throw duplicateAdvertisement(name);
}
if (!StringUtils.isEmptyOrNull(symRefTarget)) {
symRefs.put(name, symRefTarget);
}
}
/**
* Updates the given refMap with {@link SymbolicRef}s defined by the given
* symRefs.
* <p>
* For each entry, symRef, in symRefs, whose value is a key in refMap, adds
* a new entry to refMap with that same key and value of a new
* {@link SymbolicRef} with source=symRef.key and
* target=refMap.get(symRef.value), then removes that entry from symRefs.
* <p>
* If refMap already contains an entry for symRef.key, it is replaced.
* </p>
* </p>
* <p>
* For example, given:
* </p>
*
* <pre>
* refMap.put("refs/heads/main", ref);
* symRefs.put("HEAD", "refs/heads/main");
* </pre>
*
* then:
*
* <pre>
* updateWithSymRefs(refMap, symRefs);
* </pre>
*
* has the <em>effect</em> of:
*
* <pre>
* refMap.put("HEAD",
* new SymbolicRef("HEAD", refMap.get(symRefs.remove("HEAD"))))
* </pre>
* <p>
* Any entry in symRefs whose value is not a key in refMap is ignored. Any
* circular symRefs are ignored.
* </p>
* <p>
* Upon completion, symRefs will contain only any unresolvable entries.
* </p>
*
* @param refMap
* a non-null, modifiable, Map to update, and the provider of
* symref targets.
* @param symRefs
* a non-null, modifiable, Map of symrefs.
* @throws NullPointerException
* if refMap or symRefs is null
*/
static void updateWithSymRefs(Map<String, Ref> refMap, Map<String, String> symRefs) {
boolean haveNewRefMapEntries = !refMap.isEmpty();
while (!symRefs.isEmpty() && haveNewRefMapEntries) {
haveNewRefMapEntries = false;
final Iterator<Map.Entry<String, String>> iterator = symRefs.entrySet().iterator();
while (iterator.hasNext()) {
final Map.Entry<String, String> symRef = iterator.next();
if (!symRefs.containsKey(symRef.getValue())) { // defer forward reference
final Ref r = refMap.get(symRef.getValue());
if (r != null) {
refMap.put(symRef.getKey(), new SymbolicRef(symRef.getKey(), r));
haveNewRefMapEntries = true;
iterator.remove();
}
}
}
}
// If HEAD is still in the symRefs map here, the real ref was not
// reported, but we know it must point to the object reported for HEAD.
// So fill it in in the refMap.
String headRefName = symRefs.get(Constants.HEAD);
if (headRefName != null && !refMap.containsKey(headRefName)) {
Ref headRef = refMap.get(Constants.HEAD);
if (headRef != null) {
ObjectId headObj = headRef.getObjectId();
headRef = new ObjectIdRef.PeeledNonTag(Ref.Storage.NETWORK,
headRefName, headObj);
refMap.put(headRefName, headRef);
headRef = new SymbolicRef(Constants.HEAD, headRef);
refMap.put(Constants.HEAD, headRef);
symRefs.remove(Constants.HEAD);
}
}
}
/**
* Create an exception to indicate problems finding a remote repository. The
* caller is expected to throw the returned exception.
*
* Subclasses may override this method to provide better diagnostics.
*
* @return a TransportException saying a repository cannot be found and
* possibly why.
*/
protected TransportException noRepository() {
return new NoRemoteRepositoryException(uri, JGitText.get().notFound);
}
/**
* Whether this option is supported
*
* @param option
* option string
* @return whether this option is supported
*/
protected boolean isCapableOf(String option) {
return remoteCapabilities.containsKey(option);
}
/**
* Request capability
*
* @param b
* buffer
* @param option
* option we want
* @return {@code true} if the requested option is supported
*/
protected boolean wantCapability(StringBuilder b, String option) {
if (!isCapableOf(option))
return false;
b.append(' ');
b.append(option);
return true;
}
/**
* Return a capability value.
*
* @param option
* to get
* @return the value stored, if any.
*/
protected String getCapability(String option) {
return remoteCapabilities.get(option);
}
/**
* Add user agent capability
*
* @param b
* a {@link java.lang.StringBuilder} object.
*/
protected void addUserAgentCapability(StringBuilder b) {
String a = UserAgent.get();
if (a != null && remoteCapabilities.get(OPTION_AGENT) != null) {
b.append(' ').append(OPTION_AGENT).append('=').append(a);
}
}
/** {@inheritDoc} */
@Override
public String getPeerUserAgent() {
String agent = remoteCapabilities.get(OPTION_AGENT);
return agent != null ? agent : super.getPeerUserAgent();
}
private PackProtocolException duplicateAdvertisement(String name) {
return new PackProtocolException(uri, MessageFormat.format(JGitText.get().duplicateAdvertisementsOf, name));
}
private PackProtocolException invalidRefAdvertisementLine(String line) {
return new PackProtocolException(uri, MessageFormat.format(JGitText.get().invalidRefAdvertisementLine, line));
}
/** {@inheritDoc} */
@Override
public void close() {
if (out != null) {
try {
if (outNeedsEnd) {
outNeedsEnd = false;
pckOut.end();
}
out.close();
} catch (IOException err) {
// Ignore any close errors.
} finally {
out = null;
pckOut = null;
}
}
if (in != null) {
try {
in.close();
} catch (IOException err) {
// Ignore any close errors.
} finally {
in = null;
pckIn = null;
}
}
if (myTimer != null) {
try {
myTimer.terminate();
} finally {
myTimer = null;
timeoutIn = null;
timeoutOut = null;
}
}
}
/**
* Tell the peer we are disconnecting, if it cares to know.
*/
protected void endOut() {
if (outNeedsEnd && out != null) {
try {
outNeedsEnd = false;
pckOut.end();
} catch (IOException e) {
try {
out.close();
} catch (IOException err) {
// Ignore any close errors.
} finally {
out = null;
pckOut = null;
}
}
}
}
}