git/fetch-pack.c

#include "git-compat-util.h"
#include "repository.h"
#include "config.h"
#include "date.h"
#include "environment.h"
#include "gettext.h"
#include "hex.h"
#include "lockfile.h"
#include "refs.h"
#include "pkt-line.h"
#include "commit.h"
#include "tag.h"
#include "exec-cmd.h"
#include "pack.h"
#include "sideband.h"
#include "fetch-pack.h"
#include "remote.h"
#include "run-command.h"
#include "connect.h"
#include "trace2.h"
#include "transport.h"
#include "version.h"
#include "oid-array.h"
#include "oidset.h"
#include "packfile.h"
#include "object-store-ll.h"
#include "path.h"
#include "connected.h"
#include "fetch-negotiator.h"
#include "fsck.h"
#include "shallow.h"
#include "commit-reach.h"
#include "commit-graph.h"
#include "sigchain.h"
#include "mergesort.h"

static int transfer_unpack_limit = -1;
static int fetch_unpack_limit = -1;
static int unpack_limit = 100;
static int prefer_ofs_delta = 1;
static int no_done;
static int deepen_since_ok;
static int deepen_not_ok;
static int fetch_fsck_objects = -1;
static int transfer_fsck_objects = -1;
static int agent_supported;
static int server_supports_filtering;
static int advertise_sid;
static struct shallow_lock shallow_lock;
static const char *alternate_shallow_file;
static struct fsck_options fsck_options = FSCK_OPTIONS_MISSING_GITMODULES;
static struct strbuf fsck_msg_types = STRBUF_INIT;
static struct string_list uri_protocols = STRING_LIST_INIT_DUP;

/* Remember to update object flag allocation in object.h */
#define COMPLETE	(1U << 0)
#define ALTERNATE	(1U << 1)
#define COMMON		(1U << 6)
#define REACH_SCRATCH	(1U << 7)

/*
 * After sending this many "have"s if we do not get any new ACK , we
 * give up traversing our history.
 */
#define MAX_IN_VAIN 256

static int multi_ack, use_sideband;
/* Allow specifying sha1 if it is a ref tip. */
#define ALLOW_TIP_SHA1	01
/* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
#define ALLOW_REACHABLE_SHA1	02
static unsigned int allow_unadvertised_object_request;

__attribute__((format (printf, 2, 3)))
static inline void print_verbose(const struct fetch_pack_args *args,
				 const char *fmt, ...)
{
	va_list params;

	if (!args->verbose)
		return;

	va_start(params, fmt);
	vfprintf(stderr, fmt, params);
	va_end(params);
	fputc('\n', stderr);
}

struct alternate_object_cache {
	struct object **items;
	size_t nr, alloc;
};

static void cache_one_alternate(const struct object_id *oid,
				void *vcache)
{
	struct alternate_object_cache *cache = vcache;
	struct object *obj = parse_object(the_repository, oid);

	if (!obj || (obj->flags & ALTERNATE))
		return;

	obj->flags |= ALTERNATE;
	ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
	cache->items[cache->nr++] = obj;
}

static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
				      void (*cb)(struct fetch_negotiator *,
						 struct object *))
{
	static int initialized;
	static struct alternate_object_cache cache;
	size_t i;

	if (!initialized) {
		for_each_alternate_ref(cache_one_alternate, &cache);
		initialized = 1;
	}

	for (i = 0; i < cache.nr; i++)
		cb(negotiator, cache.items[i]);
}

static struct commit *deref_without_lazy_fetch_extended(const struct object_id *oid,
							int mark_tags_complete,
							enum object_type *type,
							unsigned int oi_flags)
{
	struct object_info info = { .typep = type };
	struct commit *commit;

	commit = lookup_commit_in_graph(the_repository, oid);
	if (commit)
		return commit;

	while (1) {
		if (oid_object_info_extended(the_repository, oid, &info,
					     oi_flags))
			return NULL;
		if (*type == OBJ_TAG) {
			struct tag *tag = (struct tag *)
				parse_object(the_repository, oid);

			if (!tag->tagged)
				return NULL;
			if (mark_tags_complete)
				tag->object.flags |= COMPLETE;
			oid = &tag->tagged->oid;
		} else {
			break;
		}
	}

	if (*type == OBJ_COMMIT) {
		struct commit *commit = lookup_commit(the_repository, oid);
		if (!commit || repo_parse_commit(the_repository, commit))
			return NULL;
		return commit;
	}

	return NULL;
}


static struct commit *deref_without_lazy_fetch(const struct object_id *oid,
					       int mark_tags_complete)
{
	enum object_type type;
	unsigned flags = OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK;
	return deref_without_lazy_fetch_extended(oid, mark_tags_complete,
						 &type, flags);
}

static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
			       const struct object_id *oid)
{
	struct commit *c = deref_without_lazy_fetch(oid, 0);

	if (c)
		negotiator->add_tip(negotiator, c);
	return 0;
}

static int rev_list_insert_ref_oid(const char *refname UNUSED,
				   const struct object_id *oid,
				   int flag UNUSED,
				   void *cb_data)
{
	return rev_list_insert_ref(cb_data, oid);
}

enum ack_type {
	NAK = 0,
	ACK,
	ACK_continue,
	ACK_common,
	ACK_ready
};

static void consume_shallow_list(struct fetch_pack_args *args,
				 struct packet_reader *reader)
{
	if (args->stateless_rpc && args->deepen) {
		/* If we sent a depth we will get back "duplicate"
		 * shallow and unshallow commands every time there
		 * is a block of have lines exchanged.
		 */
		while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
			if (starts_with(reader->line, "shallow "))
				continue;
			if (starts_with(reader->line, "unshallow "))
				continue;
			die(_("git fetch-pack: expected shallow list"));
		}
		if (reader->status != PACKET_READ_FLUSH)
			die(_("git fetch-pack: expected a flush packet after shallow list"));
	}
}

static enum ack_type get_ack(struct packet_reader *reader,
			     struct object_id *result_oid)
{
	int len;
	const char *arg;

	if (packet_reader_read(reader) != PACKET_READ_NORMAL)
		die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
	len = reader->pktlen;

	if (!strcmp(reader->line, "NAK"))
		return NAK;
	if (skip_prefix(reader->line, "ACK ", &arg)) {
		const char *p;
		if (!parse_oid_hex(arg, result_oid, &p)) {
			len -= p - reader->line;
			if (len < 1)
				return ACK;
			if (strstr(p, "continue"))
				return ACK_continue;
			if (strstr(p, "common"))
				return ACK_common;
			if (strstr(p, "ready"))
				return ACK_ready;
			return ACK;
		}
	}
	die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
}

static void send_request(struct fetch_pack_args *args,
			 int fd, struct strbuf *buf)
{
	if (args->stateless_rpc) {
		send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
		packet_flush(fd);
	} else {
		if (write_in_full(fd, buf->buf, buf->len) < 0)
			die_errno(_("unable to write to remote"));
	}
}

static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
					struct object *obj)
{
	rev_list_insert_ref(negotiator, &obj->oid);
}

#define INITIAL_FLUSH 16
#define PIPESAFE_FLUSH 32
#define LARGE_FLUSH 16384

static int next_flush(int stateless_rpc, int count)
{
	if (stateless_rpc) {
		if (count < LARGE_FLUSH)
			count <<= 1;
		else
			count = count * 11 / 10;
	} else {
		if (count < PIPESAFE_FLUSH)
			count <<= 1;
		else
			count += PIPESAFE_FLUSH;
	}
	return count;
}

static void mark_tips(struct fetch_negotiator *negotiator,
		      const struct oid_array *negotiation_tips)
{
	int i;

	if (!negotiation_tips) {
		for_each_rawref(rev_list_insert_ref_oid, negotiator);
		return;
	}

	for (i = 0; i < negotiation_tips->nr; i++)
		rev_list_insert_ref(negotiator, &negotiation_tips->oid[i]);
	return;
}

static void send_filter(struct fetch_pack_args *args,
			struct strbuf *req_buf,
			int server_supports_filter)
{
	if (args->filter_options.choice) {
		const char *spec =
			expand_list_objects_filter_spec(&args->filter_options);
		if (server_supports_filter) {
			print_verbose(args, _("Server supports filter"));
			packet_buf_write(req_buf, "filter %s", spec);
			trace2_data_string("fetch", the_repository,
					   "filter/effective", spec);
		} else {
			warning("filtering not recognized by server, ignoring");
			trace2_data_string("fetch", the_repository,
					   "filter/unsupported", spec);
		}
	} else {
		trace2_data_string("fetch", the_repository,
				   "filter/none", "");
	}
}

static int find_common(struct fetch_negotiator *negotiator,
		       struct fetch_pack_args *args,
		       int fd[2], struct object_id *result_oid,
		       struct ref *refs)
{
	int fetching;
	int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
	int negotiation_round = 0, haves = 0;
	const struct object_id *oid;
	unsigned in_vain = 0;
	int got_continue = 0;
	int got_ready = 0;
	struct strbuf req_buf = STRBUF_INIT;
	size_t state_len = 0;
	struct packet_reader reader;

	if (args->stateless_rpc && multi_ack == 1)
		die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed");

	packet_reader_init(&reader, fd[0], NULL, 0,
			   PACKET_READ_CHOMP_NEWLINE |
			   PACKET_READ_DIE_ON_ERR_PACKET);

	mark_tips(negotiator, args->negotiation_tips);
	for_each_cached_alternate(negotiator, insert_one_alternate_object);

	fetching = 0;
	for ( ; refs ; refs = refs->next) {
		struct object_id *remote = &refs->old_oid;
		const char *remote_hex;
		struct object *o;

		if (!args->refetch) {
			/*
			* If that object is complete (i.e. it is an ancestor of a
			* local ref), we tell them we have it but do not have to
			* tell them about its ancestors, which they already know
			* about.
			*
			* We use lookup_object here because we are only
			* interested in the case we *know* the object is
			* reachable and we have already scanned it.
			*/
			if (((o = lookup_object(the_repository, remote)) != NULL) &&
					(o->flags & COMPLETE)) {
				continue;
			}
		}

		remote_hex = oid_to_hex(remote);
		if (!fetching) {
			struct strbuf c = STRBUF_INIT;
			if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
			if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
			if (no_done)            strbuf_addstr(&c, " no-done");
			if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
			if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
			if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
			if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
			if (args->no_progress)   strbuf_addstr(&c, " no-progress");
			if (args->include_tag)   strbuf_addstr(&c, " include-tag");
			if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
			if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
			if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
			if (agent_supported)    strbuf_addf(&c, " agent=%s",
							    git_user_agent_sanitized());
			if (advertise_sid)
				strbuf_addf(&c, " session-id=%s", trace2_session_id());
			if (args->filter_options.choice)
				strbuf_addstr(&c, " filter");
			packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
			strbuf_release(&c);
		} else
			packet_buf_write(&req_buf, "want %s\n", remote_hex);
		fetching++;
	}

	if (!fetching) {
		strbuf_release(&req_buf);
		packet_flush(fd[1]);
		return 1;
	}

	if (is_repository_shallow(the_repository))
		write_shallow_commits(&req_buf, 1, NULL);
	if (args->depth > 0)
		packet_buf_write(&req_buf, "deepen %d", args->depth);
	if (args->deepen_since) {
		timestamp_t max_age = approxidate(args->deepen_since);
		packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
	}
	if (args->deepen_not) {
		int i;
		for (i = 0; i < args->deepen_not->nr; i++) {
			struct string_list_item *s = args->deepen_not->items + i;
			packet_buf_write(&req_buf, "deepen-not %s", s->string);
		}
	}
	send_filter(args, &req_buf, server_supports_filtering);
	packet_buf_flush(&req_buf);
	state_len = req_buf.len;

	if (args->deepen) {
		const char *arg;
		struct object_id oid;

		send_request(args, fd[1], &req_buf);
		while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
			if (skip_prefix(reader.line, "shallow ", &arg)) {
				if (get_oid_hex(arg, &oid))
					die(_("invalid shallow line: %s"), reader.line);
				register_shallow(the_repository, &oid);
				continue;
			}
			if (skip_prefix(reader.line, "unshallow ", &arg)) {
				if (get_oid_hex(arg, &oid))
					die(_("invalid unshallow line: %s"), reader.line);
				if (!lookup_object(the_repository, &oid))
					die(_("object not found: %s"), reader.line);
				/* make sure that it is parsed as shallow */
				if (!parse_object(the_repository, &oid))
					die(_("error in object: %s"), reader.line);
				if (unregister_shallow(&oid))
					die(_("no shallow found: %s"), reader.line);
				continue;
			}
			die(_("expected shallow/unshallow, got %s"), reader.line);
		}
	} else if (!args->stateless_rpc)
		send_request(args, fd[1], &req_buf);

	if (!args->stateless_rpc) {
		/* If we aren't using the stateless-rpc interface
		 * we don't need to retain the headers.
		 */
		strbuf_setlen(&req_buf, 0);
		state_len = 0;
	}

	trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository);
	flushes = 0;
	retval = -1;
	while ((oid = negotiator->next(negotiator))) {
		packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
		print_verbose(args, "have %s", oid_to_hex(oid));
		in_vain++;
		haves++;
		if (flush_at <= ++count) {
			int ack;

			negotiation_round++;
			trace2_region_enter_printf("negotiation_v0_v1", "round",
						   the_repository, "%d",
						   negotiation_round);
			trace2_data_intmax("negotiation_v0_v1", the_repository,
					   "haves_added", haves);
			trace2_data_intmax("negotiation_v0_v1", the_repository,
					   "in_vain", in_vain);
			haves = 0;
			packet_buf_flush(&req_buf);
			send_request(args, fd[1], &req_buf);
			strbuf_setlen(&req_buf, state_len);
			flushes++;
			flush_at = next_flush(args->stateless_rpc, count);

			/*
			 * We keep one window "ahead" of the other side, and
			 * will wait for an ACK only on the next one
			 */
			if (!args->stateless_rpc && count == INITIAL_FLUSH)
				continue;

			consume_shallow_list(args, &reader);
			do {
				ack = get_ack(&reader, result_oid);
				if (ack)
					print_verbose(args, _("got %s %d %s"), "ack",
						      ack, oid_to_hex(result_oid));
				switch (ack) {
				case ACK:
					trace2_region_leave_printf("negotiation_v0_v1", "round",
								   the_repository, "%d",
								   negotiation_round);
					flushes = 0;
					multi_ack = 0;
					retval = 0;
					goto done;
				case ACK_common:
				case ACK_ready:
				case ACK_continue: {
					struct commit *commit =
						lookup_commit(the_repository,
							      result_oid);
					int was_common;

					if (!commit)
						die(_("invalid commit %s"), oid_to_hex(result_oid));
					was_common = negotiator->ack(negotiator, commit);
					if (args->stateless_rpc
					 && ack == ACK_common
					 && !was_common) {
						/* We need to replay the have for this object
						 * on the next RPC request so the peer knows
						 * it is in common with us.
						 */
						const char *hex = oid_to_hex(result_oid);
						packet_buf_write(&req_buf, "have %s\n", hex);
						state_len = req_buf.len;
						haves++;
						/*
						 * Reset in_vain because an ack
						 * for this commit has not been
						 * seen.
						 */
						in_vain = 0;
					} else if (!args->stateless_rpc
						   || ack != ACK_common)
						in_vain = 0;
					retval = 0;
					got_continue = 1;
					if (ack == ACK_ready)
						got_ready = 1;
					break;
					}
				}