git/merge-ort.c

/*
 * "Ostensibly Recursive's Twin" merge strategy, or "ort" for short.  Meant
 * as a drop-in replacement for the "recursive" merge strategy, allowing one
 * to replace
 *
 *   git merge [-s recursive]
 *
 * with
 *
 *   git merge -s ort
 *
 * Note: git's parser allows the space between '-s' and its argument to be
 * missing.  (Should I have backronymed "ham", "alsa", "kip", "nap, "alvo",
 * "cale", "peedy", or "ins" instead of "ort"?)
 */

#include "cache.h"
#include "merge-ort.h"

#include "alloc.h"
#include "attr.h"
#include "blob.h"
#include "cache-tree.h"
#include "commit.h"
#include "commit-reach.h"
#include "diff.h"
#include "diffcore.h"
#include "dir.h"
#include "entry.h"
#include "ll-merge.h"
#include "object-store.h"
#include "promisor-remote.h"
#include "revision.h"
#include "strmap.h"
#include "submodule-config.h"
#include "submodule.h"
#include "tree.h"
#include "unpack-trees.h"
#include "xdiff-interface.h"

/*
 * We have many arrays of size 3.  Whenever we have such an array, the
 * indices refer to one of the sides of the three-way merge.  This is so
 * pervasive that the constants 0, 1, and 2 are used in many places in the
 * code (especially in arithmetic operations to find the other side's index
 * or to compute a relevant mask), but sometimes these enum names are used
 * to aid code clarity.
 *
 * See also 'filemask' and 'dirmask' in struct conflict_info; the "ith side"
 * referred to there is one of these three sides.
 */
enum merge_side {
	MERGE_BASE = 0,
	MERGE_SIDE1 = 1,
	MERGE_SIDE2 = 2
};

static unsigned RESULT_INITIALIZED = 0x1abe11ed; /* unlikely accidental value */

struct traversal_callback_data {
	unsigned long mask;
	unsigned long dirmask;
	struct name_entry names[3];
};

struct deferred_traversal_data {
	/*
	 * possible_trivial_merges: directories to be explored only when needed
	 *
	 * possible_trivial_merges is a map of directory names to
	 * dir_rename_mask.  When we detect that a directory is unchanged on
	 * one side, we can sometimes resolve the directory without recursing
	 * into it.  Renames are the only things that can prevent such an
	 * optimization.  However, for rename sources:
	 *   - If no parent directory needed directory rename detection, then
	 *     no path under such a directory can be a relevant_source.
	 * and for rename destinations:
	 *   - If no cached rename has a target path under the directory AND
	 *   - If there are no unpaired relevant_sources elsewhere in the
	 *     repository
	 * then we don't need any path under this directory for a rename
	 * destination.  The only way to know the last item above is to defer
	 * handling such directories until the end of collect_merge_info(),
	 * in handle_deferred_entries().
	 *
	 * For each we store dir_rename_mask, since that's the only bit of
	 * information we need, other than the path, to resume the recursive
	 * traversal.
	 */
	struct strintmap possible_trivial_merges;

	/*
	 * trivial_merges_okay: if trivial directory merges are okay
	 *
	 * See possible_trivial_merges above.  The "no unpaired
	 * relevant_sources elsewhere in the repository" is a single boolean
	 * per merge side, which we store here.  Note that while 0 means no,
	 * 1 only means "maybe" rather than "yes"; we optimistically set it
	 * to 1 initially and only clear when we determine it is unsafe to
	 * do trivial directory merges.
	 */
	unsigned trivial_merges_okay;

	/*
	 * target_dirs: ancestor directories of rename targets
	 *
	 * target_dirs contains all directory names that are an ancestor of
	 * any rename destination.
	 */
	struct strset target_dirs;
};

struct rename_info {
	/*
	 * All variables that are arrays of size 3 correspond to data tracked
	 * for the sides in enum merge_side.  Index 0 is almost always unused
	 * because we often only need to track information for MERGE_SIDE1 and
	 * MERGE_SIDE2 (MERGE_BASE can't have rename information since renames
	 * are determined relative to what changed since the MERGE_BASE).
	 */

	/*
	 * pairs: pairing of filenames from diffcore_rename()
	 */
	struct diff_queue_struct pairs[3];

	/*
	 * dirs_removed: directories removed on a given side of history.
	 *
	 * The keys of dirs_removed[side] are the directories that were removed
	 * on the given side of history.  The value of the strintmap for each
	 * directory is a value from enum dir_rename_relevance.
	 */
	struct strintmap dirs_removed[3];

	/*
	 * dir_rename_count: tracking where parts of a directory were renamed to
	 *
	 * When files in a directory are renamed, they may not all go to the
	 * same location.  Each strmap here tracks:
	 *      old_dir => {new_dir => int}
	 * That is, dir_rename_count[side] is a strmap to a strintmap.
	 */
	struct strmap dir_rename_count[3];

	/*
	 * dir_renames: computed directory renames
	 *
	 * This is a map of old_dir => new_dir and is derived in part from
	 * dir_rename_count.
	 */
	struct strmap dir_renames[3];

	/*
	 * relevant_sources: deleted paths wanted in rename detection, and why
	 *
	 * relevant_sources is a set of deleted paths on each side of
	 * history for which we need rename detection.  If a path is deleted
	 * on one side of history, we need to detect if it is part of a
	 * rename if either
	 *    * the file is modified/deleted on the other side of history
	 *    * we need to detect renames for an ancestor directory
	 * If neither of those are true, we can skip rename detection for
	 * that path.  The reason is stored as a value from enum
	 * file_rename_relevance, as the reason can inform the algorithm in
	 * diffcore_rename_extended().
	 */
	struct strintmap relevant_sources[3];

	struct deferred_traversal_data deferred[3];

	/*
	 * dir_rename_mask:
	 *   0: optimization removing unmodified potential rename source okay
	 *   2 or 4: optimization okay, but must check for files added to dir
	 *   7: optimization forbidden; need rename source in case of dir rename
	 */
	unsigned dir_rename_mask:3;

	/*
	 * callback_data_*: supporting data structures for alternate traversal
	 *
	 * We sometimes need to be able to traverse through all the files
	 * in a given tree before all immediate subdirectories within that
	 * tree.  Since traverse_trees() doesn't do that naturally, we have
	 * a traverse_trees_wrapper() that stores any immediate
	 * subdirectories while traversing files, then traverses the
	 * immediate subdirectories later.  These callback_data* variables
	 * store the information for the subdirectories so that we can do
	 * that traversal order.
	 */
	struct traversal_callback_data *callback_data;
	int callback_data_nr, callback_data_alloc;
	char *callback_data_traverse_path;

	/*
	 * merge_trees: trees passed to the merge algorithm for the merge
	 *
	 * merge_trees records the trees passed to the merge algorithm.  But,
	 * this data also is stored in merge_result->priv.  If a sequence of
	 * merges are being done (such as when cherry-picking or rebasing),
	 * the next merge can look at this and re-use information from
	 * previous merges under certain circumstances.
	 *
	 * See also all the cached_* variables.
	 */
	struct tree *merge_trees[3];

	/*
	 * cached_pairs_valid_side: which side's cached info can be reused
	 *
	 * See the description for merge_trees.  For repeated merges, at most
	 * only one side's cached information can be used.  Valid values:
	 *   MERGE_SIDE2: cached data from side2 can be reused
	 *   MERGE_SIDE1: cached data from side1 can be reused
	 *   0:           no cached data can be reused
	 *   -1:          See redo_after_renames; both sides can be reused.
	 */
	int cached_pairs_valid_side;

	/*
	 * cached_pairs: Caching of renames and deletions.
	 *
	 * These are mappings recording renames and deletions of individual
	 * files (not directories).  They are thus a map from an old
	 * filename to either NULL (for deletions) or a new filename (for
	 * renames).
	 */
	struct strmap cached_pairs[3];

	/*
	 * cached_target_names: just the destinations from cached_pairs
	 *
	 * We sometimes want a fast lookup to determine if a given filename
	 * is one of the destinations in cached_pairs.  cached_target_names
	 * is thus duplicative information, but it provides a fast lookup.
	 */
	struct strset cached_target_names[3];

	/*
	 * cached_irrelevant: Caching of rename_sources that aren't relevant.
	 *
	 * If we try to detect a rename for a source path and succeed, it's
	 * part of a rename.  If we try to detect a rename for a source path
	 * and fail, then it's a delete.  If we do not try to detect a rename
	 * for a path, then we don't know if it's a rename or a delete.  If
	 * merge-ort doesn't think the path is relevant, then we just won't
	 * cache anything for that path.  But there's a slight problem in
	 * that merge-ort can think a path is RELEVANT_LOCATION, but due to
	 * commit 9bd342137e ("diffcore-rename: determine which
	 * relevant_sources are no longer relevant", 2021-03-13),
	 * diffcore-rename can downgrade the path to RELEVANT_NO_MORE.  To
	 * avoid excessive calls to diffcore_rename_extended() we still need
	 * to cache such paths, though we cannot record them as either
	 * renames or deletes.  So we cache them here as a "turned out to be
	 * irrelevant *for this commit*" as they are often also irrelevant
	 * for subsequent commits, though we will have to do some extra
	 * checking to see whether such paths become relevant for rename
	 * detection when cherry-picking/rebasing subsequent commits.
	 */
	struct strset cached_irrelevant[3];

	/*
	 * redo_after_renames: optimization flag for "restarting" the merge
	 *
	 * Sometimes it pays to detect renames, cache them, and then
	 * restart the merge operation from the beginning.  The reason for
	 * this is that when we know where all the renames are, we know
	 * whether a certain directory has any paths under it affected --
	 * and if a directory is not affected then it permits us to do
	 * trivial tree merging in more cases.  Doing trivial tree merging
	 * prevents the need to run process_entry() on every path
	 * underneath trees that can be trivially merged, and
	 * process_entry() is more expensive than collect_merge_info() --
	 * plus, the second collect_merge_info() will be much faster since
	 * it doesn't have to recurse into the relevant trees.
	 *
	 * Values for this flag:
	 *   0 = don't bother, not worth it (or conditions not yet checked)
	 *   1 = conditions for optimization met, optimization worthwhile
	 *   2 = we already did it (don't restart merge yet again)
	 */
	unsigned redo_after_renames;

	/*
	 * needed_limit: value needed for inexact rename detection to run
	 *
	 * If the current rename limit wasn't high enough for inexact
	 * rename detection to run, this records the limit needed.  Otherwise,
	 * this value remains 0.
	 */
	int needed_limit;
};

struct merge_options_internal {
	/*
	 * paths: primary data structure in all of merge ort.
	 *
	 * The keys of paths:
	 *   * are full relative paths from the toplevel of the repository
	 *     (e.g. "drivers/firmware/raspberrypi.c").
	 *   * store all relevant paths in the repo, both directories and
	 *     files (e.g. drivers, drivers/firmware would also be included)
	 *   * these keys serve to intern all the path strings, which allows
	 *     us to do pointer comparison on directory names instead of
	 *     strcmp; we just have to be careful to use the interned strings.
	 *
	 * The values of paths:
	 *   * either a pointer to a merged_info, or a conflict_info struct
	 *   * merged_info contains all relevant information for a
	 *     non-conflicted entry.
	 *   * conflict_info contains a merged_info, plus any additional
	 *     information about a conflict such as the higher orders stages
	 *     involved and the names of the paths those came from (handy
	 *     once renames get involved).
	 *   * a path may start "conflicted" (i.e. point to a conflict_info)
	 *     and then a later step (e.g. three-way content merge) determines
	 *     it can be cleanly merged, at which point it'll be marked clean
	 *     and the algorithm will ignore any data outside the contained
	 *     merged_info for that entry
	 *   * If an entry remains conflicted, the merged_info portion of a
	 *     conflict_info will later be filled with whatever version of
	 *     the file should be placed in the working directory (e.g. an
	 *     as-merged-as-possible variation that contains conflict markers).
	 */
	struct strmap paths;

	/*
	 * conflicted: a subset of keys->values from "paths"
	 *
	 * conflicted is basically an optimization between process_entries()
	 * and record_conflicted_index_entries(); the latter could loop over
	 * ALL the entries in paths AGAIN and look for the ones that are
	 * still conflicted, but since process_entries() has to loop over
	 * all of them, it saves the ones it couldn't resolve in this strmap
	 * so that record_conflicted_index_entries() can iterate just the
	 * relevant entries.
	 */
	struct strmap conflicted;

	/*
	 * pool: memory pool for fast allocation/deallocation
	 *
	 * We allocate room for lots of filenames and auxiliary data
	 * structures in merge_options_internal, and it tends to all be
	 * freed together too.  Using a memory pool for these provides a
	 * nice speedup.
	 */
	struct mem_pool pool;

	/*
	 * output: special messages and conflict notices for various paths
	 *
	 * This is a map of pathnames (a subset of the keys in "paths" above)
	 * to strbufs.  It gathers various warning/conflict/notice messages
	 * for later processing.
	 */
	struct strmap output;

	/*
	 * renames: various data relating to rename detection
	 */
	struct rename_info renames;

	/*
	 * attr_index: hacky minimal index used for renormalization
	 *
	 * renormalization code _requires_ an index, though it only needs to
	 * find a .gitattributes file within the index.  So, when
	 * renormalization is important, we create a special index with just
	 * that one file.
	 */
	struct index_state attr_index;

	/*
	 * current_dir_name, toplevel_dir: temporary vars
	 *
	 * These are used in collect_merge_info_callback(), and will set the
	 * various merged_info.directory_name for the various paths we get;
	 * see documentation for that variable and the requirements placed on
	 * that field.
	 */
	const char *current_dir_name;
	const char *toplevel_dir;

	/* call_depth: recursion level counter for merging merge bases */
	int call_depth;
};

struct version_info {
	struct object_id oid;
	unsigned short mode;
};

struct merged_info {
	/* if is_null, ignore result.  otherwise result has oid & mode */
	struct version_info result;
	unsigned is_null:1;

	/*
	 * clean: whether the path in question is cleanly merged.
	 *
	 * see conflict_info.merged for more details.
	 */
	unsigned clean:1;

	/*
	 * basename_offset: offset of basename of path.
	 *
	 * perf optimization to avoid recomputing offset of final '/'
	 * character in pathname (0 if no '/' in pathname).
	 */
	size_t basename_offset;

	 /*
	  * directory_name: containing directory name.
	  *
	  * Note that we assume directory_name is constructed such that
	  *    strcmp(dir1_name, dir2_name) == 0 iff dir1_name == dir2_name,
	  * i.e. string equality is equivalent to pointer equality.  For this
	  * to hold, we have to be careful setting directory_name.
	  */
	const char *directory_name;
};

struct conflict_info {
	/*
	 * merged: the version of the path that will be written to working tree
	 *
	 * WARNING: It is critical to check merged.clean and ensure it is 0
	 * before reading any conflict_info fields outside of merged.
	 * Allocated merge_info structs will always have clean set to 1.
	 * Allocated conflict_info structs will have merged.clean set to 0
	 * initially.  The merged.clean field is how we know if it is safe
	 * to access other parts of conflict_info besides merged; if a
	 * conflict_info's merged.clean is changed to 1, the rest of the
	 * algorithm is not allowed to look at anything outside of the
	 * merged member anymore.
	 */
	struct merged_info merged;

	/* oids & modes from each of the three trees for this path */
	struct version_info stages[3];

	/* pathnames for each stage; may differ due to rename detection */
	const char *pathnames[3];

	/* Whether this path is/was involved in a directory/file conflict */
	unsigned df_conflict:1;

	/*
	 * Whether this path is/was involved in a non-content conflict other
	 * than a directory/file conflict (e.g. rename/rename, rename/delete,
	 * file location based on possible directory rename).
	 */
	unsigned path_conflict:1;

	/*
	 * For filemask and dirmask, the ith bit corresponds to whether the
	 * ith entry is a file (filemask) or a directory (dirmask).  Thus,
	 * filemask & dirmask is always zero, and filemask | dirmask is at
	 * most 7 but can be less when a path does not appear as either a
	 * file or a directory on at least one side of history.
	 *
	 * Note that these masks are related to enum merge_side, as the ith
	 * entry corresponds to side i.
	 *
	 * These values come from a traverse_trees() call; more info may be
	 * found looking at tree-walk.h's struct traverse_info,
	 * particularly the documentation above the "fn" member (note that
	 * filemask = mask & ~dirmask from that documentation).
	 */
	unsigned filemask:3;
	unsigned dirmask:3;

	/*
	 * Optimization to track which stages match, to avoid the need to
	 * recompute it in multiple steps. Either 0 or at least 2 bits are
	 * set; if at least 2 bits are set, their corresponding stages match.
	 */
	unsigned match_mask:3;
};

/*** Function Grouping: various utility functions ***/

/*
 * For the next three macros, see warning for conflict_info.merged.
 *
 * In each of the below, mi is a struct merged_info*, and ci was defined
 * as a struct conflict_info* (but we need to verify ci isn't actually
 * pointed at a struct merged_info*).
 *
 * INITIALIZE_CI: Assign ci to mi but only if it's safe; set to NULL otherwise.
 * VERIFY_CI: Ensure that something we assigned to a conflict_info* is one.
 * ASSIGN_AND_VERIFY_CI: Similar to VERIFY_CI but do assignment first.
 */
#define INITIALIZE_CI(ci, mi) do {                                           \
	(ci) = (!(mi) || (mi)->clean) ? NULL : (struct conflict_info *)(mi); \
} while (0)
#define VERIFY_CI(ci) assert(ci && !ci->merged.clean);
#define ASSIGN_AND_VERIFY_CI(ci, mi) do {    \
	(ci) = (struct conflict_info *)(mi);  \
	assert((ci) && !(mi)->clean);        \
} while (0)

static void free_strmap_strings(struct strmap *map)
{
	struct hashmap_iter iter;
	struct strmap_entry *entry;

	strmap_for_each_entry(map, &iter, entry) {
		free((char*)entry->key);
	}
}

static void clear_or_reinit_internal_opts(struct merge_options_internal *opti,
					  int reinitialize)
{
	struct rename_info *renames = &opti->renames;
	int i;
	void (*strmap_clear_func)(struct strmap *, int) =
		reinitialize ? strmap_partial_clear : strmap_clear;
	void (*strintmap_clear_func)(struct strintmap *) =
		reinitialize ? strintmap_partial_clear : strintmap_clear;
	void (*strset_clear_func)(struct strset *) =
		reinitialize ? strset_partial_clear : strset_clear;

	strmap_clear_func(&opti->paths, 0);

	/*
	 * All keys and values in opti->conflicted are a subset of those in
	 * opti->paths.  We don't want to deallocate anything twice, so we
	 * don't free the keys and we pass 0 for free_values.
	 */
	strmap_clear_func(&opti->conflicted, 0);

	if (opti->attr_index.cache_nr) /* true iff opt->renormalize */
		discard_index(&opti->attr_index);

	/* Free memory used by various renames maps */
	for (i = MERGE_SIDE1; i <= MERGE_SIDE2; ++i) {
		strintmap_clear_func(&renames->dirs_removed[i]);
		strmap_clear_func(&renames->dir_renames[i], 0);
		strintmap_clear_func(&renames->relevant_sources[i]);
		if (!reinitialize)
			assert(renames->cached_pairs_valid_side == 0);
		if (i != renames->cached_pairs_valid_side &&
		    -1 != renames->cached_pairs_valid_side) {
			strset_clear_func(&renames->cached_target_names[i]);
			strmap_clear_func(&renames->cached_pairs[i], 1);
			strset_clear_func(&renames->cached_irrelevant[i]);
			partial_clear_dir_rename_count(&renames->dir_rename_count[i]);
			if (!reinitialize)
				strmap_clear(&renames->dir_rename_count[i], 1);
		}
	}
	for (i = MERGE_SIDE1; i <= MERGE_SIDE2; ++i) {
		strintmap_clear_func(&renames->deferred[i].possible_trivial_merges);
		strset_clear_func(&renames->deferred[i].target_dirs);
		renames->deferred[i].trivial_merges_okay = 1; /* 1 == maybe */
	}
	renames->cached_pairs_valid_side = 0;
	renames->dir_rename_mask = 0;

	if (!reinitialize) {
		struct hashmap_iter iter;
		struct strmap_entry *e;

		/* Release and free each strbuf found in output */
		strmap_for_each_entry(&opti->output, &iter, e) {
			struct strbuf *sb = e->value;
			strbuf_release(sb);
			/*
			 * While strictly speaking we don't need to free(sb)
			 * here because we could pass free_values=1 when
			 * calling strmap_clear() on opti->output, that would
			 * require strmap_clear to do another
			 * strmap_for_each_entry() loop, so we just free it
			 * while we're iterating anyway.
			 */
			free(sb);
		}
		strmap_clear(&opti->output, 0);
	}

	mem_pool_discard(&opti->pool, 0);

	/* Clean out callback_data as well. */
	FREE_AND_NULL(renames->callback_data);
	renames->callback_data_nr = renames->callback_data_alloc = 0;
}

__attribute__((format (printf, 2, 3)))
static int err(struct merge_options *opt, const char *err, ...)
{
	va_list params;
	struct strbuf sb = STRBUF_INIT;

	strbuf_addstr(&sb, "error: ");
	va_start(params, err);
	strbuf_vaddf(&sb, err, params);
	va_end(params);

	error("%s", sb.buf);
	strbuf_release(&sb);

	return -1;
}

static void format_commit(struct strbuf *sb,
			  int indent,
			  struct repository *repo,
			  struct commit *commit)
{
	struct merge_remote_desc *desc;
	struct pretty_print_context ctx = {0};
	ctx.abbrev = DEFAULT_ABBREV;

	strbuf_addchars(sb, ' ', indent);
	desc = merge_remote_util(commit);
	if (desc) {
		strbuf_addf(sb, "virtual %s\n", desc->name);
		return;
	}

	repo_format_commit_message(repo, commit, "%h %s", sb, &ctx);
	strbuf_addch(sb, '\n');
}

__attribute__((format (printf, 4, 5)))
static void path_msg(struct merge_options *opt,
		     const char *path,
		     int omittable_hint, /* skippable under --remerge-diff */
		     const char *fmt, ...)
{
	va_list ap;
	struct strbuf *sb, *dest;
	struct strbuf tmp = STRBUF_INIT;

	if (opt->record_conflict_msgs_as_headers && omittable_hint)
		return; /* Do not record mere hints in headers */
	if (opt->priv->call_depth && opt->verbosity < 5)
		return; /* Ignore messages from inner merges */

	sb = strmap_get(&opt->priv->output, path);