Browse Source

Merge branch 'rc/fetch-refetch'

"git fetch --refetch" learned to fetch everything without telling
the other side what we already have, which is useful when you
cannot trust what you have in the local object store.

* rc/fetch-refetch:
  docs: mention --refetch fetch option
  fetch: after refetch, encourage auto gc repacking
  t5615-partial-clone: add test for fetch --refetch
  fetch: add --refetch option
  builtin/fetch-pack: add --refetch option
  fetch-pack: add refetch
  fetch-negotiator: add specific noop initializer
pull/1232/head
Junio C Hamano 5 months ago
parent
commit
0f5e885173
  1. 6
      Documentation/config/remote.txt
  2. 10
      Documentation/fetch-options.txt
  3. 4
      Documentation/git-fetch-pack.txt
  4. 3
      Documentation/technical/partial-clone.txt
  5. 4
      builtin/fetch-pack.c
  6. 34
      builtin/fetch.c
  7. 5
      fetch-negotiator.c
  8. 8
      fetch-negotiator.h
  9. 46
      fetch-pack.c
  10. 1
      fetch-pack.h
  11. 6
      remote-curl.c
  12. 81
      t/t5616-partial-clone.sh
  13. 3
      transport-helper.c
  14. 4
      transport.c
  15. 4
      transport.h

6
Documentation/config/remote.txt

@ -82,5 +82,7 @@ remote.<name>.promisor::
objects.
remote.<name>.partialclonefilter::
The filter that will be applied when fetching from this
promisor remote.
The filter that will be applied when fetching from this promisor remote.
Changing or clearing this value will only affect fetches for new commits.
To fetch associated objects for commits already present in the local object
database, use the `--refetch` option of linkgit:git-fetch[1].

10
Documentation/fetch-options.txt

@ -163,6 +163,16 @@ endif::git-pull[]
behavior for a remote may be specified with the remote.<name>.tagOpt
setting. See linkgit:git-config[1].
ifndef::git-pull[]
--refetch::
Instead of negotiating with the server to avoid transferring commits and
associated objects that are already present locally, this option fetches
all objects as a fresh clone would. Use this to reapply a partial clone
filter from configuration or using `--filter=` when the filter
definition has changed. Automatic post-fetch maintenance will perform
object database pack consolidation to remove any duplicate objects.
endif::git-pull[]
--refmap=<refspec>::
When fetching refs listed on the command line, use the
specified refspec (can be given more than once) to map the

4
Documentation/git-fetch-pack.txt

@ -101,6 +101,10 @@ be in a separate packet, and the list must end with a flush packet.
current shallow boundary instead of from the tip of each
remote branch history.
--refetch::
Skips negotiating commits with the server in order to fetch all matching
objects. Use to reapply a new partial clone blob/tree filter.
--no-progress::
Do not show the progress.

3
Documentation/technical/partial-clone.txt

@ -181,6 +181,9 @@ Fetching Missing Objects
currently fetches all objects referred to by the requested objects, even
though they are not necessary.
- Fetching with `--refetch` will request a complete new filtered packfile from
the remote, which can be used to change a filter without needing to
dynamically fetch missing objects.
Using many promisor remotes
---------------------------

4
builtin/fetch-pack.c

@ -153,6 +153,10 @@ int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
args.from_promisor = 1;
continue;
}
if (!strcmp("--refetch", arg)) {
args.refetch = 1;
continue;
}
if (skip_prefix(arg, ("--filter="), &arg)) {
parse_list_objects_filter(&args.filter_options, arg);
continue;

34
builtin/fetch.c

@ -59,7 +59,7 @@ static int prune_tags = -1; /* unspecified */
static int all, append, dry_run, force, keep, multiple, update_head_ok;
static int write_fetch_head = 1;
static int verbosity, deepen_relative, set_upstream;
static int verbosity, deepen_relative, set_upstream, refetch;
static int progress = -1;
static int enable_auto_gc = 1;
static int tags = TAGS_DEFAULT, unshallow, update_shallow, deepen;
@ -190,6 +190,9 @@ static struct option builtin_fetch_options[] = {
OPT_SET_INT_F(0, "unshallow", &unshallow,
N_("convert to a complete repository"),
1, PARSE_OPT_NONEG),
OPT_SET_INT_F(0, "refetch", &refetch,
N_("re-fetch without negotiating common commits"),
1, PARSE_OPT_NONEG),
{ OPTION_STRING, 0, "submodule-prefix", &submodule_prefix, N_("dir"),
N_("prepend this to submodule path output"), PARSE_OPT_HIDDEN },
OPT_CALLBACK_F(0, "recurse-submodules-default",
@ -1304,6 +1307,14 @@ static int check_exist_and_connected(struct ref *ref_map)
if (deepen)
return -1;
/*
* Similarly, if we need to refetch, we always want to perform a full
* fetch ignoring existing objects.
*/
if (refetch)
return -1;
/*
* check_connected() allows objects to merely be promised, but
* we need all direct targets to exist.
@ -1517,6 +1528,8 @@ static struct transport *prepare_transport(struct remote *remote, int deepen)
set_option(transport, TRANS_OPT_DEEPEN_RELATIVE, "yes");
if (update_shallow)
set_option(transport, TRANS_OPT_UPDATE_SHALLOW, "yes");
if (refetch)
set_option(transport, TRANS_OPT_REFETCH, "yes");
if (filter_options.choice) {
const char *spec =
expand_list_objects_filter_spec(&filter_options);
@ -2293,8 +2306,25 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
NULL);
}
if (enable_auto_gc)
if (enable_auto_gc) {
if (refetch) {
/*
* Hint auto-maintenance strongly to encourage repacking,
* but respect config settings disabling it.
*/
int opt_val;
if (git_config_get_int("gc.autopacklimit", &opt_val))
opt_val = -1;
if (opt_val != 0)
git_config_push_parameter("gc.autoPackLimit=1");
if (git_config_get_int("maintenance.incremental-repack.auto", &opt_val))
opt_val = -1;
if (opt_val != 0)
git_config_push_parameter("maintenance.incremental-repack.auto=-1");
}
run_auto_maintenance(verbosity < 0);
}
cleanup:
string_list_clear(&list, 0);

5
fetch-negotiator.c

@ -23,3 +23,8 @@ void fetch_negotiator_init(struct repository *r,
return;
}
}
void fetch_negotiator_init_noop(struct fetch_negotiator *negotiator)
{
noop_negotiator_init(negotiator);
}

8
fetch-negotiator.h

@ -53,7 +53,15 @@ struct fetch_negotiator {
void *data;
};
/*
* Initialize a negotiator based on the repository settings.
*/
void fetch_negotiator_init(struct repository *r,
struct fetch_negotiator *negotiator);
/*
* Initialize a noop negotiator.
*/
void fetch_negotiator_init_noop(struct fetch_negotiator *negotiator);
#endif

46
fetch-pack.c

@ -312,19 +312,21 @@ static int find_common(struct fetch_negotiator *negotiator,
const char *remote_hex;
struct object *o;
/*
* If that object is complete (i.e. it is an ancestor of a
* local ref), we tell them we have it but do not have to
* tell them about its ancestors, which they already know
* about.
*
* We use lookup_object here because we are only
* interested in the case we *know* the object is
* reachable and we have already scanned it.
*/
if (((o = lookup_object(the_repository, remote)) != NULL) &&
(o->flags & COMPLETE)) {
continue;
if (!args->refetch) {
/*
* If that object is complete (i.e. it is an ancestor of a
* local ref), we tell them we have it but do not have to
* tell them about its ancestors, which they already know
* about.
*
* We use lookup_object here because we are only
* interested in the case we *know* the object is
* reachable and we have already scanned it.
*/
if (((o = lookup_object(the_repository, remote)) != NULL) &&
(o->flags & COMPLETE)) {
continue;
}
}
remote_hex = oid_to_hex(remote);
@ -692,6 +694,9 @@ static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
int old_save_commit_buffer = save_commit_buffer;
timestamp_t cutoff = 0;
if (args->refetch)
return;
save_commit_buffer = 0;
trace2_region_enter("fetch-pack", "parse_remote_refs_and_find_cutoff", NULL);
@ -1028,7 +1033,11 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
struct fetch_negotiator *negotiator;
negotiator = &negotiator_alloc;
fetch_negotiator_init(r, negotiator);
if (args->refetch) {
fetch_negotiator_init_noop(negotiator);
} else {
fetch_negotiator_init(r, negotiator);
}
sort_ref_list(&ref, ref_compare_name);
QSORT(sought, nr_sought, cmp_ref_by_name);
@ -1121,7 +1130,7 @@ static struct ref *do_fetch_pack(struct fetch_pack_args *args,
mark_complete_and_common_ref(negotiator, args, &ref);
filter_refs(args, &ref, sought, nr_sought);
if (everything_local(args, &ref)) {
if (!args->refetch && everything_local(args, &ref)) {
packet_flush(fd[1]);
goto all_done;
}
@ -1587,7 +1596,10 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
struct strvec index_pack_args = STRVEC_INIT;
negotiator = &negotiator_alloc;
fetch_negotiator_init(r, negotiator);
if (args->refetch)
fetch_negotiator_init_noop(negotiator);
else
fetch_negotiator_init(r, negotiator);
packet_reader_init(&reader, fd[0], NULL, 0,
PACKET_READ_CHOMP_NEWLINE |
@ -1613,7 +1625,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
/* Filter 'ref' by 'sought' and those that aren't local */
mark_complete_and_common_ref(negotiator, args, &ref);
filter_refs(args, &ref, sought, nr_sought);
if (everything_local(args, &ref))
if (!args->refetch && everything_local(args, &ref))
state = FETCH_DONE;
else
state = FETCH_SEND_REQUEST;

1
fetch-pack.h

@ -42,6 +42,7 @@ struct fetch_pack_args {
unsigned update_shallow:1;
unsigned reject_shallow_remote:1;
unsigned deepen:1;
unsigned refetch:1;
/*
* Indicate that the remote of this request is a promisor remote. The

6
remote-curl.c

@ -43,6 +43,7 @@ struct options {
/* see documentation of corresponding flag in fetch-pack.h */
from_promisor : 1,
refetch : 1,
atomic : 1,
object_format : 1,
force_if_includes : 1;
@ -198,6 +199,9 @@ static int set_option(const char *name, const char *value)
} else if (!strcmp(name, "from-promisor")) {
options.from_promisor = 1;
return 0;
} else if (!strcmp(name, "refetch")) {
options.refetch = 1;
return 0;
} else if (!strcmp(name, "filter")) {
options.filter = xstrdup(value);
return 0;
@ -1182,6 +1186,8 @@ static int fetch_git(struct discovery *heads,
strvec_push(&args, "--deepen-relative");
if (options.from_promisor)
strvec_push(&args, "--from-promisor");
if (options.refetch)
strvec_push(&args, "--refetch");
if (options.filter)
strvec_pushf(&args, "--filter=%s", options.filter);
strvec_push(&args, url.buf);

81
t/t5616-partial-clone.sh

@ -166,6 +166,85 @@ test_expect_success 'manual prefetch of missing objects' '
test_line_count = 0 observed.oids
'
# create new commits in "src" repo to establish a history on file.4.txt
# and push to "srv.bare".
test_expect_success 'push new commits to server for file.4.txt' '
for x in a b c d e f
do
echo "Mod file.4.txt $x" >src/file.4.txt &&
if list_contains "a,b" "$x"; then
printf "%10000s" X >>src/file.4.txt
fi &&
if list_contains "c,d" "$x"; then
printf "%20000s" X >>src/file.4.txt
fi &&
git -C src add file.4.txt &&
git -C src commit -m "mod $x" || return 1
done &&
git -C src push -u srv main
'
# Do partial fetch to fetch smaller files; then verify that without --refetch
# applying a new filter does not refetch missing large objects. Then use
# --refetch to apply the new filter on existing commits. Test it under both
# protocol v2 & v0.
test_expect_success 'apply a different filter using --refetch' '
git -C pc1 fetch --filter=blob:limit=999 origin &&
git -C pc1 rev-list --quiet --objects --missing=print \
main..origin/main >observed &&
test_line_count = 4 observed &&
git -C pc1 fetch --filter=blob:limit=19999 --refetch origin &&
git -C pc1 rev-list --quiet --objects --missing=print \
main..origin/main >observed &&
test_line_count = 2 observed &&
git -c protocol.version=0 -C pc1 fetch --filter=blob:limit=29999 \
--refetch origin &&
git -C pc1 rev-list --quiet --objects --missing=print \
main..origin/main >observed &&
test_line_count = 0 observed
'
test_expect_success 'fetch --refetch works with a shallow clone' '
git clone --no-checkout --depth=1 --filter=blob:none "file://$(pwd)/srv.bare" pc1s &&
git -C pc1s rev-list --objects --missing=print HEAD >observed &&
test_line_count = 6 observed &&
GIT_TRACE=1 git -C pc1s fetch --filter=blob:limit=999 --refetch origin &&
git -C pc1s rev-list --objects --missing=print HEAD >observed &&
test_line_count = 6 observed
'
test_expect_success 'fetch --refetch triggers repacking' '
GIT_TRACE2_CONFIG_PARAMS=gc.autoPackLimit,maintenance.incremental-repack.auto &&
export GIT_TRACE2_CONFIG_PARAMS &&
GIT_TRACE2_EVENT="$PWD/trace1.event" \
git -C pc1 fetch --refetch origin &&
test_subcommand git maintenance run --auto --no-quiet <trace1.event &&
grep \"param\":\"gc.autopacklimit\",\"value\":\"1\" trace1.event &&
grep \"param\":\"maintenance.incremental-repack.auto\",\"value\":\"-1\" trace1.event &&
GIT_TRACE2_EVENT="$PWD/trace2.event" \
git -c protocol.version=0 \
-c gc.autoPackLimit=0 \
-c maintenance.incremental-repack.auto=1234 \
-C pc1 fetch --refetch origin &&
test_subcommand git maintenance run --auto --no-quiet <trace2.event &&
grep \"param\":\"gc.autopacklimit\",\"value\":\"0\" trace2.event &&
grep \"param\":\"maintenance.incremental-repack.auto\",\"value\":\"-1\" trace2.event &&
GIT_TRACE2_EVENT="$PWD/trace3.event" \
git -c protocol.version=0 \
-c gc.autoPackLimit=1234 \
-c maintenance.incremental-repack.auto=0 \
-C pc1 fetch --refetch origin &&
test_subcommand git maintenance run --auto --no-quiet <trace3.event &&
grep \"param\":\"gc.autopacklimit\",\"value\":\"1\" trace3.event &&
grep \"param\":\"maintenance.incremental-repack.auto\",\"value\":\"0\" trace3.event
'
test_expect_success 'partial clone with transfer.fsckobjects=1 works with submodules' '
test_create_repo submodule &&
test_commit -C submodule mycommit &&
@ -225,7 +304,7 @@ test_expect_success 'use fsck before and after manually fetching a missing subtr
# Auto-fetch all remaining trees and blobs with --missing=error
git -C dst rev-list --missing=error --objects main >fetched_objects &&
test_line_count = 70 fetched_objects &&
test_line_count = 88 fetched_objects &&
awk -f print_1.awk fetched_objects |
xargs -n1 git -C dst cat-file -t >fetched_types &&

3
transport-helper.c

@ -715,6 +715,9 @@ static int fetch_refs(struct transport *transport,
if (data->transport_options.update_shallow)
set_helper_option(transport, "update-shallow", "true");
if (data->transport_options.refetch)
set_helper_option(transport, "refetch", "true");
if (data->transport_options.filter_options.choice) {
const char *spec = expand_list_objects_filter_spec(
&data->transport_options.filter_options);

4
transport.c

@ -250,6 +250,9 @@ static int set_git_option(struct git_transport_options *opts,
list_objects_filter_die_if_populated(&opts->filter_options);
parse_list_objects_filter(&opts->filter_options, value);
return 0;
} else if (!strcmp(name, TRANS_OPT_REFETCH)) {
opts->refetch = !!value;
return 0;
} else if (!strcmp(name, TRANS_OPT_REJECT_SHALLOW)) {
opts->reject_shallow = !!value;
return 0;
@ -384,6 +387,7 @@ static int fetch_refs_via_pack(struct transport *transport,
args.update_shallow = data->options.update_shallow;
args.from_promisor = data->options.from_promisor;
args.filter_options = data->options.filter_options;
args.refetch = data->options.refetch;
args.stateless_rpc = transport->stateless_rpc;
args.server_options = transport->server_options;
args.negotiation_tips = data->options.negotiation_tips;

4
transport.h

@ -16,6 +16,7 @@ struct git_transport_options {
unsigned update_shallow : 1;
unsigned reject_shallow : 1;
unsigned deepen_relative : 1;
unsigned refetch : 1;
/* see documentation of corresponding flag in fetch-pack.h */
unsigned from_promisor : 1;
@ -216,6 +217,9 @@ void transport_check_allowed(const char *type);
/* Filter objects for partial clone and fetch */
#define TRANS_OPT_LIST_OBJECTS_FILTER "filter"
/* Refetch all objects without negotiating */
#define TRANS_OPT_REFETCH "refetch"
/* Request atomic (all-or-nothing) updates when pushing */
#define TRANS_OPT_ATOMIC "atomic"

Loading…
Cancel
Save