2018-07-12 21:39:21 +02:00
|
|
|
#include "cache.h"
|
2018-07-12 21:39:33 +02:00
|
|
|
#include "config.h"
|
2018-07-12 21:39:22 +02:00
|
|
|
#include "csum-file.h"
|
2018-07-12 21:39:26 +02:00
|
|
|
#include "dir.h"
|
2018-07-12 21:39:22 +02:00
|
|
|
#include "lockfile.h"
|
2018-07-12 21:39:26 +02:00
|
|
|
#include "packfile.h"
|
2018-07-12 21:39:23 +02:00
|
|
|
#include "object-store.h"
|
2020-12-31 12:56:23 +01:00
|
|
|
#include "hash-lookup.h"
|
2018-07-12 21:39:21 +02:00
|
|
|
#include "midx.h"
|
2018-09-13 20:02:26 +02:00
|
|
|
#include "progress.h"
|
2019-03-21 20:36:13 +01:00
|
|
|
#include "trace2.h"
|
midx: implement midx_repack()
To repack with a non-zero batch-size, first sort all pack-files by
their modified time. Second, walk those pack-files from oldest
to newest, compute their expected size, and add the packs to a list
if they are smaller than the given batch-size. Stop when the total
expected size is at least the batch size.
If the batch size is zero, select all packs in the multi-pack-index.
Finally, collect the objects from the multi-pack-index that are in
the selected packs and send them to 'git pack-objects'. Write a new
multi-pack-index that includes the new pack.
Using a batch size of zero is very similar to a standard 'git repack'
command, except that we do not delete the old packs and instead rely
on the new multi-pack-index to prevent new processes from reading the
old packs. This does not disrupt other Git processes that are currently
reading the old packs based on the old multi-pack-index.
While first designing a 'git multi-pack-index repack' operation, I
started by collecting the batches based on the actual size of the
objects instead of the size of the pack-files. This allows repacking
a large pack-file that has very few referencd objects. However, this
came at a significant cost of parsing pack-files instead of simply
reading the multi-pack-index and getting the file information for
the pack-files. The "expected size" version provides similar
behavior, but could skip a pack-file if the average object size is
much larger than the actual size of the referenced objects, or
can create a large pack if the actual size of the referenced objects
is larger than the expected size.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-06-11 01:35:27 +02:00
|
|
|
#include "run-command.h"
|
2020-09-25 14:33:34 +02:00
|
|
|
#include "repository.h"
|
2021-02-18 15:07:33 +01:00
|
|
|
#include "chunk-format.h"
|
pack-revindex: write multi-pack reverse indexes
Implement the writing half of multi-pack reverse indexes. This is
nothing more than the format describe a few patches ago, with a new set
of helper functions that will be used to clear out stale .rev files
corresponding to old MIDXs.
Unfortunately, a very similar comparison function as the one implemented
recently in pack-revindex.c is reimplemented here, this time accepting a
MIDX-internal type. An effort to DRY these up would create more
indirection and overhead than is necessary, so it isn't pursued here.
Currently, there are no callers which pass the MIDX_WRITE_REV_INDEX
flag, meaning that this is all dead code. But, that won't be the case
for long, since subsequent patches will introduce the multi-pack bitmap,
which will begin passing this field.
(In midx.c:write_midx_internal(), the two adjacent if statements share a
conditional, but are written separately since the first one will
eventually also handle the MIDX_WRITE_BITMAP flag, which does not yet
exist.)
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-03-30 17:04:32 +02:00
|
|
|
#include "pack.h"
|
2021-08-31 22:52:24 +02:00
|
|
|
#include "pack-bitmap.h"
|
|
|
|
|
#include "refs.h"
|
|
|
|
|
#include "revision.h"
|
|
|
|
|
#include "list-objects.h"
|
2018-07-12 21:39:21 +02:00
|
|
|
|
2018-07-12 21:39:22 +02:00
|
|
|
#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
|
|
|
|
|
#define MIDX_VERSION 1
|
2018-07-12 21:39:23 +02:00
|
|
|
#define MIDX_BYTE_FILE_VERSION 4
|
|
|
|
|
#define MIDX_BYTE_HASH_VERSION 5
|
|
|
|
|
#define MIDX_BYTE_NUM_CHUNKS 6
|
|
|
|
|
#define MIDX_BYTE_NUM_PACKS 8
|
2018-07-12 21:39:22 +02:00
|
|
|
#define MIDX_HEADER_SIZE 12
|
2019-08-18 22:04:27 +02:00
|
|
|
#define MIDX_MIN_SIZE (MIDX_HEADER_SIZE + the_hash_algo->rawsz)
|
2018-07-12 21:39:22 +02:00
|
|
|
|
2018-07-12 21:39:27 +02:00
|
|
|
#define MIDX_CHUNK_ALIGNMENT 4
|
|
|
|
|
#define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
|
2018-07-12 21:39:31 +02:00
|
|
|
#define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
|
2018-07-12 21:39:30 +02:00
|
|
|
#define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
|
2018-07-12 21:39:32 +02:00
|
|
|
#define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
|
|
|
|
|
#define MIDX_CHUNKID_LARGEOFFSETS 0x4c4f4646 /* "LOFF" */
|
midx.c: make changing the preferred pack safe
The previous patch demonstrates a bug where a MIDX's auxiliary object
order can become out of sync with a MIDX bitmap.
This is because of two confounding factors:
- First, the object order is stored in a file which is named according
to the multi-pack index's checksum, and the MIDX does not store the
object order. This means that the object order can change without
altering the checksum.
- But the .rev file is moved into place with finalize_object_file(),
which link(2)'s the file into place instead of renaming it. For us,
that means that a modified .rev file will not be moved into place if
MIDX's checksum was unchanged.
This fix is to force the MIDX's checksum to change when the preferred
pack changes but the set of packs contained in the MIDX does not. In
other words, when the object order changes, the MIDX's checksum needs to
change with it (regardless of whether the MIDX is tracking the same or
different packs).
This prevents a race whereby changing the object order (but not the
packs themselves) enables a reader to see the new .rev file with the old
MIDX, or similarly seeing the new bitmap with the old object order.
But why can't we just stop hardlinking the .rev into place instead
adding additional data to the MIDX? Suppose that's what we did. Then
when we go to generate the new bitmap, we'll load the old MIDX bitmap,
along with the MIDX that it references. That's fine, since the new MIDX
isn't moved into place until after the new bitmap is generated. But the
new object order *has* been moved into place. So we'll read the old
bitmaps in the new order when generating the new bitmap file, meaning
that without this secondary change, bitmap generation itself would
become a victim of the race described here.
This can all be prevented by forcing the MIDX's checksum to change when
the object order does. By embedding the entire object order into the
MIDX, we do just that. That is, the MIDX's checksum will change in
response to any perturbation of the underlying object order. In t5326,
this will cause the MIDX's checksum to update (even without changing the
set of packs in the MIDX), preventing the stale read problem.
Note that this makes it safe to continue to link(2) the MIDX .rev file
into place, since it is now impossible to have a .rev file that is
out-of-sync with the MIDX whose checksum it references. (But we will do
away with MIDX .rev files later in this series anyway, so this is
somewhat of a moot point).
In theory, it is possible to store a "fingerprint" of the full object
order here, so long as that fingerprint changes at least as often as the
full object order does. Some possibilities here include storing the
identity of the preferred pack, along with the mtimes of the
non-preferred packs in a consistent order. But storing a limited part of
the information makes it difficult to reason about whether or not there
are gaps between the two that would cause us to get bitten by this bug
again.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Reviewed-by: Derrick Stolee <dstolee@microsoft.com>
Reviewed-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-01-25 23:41:03 +01:00
|
|
|
#define MIDX_CHUNKID_REVINDEX 0x52494458 /* "RIDX" */
|
2018-07-12 21:39:31 +02:00
|
|
|
#define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
|
2018-07-12 21:39:32 +02:00
|
|
|
#define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
|
|
|
|
|
#define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
|
|
|
|
|
#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
|
2018-07-12 21:39:27 +02:00
|
|
|
|
2019-06-11 01:35:25 +02:00
|
|
|
#define PACK_EXPIRED UINT_MAX
|
|
|
|
|
|
2020-08-17 16:04:48 +02:00
|
|
|
static uint8_t oid_version(void)
|
|
|
|
|
{
|
|
|
|
|
switch (hash_algo_by_ptr(the_hash_algo)) {
|
|
|
|
|
case GIT_HASH_SHA1:
|
|
|
|
|
return 1;
|
|
|
|
|
case GIT_HASH_SHA256:
|
|
|
|
|
return 2;
|
|
|
|
|
default:
|
|
|
|
|
die(_("invalid hash version"));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-08-31 22:52:21 +02:00
|
|
|
const unsigned char *get_midx_checksum(struct multi_pack_index *m)
|
pack-revindex: read multi-pack reverse indexes
Implement reading for multi-pack reverse indexes, as described in the
previous patch.
Note that these functions don't yet have any callers, and won't until
multi-pack reachability bitmaps are introduced in a later patch series.
In the meantime, this patch implements some of the infrastructure
necessary to support multi-pack bitmaps.
There are three new functions exposed by the revindex API:
- load_midx_revindex(): loads the reverse index corresponding to the
given multi-pack index.
- midx_to_pack_pos() and pack_pos_to_midx(): these convert between the
multi-pack index and pseudo-pack order.
load_midx_revindex() and pack_pos_to_midx() are both relatively
straightforward.
load_midx_revindex() needs a few functions to be exposed from the midx
API. One to get the checksum of a midx, and another to get the .rev's
filename. Similar to recent changes in the packed_git struct, three new
fields are added to the multi_pack_index struct: one to keep track of
the size, one to keep track of the mmap'd pointer, and another to point
past the header and at the reverse index's data.
pack_pos_to_midx() simply reads the corresponding entry out of the
table.
midx_to_pack_pos() is the trickiest, since it needs to find an object's
position in the psuedo-pack order, but that order can only be recovered
in the .rev file itself. This mapping can be implemented with a binary
search, but note that the thing we're binary searching over isn't an
array of values, but rather a permuted order of those values.
So, when comparing two items, it's helpful to keep in mind the
difference. Instead of a traditional binary search, where you are
comparing two things directly, here we're comparing a (pack, offset)
tuple with an index into the multi-pack index. That index describes
another (pack, offset) tuple, and it is _those_ two tuples that are
compared.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-03-30 17:04:26 +02:00
|
|
|
{
|
|
|
|
|
return m->data + m->data_len - the_hash_algo->rawsz;
|
|
|
|
|
}
|
|
|
|
|
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
void get_midx_filename(struct strbuf *out, const char *object_dir)
|
2018-07-12 21:39:22 +02:00
|
|
|
{
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
strbuf_addf(out, "%s/pack/multi-pack-index", object_dir);
|
2018-07-12 21:39:22 +02:00
|
|
|
}
|
|
|
|
|
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
void get_midx_rev_filename(struct strbuf *out, struct multi_pack_index *m)
|
pack-revindex: read multi-pack reverse indexes
Implement reading for multi-pack reverse indexes, as described in the
previous patch.
Note that these functions don't yet have any callers, and won't until
multi-pack reachability bitmaps are introduced in a later patch series.
In the meantime, this patch implements some of the infrastructure
necessary to support multi-pack bitmaps.
There are three new functions exposed by the revindex API:
- load_midx_revindex(): loads the reverse index corresponding to the
given multi-pack index.
- midx_to_pack_pos() and pack_pos_to_midx(): these convert between the
multi-pack index and pseudo-pack order.
load_midx_revindex() and pack_pos_to_midx() are both relatively
straightforward.
load_midx_revindex() needs a few functions to be exposed from the midx
API. One to get the checksum of a midx, and another to get the .rev's
filename. Similar to recent changes in the packed_git struct, three new
fields are added to the multi_pack_index struct: one to keep track of
the size, one to keep track of the mmap'd pointer, and another to point
past the header and at the reverse index's data.
pack_pos_to_midx() simply reads the corresponding entry out of the
table.
midx_to_pack_pos() is the trickiest, since it needs to find an object's
position in the psuedo-pack order, but that order can only be recovered
in the .rev file itself. This mapping can be implemented with a binary
search, but note that the thing we're binary searching over isn't an
array of values, but rather a permuted order of those values.
So, when comparing two items, it's helpful to keep in mind the
difference. Instead of a traditional binary search, where you are
comparing two things directly, here we're comparing a (pack, offset)
tuple with an index into the multi-pack index. That index describes
another (pack, offset) tuple, and it is _those_ two tuples that are
compared.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-03-30 17:04:26 +02:00
|
|
|
{
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
get_midx_filename(out, m->object_dir);
|
|
|
|
|
strbuf_addf(out, "-%s.rev", hash_to_hex(get_midx_checksum(m)));
|
pack-revindex: read multi-pack reverse indexes
Implement reading for multi-pack reverse indexes, as described in the
previous patch.
Note that these functions don't yet have any callers, and won't until
multi-pack reachability bitmaps are introduced in a later patch series.
In the meantime, this patch implements some of the infrastructure
necessary to support multi-pack bitmaps.
There are three new functions exposed by the revindex API:
- load_midx_revindex(): loads the reverse index corresponding to the
given multi-pack index.
- midx_to_pack_pos() and pack_pos_to_midx(): these convert between the
multi-pack index and pseudo-pack order.
load_midx_revindex() and pack_pos_to_midx() are both relatively
straightforward.
load_midx_revindex() needs a few functions to be exposed from the midx
API. One to get the checksum of a midx, and another to get the .rev's
filename. Similar to recent changes in the packed_git struct, three new
fields are added to the multi_pack_index struct: one to keep track of
the size, one to keep track of the mmap'd pointer, and another to point
past the header and at the reverse index's data.
pack_pos_to_midx() simply reads the corresponding entry out of the
table.
midx_to_pack_pos() is the trickiest, since it needs to find an object's
position in the psuedo-pack order, but that order can only be recovered
in the .rev file itself. This mapping can be implemented with a binary
search, but note that the thing we're binary searching over isn't an
array of values, but rather a permuted order of those values.
So, when comparing two items, it's helpful to keep in mind the
difference. Instead of a traditional binary search, where you are
comparing two things directly, here we're comparing a (pack, offset)
tuple with an index into the multi-pack index. That index describes
another (pack, offset) tuple, and it is _those_ two tuples that are
compared.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-03-30 17:04:26 +02:00
|
|
|
}
|
|
|
|
|
|
2021-02-18 15:07:36 +01:00
|
|
|
static int midx_read_oid_fanout(const unsigned char *chunk_start,
|
|
|
|
|
size_t chunk_size, void *data)
|
|
|
|
|
{
|
|
|
|
|
struct multi_pack_index *m = data;
|
|
|
|
|
m->chunk_oid_fanout = (uint32_t *)chunk_start;
|
|
|
|
|
|
|
|
|
|
if (chunk_size != 4 * 256) {
|
|
|
|
|
error(_("multi-pack-index OID fanout is of the wrong size"));
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-08-20 18:51:55 +02:00
|
|
|
struct multi_pack_index *load_multi_pack_index(const char *object_dir, int local)
|
2018-07-12 21:39:23 +02:00
|
|
|
{
|
|
|
|
|
struct multi_pack_index *m = NULL;
|
|
|
|
|
int fd;
|
|
|
|
|
struct stat st;
|
|
|
|
|
size_t midx_size;
|
|
|
|
|
void *midx_map = NULL;
|
|
|
|
|
uint32_t hash_version;
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
struct strbuf midx_name = STRBUF_INIT;
|
2018-07-12 21:39:27 +02:00
|
|
|
uint32_t i;
|
2018-07-12 21:39:28 +02:00
|
|
|
const char *cur_pack_name;
|
2021-02-18 15:07:36 +01:00
|
|
|
struct chunkfile *cf = NULL;
|
2018-07-12 21:39:23 +02:00
|
|
|
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
get_midx_filename(&midx_name, object_dir);
|
|
|
|
|
|
|
|
|
|
fd = git_open(midx_name.buf);
|
2018-07-12 21:39:23 +02:00
|
|
|
|
|
|
|
|
if (fd < 0)
|
|
|
|
|
goto cleanup_fail;
|
|
|
|
|
if (fstat(fd, &st)) {
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
error_errno(_("failed to read %s"), midx_name.buf);
|
2018-07-12 21:39:23 +02:00
|
|
|
goto cleanup_fail;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
midx_size = xsize_t(st.st_size);
|
|
|
|
|
|
|
|
|
|
if (midx_size < MIDX_MIN_SIZE) {
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
error(_("multi-pack-index file %s is too small"), midx_name.buf);
|
2018-07-12 21:39:23 +02:00
|
|
|
goto cleanup_fail;
|
|
|
|
|
}
|
|
|
|
|
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
strbuf_release(&midx_name);
|
2018-07-12 21:39:23 +02:00
|
|
|
|
|
|
|
|
midx_map = xmmap(NULL, midx_size, PROT_READ, MAP_PRIVATE, fd, 0);
|
2020-04-24 15:17:16 +02:00
|
|
|
close(fd);
|
2018-07-12 21:39:23 +02:00
|
|
|
|
2019-04-04 00:00:05 +02:00
|
|
|
FLEX_ALLOC_STR(m, object_dir, object_dir);
|
2018-07-12 21:39:23 +02:00
|
|
|
m->data = midx_map;
|
|
|
|
|
m->data_len = midx_size;
|
2018-08-20 18:51:55 +02:00
|
|
|
m->local = local;
|
2018-07-12 21:39:23 +02:00
|
|
|
|
|
|
|
|
m->signature = get_be32(m->data);
|
2018-09-13 20:02:15 +02:00
|
|
|
if (m->signature != MIDX_SIGNATURE)
|
|
|
|
|
die(_("multi-pack-index signature 0x%08x does not match signature 0x%08x"),
|
2018-07-12 21:39:23 +02:00
|
|
|
m->signature, MIDX_SIGNATURE);
|
|
|
|
|
|
|
|
|
|
m->version = m->data[MIDX_BYTE_FILE_VERSION];
|
2018-09-13 20:02:15 +02:00
|
|
|
if (m->version != MIDX_VERSION)
|
|
|
|
|
die(_("multi-pack-index version %d not recognized"),
|
2018-07-12 21:39:23 +02:00
|
|
|
m->version);
|
|
|
|
|
|
|
|
|
|
hash_version = m->data[MIDX_BYTE_HASH_VERSION];
|
2020-08-17 16:04:48 +02:00
|
|
|
if (hash_version != oid_version()) {
|
|
|
|
|
error(_("multi-pack-index hash version %u does not match version %u"),
|
|
|
|
|
hash_version, oid_version());
|
|
|
|
|
goto cleanup_fail;
|
|
|
|
|
}
|
2019-08-18 22:04:27 +02:00
|
|
|
m->hash_len = the_hash_algo->rawsz;
|
2018-07-12 21:39:23 +02:00
|
|
|
|
|
|
|
|
m->num_chunks = m->data[MIDX_BYTE_NUM_CHUNKS];
|
|
|
|
|
|
|
|
|
|
m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
|
|
|
|
|
|
2021-02-18 15:07:36 +01:00
|
|
|
cf = init_chunkfile(NULL);
|
2018-07-12 21:39:27 +02:00
|
|
|
|
2021-02-18 15:07:36 +01:00
|
|
|
if (read_table_of_contents(cf, m->data, midx_size,
|
|
|
|
|
MIDX_HEADER_SIZE, m->num_chunks))
|
|
|
|
|
goto cleanup_fail;
|
2018-07-12 21:39:27 +02:00
|
|
|
|
2021-02-18 15:07:36 +01:00
|
|
|
if (pair_chunk(cf, MIDX_CHUNKID_PACKNAMES, &m->chunk_pack_names) == CHUNK_NOT_FOUND)
|
2018-07-12 21:39:27 +02:00
|
|
|
die(_("multi-pack-index missing required pack-name chunk"));
|
2021-02-18 15:07:36 +01:00
|
|
|
if (read_chunk(cf, MIDX_CHUNKID_OIDFANOUT, midx_read_oid_fanout, m) == CHUNK_NOT_FOUND)
|
2018-07-12 21:39:31 +02:00
|
|
|
die(_("multi-pack-index missing required OID fanout chunk"));
|
2021-02-18 15:07:36 +01:00
|
|
|
if (pair_chunk(cf, MIDX_CHUNKID_OIDLOOKUP, &m->chunk_oid_lookup) == CHUNK_NOT_FOUND)
|
2018-07-12 21:39:30 +02:00
|
|
|
die(_("multi-pack-index missing required OID lookup chunk"));
|
2021-02-18 15:07:36 +01:00
|
|
|
if (pair_chunk(cf, MIDX_CHUNKID_OBJECTOFFSETS, &m->chunk_object_offsets) == CHUNK_NOT_FOUND)
|
2018-07-12 21:39:32 +02:00
|
|
|
die(_("multi-pack-index missing required object offsets chunk"));
|
2018-07-12 21:39:27 +02:00
|
|
|
|
2021-02-18 15:07:36 +01:00
|
|
|
pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets);
|
|
|
|
|
|
midx: read `RIDX` chunk when present
When a MIDX contains the new `RIDX` chunk, ensure that the reverse index
is read from it instead of the on-disk .rev file. Since we need to
encode the object order in the MIDX itself for correctness reasons,
there is no point in storing the same data again outside of the MIDX.
So, this patch stops writing separate .rev files, and reads it out of
the MIDX itself. This is possible to do with relatively little new code,
since the format of the RIDX chunk is identical to the data in the .rev
file. In other words, we can implement this by pointing the
`revindex_data` field at the reverse index chunk of the MIDX instead of
the .rev file without any other changes.
Note that we have two knobs that are adjusted for the new tests:
GIT_TEST_MIDX_WRITE_REV and GIT_TEST_MIDX_READ_RIDX. The former controls
whether the MIDX .rev is written at all, and the latter controls whether
we read the MIDX's RIDX chunk.
Both are necessary to ensure that the test added at the beginning of
this series continues to work. This is because we always need to write
the RIDX chunk in the MIDX in order to change its checksum, but we want
to make sure reading the existing .rev file still works (since the RIDX
chunk takes precedence by default).
Arguably this isn't a very interesting mode to test, because the
precedence rules mean that we'll always read the RIDX chunk over the
.rev file. But it makes it impossible for a user to induce corruption in
their repository by adjusting the test knobs (since if we had an
either/or knob they could stop writing the RIDX chunk, allowing them to
tweak the MIDX's object order without changing its checksum).
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Reviewed-by: Derrick Stolee <dstolee@microsoft.com>
Reviewed-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2022-01-25 23:41:17 +01:00
|
|
|
if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1))
|
|
|
|
|
pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex);
|
|
|
|
|
|
2018-07-12 21:39:31 +02:00
|
|
|
m->num_objects = ntohl(m->chunk_oid_fanout[255]);
|
|
|
|
|
|
2021-03-13 17:17:22 +01:00
|
|
|
CALLOC_ARRAY(m->pack_names, m->num_packs);
|
|
|
|
|
CALLOC_ARRAY(m->packs, m->num_packs);
|
2018-07-12 21:39:28 +02:00
|
|
|
|
|
|
|
|
cur_pack_name = (const char *)m->chunk_pack_names;
|
|
|
|
|
for (i = 0; i < m->num_packs; i++) {
|
|
|
|
|
m->pack_names[i] = cur_pack_name;
|
|
|
|
|
|
|
|
|
|
cur_pack_name += strlen(cur_pack_name) + 1;
|
|
|
|
|
|
2018-09-13 20:02:18 +02:00
|
|
|
if (i && strcmp(m->pack_names[i], m->pack_names[i - 1]) <= 0)
|
|
|
|
|
die(_("multi-pack-index pack names out of order: '%s' before '%s'"),
|
2018-07-12 21:39:28 +02:00
|
|
|
m->pack_names[i - 1],
|
|
|
|
|
m->pack_names[i]);
|
|
|
|
|
}
|
|
|
|
|
|
2019-03-21 20:36:13 +01:00
|
|
|
trace2_data_intmax("midx", the_repository, "load/num_packs", m->num_packs);
|
|
|
|
|
trace2_data_intmax("midx", the_repository, "load/num_objects", m->num_objects);
|
|
|
|
|
|
2021-10-21 05:39:47 +02:00
|
|
|
free_chunkfile(cf);
|
2018-07-12 21:39:23 +02:00
|
|
|
return m;
|
|
|
|
|
|
|
|
|
|
cleanup_fail:
|
|
|
|
|
free(m);
|
midx.c: write MIDX filenames to strbuf
To ask for the name of a MIDX and its corresponding .rev file, callers
invoke get_midx_filename() and get_midx_rev_filename(), respectively.
These both invoke xstrfmt(), allocating a chunk of memory which must be
freed later on.
This makes callers in pack-bitmap.c somewhat awkward. Specifically,
midx_bitmap_filename(), which is implemented like:
return xstrfmt("%s-%s.bitmap",
get_midx_filename(midx->object_dir),
hash_to_hex(get_midx_checksum(midx)));
this leaks the second argument to xstrfmt(), which itself was allocated
with xstrfmt(). This caller could assign both the result of
get_midx_filename() and the outer xstrfmt() to a temporary variable,
remembering to free() the former before returning. But that involves a
wasteful copy.
Instead, get_midx_filename() and get_midx_rev_filename() take a strbuf
as an output parameter. This way midx_bitmap_filename() can manipulate
and pass around a temporary buffer which it detaches back to its caller.
That allows us to implement the function without copying or open-coding
get_midx_filename() in a way that doesn't leak.
Update the other callers of get_midx_filename() and
get_midx_rev_filename() accordingly.
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-10-26 23:01:21 +02:00
|
|
|
strbuf_release(&midx_name);
|
2021-10-21 05:39:47 +02:00
|
|
|
free_chunkfile(cf);
|
2018-07-12 21:39:23 +02:00
|
|
|
if (midx_map)
|
|
|
|
|
munmap(midx_map, midx_size);
|
|
|
|
|
if (0 <= fd)
|
|
|
|
|
close(fd);
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2018-10-12 19:34:19 +02:00
|
|
|
void close_midx(struct multi_pack_index *m)
|
2018-07-12 21:39:36 +02:00
|
|
|
{
|
|
|
|
|
uint32_t i;
|
2018-10-12 19:34:19 +02:00
|
|
|
|
|
|
|
|
if (!m)
|
|
|
|
|
return;
|
|
|
|
|
|
2021-08-31 22:52:07 +02:00
|
|
|
close_midx(m->next);
|
|
|
|
|
|
2018-07-12 21:39:36 +02:00
|
|
|
munmap((unsigned char *)m->data, m->data_len);
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < m->num_packs; i++) {
|
midx: add packs to packed_git linked list
The multi-pack-index allows searching for objects across multiple
packs using one object list. The original design gains many of
these performance benefits by keeping the packs in the
multi-pack-index out of the packed_git list.
Unfortunately, this has one major drawback. If the multi-pack-index
covers thousands of packs, and a command loads many of those packs,
then we can hit the limit for open file descriptors. The
close_one_pack() method is used to limit this resource, but it
only looks at the packed_git list, and uses an LRU cache to prevent
thrashing.
Instead of complicating this close_one_pack() logic to include
direct references to the multi-pack-index, simply add the packs
opened by the multi-pack-index to the packed_git list. This
immediately solves the file-descriptor limit problem, but requires
some extra steps to avoid performance issues or other problems:
1. Create a multi_pack_index bit in the packed_git struct that is
one if and only if the pack was loaded from a multi-pack-index.
2. Skip packs with the multi_pack_index bit when doing object
lookups and abbreviations. These algorithms already check the
multi-pack-index before the packed_git struct. This has a very
small performance hit, as we need to walk more packed_git
structs. This is acceptable, since these operations run binary
search on the other packs, so this walk-and-ignore logic is
very fast by comparison.
3. When closing a multi-pack-index file, do not close its packs,
as those packs will be closed using close_all_packs(). In some
cases, such as 'git repack', we run 'close_midx()' without also
closing the packs, so we need to un-set the multi_pack_index bit
in those packs. This is necessary, and caught by running
t6501-freshen-objects.sh with GIT_TEST_MULTI_PACK_INDEX=1.
To manually test this change, I inserted trace2 logging into
close_pack_fd() and set pack_max_fds to 10, then ran 'git rev-list
--all --objects' on a copy of the Git repo with 300+ pack-files and
a multi-pack-index. The logs verified the packs are closed as
we read them beyond the file descriptor limit.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-29 18:18:56 +02:00
|
|
|
if (m->packs[i])
|
|
|
|
|
m->packs[i]->multi_pack_index = 0;
|
2018-07-12 21:39:36 +02:00
|
|
|
}
|
|
|
|
|
FREE_AND_NULL(m->packs);
|
|
|
|
|
FREE_AND_NULL(m->pack_names);
|
2021-08-31 22:52:07 +02:00
|
|
|
free(m);
|
2018-07-12 21:39:36 +02:00
|
|
|
}
|
|
|
|
|
|
2019-04-29 18:18:55 +02:00
|
|
|
int prepare_midx_pack(struct repository *r, struct multi_pack_index *m, uint32_t pack_int_id)
|
2018-07-12 21:39:34 +02:00
|
|
|
{
|
|
|
|
|
struct strbuf pack_name = STRBUF_INIT;
|
midx: add packs to packed_git linked list
The multi-pack-index allows searching for objects across multiple
packs using one object list. The original design gains many of
these performance benefits by keeping the packs in the
multi-pack-index out of the packed_git list.
Unfortunately, this has one major drawback. If the multi-pack-index
covers thousands of packs, and a command loads many of those packs,
then we can hit the limit for open file descriptors. The
close_one_pack() method is used to limit this resource, but it
only looks at the packed_git list, and uses an LRU cache to prevent
thrashing.
Instead of complicating this close_one_pack() logic to include
direct references to the multi-pack-index, simply add the packs
opened by the multi-pack-index to the packed_git list. This
immediately solves the file-descriptor limit problem, but requires
some extra steps to avoid performance issues or other problems:
1. Create a multi_pack_index bit in the packed_git struct that is
one if and only if the pack was loaded from a multi-pack-index.
2. Skip packs with the multi_pack_index bit when doing object
lookups and abbreviations. These algorithms already check the
multi-pack-index before the packed_git struct. This has a very
small performance hit, as we need to walk more packed_git
structs. This is acceptable, since these operations run binary
search on the other packs, so this walk-and-ignore logic is
very fast by comparison.
3. When closing a multi-pack-index file, do not close its packs,
as those packs will be closed using close_all_packs(). In some
cases, such as 'git repack', we run 'close_midx()' without also
closing the packs, so we need to un-set the multi_pack_index bit
in those packs. This is necessary, and caught by running
t6501-freshen-objects.sh with GIT_TEST_MULTI_PACK_INDEX=1.
To manually test this change, I inserted trace2 logging into
close_pack_fd() and set pack_max_fds to 10, then ran 'git rev-list
--all --objects' on a copy of the Git repo with 300+ pack-files and
a multi-pack-index. The logs verified the packs are closed as
we read them beyond the file descriptor limit.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-29 18:18:56 +02:00
|
|
|
struct packed_git *p;
|
2018-07-12 21:39:34 +02:00
|
|
|
|
|
|
|
|
if (pack_int_id >= m->num_packs)
|
2018-11-28 22:43:09 +01:00
|
|
|
die(_("bad pack-int-id: %u (%u total packs)"),
|
2018-09-13 20:02:25 +02:00
|
|
|
pack_int_id, m->num_packs);
|
2018-07-12 21:39:34 +02:00
|
|
|
|
|
|
|
|
if (m->packs[pack_int_id])
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
strbuf_addf(&pack_name, "%s/pack/%s", m->object_dir,
|
|
|
|
|
m->pack_names[pack_int_id]);
|
|
|
|
|
|
midx: add packs to packed_git linked list
The multi-pack-index allows searching for objects across multiple
packs using one object list. The original design gains many of
these performance benefits by keeping the packs in the
multi-pack-index out of the packed_git list.
Unfortunately, this has one major drawback. If the multi-pack-index
covers thousands of packs, and a command loads many of those packs,
then we can hit the limit for open file descriptors. The
close_one_pack() method is used to limit this resource, but it
only looks at the packed_git list, and uses an LRU cache to prevent
thrashing.
Instead of complicating this close_one_pack() logic to include
direct references to the multi-pack-index, simply add the packs
opened by the multi-pack-index to the packed_git list. This
immediately solves the file-descriptor limit problem, but requires
some extra steps to avoid performance issues or other problems:
1. Create a multi_pack_index bit in the packed_git struct that is
one if and only if the pack was loaded from a multi-pack-index.
2. Skip packs with the multi_pack_index bit when doing object
lookups and abbreviations. These algorithms already check the
multi-pack-index before the packed_git struct. This has a very
small performance hit, as we need to walk more packed_git
structs. This is acceptable, since these operations run binary
search on the other packs, so this walk-and-ignore logic is
very fast by comparison.
3. When closing a multi-pack-index file, do not close its packs,
as those packs will be closed using close_all_packs(). In some
cases, such as 'git repack', we run 'close_midx()' without also
closing the packs, so we need to un-set the multi_pack_index bit
in those packs. This is necessary, and caught by running
t6501-freshen-objects.sh with GIT_TEST_MULTI_PACK_INDEX=1.
To manually test this change, I inserted trace2 logging into
close_pack_fd() and set pack_max_fds to 10, then ran 'git rev-list
--all --objects' on a copy of the Git repo with 300+ pack-files and
a multi-pack-index. The logs verified the packs are closed as
we read them beyond the file descriptor limit.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-29 18:18:56 +02:00
|
|
|
p = add_packed_git(pack_name.buf, pack_name.len, m->local);
|
2018-07-12 21:39:34 +02:00
|
|
|
strbuf_release(&pack_name);
|
midx: add packs to packed_git linked list
The multi-pack-index allows searching for objects across multiple
packs using one object list. The original design gains many of
these performance benefits by keeping the packs in the
multi-pack-index out of the packed_git list.
Unfortunately, this has one major drawback. If the multi-pack-index
covers thousands of packs, and a command loads many of those packs,
then we can hit the limit for open file descriptors. The
close_one_pack() method is used to limit this resource, but it
only looks at the packed_git list, and uses an LRU cache to prevent
thrashing.
Instead of complicating this close_one_pack() logic to include
direct references to the multi-pack-index, simply add the packs
opened by the multi-pack-index to the packed_git list. This
immediately solves the file-descriptor limit problem, but requires
some extra steps to avoid performance issues or other problems:
1. Create a multi_pack_index bit in the packed_git struct that is
one if and only if the pack was loaded from a multi-pack-index.
2. Skip packs with the multi_pack_index bit when doing object
lookups and abbreviations. These algorithms already check the
multi-pack-index before the packed_git struct. This has a very
small performance hit, as we need to walk more packed_git
structs. This is acceptable, since these operations run binary
search on the other packs, so this walk-and-ignore logic is
very fast by comparison.
3. When closing a multi-pack-index file, do not close its packs,
as those packs will be closed using close_all_packs(). In some
cases, such as 'git repack', we run 'close_midx()' without also
closing the packs, so we need to un-set the multi_pack_index bit
in those packs. This is necessary, and caught by running
t6501-freshen-objects.sh with GIT_TEST_MULTI_PACK_INDEX=1.
To manually test this change, I inserted trace2 logging into
close_pack_fd() and set pack_max_fds to 10, then ran 'git rev-list
--all --objects' on a copy of the Git repo with 300+ pack-files and
a multi-pack-index. The logs verified the packs are closed as
we read them beyond the file descriptor limit.
Signed-off-by: Derrick Stolee <dstolee@microsoft.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-29 18:18:56 +02:00
|
|
|
|
|
|
|
|
if (!p)
|
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
|
|
p->multi_pack_index = 1;
|
|
|
|
|
m->packs[pack_int_id] = p;
|
|
|
|
|
install_packed_git(r, p);
|
|
|
|
|
list_add_tail(&p->mru, &r->objects->packed_git_mru);
|
|
|
|
|
|
|
|
|
|
return 0;
|
2018-07-12 21:39:34 +02:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result)
|
|
|
|
|
{
|
|
|
|
|
return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup,
|
2019-08-18 22:04:27 +02:00
|
|
|
the_hash_algo->rawsz, result);
|
2018-07-12 21:39:34 +02:00
|
|
|
}
|
|
|
|
|
|
2018-07-12 21:39:35 +02:00
|
|
|
struct object_id *nth_midxed_object_oid(struct object_id *oid,
|
|
|
|
|
struct multi_pack_index *m,
|
|
|
|
|
uint32_t n)
|
|
|
|
|
{
|
|
|
|
|
if (n >= m->num_objects)
|
|
|
|
|
return NULL;
|
|
|
|
|
|
2021-04-26 03:02:50 +02:00
|
|
|
oidread(oid, m->chunk_oid_lookup + m->hash_len * n);
|
2018-07-12 21:39:35 +02:00
|
|
|
return oid;
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-30 17:04:20 +02:00
|
|
|
off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos)
|
2018-07-12 21:39:34 +02:00
|
|
|
{
|
|
|
|
|
const unsigned char *offset_data;
|
|
|
|
|
uint32_t offset32;
|
|
|
|
|
|
2021-02-18 15:07:37 +01:00
|
|
|
offset_data = m->chunk_object_offsets + (off_t)pos * MIDX_CHUNK_OFFSET_WIDTH;
|
2018-07-12 21:39:34 +02:00
|
|
|
offset32 = get_be32(offset_data + sizeof(uint32_t));
|
|
|
|
|
|
|
|
|
|
if (m->chunk_large_offsets && offset32 & MIDX_LARGE_OFFSET_NEEDED) {
|
2018-09-13 20:02:23 +02:00
|
|
|
if (sizeof(off_t) < sizeof(uint64_t))
|
2018-07-12 21:39:34 +02:00
|
|
|
die(_("multi-pack-index stores a 64-bit offset, but off_t is too small"));
|
|
|
|
|
|
|
|
|
|
offset32 ^= MIDX_LARGE_OFFSET_NEEDED;
|
|
|
|
|
return get_be64(m->chunk_large_offsets + sizeof(uint64_t) * offset32);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return offset32;
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-30 17:04:20 +02:00
|
|
|
uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos)
|
2018-07-12 21:39:34 +02:00
|
|
|
{
|
2021-02-18 15:07:37 +01:00
|
|
|
return get_be32(m->chunk_object_offsets +
|
|
|
|
|
(off_t)pos * MIDX_CHUNK_OFFSET_WIDTH);
|
2018-07-12 21:39:34 +02:00
|
|
|
}
|
|
|
|
|
|
2021-09-11 22:39:31 +02:00
|
|
|
int fill_midx_entry(struct repository * r,
|
|
|
|
|
const struct object_id *oid,
|
|
|
|
|
struct pack_entry *e,
|
|
|
|
|
struct multi_pack_index *m)
|
2018-07-12 21:39:34 +02:00
|
|
|
{
|
2021-09-11 22:39:31 +02:00
|
|
|
uint32_t pos;
|
2018-07-12 21:39:34 +02:00
|
|
|
uint32_t pack_int_id;
|
|
|
|
|
struct packed_git *p;
|
|
|
|
|
|
2021-09-11 22:39:31 +02:00
|
|
|
if (!bsearch_midx(oid, m, &pos))
|
|
|
|
|
return 0;
|
|
|
|
|
|
2018-07-12 21:39:34 +02:00
|
|
|
if (pos >= m->num_objects)
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
pack_int_id = nth_midxed_pack_int_id(m, pos);
|
|
|
|
|
|
2019-04-29 18:18:55 +02:00
|
|
|
if (prepare_midx_pack(r, m, pack_int_id))
|
midx.c: protect against disappearing packs
When a packed object is stored in a multi-pack index, but that pack has
racily gone away, the MIDX code simply calls die(), when it could be
returning an error to the caller, which would in turn lead to
re-scanning the pack directory.
A pack can racily disappear, for example, due to a simultaneous 'git
repack -ad',
You can also reproduce this with two terminals, where one is running:
git init
while true; do
git commit -q --allow-empty -m foo
git repack -ad
git multi-pack-index write
done
(in effect, constantly writing new MIDXs), and the other is running:
obj=$(git rev-parse HEAD)
while true; do
echo $obj | git cat-file --batch-check='%(objectsize:disk)' || break
done
That will sometimes hit the error preparing packfile from
multi-pack-index message, which this patch fixes.
Right now, that path to discovering a missing pack looks something like
'find_pack_entry()' calling 'fill_midx_entry()' and eventually making
its way to call 'nth_midxed_pack_entry()'.
'nth_midxed_pack_entry()' already checks 'is_pack_valid()' and
propagates an error if the pack is invalid. So, this works if the pack
has gone away between calling 'prepare_midx_pack()' and before calling
'is_pack_valid()', but not if it disappears before then.
Catch the case where the pack has already disappeared before
'prepare_midx_pack()' by returning an error in that case, too.
Co-authored-by: Jeff King <peff@peff.net>
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-11-25 18:17:33 +01:00
|
|
|
return 0;
|
2018-07-12 21:39:34 +02:00
|
|
|
p = m->packs[pack_int_id];
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* We are about to tell the caller where they can locate the
|
|
|
|
|
* requested object. We better make sure the packfile is
|
|
|
|
|
* still here and can be accessed before supplying that
|
|
|
|
|
* answer, as it may have been deleted since the MIDX was
|
|
|
|
|
* loaded!
|
|
|
|
|
*/
|
|
|
|
|
if (!is_pack_valid(p))
|
|
|
|
|
return 0;
|
|
|
|
|
|
2021-09-11 22:43:26 +02:00
|
|
|
if (oidset_size(&p->bad_objects) &&
|
|
|
|
|
oidset_contains(&p->bad_objects, oid))
|
|
|
|
|
return 0;
|
2018-08-20 18:51:57 +02:00
|
|
|
|
2018-07-12 21:39:34 +02:00
|
|
|
e->offset = nth_midxed_offset(m, pos);
|
|
|
|
|
e->p = p;
|
|
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
midx: check both pack and index names for containment
A midx file (and the struct we parse from it) contains a list of all of
the covered packfiles, mentioned by their ".idx" names (e.g.,
"pack-1234.idx", etc). And thus calls to midx_contains_pack() expect
callers to provide the idx name.
This works for most of the calls, but the one in open_packed_git_1()
tries to feed a packed_git->pack_name, which is the ".pack" name,
meaning we'll never find a match (even if the pack is covered by the
midx).
We can fix this by converting the ".pack" to ".idx" in the caller.
However, that requires allocating a new string. Instead, let's make
midx_contains_pack() a bit friendlier, and allow it take _either_ the
.pack or .idx variant.
All cleverness in the matching code is credited to René. Bugs are mine.
There's no test here, because while this does fix _a_ bug, it's masked
by another bug in that same caller. That will be covered (with a test)
in the next patch.
Helped-by: René Scharfe <l.s.r@web.de>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-05 20:06:04 +02:00
|
|
|
/* Match "foo.idx" against either "foo.pack" _or_ "foo.idx". */
|
|
|
|
|
static int cmp_idx_or_pack_name(const char *idx_or_pack_name,
|
|
|
|
|
const char *idx_name)
|
|
|
|
|
{
|
|
|
|
|
/* Skip past any initial matching prefix. */
|
|
|
|
|
while (*idx_name && *idx_name == *idx_or_pack_name) {
|
|
|
|
|
idx_name++;
|
|
|
|
|
idx_or_pack_name++;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* If we didn't match completely, we may have matched "pack-1234." and
|
|
|
|
|
* be left with "idx" and "pack" respectively, which is also OK. We do
|
|
|
|
|
* not have to check for "idx" and "idx", because that would have been
|
|
|
|
|
* a complete match (and in that case these strcmps will be false, but
|
|
|
|
|
* we'll correctly return 0 from the final strcmp() below.
|
|
|
|
|
*
|
|
|
|
|
* Technically this matches "fooidx" and "foopack", but we'd never have
|
|
|
|
|
* such names in the first place.
|
|
|
|
|
*/
|
|
|
|
|
if (!strcmp(idx_name, "idx") && !strcmp(idx_or_pack_name, "pack"))
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* This not only checks for a complete match, but also orders based on
|
|
|
|
|
* the first non-identical character, which means our ordering will
|
|
|
|
|
* match a raw strcmp(). That makes it OK to use this to binary search
|
|
|
|
|
* a naively-sorted list.
|
|
|
|
|
*/
|
|
|
|
|
return strcmp(idx_or_pack_name, idx_name);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
|
2018-07-12 21:39:36 +02:00
|
|
|
{
|
|
|
|
|
uint32_t first = 0, last = m->num_packs;
|
|
|
|
|
|
|
|
|
|
while (first < last) {
|
|
|
|
|
uint32_t mid = first + (last - first) / 2;
|
|
|
|
|
const char *current;
|
|
|
|
|
int cmp;
|
|
|
|
|
|
|
|
|
|
current = m->pack_names[mid];
|
midx: check both pack and index names for containment
A midx file (and the struct we parse from it) contains a list of all of
the covered packfiles, mentioned by their ".idx" names (e.g.,
"pack-1234.idx", etc). And thus calls to midx_contains_pack() expect
callers to provide the idx name.
This works for most of the calls, but the one in open_packed_git_1()
tries to feed a packed_git->pack_name, which is the ".pack" name,
meaning we'll never find a match (even if the pack is covered by the
midx).
We can fix this by converting the ".pack" to ".idx" in the caller.
However, that requires allocating a new string. Instead, let's make
midx_contains_pack() a bit friendlier, and allow it take _either_ the
.pack or .idx variant.
All cleverness in the matching code is credited to René. Bugs are mine.
There's no test here, because while this does fix _a_ bug, it's masked
by another bug in that same caller. That will be covered (with a test)
in the next patch.
Helped-by: René Scharfe <l.s.r@web.de>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2019-04-05 20:06:04 +02:00
|
|
|
cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
|
2018-07-12 21:39:36 +02:00
|
|
|
if (!cmp)
|
|
|
|
|
return 1;
|
|
|
|
|
if (cmp > 0) {
|
|
|
|
|
first = mid + 1;
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
last = mid;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-08-20 18:51:55 +02:00
|
|
|
int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local)
|
2018-07-12 21:39:33 +02:00
|
|
|
{
|
2018-08-20 18:52:00 +02:00
|
|
|
struct multi_pack_index *m;
|
2018-07-12 21:39:33 +02:00
|
|
|
struct multi_pack_index *m_search;
|
|
|
|
|
|
2020-09-25 14:33:34 +02:00
|
|
|
prepare_repo_settings(r);
|
|
|
|
|
if (!r->settings.core_multi_pack_index)
|
2018-07-12 21:39:33 +02:00
|
|
|
return 0;
|
|
|
|
|
|
2018-08-20 18:52:00 +02:00
|
|
|
for (m_search = r->objects->multi_pack_index; m_search; m_search = m_search->next)
|
2018-07-12 21:39:33 +02:00
|
|
|
if (!strcmp(object_dir, m_search->object_dir))
|
|
|
|
|
return 1;
|
|
|
|
|
|
2018-08-20 18:52:00 +02:00
|
|
|
m = load_multi_pack_index(object_dir, local);
|
2018-07-12 21:39:33 +02:00
|
|
|
|
2018-08-20 18:52:00 +02:00
|
|
|
if (m) {
|
midx: traverse the local MIDX first
When a repository has an alternate object directory configured, callers
can traverse through each alternate's MIDX by walking the '->next'
pointer.
But, when 'prepare_multi_pack_index_one()' loads multiple MIDXs, it
places the new ones at the front of this pointer chain, not at the end.
This can be confusing for callers such as 'git repack -ad', causing test
failures like in t7700.6 with 'GIT_TEST_MULTI_PACK_INDEX=1'.
The occurs when dropping a pack known to the local MIDX with alternates
configured that have their own MIDX. Since the alternate's MIDX is
returned via 'get_multi_pack_index()', 'midx_contains_pack()' returns
true (which is correct, since it traverses through the '->next' pointer
to find the MIDX in the chain that does contain the requested object).
But, we call 'clear_midx_file()' on 'the_repository', which drops the
MIDX at the path of the first MIDX in the chain, which (in the case of
t7700.6 is the one in the alternate).
This patch addresses that by:
- placing the local MIDX first in the chain when calling
'prepare_multi_pack_index_one()', and
- introducing a new 'get_local_multi_pack_index()', which explicitly
returns the repository-local MIDX, if any.
Don't impose an additional order on the MIDX's '->next' pointer beyond
that the first item in the chain must be local if one exists so that we
avoid a quadratic insertion.
Likewise, use 'get_local_multi_pack_index()' in
'remove_redundant_pack()' to fix the formerly broken t7700.6 when run
with 'GIT_TEST_MULTI_PACK_INDEX=1'.
Finally, note that the MIDX ordering invariant is only preserved by the
insertion order in 'prepare_packed_git()', which traverses through the
ODB's '->next' pointer, meaning we visit the local object store first.
This fragility makes this an undesirable long-term solution if more
callers are added, but it is acceptable for now since this is the only
caller.
Helped-by: Jeff King <peff@peff.net>
Signed-off-by: Jeff King <peff@peff.net>
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2020-08-28 22:22:13 +02:00
|
|
|
struct multi_pack_index *mp = r->objects->multi_pack_index;
|
|
|
|
|
if (mp) {
|
|
|
|
|
m->next = mp->next;
|
|
|
|
|
mp->next = m;
|
|
|
|
|
} else
|
|
|
|
|
r->objects->multi_pack_index = m;
|
2018-07-12 21:39:33 +02:00
|
|
|
return 1;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 21:39:22 +02:00
|
|
|
static size_t write_midx_header(struct hashfile *f,
|
|
|
|
|
unsigned char num_chunks,
|
|
|
|
|
uint32_t num_packs)
|
|
|
|
|
{
|
|
|
|
|
hashwrite_be32(f, MIDX_SIGNATURE);
|
2020-09-06 10:59:02 +02:00
|
|
|
hashwrite_u8(f, MIDX_VERSION);
|
|
|
|
|
hashwrite_u8(f, oid_version());
|
|
|
|
|
hashwrite_u8(f, num_chunks);
|
|
|
|
|
hashwrite_u8(f, 0); /* unused */
|
2018-07-12 21:39:22 +02:00
|
|
|
hashwrite_be32(f, num_packs);
|
|
|
|
|
|
|
|
|
|
return MIDX_HEADER_SIZE;
|
|
|
|
|
}
|
|
|
|
|
|
2019-06-11 01:35:24 +02:00
|
|
|
struct pack_info {
|
|
|
|
|
uint32_t orig_pack_int_id;
|
|
|
|
|
char *pack_name;
|
|
|
|
|
struct packed_git *p;
|
2019-06-11 01:35:25 +02:00
|
|
|
unsigned expired : 1;
|
2019-06-11 01:35:24 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int pack_info_compare(const void *_a, const void *_b)
|
|
|
|
|
{
|
|
|
|
|
struct pack_info *a = (struct pack_info *)_a;
|
|
|
|
|
struct pack_info *b = (struct pack_info *)_b;
|
|
|
|
|
return strcmp(a->pack_name, b->pack_name);
|
|
|
|
|
}
|
|
|
|
|
|
2021-03-30 17:04:11 +02:00
|
|
|
static int idx_or_pack_name_cmp(const void *_va, const void *_vb)
|
|
|
|
|
{
|
|
|
|
|
const char *pack_name = _va;
|
|
|
|
|
const struct pack_info *compar = _vb;
|
|
|
|
|
|
|
|
|
|
return cmp_idx_or_pack_name(pack_name, compar->pack_name);
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-18 15:07:26 +01:00
|
|
|
struct write_midx_context {
|
2019-06-11 01:35:24 +02:00
|
|
|
struct pack_info *info;
|
2018-07-12 21:39:26 +02:00
|
|
|
uint32_t nr;
|
2019-06-11 01:35:24 +02:00
|
|
|
uint32_t alloc;
|
2018-07-12 21:39:36 +02:00
|
|
|
struct multi_pack_index *m;
|
2019-10-21 20:39:59 +02:00
|
|
|
struct progress *progress;
|
|
|
|
|
unsigned pack_paths_checked;
|
2021-02-18 15:07:28 +01:00
|
|
|
|
|
|
|
|
struct pack_midx_entry *entries;
|
|
|
|
|
uint32_t entries_nr;
|
2021-02-18 15:07:29 +01:00
|
|
|
|
|
|
|
|
uint32_t *pack_perm;
|
pack-revindex: write multi-pack reverse indexes
Implement the writing half of multi-pack reverse indexes. This is
nothing more than the format describe a few patches ago, with a new set
of helper functions that will be used to clear out stale .rev files
corresponding to old MIDXs.
Unfortunately, a very similar comparison function as the one implemented
recently in pack-revindex.c is reimplemented here, this time accepting a
MIDX-internal type. An effort to DRY these up would create more
indirection and overhead than is necessary, so it isn't pursued here.
Currently, there are no callers which pass the MIDX_WRITE_REV_INDEX
flag, meaning that this is all dead code. But, that won't be the case
for long, since subsequent patches will introduce the multi-pack bitmap,
which will begin passing this field.
(In midx.c:write_midx_internal(), the two adjacent if statements share a
conditional, but are written separately since the first one will
eventually also handle the MIDX_WRITE_BITMAP flag, which does not yet
exist.)
Signed-off-by: Taylor Blau <me@ttaylorr.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2021-03-30 17:04:32 +02:00
|
|
|
uint32_t *pack_order;
|
2021-02-18 15:07:29 +01:00
|
|
|
unsigned large_offsets_needed:1;
|
2021-02-18 15:07:30 +01:00
|
|
|
uint32_t num_large_offsets;
|
2021-03-30 17:04:11 +02:00
|
|
|
|
|
|
|
|
int preferred_pack_idx;
|
2021-09-29 03:55:01 +02:00
|
|
|
|
|
|
|
|
struct string_list *to_include;
|
2018-07-12 21:39:26 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static void add_pack_to_midx(const char *full_path, size_t full_path_len,
|
|
|
|
|
const char *file_name, void *data)
|
|
|
|
|
{
|
2021-02-18 15:07:26 +01:00
|
|
|
struct write_midx_context *ctx = data;
|
2018-07-12 21:39:26 +02:00
|
|
|
|
|
|
|
|
if (ends_with(file_name, ".idx")) {
|
2021-02-18 15:07:26 +01:00
|
|
|
display_progress(ctx->progress, ++ctx->pack_paths_checked);
|
2021-09-29 03:55:01 +02:00
|
|
|
/*
|
|
|
|
|
* Note that at most one of ctx->m and ctx->to_include are set,
|
|
|
|
|
* so we are testing midx_contains_pack() and
|
|
|
|
|
* string_list_has_string() independently (guarded by the
|
|
|
|
|
* appropriate NULL checks).
|
|
|
|
|
*
|
|
|
|
|
* We could support passing to_include while reusing an existing
|
|
|
|
|
* MIDX, but don't currently since the reuse process drags
|
|
|
|
|
* forward all packs from an existing MIDX (without checking
|
|
|
|
|
* whether or not they appear in the to_include list).
|
|
|
|
|
*
|
|
|
|
|
* If we added support for that, these next two conditional
|
|
|
|
|
* should be performed independently (likely checking
|
|
|
|
|
* to_include before the existing MIDX).
|
|
|
|
|
*/
|
2021-02-18 15:07:26 +01:00
|
|
|
if (ctx->m && midx_contains_pack(ctx->m, file_name))
|
2018-07-12 21:39:36 +02:00
|
|
|
return;
|
2021-09-29 03:55:01 +02:00
|
|
|
else if (ctx->to_include &&
|
|
|
|
|
!string_list_has_string(ctx->to_include, file_name))
|
|
|
|
|
return;
|
2018-07-12 21:39:36 +02:00
|
|
|
|
2021-02-18 15:07:26 +01:00
|
|
|
ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
|
2018-07-12 21:39:26 +02:00
|
|
|
|
2021-02-18 15:07:26 +01:00
|
|
|
ctx->info[ctx->nr].p = add_packed_git(full_path,
|
|
|
|
|
full_path_len,
|
|
|
|
|
0);
|
2018-07-12 21:39:29 +02:00
|
|
|
|
2021-02-18 15:07:26 +01:00
|
|
|
if (!ctx->info[ctx->nr].p) {
|
2018-07-12 21:39:26 +02:00
|
|
|
warning(_("failed to add packfile '%s'"),
|
|
|
|
|
full_path);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-18 15:07:26 +01:00
|
|
|
if (open_pack_index(ctx->info[ctx->nr].p)) {
|
2018-07-12 21:39:29 +02:00
|
|
|
warning(_("failed to open pack-index '%s'"),
|
|
|
|
|
full_path);
|
2021-02-18 15:07:26 +01:00
|
|
|
close_pack(ctx->info[ctx->nr].p);
|
|
|
|
|
FREE_AND_NULL(ctx->info[ctx->nr].p);
|
2018-07-12 21:39:29 +02:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
2021-02-18 15:07:26 +01:00
|
|
|
ctx->info[ctx->nr].pack_name = xstrdup(file_name);
|
|
|
|
|
ctx->info[ctx->nr].orig_pack_int_id = ctx->nr;
|
|
|
|
|
ctx->info[ctx->nr].expired = 0;
|
|
|
|
|
ctx->nr++;
|
2018-07-12 21:39:26 +02:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-07-12 21:39:29 +02:00
|
|
|
struct pack_midx_entry {
|
|
|
|
|
struct object_id oid;
|
|
|
|
|
uint32_t pack_int_id;
|
|
|
|
|
time_t pack_mtime;
|
|
|
|
|
uint64_t offset;
|
2021-03-30 17:04:11 +02:00
|
|
|
unsigned preferred : 1;
|
2018-07-12 21:39:29 +02:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int midx_oid_compare(const void *_a, const void *_b)
|
|
|
|
|
{
|
|
|
|
|
const struct pack_midx_entry *a = (const struct pack_midx_entry *)_a;
|
|
|
|
|
const struct pack_midx_entry *b = (const struct pack_midx_entry *)_b;
|
|
|
|
|
int cmp = oidcmp(&a->oid, &b->oid);
|
|
|
|
|
|
|
|
|
|
if (cmp)
|
|
|
|
|
return cmp;
|
|
|
|
|
|
2021-03-30 17:04:11 +02:00
|
|
|
/* Sort objects in a preferred pack first when multiple copies exist. */
|
|
|