Merge pull request #1017 from AladW/fetch-mirror
lib: add `aur-fetch--mirror`
This commit is contained in:
Alad Wenter
GitHub
commit
76420297a5
|
|
@ -3,8 +3,10 @@
|
|||
[[ -v AUR_DEBUG ]] && set -o xtrace
|
||||
shopt -s extglob
|
||||
argv0=fetch
|
||||
AUR_LOCATION=${AUR_LOCATION:-https://aur.archlinux.org}
|
||||
XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
|
||||
XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
|
||||
AUR_FETCH_USE_MIRROR=${AUR_FETCH_USE_MIRROR:-0}
|
||||
AUR_LOCATION=${AUR_LOCATION:-https://aur.archlinux.org}
|
||||
PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[1]}(): }'
|
||||
|
||||
# Author information for merge commits
|
||||
|
|
@ -115,6 +117,7 @@ if (( ! $# )); then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
# XXX: race with concurrent processes
|
||||
if [[ -v results_file ]]; then
|
||||
: >"$results_file" || exit 1 # truncate file
|
||||
fi
|
||||
|
|
@ -124,27 +127,75 @@ if (( ! ${#merge_args[@]} )); then
|
|||
merge_args=(--ff-only)
|
||||
fi
|
||||
|
||||
# Main loop
|
||||
if (( recurse )); then
|
||||
aur depends --pkgbase "$@" # stdin handled by aur-depends
|
||||
mapfile -t packages < <(aur depends --pkgbase "$@")
|
||||
wait "$!"
|
||||
|
||||
elif (( $# == 1 )) && [[ $1 == "-" || $1 == "/dev/stdin" ]]; then
|
||||
tee # noop
|
||||
mapfile -t packages
|
||||
else
|
||||
printf '%s\n' "$@"
|
||||
fi | while read -r pkg; do
|
||||
packages=("$@")
|
||||
set --
|
||||
fi
|
||||
|
||||
# Update revisions in local AUR mirror
|
||||
declare -A local_clones
|
||||
|
||||
# With an AUR mirror, updates are retrieved in two steps. First, updates to the
|
||||
# mirror are synchronized with `git-fetch`. Secondly, local clones of the miror
|
||||
# are created and are updated with `git-fetch` and `git-merge` as usual.
|
||||
if (( AUR_FETCH_USE_MIRROR )); then
|
||||
while read -r IFS=':' pkg head; do
|
||||
local_clones[$pkg]=$head
|
||||
done < <(aur fetch--mirror --lclone "${packages[@]}")
|
||||
wait "$!"
|
||||
fi
|
||||
|
||||
# Main loop
|
||||
for pkg in "${packages[@]}"; do
|
||||
unset -f git
|
||||
if (( AUR_FETCH_USE_MIRROR )); then # branch origin/$pkg -> master
|
||||
upstream=origin/$pkg
|
||||
else
|
||||
upstream=origin/master
|
||||
fi
|
||||
|
||||
# Verify if the repository is hosted on AUR (#959)
|
||||
if (( existing )) && ! git ls-remote --exit-code "$AUR_LOCATION/$pkg" >/dev/null; then
|
||||
printf >&2 '%s: warning: package %s is not in AUR, skipping\n' "$argv0" "$pkg"
|
||||
continue
|
||||
|
||||
elif [[ -d $pkg/.git ]]; then
|
||||
# Clone package if not existing
|
||||
elif [[ ! -d $pkg/.git ]]; then
|
||||
# Differentiate between local (fetch--mirror) and aurweb clones
|
||||
if [[ ! ${local_clones[$pkg]} ]]; then
|
||||
git clone "$AUR_LOCATION/$pkg" || exit 1
|
||||
head=$(git -C "$pkg" rev-parse --verify quiet HEAD)
|
||||
else
|
||||
printf "Cloning into '%s'\n" "$pkg"
|
||||
head=${local_clones[$pkg]}
|
||||
fi >&2
|
||||
|
||||
if [[ $head ]]; then
|
||||
git -C "$pkg" --no-pager log --pretty=reference -1
|
||||
fi >&2
|
||||
|
||||
# XXX: race with multiple instances
|
||||
if [[ -v results_file ]]; then
|
||||
results 'clone' "$git_empty_object" "${head:-$git_empty_object}" "$PWD/$pkg" "$results_file"
|
||||
fi
|
||||
|
||||
# Update existing git repository
|
||||
else
|
||||
# Per-package lock
|
||||
exec {fd}< "$pkg"/.git
|
||||
flock --wait 5 "$fd" || exit 1
|
||||
|
||||
# Avoid issues with filesystem boundaries (#274)
|
||||
git() { command git -C "$pkg" "$@"; }
|
||||
|
||||
# Retrieve new upstream commits
|
||||
git fetch -v origin >&2 || exit
|
||||
git fetch origin >&2 || exit
|
||||
|
||||
# Store original HEAD for --results output
|
||||
orig_head=$(git rev-parse --verify --quiet HEAD)
|
||||
|
|
@ -165,10 +216,10 @@ fi | while read -r pkg; do
|
|||
;;& # proceed to merge or rebase
|
||||
rebase)
|
||||
dest='HEAD'
|
||||
git rebase -v "${rebase_args[@]}" origin/master ;;
|
||||
git rebase "${rebase_args[@]}" "$upstream" ;;
|
||||
merge)
|
||||
dest='HEAD'
|
||||
git merge -v "${merge_args[@]}" origin/master ;;
|
||||
git merge "${merge_args[@]}" "$upstream" ;;
|
||||
reset)
|
||||
dest='master@{u}'
|
||||
git reset --hard 'master@{u}' ;;
|
||||
|
|
@ -184,20 +235,7 @@ fi | while read -r pkg; do
|
|||
if [[ -v results_file ]]; then
|
||||
results "$sync_pkg" "$orig_head" "$head" "$PWD/$pkg" "$results_file"
|
||||
fi
|
||||
|
||||
# Otherwise, try to clone anew
|
||||
elif git clone "$AUR_LOCATION/$pkg" >&2; then
|
||||
if head=$(git -C "$pkg" rev-parse --verify --quiet HEAD); then
|
||||
git -C "$pkg" --no-pager log --pretty=reference -1 >&2
|
||||
fi
|
||||
head=${head:-$git_empty_object}
|
||||
|
||||
if [[ -v results_file ]]; then
|
||||
results 'clone' "$git_empty_object" "$head" "$PWD/$pkg" "$results_file"
|
||||
fi
|
||||
else
|
||||
printf >&2 '%s: failed to clone repository %s\n' "$argv0" "$pkg"
|
||||
exit 1
|
||||
exec {fd}<&- # release lock
|
||||
fi
|
||||
done
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,113 @@
|
|||
#!/bin/bash
|
||||
set -o errexit
|
||||
[[ -v AUR_DEBUG ]] && set -o xtrace
|
||||
#argv0=fetch--mirror
|
||||
XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
|
||||
AUR_MIRROR=${AUR_MIRROR:-https://github.com/archlinux/aur}
|
||||
AUR_MIRROR_TTL=${AUR_MIRROR_TTL:-300}
|
||||
AUR_ROOT=${AUR_ROOT:-$XDG_CACHE_HOME/aur}
|
||||
startdir=$PWD
|
||||
local_clones=0
|
||||
|
||||
if [[ $1 = '--lclone' ]]; then
|
||||
local_clones=1
|
||||
shift
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# Step 1. Create a sparse checkout of the AUR mirror
|
||||
git_active_remotes() {
|
||||
git config --get-all remote.origin.fetch | awk -F'[/:]' '{print $3}'
|
||||
}
|
||||
|
||||
# XXX: this includes refs/heads (which we do not use) but unlike
|
||||
# git-config this accepts multiple arguments in one call
|
||||
git_set_branches() {
|
||||
git remote set-branches origin --add main # ensure at least 1 remote
|
||||
sort -u | xargs -d '\n' git remote set-branches origin
|
||||
}
|
||||
|
||||
git_ls_remote_ttl() {
|
||||
local cachetime now ttl=$AUR_MIRROR_TTL
|
||||
printf -v now '%(%s)T' -1
|
||||
|
||||
if ! cachetime=$(stat -c %Y "$1" 2>/dev/null) || (( now > (cachetime + ttl) )); then
|
||||
git ls-remote origin 'refs/heads/*' | awk -F/ '{print $3}' >"$1"
|
||||
fi
|
||||
}
|
||||
|
||||
exec {fd}< "$AUR_ROOT"
|
||||
{ flock --wait 5 "$fd" || exit 1
|
||||
|
||||
# Shallow clone
|
||||
[[ ! -d $AUR_ROOT ]] && git clone --depth=1 --bare "$AUR_MIRROR" "$AUR_ROOT"
|
||||
cd "$AUR_ROOT"
|
||||
|
||||
# Keep a refcache to filter out packages which are not in AUR. This is
|
||||
# includes unlisted packages (unlike `aur pkglist --pkgbase`)
|
||||
git_ls_remote_ttl 'remote-pkgbase'
|
||||
|
||||
# Retrieve list of active remotes
|
||||
mapfile -t active_remotes < <(git_active_remotes | grep -v 'main')
|
||||
|
||||
# Only consider AUR targets
|
||||
mapfile -t target_remotes < <(printf '%s\n' "$@" | grep -Fxf 'remote-pkgbase')
|
||||
|
||||
# Set remote branches as union of active remotes and arguments. If '*' is set
|
||||
# instead, `git fetch origin` in $AUR_ROOT will pull in the entire history.
|
||||
printf '%s\n' "${target_remotes[@]}" "${active_remotes[@]}" | git_set_branches
|
||||
|
||||
# Retrieve objects
|
||||
git fetch origin "${target_remotes[@]}"
|
||||
|
||||
# Resetting branches is only needed when .git/config contains both refs/heads
|
||||
# and refs/remotes/origin, i.e. when using git-clone. `--lclone` uses git-init
|
||||
# and refspec manipulation instead.
|
||||
# for b in "${target_remotes[@]}"; do
|
||||
# git branch --force "$b" origin/"$b"
|
||||
# done
|
||||
} >&2
|
||||
|
||||
# ------------------------------------------------------------
|
||||
# Step 2. Propagate each branch in the mirror to a local clone
|
||||
(( ! local_clones )) && exit
|
||||
results=()
|
||||
|
||||
git_srcdest_origin() {
|
||||
local refspec=refs/remotes/origin/$1
|
||||
|
||||
git config remote.origin.fetch "+$refspec:$refspec"
|
||||
git fetch --quiet
|
||||
git checkout -B master "$refspec"
|
||||
}
|
||||
|
||||
for b in "${target_remotes[@]}"; do
|
||||
cd "$startdir"
|
||||
|
||||
# XXX: "Cloning into an existing directory is only allowed if the
|
||||
# directory is empty"
|
||||
if [[ ! -d $b/.git ]] && mkdir "$b"; then
|
||||
cd "$b"
|
||||
git init
|
||||
git remote add origin "$AUR_ROOT"
|
||||
|
||||
# Compared to git clone --local, using $refspec as both source and
|
||||
# destination allows updates from `git-fetch` in $AUR_ROOT to be
|
||||
# directly visible in local clones. $AUR_ROOT does not require
|
||||
# local branches or refs/heads for this to work.
|
||||
git_srcdest_origin "$b"
|
||||
|
||||
# Export results to aur-fetch
|
||||
head=$(git rev-parse --verify --quiet HEAD)
|
||||
results+=("$b:$head")
|
||||
fi >&2
|
||||
done
|
||||
|
||||
# The lock is only released here, because we do not want an instance to modify
|
||||
# AUR_ROOT while local git-clones are still underway. Additionally, local clones
|
||||
# use cp(1) and are thus a cheap operation. See --local in git-clone(1).
|
||||
exec {fd}<&-
|
||||
|
||||
if (( ${#results[@]} )); then
|
||||
printf '%s\n' "${results[@]}"
|
||||
fi
|
||||
|
|
@ -5,6 +5,12 @@
|
|||
- `pacman -Syu --config` is replaced by `pacsync <repo>` and `pacman -S <repo>/<pkg>`
|
||||
- local repository upgrades are now unaffected by `--pacman-conf`
|
||||
|
||||
* `aur-fetch`
|
||||
+ add `aur-fetch--mirror` helper for `aur.git` mirrors
|
||||
- defaults to `github.com/archlinux/aur`
|
||||
- enabled with `AUR_FETCH_USE_MIRROR=1`
|
||||
+ run flock(1) on existing repositories
|
||||
|
||||
* `aur-graph`
|
||||
+ selectively disable/enable depends with `aur graph -v <TYPE>=[0|1]`
|
||||
- supported types: `DEPENDS`, `MAKEDEPENDS`, `CHECKDEPENDS`, `OPTDEPENDS`
|
||||
|
|
|
|||
|
|
@ -195,6 +195,57 @@ and
|
|||
.I <head_to>
|
||||
will be identical.
|
||||
.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B AUR_LOCATION
|
||||
URI where repositories are cloned from. Defaults to
|
||||
.IR https://aur.archlinux.org .
|
||||
.
|
||||
.TP
|
||||
.B AUR_MIRROR
|
||||
URI to an AUR mirror, a
|
||||
.BR git (1)
|
||||
repository where each package is contained in a
|
||||
.BR git\-branch (1).
|
||||
Defaults to
|
||||
.MT https://github.com/archlinux/aur
|
||||
.ME .
|
||||
.
|
||||
.TP
|
||||
.B AUR_MIRROR_TTL
|
||||
When updating an AUR mirror, a list of all branches is retrieved with
|
||||
.BR git\-ls\-remote
|
||||
to validate arguments. The
|
||||
.B AUR_MIRROR_TTL
|
||||
variable defines (in seconds) how often this command is run. For accurate
|
||||
results, this should roughly equal the synchronization interval between AUR and
|
||||
its mirrors. Defaults to
|
||||
.IR 300 .
|
||||
.
|
||||
.TP
|
||||
.B AUR_FETCH_USE_MIRROR
|
||||
If this variable is set to a positive value, clone repositories from an AUR
|
||||
mirror instead of
|
||||
.BR AUR_LOCATION .
|
||||
Upstream changes are merged from
|
||||
.I origin/<package>
|
||||
instead of
|
||||
.IR origin/master .
|
||||
.IP
|
||||
Remotes of existing
|
||||
.BR git (1)
|
||||
repositories are preserved. When enabling or disabling
|
||||
.BR AUR_FETCH_MIRROR ,
|
||||
repositories should be converted manually with
|
||||
.B git\-remote set\-url
|
||||
and
|
||||
.BR "git\-remote set\-branches origin" .
|
||||
.
|
||||
.TP
|
||||
.B AUR_ROOT
|
||||
Directory where an AUR mirror is cloned to. Defaults to
|
||||
.IR XDG_CACHE_HOME/aur .
|
||||
.
|
||||
.SH SEE ALSO
|
||||
.ad l
|
||||
.nh
|
||||
|
|
|
|||
Loading…
Reference in New Issue