Merge pull request #1017 from AladW/fetch-mirror

lib: add `aur-fetch--mirror`
pull/1018/head
Alad Wenter 2 months ago committed by GitHub
commit 76420297a5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 86
      lib/aur-fetch
  2. 113
      lib/aur-fetch--mirror
  3. 6
      makepkg/aurutils.changelog
  4. 51
      man1/aur-fetch.1

@ -3,8 +3,10 @@
[[ -v AUR_DEBUG ]] && set -o xtrace
shopt -s extglob
argv0=fetch
AUR_LOCATION=${AUR_LOCATION:-https://aur.archlinux.org}
XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
XDG_CONFIG_HOME=${XDG_CONFIG_HOME:-$HOME/.config}
AUR_FETCH_USE_MIRROR=${AUR_FETCH_USE_MIRROR:-0}
AUR_LOCATION=${AUR_LOCATION:-https://aur.archlinux.org}
PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[1]}(): }'
# Author information for merge commits
@ -115,6 +117,7 @@ if (( ! $# )); then
exit 1
fi
# XXX: race with concurrent processes
if [[ -v results_file ]]; then
: >"$results_file" || exit 1 # truncate file
fi
@ -124,27 +127,75 @@ if (( ! ${#merge_args[@]} )); then
merge_args=(--ff-only)
fi
# Main loop
if (( recurse )); then
aur depends --pkgbase "$@" # stdin handled by aur-depends
mapfile -t packages < <(aur depends --pkgbase "$@")
wait "$!"
elif (( $# == 1 )) && [[ $1 == "-" || $1 == "/dev/stdin" ]]; then
tee # noop
mapfile -t packages
else
printf '%s\n' "$@"
fi | while read -r pkg; do
packages=("$@")
set --
fi
# Update revisions in local AUR mirror
declare -A local_clones
# With an AUR mirror, updates are retrieved in two steps. First, updates to the
# mirror are synchronized with `git-fetch`. Secondly, local clones of the miror
# are created and are updated with `git-fetch` and `git-merge` as usual.
if (( AUR_FETCH_USE_MIRROR )); then
while read -r IFS=':' pkg head; do
local_clones[$pkg]=$head
done < <(aur fetch--mirror --lclone "${packages[@]}")
wait "$!"
fi
# Main loop
for pkg in "${packages[@]}"; do
unset -f git
if (( AUR_FETCH_USE_MIRROR )); then # branch origin/$pkg -> master
upstream=origin/$pkg
else
upstream=origin/master
fi
# Verify if the repository is hosted on AUR (#959)
if (( existing )) && ! git ls-remote --exit-code "$AUR_LOCATION/$pkg" >/dev/null; then
printf >&2 '%s: warning: package %s is not in AUR, skipping\n' "$argv0" "$pkg"
continue
elif [[ -d $pkg/.git ]]; then
# Clone package if not existing
elif [[ ! -d $pkg/.git ]]; then
# Differentiate between local (fetch--mirror) and aurweb clones
if [[ ! ${local_clones[$pkg]} ]]; then
git clone "$AUR_LOCATION/$pkg" || exit 1
head=$(git -C "$pkg" rev-parse --verify quiet HEAD)
else
printf "Cloning into '%s'\n" "$pkg"
head=${local_clones[$pkg]}
fi >&2
if [[ $head ]]; then
git -C "$pkg" --no-pager log --pretty=reference -1
fi >&2
# XXX: race with multiple instances
if [[ -v results_file ]]; then
results 'clone' "$git_empty_object" "${head:-$git_empty_object}" "$PWD/$pkg" "$results_file"
fi
# Update existing git repository
else
# Per-package lock
exec {fd}< "$pkg"/.git
flock --wait 5 "$fd" || exit 1
# Avoid issues with filesystem boundaries (#274)
git() { command git -C "$pkg" "$@"; }
# Retrieve new upstream commits
git fetch -v origin >&2 || exit
git fetch origin >&2 || exit
# Store original HEAD for --results output
orig_head=$(git rev-parse --verify --quiet HEAD)
@ -165,10 +216,10 @@ fi | while read -r pkg; do
;;& # proceed to merge or rebase
rebase)
dest='HEAD'
git rebase -v "${rebase_args[@]}" origin/master ;;
git rebase "${rebase_args[@]}" "$upstream" ;;
merge)
dest='HEAD'
git merge -v "${merge_args[@]}" origin/master ;;
git merge "${merge_args[@]}" "$upstream" ;;
reset)
dest='master@{u}'
git reset --hard 'master@{u}' ;;
@ -184,20 +235,7 @@ fi | while read -r pkg; do
if [[ -v results_file ]]; then
results "$sync_pkg" "$orig_head" "$head" "$PWD/$pkg" "$results_file"
fi
# Otherwise, try to clone anew
elif git clone "$AUR_LOCATION/$pkg" >&2; then
if head=$(git -C "$pkg" rev-parse --verify --quiet HEAD); then
git -C "$pkg" --no-pager log --pretty=reference -1 >&2
fi
head=${head:-$git_empty_object}
if [[ -v results_file ]]; then
results 'clone' "$git_empty_object" "$head" "$PWD/$pkg" "$results_file"
fi
else
printf >&2 '%s: failed to clone repository %s\n' "$argv0" "$pkg"
exit 1
exec {fd}<&- # release lock
fi
done

@ -0,0 +1,113 @@
#!/bin/bash
set -o errexit
[[ -v AUR_DEBUG ]] && set -o xtrace
#argv0=fetch--mirror
XDG_CACHE_HOME=${XDG_CACHE_HOME:-$HOME/.cache}
AUR_MIRROR=${AUR_MIRROR:-https://github.com/archlinux/aur}
AUR_MIRROR_TTL=${AUR_MIRROR_TTL:-300}
AUR_ROOT=${AUR_ROOT:-$XDG_CACHE_HOME/aur}
startdir=$PWD
local_clones=0
if [[ $1 = '--lclone' ]]; then
local_clones=1
shift
fi
# --------------------------------------------------
# Step 1. Create a sparse checkout of the AUR mirror
git_active_remotes() {
git config --get-all remote.origin.fetch | awk -F'[/:]' '{print $3}'
}
# XXX: this includes refs/heads (which we do not use) but unlike
# git-config this accepts multiple arguments in one call
git_set_branches() {
git remote set-branches origin --add main # ensure at least 1 remote
sort -u | xargs -d '\n' git remote set-branches origin
}
git_ls_remote_ttl() {
local cachetime now ttl=$AUR_MIRROR_TTL
printf -v now '%(%s)T' -1
if ! cachetime=$(stat -c %Y "$1" 2>/dev/null) || (( now > (cachetime + ttl) )); then
git ls-remote origin 'refs/heads/*' | awk -F/ '{print $3}' >"$1"
fi
}
exec {fd}< "$AUR_ROOT"
{ flock --wait 5 "$fd" || exit 1
# Shallow clone
[[ ! -d $AUR_ROOT ]] && git clone --depth=1 --bare "$AUR_MIRROR" "$AUR_ROOT"
cd "$AUR_ROOT"
# Keep a refcache to filter out packages which are not in AUR. This is
# includes unlisted packages (unlike `aur pkglist --pkgbase`)
git_ls_remote_ttl 'remote-pkgbase'
# Retrieve list of active remotes
mapfile -t active_remotes < <(git_active_remotes | grep -v 'main')
# Only consider AUR targets
mapfile -t target_remotes < <(printf '%s\n' "$@" | grep -Fxf 'remote-pkgbase')
# Set remote branches as union of active remotes and arguments. If '*' is set
# instead, `git fetch origin` in $AUR_ROOT will pull in the entire history.
printf '%s\n' "${target_remotes[@]}" "${active_remotes[@]}" | git_set_branches
# Retrieve objects
git fetch origin "${target_remotes[@]}"
# Resetting branches is only needed when .git/config contains both refs/heads
# and refs/remotes/origin, i.e. when using git-clone. `--lclone` uses git-init
# and refspec manipulation instead.
# for b in "${target_remotes[@]}"; do
# git branch --force "$b" origin/"$b"
# done
} >&2
# ------------------------------------------------------------
# Step 2. Propagate each branch in the mirror to a local clone
(( ! local_clones )) && exit
results=()
git_srcdest_origin() {
local refspec=refs/remotes/origin/$1
git config remote.origin.fetch "+$refspec:$refspec"
git fetch --quiet
git checkout -B master "$refspec"
}
for b in "${target_remotes[@]}"; do
cd "$startdir"
# XXX: "Cloning into an existing directory is only allowed if the
# directory is empty"
if [[ ! -d $b/.git ]] && mkdir "$b"; then
cd "$b"
git init
git remote add origin "$AUR_ROOT"
# Compared to git clone --local, using $refspec as both source and
# destination allows updates from `git-fetch` in $AUR_ROOT to be
# directly visible in local clones. $AUR_ROOT does not require
# local branches or refs/heads for this to work.
git_srcdest_origin "$b"
# Export results to aur-fetch
head=$(git rev-parse --verify --quiet HEAD)
results+=("$b:$head")
fi >&2
done
# The lock is only released here, because we do not want an instance to modify
# AUR_ROOT while local git-clones are still underway. Additionally, local clones
# use cp(1) and are thus a cheap operation. See --local in git-clone(1).
exec {fd}<&-
if (( ${#results[@]} )); then
printf '%s\n' "${results[@]}"
fi

@ -5,6 +5,12 @@
- `pacman -Syu --config` is replaced by `pacsync <repo>` and `pacman -S <repo>/<pkg>`
- local repository upgrades are now unaffected by `--pacman-conf`
* `aur-fetch`
+ add `aur-fetch--mirror` helper for `aur.git` mirrors
- defaults to `github.com/archlinux/aur`
- enabled with `AUR_FETCH_USE_MIRROR=1`
+ run flock(1) on existing repositories
* `aur-graph`
+ selectively disable/enable depends with `aur graph -v <TYPE>=[0|1]`
- supported types: `DEPENDS`, `MAKEDEPENDS`, `CHECKDEPENDS`, `OPTDEPENDS`

@ -195,6 +195,57 @@ and
.I <head_to>
will be identical.
.
.SH ENVIRONMENT
.TP
.B AUR_LOCATION
URI where repositories are cloned from. Defaults to
.IR https://aur.archlinux.org .
.
.TP
.B AUR_MIRROR
URI to an AUR mirror, a
.BR git (1)
repository where each package is contained in a
.BR git\-branch (1).
Defaults to
.MT https://github.com/archlinux/aur
.ME .
.
.TP
.B AUR_MIRROR_TTL
When updating an AUR mirror, a list of all branches is retrieved with
.BR git\-ls\-remote
to validate arguments. The
.B AUR_MIRROR_TTL
variable defines (in seconds) how often this command is run. For accurate
results, this should roughly equal the synchronization interval between AUR and
its mirrors. Defaults to
.IR 300 .
.
.TP
.B AUR_FETCH_USE_MIRROR
If this variable is set to a positive value, clone repositories from an AUR
mirror instead of
.BR AUR_LOCATION .
Upstream changes are merged from
.I origin/<package>
instead of
.IR origin/master .
.IP
Remotes of existing
.BR git (1)
repositories are preserved. When enabling or disabling
.BR AUR_FETCH_MIRROR ,
repositories should be converted manually with
.B git\-remote set\-url
and
.BR "git\-remote set\-branches origin" .
.
.TP
.B AUR_ROOT
Directory where an AUR mirror is cloned to. Defaults to
.IR XDG_CACHE_HOME/aur .
.
.SH SEE ALSO
.ad l
.nh

Loading…
Cancel
Save