1//! Handles the download of package source repositories.
2//!
3//! This requires interaction with `git` and the official Arch Linux Gitlab, where all of the
4//! package source repositories for the official packages are located.
56use std::{
7 collections::{HashMap, HashSet},
8 fs::remove_dir_all,
9 path::{Path, PathBuf},
10 process::Command,
11};
1213use anyhow::{Context, Result};
14use log::{error, info, trace};
15use rayon::prelude::*;
16use strum::Display;
1718use super::filenames_in_dir;
19use crate::{cmd::ensure_success, ui::get_progress_bar};
2021const PKGBASE_MAINTAINER_URL: &str = "https://archlinux.org/packages/pkgbase-maintainer";
22const SSH_HOST: &str = "git@gitlab.archlinux.org";
23const REPO_BASE_URL: &str = "archlinux/packaging/packages";
2425/// Some package repositories' names differ from the name of the package.
26/// These are only few and need to be handled separately.
27const PACKAGE_REPO_RENAMES: [(&str, &str); 3] = [
28 ("gtk2+extra", "gtk2-extra"),
29 ("dvd+rw-tools", "dvd-rw-tools"),
30 ("tree", "unix-tree"),
31];
3233/// This struct is the entry point for downloading package source repositories from ArchLinux's
34/// Gitlab.
35///
36/// Look at [Self::download_package_source_repositories] for more information.
37#[derive(Clone, Debug)]
38pub struct PkgSrcDownloader {
39/// The destination folder into which files should be downloaded.
40pub dest: PathBuf,
41}
4243impl PkgSrcDownloader {
44/// Download all official package source git repositories.
45pub fn download_package_source_repositories(&self) -> Result<()> {
46// Query the arch web API to get a list all official active repositories
47 // The returned json is a map where the keys are the package names
48 // and the value is a list of maintainer names.
49let repos = reqwest::blocking::get(PKGBASE_MAINTAINER_URL)
50 .context("Failed to query pkgbase url.")?
51.json::<HashMap<String, Vec<String>>>()
52 .context("Failed to deserialize archweb pkglist.")?;
5354let all_repo_names: Vec<String> = repos.keys().map(String::from).collect();
55info!("Found {} official packages.", all_repo_names.len());
5657let download_dir = self.dest.join("download/pkgsrc");
5859// Remove all old repos before trying to update them.
60self.remove_old_repos(&all_repo_names, &download_dir)?;
6162// Copy all .SRCINFO files to the target directory
63self.parallel_update_or_clone(&all_repo_names, &download_dir)?;
6465// Copy .SRCINFO and PKGBUILD files to the target directory
66for repo in all_repo_names {
67let download_path = download_dir.join(&repo);
68for file in [".SRCINFO", "PKGBUILD"] {
69if download_path.join(file).exists() {
70let target_dir = self.dest.join("pkgsrc").join(&repo);
71 std::fs::create_dir_all(&target_dir)?;
72 std::fs::copy(download_path.join(file), target_dir.join(file))?;
73 }
74 }
75 }
7677Ok(())
78 }
7980/// Remove all local repositories for packages that no longer exist in the official
81 /// repositories.
82 ///
83 /// Get the list of all locally available pkgsrc repositories.
84 /// If we find any that are not in the list of official packages, we remove them.
85fn remove_old_repos(&self, repos: &[String], download_dir: &Path) -> Result<()> {
86// First up, read the names of all repositories in the local download folder.
87let local_repositories = filenames_in_dir(download_dir)?;
8889// Get the list of packages that no longer exist on the mirrors (and thereby archweb).
90let remote_pkgs: HashSet<String> = HashSet::from_iter(repos.iter().map(String::from));
9192// Now remove all local repositories for which there's no longer an entry in the archweb
93 // response, as those packages are no longer served by the official mirrors.
94let removed_pkgs: Vec<&String> = local_repositories.difference(&remote_pkgs).collect();
9596// Delete the repositories
97if !removed_pkgs.is_empty() {
98info!("Found {} repositories for cleanup:", removed_pkgs.len());
99for removed in removed_pkgs {
100 remove_dir_all(download_dir.join(removed))
101 .context("Failed to remove local repository {removed}")?;
102 }
103 }
104105Ok(())
106 }
107108/// Update/clone all git repositories in parallel with rayon.
109 ///
110 /// A progress bar is added for progress indication.
111fn parallel_update_or_clone(&self, repos: &[String], download_dir: &Path) -> Result<()> {
112let progress_bar = get_progress_bar(repos.len() as u64);
113114// Prepare a ssh session for better performance.
115warmup_ssh_session()?;
116117// Clone/update all repositories in parallel
118let results: Vec<Result<(), RepoUpdateError>> = repos
119 .par_iter()
120 .map(|repo| {
121let target_dir = download_dir.join(repo);
122123// If the repo already exists, only pull it.
124 // Otherwise do a clone.
125let result = if target_dir.exists() {
126 update_repo(repo, &target_dir)
127 } else {
128 clone_repo(repo.to_string(), &target_dir)
129 };
130131// Increment the counter
132progress_bar.inc(1);
133 result
134 })
135 .collect();
136137// Finish the spinner
138progress_bar.finish_with_message("All repositories cloned or updated.");
139140// Display any errors during cloning/updating to the user.
141let mut error_iter = results.into_iter().filter_map(Result::err).peekable();
142if error_iter.peek().is_some() {
143error!("The command failed for the following repositories:");
144for error in error_iter {
145error!(
146"{} failed for repo {} with error:\n{:?}",
147 error.operation, error.repo, error.inner
148 );
149 }
150 }
151152Ok(())
153 }
154}
155156/// Create a new ssh connection that doesn't get bound to a given session.
157/// This allows that session to be reused, effectively eliminating the need to authenticate every
158/// time a git repository is cloned/pulled.
159///
160/// This is especially necessary for users that have their SSH key on a physical device, such as a
161/// NitroKey, as authentications with such devices are sequential and take quite some time.
162pub fn warmup_ssh_session() -> Result<()> {
163let mut ssh_command = Command::new("ssh");
164 ssh_command.args(vec!["-T", SSH_HOST]);
165trace!("Running command: {ssh_command:?}");
166let output = &ssh_command
167 .output()
168 .context("Failed to start ssh warmup command")?;
169170 ensure_success(output).context("Failed to run ssh warmup command:")
171}
172173#[derive(Display)]
174enum RepoUpdateOperation {
175 Clone,
176 Update,
177}
178179struct RepoUpdateError {
180 repo: String,
181 operation: RepoUpdateOperation,
182 inner: anyhow::Error,
183}
184185/// Update a local git repository to the newest state.
186/// Resets any local changes in case in each repository beforehand to prevent any conflicts.
187fn update_repo(repo: &str, target_dir: &Path) -> Result<(), RepoUpdateError> {
188// Reset any possible local changes.
189let output = Command::new("git")
190 .current_dir(target_dir)
191 .args(vec!["reset", "--hard"])
192 .output()
193 .map_err(|err| RepoUpdateError {
194 repo: repo.to_string(),
195 operation: RepoUpdateOperation::Update,
196 inner: err.into(),
197 })?;
198199 ensure_success(&output).map_err(|err| RepoUpdateError {
200 repo: repo.to_string(),
201 operation: RepoUpdateOperation::Update,
202 inner: err,
203 })?;
204205let output = &Command::new("git")
206 .current_dir(target_dir)
207 .args(["pull", "--force"])
208 .output()
209 .map_err(|err| RepoUpdateError {
210 repo: repo.to_string(),
211 operation: RepoUpdateOperation::Update,
212 inner: err.into(),
213 })?;
214215 ensure_success(output).map_err(|err| RepoUpdateError {
216 repo: repo.to_string(),
217 operation: RepoUpdateOperation::Update,
218 inner: err,
219 })
220}
221222/// Clone a git repository into a target directory.
223fn clone_repo(mut repo: String, target_dir: &Path) -> Result<(), RepoUpdateError> {
224// Check if this is one of the few packages that needs to be replaced.
225for (to_replace, replace_with) in PACKAGE_REPO_RENAMES {
226if repo == to_replace {
227 repo = replace_with.to_string();
228 }
229 }
230231// Arch linux replaces the literal `+` chars with spelled out `plus` equivalents in their
232 // repository urls. This is to prevent any issues with external tooling and such.
233repo = repo.replace("+", "plus");
234235let ssh_url = format!("{SSH_HOST}:{REPO_BASE_URL}/{repo}.git");
236237let output = &Command::new("git")
238 .arg("clone")
239 .arg(&ssh_url)
240 .arg(target_dir)
241 .output()
242 .map_err(|err| RepoUpdateError {
243 repo: repo.to_string(),
244 operation: RepoUpdateOperation::Clone,
245 inner: err.into(),
246 })?;
247248 ensure_success(output).map_err(|err| RepoUpdateError {
249 repo: repo.to_string(),
250 operation: RepoUpdateOperation::Clone,
251 inner: err,
252 })
253}