dev_scripts/sync/
aur.rs

1//! Handles the download of packages from the AUR.
2//!
3//! This requires interaction with `git` and the experimental aur.git GitHub mirror.
4
5use std::{
6    fs::{File, create_dir_all},
7    io::Read,
8    path::PathBuf,
9    process::{Command, Stdio},
10};
11
12use alpm_types::{PKGBUILD_FILE_NAME, SRCINFO_FILE_NAME};
13use flate2::read::GzDecoder;
14use log::{error, info};
15use rayon::prelude::*;
16use reqwest::blocking::get;
17
18use crate::{
19    CacheDir,
20    Error,
21    cmd::ensure_success,
22    consts::{AUR_DIR, DOWNLOAD_DIR},
23    ui::get_progress_bar,
24};
25
26const AUR_PKGBASE_URL: &str = "https://aur.archlinux.org/pkgbase.gz";
27const AUR_GIT_MIRROR_URL: &str = "https://github.com/archlinux/aur.git";
28
29/// The entry point for downloading packages from the AUR.
30///
31/// See [`AurDownloader::download_packages`] for more information.
32#[derive(Clone, Debug)]
33pub struct AurDownloader {
34    /// The destination folder into which files should be downloaded.
35    pub cache_dir: CacheDir,
36}
37
38impl AurDownloader {
39    /// Clone (or update) the AUR git repo and extract
40    /// .SRCINFO + PKGBUILD for every package
41    pub fn download_packages(&self) -> Result<(), Error> {
42        self.update_or_clone()?;
43        self.parallel_extract_files()?;
44        Ok(())
45    }
46
47    /// Ensure we have a bare clone of aur.git locally. Update if already present.
48    fn update_or_clone(&self) -> Result<(), Error> {
49        if self.repo_dir().exists() {
50            info!("Updating aur.git mirror...");
51            let output = Command::new("git")
52                .arg("-C")
53                .arg(self.repo_dir())
54                .args(["fetch", "--depth=1", "--prune"])
55                .output()
56                .map_err(|source| Error::Io {
57                    context: "fetching latest AUR git sources".to_string(),
58                    source,
59                })?;
60            ensure_success(&output, "Fetching latest AUR git sources".to_string())?;
61        } else {
62            create_dir_all(self.download_dir()).map_err(|source| Error::IoPath {
63                path: self.download_dir(),
64                context: "recursively creating the directory".to_string(),
65                source,
66            })?;
67            info!("Cloning aur.git mirror...");
68            let output = Command::new("git")
69                .args([
70                    // We want all remote branches locally,
71                    // using a mirror clone is the simplest solution.
72                    "clone",
73                    "--mirror",
74                    "--depth=1",
75                    "--no-single-branch",
76                    AUR_GIT_MIRROR_URL,
77                ])
78                .arg(self.repo_dir())
79                .output()
80                .map_err(|source| Error::Io {
81                    context: "cloning AUR git sources".to_string(),
82                    source,
83                })?;
84            ensure_success(&output, "Cloning AUR git sources".to_string())?;
85        }
86        Ok(())
87    }
88
89    /// Extract .SRCINFO and PKGBUILD files from aur.git branches.
90    fn parallel_extract_files(&self) -> Result<(), Error> {
91        let packages: Vec<String> = get_packages_list()?;
92
93        let progress_bar = get_progress_bar(packages.len() as u64);
94
95        create_dir_all(self.target_dir()).map_err(|source| Error::IoPath {
96            path: self.download_dir(),
97            context: "recursively creating the directory".to_string(),
98            source,
99        })?;
100
101        let results: Vec<Result<(), Error>> = packages
102            .par_iter()
103            .map(|pkg| {
104                let pkg_dir = self.target_dir().join(pkg);
105                create_dir_all(&pkg_dir).map_err(|source| Error::IoPath {
106                    path: pkg_dir.clone(),
107                    context: "recursively creating the directory".to_string(),
108                    source,
109                })?;
110
111                for file_type in [SRCINFO_FILE_NAME, PKGBUILD_FILE_NAME] {
112                    let out_file =
113                        File::create(pkg_dir.join(file_type)).map_err(|source| Error::IoPath {
114                            path: pkg_dir.join(file_type),
115                            context: "recursively creating the directory".to_string(),
116                            source,
117                        })?;
118                    let output = Command::new("git")
119                        .arg("show")
120                        .arg(format!("{pkg}:{file_type}"))
121                        .current_dir(self.repo_dir())
122                        .stdout(Stdio::from(out_file))
123                        .output()
124                        .map_err(|source| Error::Io {
125                            context: format!("extracting {file_type:?} from AUR repo {pkg:?}"),
126                            source,
127                        })?;
128                    ensure_success(
129                        &output,
130                        format!("Extracting {file_type:?} from AUR repo {pkg:?}"),
131                    )?;
132                }
133                progress_bar.inc(1);
134                Ok(())
135            })
136            .collect();
137
138        progress_bar.finish_with_message("All files extracted.");
139
140        // Log all errors during parallel extraction.
141        for error in results.into_iter().filter_map(Result::err) {
142            error!("{error:?}");
143        }
144
145        Ok(())
146    }
147
148    fn download_dir(&self) -> PathBuf {
149        self.cache_dir.as_ref().join(DOWNLOAD_DIR)
150    }
151
152    fn repo_dir(&self) -> PathBuf {
153        self.download_dir().join(AUR_DIR)
154    }
155
156    fn target_dir(&self) -> PathBuf {
157        self.cache_dir.as_ref().join(AUR_DIR)
158    }
159}
160
161/// Downloads pkgbase.gz from aurweb and extracts the package names.
162fn get_packages_list() -> Result<Vec<String>, Error> {
163    let resp = get(AUR_PKGBASE_URL)
164        .map_err(|source| Error::HttpQueryFailed {
165            context: "retrieving the list of AUR packages".to_string(),
166            source,
167        })?
168        .error_for_status()
169        .map_err(|source| Error::HttpQueryFailed {
170            context: "retrieving the list of AUR packages".to_string(),
171            source,
172        })?;
173    let bytes = resp.bytes().map_err(|source| Error::HttpQueryFailed {
174        context: "retrieving the response body as bytes for the list of AUR packages".to_string(),
175        source,
176    })?;
177    let mut decoder = GzDecoder::new(&bytes[..]);
178    let mut aur_packages_raw = String::new();
179    decoder
180        .read_to_string(&mut aur_packages_raw)
181        .map_err(|source| Error::Io {
182            context:
183                "reading the gzip decoded response body for the list of AUR packages as string"
184                    .to_string(),
185            source,
186        })?;
187    Ok(aur_packages_raw.lines().map(|s| s.to_string()).collect())
188}
dev_scripts/sync/aur.rs

dev_scripts/sync/
aur.rs
