-
-
Notifications
You must be signed in to change notification settings - Fork 1.1k
feat(lockfile): add multi-platform checksums without downloading tarballs #7113
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
34cd0fb
fc7aaaa
c43337d
fa9d0fe
fc2bdbb
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
…alls Enable `mise lock` to include SHA256 checksums for all platforms by leveraging: 1. **GitHub API digests** - GitHub provides SHA256 digests for release assets via their API, available without downloading files 2. **Aqua checksum files** - Download small checksum files (~1KB) that contain checksums for all platforms, parsed for the target platform's asset Changes: - aqua.rs: Use GitHub API digest in resolve_lock_info instead of discarding it; add fetch_checksum_from_file() for tools with checksum config - github.rs: Override resolve_lock_info to include digest from GitHub API; add resolve_asset_url_for_target() for cross-platform resolution - static_helpers.rs: Add lookup_platform_key_for_target() helper - asset_detector.rs: Add detect_asset_for_target() helper - Export AquaChecksum type from aqua-registry crate 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude <noreply@anthropic.com>
- Loading branch information
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -14,7 +14,8 @@ use crate::registry::REGISTRY; | |
| use crate::toolset::ToolVersion; | ||
| use crate::{ | ||
| aqua::aqua_registry_wrapper::{ | ||
| AQUA_REGISTRY, AquaChecksumType, AquaMinisignType, AquaPackage, AquaPackageType, | ||
| AQUA_REGISTRY, AquaChecksum, AquaChecksumType, AquaMinisignType, AquaPackage, | ||
| AquaPackageType, | ||
| }, | ||
| cache::{CacheManager, CacheManagerBuilder}, | ||
| }; | ||
|
|
@@ -336,35 +337,46 @@ impl Backend for AquaBackend { | |
| return Ok(PlatformInfo::default()); | ||
| } | ||
|
|
||
| // Get URL for the target platform | ||
| let url = match pkg.r#type { | ||
| // Get URL and checksum for the target platform | ||
| let (url, checksum) = match pkg.r#type { | ||
| AquaPackageType::GithubRelease => { | ||
| // For GitHub releases, we need to find the asset for the target platform | ||
| let asset_strs = pkg.asset_strs(&v, target_os, target_arch)?; | ||
| match self.github_release_asset(&pkg, &v, asset_strs).await { | ||
| Ok((url, _digest)) => Some(url), | ||
| Ok((url, digest)) => (Some(url), digest), | ||
| Err(e) => { | ||
| debug!( | ||
| "Failed to get GitHub release asset for {} on {}: {}", | ||
| self.id, | ||
| target.to_key(), | ||
| e | ||
| ); | ||
| None | ||
| (None, None) | ||
| } | ||
| } | ||
| } | ||
| AquaPackageType::GithubArchive | AquaPackageType::GithubContent => { | ||
| Some(self.github_archive_url(&pkg, &v)) | ||
| (Some(self.github_archive_url(&pkg, &v)), None) | ||
| } | ||
| AquaPackageType::Http => pkg.url(&v, target_os, target_arch).ok(), | ||
| _ => None, | ||
| AquaPackageType::Http => (pkg.url(&v, target_os, target_arch).ok(), None), | ||
| _ => (None, None), | ||
| }; | ||
|
|
||
| let name = url.as_ref().map(|u| get_filename_from_url(u)); | ||
|
|
||
| // Try to get checksum from checksum file if not available from GitHub API | ||
| let checksum = match checksum { | ||
| Some(c) => Some(c), | ||
| None => self | ||
| .fetch_checksum_from_file(&pkg, &v, target_os, target_arch, name.as_deref()) | ||
| .await | ||
| .ok() | ||
| .flatten(), | ||
| }; | ||
|
|
||
| Ok(PlatformInfo { | ||
| url, | ||
| checksum: None, // Checksums require downloading checksum files - done during install | ||
| checksum, | ||
| name, | ||
| size: None, | ||
| url_api: None, | ||
|
|
@@ -491,6 +503,127 @@ impl AquaBackend { | |
| format!("https://github.com/{gh_id}/archive/refs/tags/{v}.tar.gz") | ||
| } | ||
|
|
||
| /// Fetch checksum from a checksum file without downloading the actual tarball. | ||
| /// This is used for cross-platform lockfile generation. | ||
| async fn fetch_checksum_from_file( | ||
| &self, | ||
| pkg: &AquaPackage, | ||
| v: &str, | ||
| target_os: &str, | ||
| target_arch: &str, | ||
| filename: Option<&str>, | ||
| ) -> Result<Option<String>> { | ||
| let Some(checksum_config) = &pkg.checksum else { | ||
| return Ok(None); | ||
| }; | ||
| if !checksum_config.enabled() { | ||
| return Ok(None); | ||
| } | ||
| let Some(filename) = filename else { | ||
| return Ok(None); | ||
| }; | ||
|
|
||
| // Get the checksum file URL | ||
| let url = match checksum_config._type() { | ||
| AquaChecksumType::GithubRelease => { | ||
| let asset_strs = checksum_config.asset_strs(pkg, v, target_os, target_arch)?; | ||
| match self.github_release_asset(pkg, v, asset_strs).await { | ||
| Ok((url, _)) => url, | ||
| Err(e) => { | ||
| debug!("Failed to get checksum file asset: {}", e); | ||
| return Ok(None); | ||
| } | ||
| } | ||
| } | ||
| AquaChecksumType::Http => checksum_config.url(pkg, v, target_os, target_arch)?, | ||
| }; | ||
|
|
||
| // Download checksum file content | ||
| let checksum_content = match HTTP.get_text(&url).await { | ||
| Ok(content) => content, | ||
| Err(e) => { | ||
| debug!("Failed to download checksum file {}: {}", url, e); | ||
| return Ok(None); | ||
| } | ||
| }; | ||
|
|
||
| // Parse checksum from file content | ||
| let checksum_str = | ||
| self.parse_checksum_from_content(&checksum_content, checksum_config, filename)?; | ||
|
|
||
| Ok(Some(format!( | ||
| "{}:{}", | ||
| checksum_config.algorithm(), | ||
| checksum_str | ||
| ))) | ||
| } | ||
|
|
||
| /// Parse a checksum from checksum file content for a specific filename. | ||
| fn parse_checksum_from_content( | ||
| &self, | ||
| content: &str, | ||
| checksum_config: &AquaChecksum, | ||
| filename: &str, | ||
| ) -> Result<String> { | ||
| let mut checksum_file = content.to_string(); | ||
|
|
||
| if checksum_config.file_format() == "regexp" { | ||
| let pattern = checksum_config.pattern(); | ||
| if let Some(file_pattern) = &pattern.file { | ||
| let re = regex::Regex::new(file_pattern.as_str())?; | ||
| if let Some(line) = checksum_file | ||
| .lines() | ||
| .find(|l| re.captures(l).is_some_and(|c| c[1].to_string() == filename)) | ||
| { | ||
| checksum_file = line.to_string(); | ||
| } else { | ||
| debug!( | ||
| "no line found matching {} in checksum file for {}", | ||
| file_pattern, filename | ||
| ); | ||
| } | ||
| } | ||
| let re = regex::Regex::new(pattern.checksum.as_str())?; | ||
| if let Some(caps) = re.captures(checksum_file.as_str()) { | ||
| checksum_file = caps[1].to_string(); | ||
| } else { | ||
| debug!( | ||
| "no checksum found matching {} in checksum file", | ||
| pattern.checksum | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| // Standard format: "<hash> <filename>" or "<hash> *<filename>" | ||
| let checksum_str = checksum_file | ||
| .lines() | ||
| .filter_map(|l| { | ||
| let split = l.split_whitespace().collect_vec(); | ||
| if split.len() == 2 { | ||
| Some(( | ||
| split[0].to_string(), | ||
| split[1] | ||
| .rsplit_once('/') | ||
| .map(|(_, f)| f) | ||
| .unwrap_or(split[1]) | ||
| .trim_matches('*') | ||
| .to_string(), | ||
| )) | ||
| } else { | ||
| None | ||
| } | ||
| }) | ||
| .find(|(_, f)| f == filename) | ||
| .map(|(c, _)| c) | ||
| .unwrap_or(checksum_file); | ||
|
Comment on lines
+592
to
+612
|
||
|
|
||
| let checksum_str = checksum_str | ||
| .split_whitespace() | ||
| .next() | ||
| .unwrap_or(&checksum_str); | ||
| Ok(checksum_str.to_string()) | ||
| } | ||
|
|
||
| async fn download( | ||
| &self, | ||
| ctx: &InstallContext, | ||
|
|
||
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| @@ -1,6 +1,8 @@ | ||||||||||
| use eyre::Result; | ||||||||||
| use regex::Regex; | ||||||||||
| use std::sync::LazyLock; | ||||||||||
|
|
||||||||||
| use crate::backend::platform_target::PlatformTarget; | ||||||||||
| use crate::backend::static_helpers::get_filename_from_url; | ||||||||||
|
|
||||||||||
| /// Platform detection patterns | ||||||||||
|
|
@@ -359,6 +361,30 @@ pub fn detect_platform_from_url(url: &str) -> Option<DetectedPlatform> { | |||||||||
| } | ||||||||||
| } | ||||||||||
|
|
||||||||||
| /// Detects the best asset for a given target platform | ||||||||||
| /// Used for cross-platform lockfile generation | ||||||||||
| pub fn detect_asset_for_target(assets: &[String], target: &PlatformTarget) -> Result<String> { | ||||||||||
| let target_os = match target.os_name() { | ||||||||||
| "macos" => "darwin", | ||||||||||
| other => other, | ||||||||||
| }; | ||||||||||
| let target_arch = match target.arch_name() { | ||||||||||
| "x64" => "x86_64", | ||||||||||
| "arm64" => "aarch64", | ||||||||||
| other => other, | ||||||||||
| }; | ||||||||||
|
|
||||||||||
| let picker = AssetPicker::new(target_os.to_string(), target_arch.to_string()); | ||||||||||
| picker.pick_best_asset(assets).ok_or_else(|| { | ||||||||||
|
Comment on lines
+377
to
+378
|
||||||||||
| let picker = AssetPicker::new(target_os.to_string(), target_arch.to_string()); | |
| picker.pick_best_asset(assets).ok_or_else(|| { | |
| let asset_picker = AssetPicker::new(target_os.to_string(), target_arch.to_string()); | |
| asset_picker.pick_best_asset(assets).ok_or_else(|| { |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
The variable name
checksum_fileis misleading as it contains the content of the checksum file, not the file itself. Consider renaming it tochecksum_contentorchecksum_data.