Skip to content

Add support for --incompatible_compact_repo_mapping_manifest #301

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Add support for --incompatible_compact_repo_mapping_manifest #301

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
04be940
Add impl
fmeum Jun 11, 2025
678ea6d
Add test
fmeum Jun 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 72 additions & 41 deletions java/runfiles/src/main/java/com/google/devtools/build/runfiles/Runfiles.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.NavigableMap;
import java.util.Objects;
import java.util.stream.Collectors;
import java.util.TreeMap;

/**
* Runfiles lookup library for Bazel-built Java binaries and tests.
Expand All @@ -45,7 +46,7 @@
* <p>2. Import the runfiles library.
*
* <pre>
* import com.google.devtools.build.runfiles.Runfiles;
* import Runfiles;
* </pre>
*
* <p>3. Create a {@link Runfiles.Preloaded} object:
Expand All @@ -59,11 +60,11 @@
* <p>4. To look up a runfile, use either of the following approaches:
*
* <p>4a. Annotate the class from which runfiles should be looked up with {@link
* AutoBazelRepository} and obtain the name of the Bazel repository containing the class from a
* constant generated by this annotation:
* AutoBazelRepository} and obtain the name of the Bazel
* repository containing the class from a constant generated by this annotation:
*
* <pre>
* import com.google.devtools.build.runfiles.AutoBazelRepository;
* import AutoBazelRepository;
* &#64;AutoBazelRepository
* public class MyClass {
* public void myFunction() {
Expand Down Expand Up @@ -95,7 +96,7 @@
*
* </pre>
*
* For more details on why it is required to pass in the current repository name, see {@see
* <p>For more details on why it is required to pass in the current repository name, see {@see
* https://bazel.build/build/bzlmod#repository-names}.
*
* <h3>Subprocesses</h3>
Expand Down Expand Up @@ -204,7 +205,7 @@ public final Runfiles unmapped() {

protected abstract String rlocationChecked(String path);

protected abstract Map<RepoMappingKey, String> getRepoMapping();
protected abstract RepositoryMapping getRepoMapping();

// Private constructor, so only nested classes may extend it.
private Preloaded() {}
Expand Down Expand Up @@ -406,47 +407,19 @@ private static String getRunfilesDir(Map<String, String> env) throws IOException
return value;
}

private static Map<Preloaded.RepoMappingKey, String> loadRepositoryMapping(String path)
throws IOException {
if (path == null || !new File(path).exists()) {
return Collections.emptyMap();
}

try (BufferedReader r =
new BufferedReader(
new InputStreamReader(new FileInputStream(path), StandardCharsets.UTF_8))) {
return Collections.unmodifiableMap(
r.lines()
.filter(line -> !line.isEmpty())
.map(
line -> {
String[] split = line.split(",");
if (split.length != 3) {
throw new IllegalArgumentException(
"Invalid line in repository mapping: '" + line + "'");
}
return split;
})
.collect(
Collectors.toMap(
split -> new Preloaded.RepoMappingKey(split[0], split[1]),
split -> split[2])));
}
}

/** {@link Runfiles} implementation that parses a runfiles-manifest file to look up runfiles. */
private static final class ManifestBased extends Preloaded {

private final Map<String, String> runfiles;
private final String manifestPath;
private final Map<RepoMappingKey, String> repoMapping;
private final RepositoryMapping repoMapping;

ManifestBased(String manifestPath) throws IOException {
Util.checkArgument(manifestPath != null);
Util.checkArgument(!manifestPath.isEmpty());
this.manifestPath = manifestPath;
this.runfiles = loadRunfiles(manifestPath);
this.repoMapping = loadRepositoryMapping(rlocationChecked("_repo_mapping"));
this.repoMapping = RepositoryMapping.readFromFile(rlocationChecked("_repo_mapping"));
}

@Override
Expand Down Expand Up @@ -481,7 +454,7 @@ protected Map<String, String> getEnvVars() {
}

@Override
protected Map<RepoMappingKey, String> getRepoMapping() {
protected RepositoryMapping getRepoMapping() {
return repoMapping;
}

Expand Down Expand Up @@ -543,13 +516,13 @@ private static String findRunfilesDir(String manifest) {
private static final class DirectoryBased extends Preloaded {

private final String runfilesRoot;
private final Map<RepoMappingKey, String> repoMapping;
private final RepositoryMapping repoMapping;

DirectoryBased(String runfilesDir) throws IOException {
Util.checkArgument(!Util.isNullOrEmpty(runfilesDir));
Util.checkArgument(new File(runfilesDir).isDirectory());
this.runfilesRoot = runfilesDir;
this.repoMapping = loadRepositoryMapping(rlocationChecked("_repo_mapping"));
this.repoMapping = RepositoryMapping.readFromFile(rlocationChecked("_repo_mapping"));
}

@Override
Expand All @@ -558,7 +531,7 @@ protected String rlocationChecked(String path) {
}

@Override
protected Map<RepoMappingKey, String> getRepoMapping() {
protected RepositoryMapping getRepoMapping() {
return repoMapping;
}

Expand All @@ -579,4 +552,62 @@ static Preloaded createManifestBasedForTesting(String manifestPath) throws IOExc
static Preloaded createDirectoryBasedForTesting(String runfilesDir) throws IOException {
return new DirectoryBased(runfilesDir);
}

static final class RepositoryMapping {
private final Map<Preloaded.RepoMappingKey, String> exactMappings;
private final NavigableMap<String, Map<String, String>> wildcardMappings;

private RepositoryMapping(
Map<Preloaded.RepoMappingKey, String> exactMappings,
NavigableMap<String, Map<String, String>> wildcardMappings) {
this.exactMappings = exactMappings;
this.wildcardMappings = wildcardMappings;
}

static RepositoryMapping readFromFile(String path) throws IOException {
if (path == null || !new File(path).exists()) {
return new RepositoryMapping(Collections.emptyMap(), Collections.emptyNavigableMap());
}

try (BufferedReader r =
new BufferedReader(
new InputStreamReader(new FileInputStream(path), StandardCharsets.UTF_8))) {
Map<Preloaded.RepoMappingKey, String> exactMappings = new HashMap<>();
NavigableMap<String, Map<String, String>> wildcardMappings = new TreeMap<>();
String line;
while ((line = r.readLine()) != null) {
if (line.isEmpty()) {
continue;
}
String[] split = line.split(",");
if (split.length != 3) {
throw new IllegalArgumentException(
"Invalid line in repository mapping: '" + line + "'");
}
if (split[0].endsWith("*")) {
Map<String, String> targetMap =
wildcardMappings.computeIfAbsent(
split[0].substring(0, split[0].length() - 1), k -> new HashMap<>());
targetMap.put(split[1], split[2]);
} else {
exactMappings.put(new Preloaded.RepoMappingKey(split[0], split[1]), split[2]);
}
}
return new RepositoryMapping(exactMappings, wildcardMappings);
}
}

String getOrDefault(Preloaded.RepoMappingKey key, String defaultValue) {
String exactMatch = exactMappings.get(key);
if (exactMatch != null) {
return exactMatch;
}
Map.Entry<String, Map<String, String>> floorEntry =
wildcardMappings.floorEntry(key.sourceRepo);
if (floorEntry == null || !key.sourceRepo.startsWith(floorEntry.getKey())) {
return defaultValue;
}
return floorEntry.getValue().getOrDefault(key.targetRepoApparentName, defaultValue);
}
}
}
59 changes: 58 additions & 1 deletion test/java/runfiles/src/test/java/com/google/devtools/build/runfiles/RunfilesTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -427,6 +427,40 @@ public void testManifestBasedRlocationWithRepoMapping_fromOtherRepo() throws Exc
assertThat(r.rlocation("protobuf")).isNull();
}

@Test
public void testManifestBasedRlocationWithRepoMapping_fromExtensionRepo() throws Exception {
Path rm =
tempFile(
"foo.repo_mapping",
ImmutableList.of(
",config.json,config.json+1.2.3",
",my_module,_main",
",my_protobuf,protobuf+3.19.2",
",my_workspace,_main",
"my_module++ext+*,my_module,my_module+",
"my_module++ext+*,repo1,my_module++ext+repo1"));
Path mf =
tempFile(
"foo.runfiles/MANIFEST",
ImmutableList.of(
"_repo_mapping " + rm,
"config.json /etc/config.json",
"protobuf+3.19.2/foo/runfile C:/Actual Path\\protobuf\\runfile",
"_main/bar/runfile /the/path/./to/other//other runfile.txt",
"protobuf+3.19.2/bar/dir E:\\Actual Path\\Directory",
"my_module+/foo/runfile /the/path/to/my_module+/runfile",
"my_module++ext+repo1/foo/runfile /the/path/to/my_module++ext+repo1/runfile",
"repo2+/foo/runfile /the/path/to/repo2+/runfile"));
Runfiles r =
Runfiles.createManifestBasedForTesting(mf.toString())
.withSourceRepository("my_module++ext+repo1");

assertThat(r.rlocation("my_module/foo/runfile")).isEqualTo("/the/path/to/my_module+/runfile");
assertThat(r.rlocation("repo1/foo/runfile"))
.isEqualTo("/the/path/to/my_module++ext+repo1/runfile");
assertThat(r.rlocation("repo2+/foo/runfile")).isEqualTo("/the/path/to/repo2+/runfile");
}

@Test
public void testDirectoryBasedRlocationWithRepoMapping_fromMain() throws Exception {
Path dir = tempDir.newFolder("foo.runfiles").toPath();
Expand Down Expand Up @@ -549,6 +583,28 @@ public void testDirectoryBasedRlocationWithRepoMapping_fromOtherRepo() throws Ex
assertThat(r.rlocation("config.json")).isEqualTo(dir + "/config.json");
}

@Test
public void testDirectoryBasedRlocationWithRepoMapping_fromExtensionRepo() throws Exception {
Path dir = tempDir.newFolder("foo.runfiles").toPath();
Path unused =
tempFile(
dir.resolve("_repo_mapping").toString(),
ImmutableList.of(
",config.json,config.json+1.2.3",
",my_module,_main",
",my_protobuf,protobuf+3.19.2",
",my_workspace,_main",
"my_module++ext+*,my_module,my_module+",
"my_module++ext+*,repo1,my_module++ext+repo1"));
Runfiles r =
Runfiles.createDirectoryBasedForTesting(dir.toString())
.withSourceRepository("my_module++ext+repo1");

assertThat(r.rlocation("my_module/foo")).isEqualTo(dir + "/my_module+/foo");
assertThat(r.rlocation("repo1/foo")).isEqualTo(dir + "/my_module++ext+repo1/foo");
assertThat(r.rlocation("repo2+/foo")).isEqualTo(dir + "/repo2+/foo");
}

@Test
public void testDirectoryBasedCtorArgumentValidation() throws IOException {
assertThrows(
Expand Down Expand Up @@ -581,7 +637,8 @@ public void testManifestBasedCtorArgumentValidation() throws Exception {
() -> Runfiles.createManifestBasedForTesting("").withSourceRepository(""));

Path mf = tempFile("foobar", ImmutableList.of("a b"));
Runfiles unused = Runfiles.createManifestBasedForTesting(mf.toString()).withSourceRepository("");
Runfiles unused =
Runfiles.createManifestBasedForTesting(mf.toString()).withSourceRepository("");
}

@Test
Expand Down