-
Notifications
You must be signed in to change notification settings - Fork 2.3k
Implement pagination for snapshot indices status #19251
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
omricohenn
wants to merge
1
commit into
opensearch-project:main
Choose a base branch
from
omricohenn:paginate-snapshot-indices-status-fetching-16985
base: main
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
28 changes: 28 additions & 0 deletions
28
...in/java/org/opensearch/action/admin/cluster/snapshots/list/SnapshotIndicesListAction.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.admin.cluster.snapshots.list; | ||
|
|
||
| import org.opensearch.action.ActionType; | ||
|
|
||
| /** | ||
| * ActionType for listing snapshot indices with pagination. | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class SnapshotIndicesListAction extends ActionType<SnapshotIndicesListResponse> { | ||
|
|
||
| public static final SnapshotIndicesListAction INSTANCE = new SnapshotIndicesListAction(); | ||
| public static final String NAME = "cluster:admin/snapshot/list/indices"; | ||
|
|
||
| private SnapshotIndicesListAction() { | ||
| super(NAME, SnapshotIndicesListResponse::new); | ||
| } | ||
| } | ||
|
|
||
|
|
80 changes: 80 additions & 0 deletions
80
...n/java/org/opensearch/action/admin/cluster/snapshots/list/SnapshotIndicesListRequest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.admin.cluster.snapshots.list; | ||
|
|
||
| import org.opensearch.action.ActionRequestValidationException; | ||
| import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest; | ||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
|
|
||
| import java.io.IOException; | ||
|
|
||
| /** | ||
| * Request for listing snapshot indices with pagination. | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class SnapshotIndicesListRequest extends ClusterManagerNodeRequest<SnapshotIndicesListRequest> { | ||
|
|
||
| private String repository; | ||
| private String snapshot; | ||
| private int from; | ||
| private int size; | ||
|
|
||
| public SnapshotIndicesListRequest() {} | ||
|
|
||
| public SnapshotIndicesListRequest(StreamInput in) throws IOException { | ||
| super(in); | ||
| this.repository = in.readString(); | ||
| this.snapshot = in.readString(); | ||
| this.from = in.readVInt(); | ||
| this.size = in.readVInt(); | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| super.writeTo(out); | ||
| out.writeString(repository); | ||
| out.writeString(snapshot); | ||
| out.writeVInt(from); | ||
| out.writeVInt(size); | ||
| } | ||
|
|
||
| @Override | ||
| public ActionRequestValidationException validate() { | ||
| return null; | ||
| } | ||
|
|
||
| public String repository() { return repository; } | ||
| public String snapshot() { return snapshot; } | ||
| public int from() { return from; } | ||
| public int size() { return size; } | ||
|
|
||
| public SnapshotIndicesListRequest repository(String repository) { | ||
| this.repository = repository; | ||
| return this; | ||
| } | ||
|
|
||
| public SnapshotIndicesListRequest snapshot(String snapshot) { | ||
| this.snapshot = snapshot; | ||
| return this; | ||
| } | ||
|
|
||
| public SnapshotIndicesListRequest from(int from) { | ||
| this.from = from; | ||
| return this; | ||
| } | ||
|
|
||
| public SnapshotIndicesListRequest size(int size) { | ||
| this.size = size; | ||
| return this; | ||
| } | ||
| } | ||
|
|
||
|
|
||
84 changes: 84 additions & 0 deletions
84
.../java/org/opensearch/action/admin/cluster/snapshots/list/SnapshotIndicesListResponse.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.admin.cluster.snapshots.list; | ||
|
|
||
| import org.opensearch.core.action.ActionResponse; | ||
| import org.opensearch.core.common.io.stream.StreamInput; | ||
| import org.opensearch.core.common.io.stream.StreamOutput; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| /** | ||
| * Response payload for snapshot indices list. | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| public class SnapshotIndicesListResponse extends ActionResponse { | ||
|
|
||
| public static class IndexRow { | ||
| public String index; | ||
| public int shardsTotal; | ||
| public int shardsDone; | ||
| public int shardsFailed; | ||
| public long fileCount; | ||
| public long sizeInBytes; | ||
| public long startTimeInMillis; | ||
| public long timeInMillis; | ||
|
|
||
| public IndexRow() {} | ||
|
|
||
| public IndexRow(StreamInput in) throws IOException { | ||
| this.index = in.readString(); | ||
| this.shardsTotal = in.readVInt(); | ||
| this.shardsDone = in.readVInt(); | ||
| this.shardsFailed = in.readVInt(); | ||
| this.fileCount = in.readVLong(); | ||
| this.sizeInBytes = in.readVLong(); | ||
| this.startTimeInMillis = in.readVLong(); | ||
| this.timeInMillis = in.readVLong(); | ||
| } | ||
|
|
||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeString(index); | ||
| out.writeVInt(shardsTotal); | ||
| out.writeVInt(shardsDone); | ||
| out.writeVInt(shardsFailed); | ||
| out.writeVLong(fileCount); | ||
| out.writeVLong(sizeInBytes); | ||
| out.writeVLong(startTimeInMillis); | ||
| out.writeVLong(timeInMillis); | ||
| } | ||
| } | ||
|
|
||
| private List<IndexRow> rows = new ArrayList<>(); | ||
|
|
||
| public SnapshotIndicesListResponse() {} | ||
|
|
||
| public SnapshotIndicesListResponse(StreamInput in) throws IOException { | ||
| super(in); | ||
| int n = in.readVInt(); | ||
| for (int i = 0; i < n; i++) { | ||
| rows.add(new IndexRow(in)); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void writeTo(StreamOutput out) throws IOException { | ||
| out.writeVInt(rows.size()); | ||
| for (IndexRow r : rows) { | ||
| r.writeTo(out); | ||
| } | ||
| } | ||
|
|
||
| public List<IndexRow> rows() { return rows; } | ||
| } | ||
|
|
||
|
|
92 changes: 92 additions & 0 deletions
92
...r/src/main/java/org/opensearch/action/admin/cluster/snapshots/list/SnapshotListUtils.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,92 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.action.admin.cluster.snapshots.list; | ||
|
|
||
| import org.opensearch.cluster.metadata.IndexMetadata; | ||
| import org.opensearch.core.index.shard.ShardId; | ||
| import org.opensearch.index.snapshots.IndexShardSnapshotStatus; | ||
| import org.opensearch.repositories.IndexId; | ||
| import org.opensearch.repositories.Repository; | ||
| import org.opensearch.repositories.RepositoryData; | ||
| import org.opensearch.snapshots.SnapshotId; | ||
| import org.opensearch.snapshots.SnapshotInfo; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.HashMap; | ||
| import java.util.List; | ||
| import java.util.Map; | ||
|
|
||
| /** | ||
| * Utilities for snapshot listing that read only page-relevant data from repository. | ||
| * | ||
| * @opensearch.internal | ||
| */ | ||
| final class SnapshotListUtils { | ||
|
|
||
| static class IndexFileStats { | ||
| static final IndexFileStats EMPTY = new IndexFileStats(0L, 0L); | ||
| final long fileCount; | ||
| final long sizeInBytes; | ||
|
|
||
| IndexFileStats(long fileCount, long sizeInBytes) { | ||
| this.fileCount = fileCount; | ||
| this.sizeInBytes = sizeInBytes; | ||
| } | ||
| } | ||
|
|
||
| private SnapshotListUtils() {} | ||
|
|
||
| static Map<String, IndexMetadata> loadIndexMetadata( | ||
| Repository repo, | ||
| RepositoryData repositoryData, | ||
| SnapshotInfo snapshotInfo, | ||
| List<String> indices | ||
| ) throws IOException { | ||
| Map<String, IndexMetadata> result = new HashMap<>(); | ||
| SnapshotId snapshotId = snapshotInfo.snapshotId(); | ||
| for (String index : indices) { | ||
| IndexId indexId = repositoryData.resolveIndexId(index); | ||
| IndexMetadata meta = repo.getSnapshotIndexMetaData(repositoryData, snapshotId, indexId); | ||
| if (meta != null) { | ||
| result.put(index, meta); | ||
| } | ||
| } | ||
| return result; | ||
| } | ||
|
|
||
| static Map<String, IndexFileStats> loadIndexFileStats( | ||
| Repository repo, | ||
| RepositoryData repositoryData, | ||
| SnapshotInfo snapshotInfo, | ||
| List<String> indices, | ||
| Map<String, IndexMetadata> indexMetadataMap | ||
| ) throws IOException { | ||
| Map<String, IndexFileStats> result = new HashMap<>(); | ||
| SnapshotId snapshotId = snapshotInfo.snapshotId(); | ||
| for (String index : indices) { | ||
| IndexMetadata meta = indexMetadataMap.get(index); | ||
| int numShards = meta == null ? 0 : meta.getNumberOfShards(); | ||
| long files = 0L; | ||
| long bytes = 0L; | ||
| if (numShards > 0) { | ||
| IndexId indexId = repositoryData.resolveIndexId(index); | ||
| for (int shard = 0; shard < numShards; shard++) { | ||
| IndexShardSnapshotStatus.Copy s = repo.getShardSnapshotStatus(snapshotId, indexId, new ShardId(meta.getIndex(), shard)) | ||
| .asCopy(); | ||
| files += s.getTotalFileCount(); | ||
| bytes += s.getTotalSize(); | ||
| } | ||
| } | ||
| result.put(index, new IndexFileStats(files, bytes)); | ||
| } | ||
| return result; | ||
| } | ||
| } | ||
|
|
||
|
|
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
would you add any validation here?