1+ /**
2+ * Licensed to the Apache Software Foundation (ASF) under one
3+ * or more contributor license agreements. See the NOTICE file
4+ * distributed with this work for additional information
5+ * regarding copyright ownership. The ASF licenses this file
6+ * to you under the Apache License, Version 2.0 (the
7+ * "License"); you may not use this file except in compliance
8+ * with the License. You may obtain a copy of the License at
9+ *
10+ * http://www.apache.org/licenses/LICENSE-2.0
11+ *
12+ * Unless required by applicable law or agreed to in writing, software
13+ * distributed under the License is distributed on an "AS IS" BASIS,
14+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+ * See the License for the specific language governing permissions and
16+ * limitations under the License.
17+ */
18+
19+ package org .apache .hadoop .hdfs ;
20+
21+ import java .util .Arrays ;
22+ import java .util .EnumSet ;
23+
24+ import org .apache .commons .logging .Log ;
25+ import org .apache .commons .logging .LogFactory ;
26+ import org .apache .hadoop .classification .InterfaceAudience ;
27+ import org .apache .hadoop .conf .Configuration ;
28+
29+ /**
30+ * A block storage policy describes how to select the storage types
31+ * for the replicas of a block.
32+ */
33+ @ InterfaceAudience .Private
34+ public class BlockStoragePolicy {
35+ public static final Log LOG = LogFactory .getLog (BlockStoragePolicy .class );
36+
37+ public static final String DFS_BLOCK_STORAGE_POLICIES_KEY
38+ = "dfs.block.storage.policies" ;
39+ public static final String DFS_BLOCK_STORAGE_POLICY_KEY_PREFIX
40+ = "dfs.block.storage.policy." ;
41+ public static final String DFS_BLOCK_STORAGE_POLICY_CREATION_FALLBACK_KEY_PREFIX
42+ = "dfs.block.storage.policy.creation-fallback." ;
43+ public static final String DFS_BLOCK_STORAGE_POLICY_REPLICATION_FALLBACK_KEY_PREFIX
44+ = "dfs.block.storage.policy.replication-fallback." ;
45+
46+ public static final int ID_BIT_LENGTH = 4 ;
47+ public static final int ID_MAX = (1 << ID_BIT_LENGTH ) - 1 ;
48+
49+ /** A 4-bit policy ID */
50+ private final byte id ;
51+ /** Policy name */
52+ private final String name ;
53+
54+ /** The storage types to store the replicas of a new block. */
55+ private final StorageType [] storageTypes ;
56+ /** The fallback storage type for block creation. */
57+ private final StorageType [] creationFallbacks ;
58+ /** The fallback storage type for replication. */
59+ private final StorageType [] replicationFallbacks ;
60+
61+ BlockStoragePolicy (byte id , String name , StorageType [] storageTypes ,
62+ StorageType [] creationFallbacks , StorageType [] replicationFallbacks ) {
63+ this .id = id ;
64+ this .name = name ;
65+ this .storageTypes = storageTypes ;
66+ this .creationFallbacks = creationFallbacks ;
67+ this .replicationFallbacks = replicationFallbacks ;
68+ }
69+
70+ /**
71+ * @return a list of {@link StorageType}s for storing the replicas of a block.
72+ */
73+ StorageType [] getStoragteTypes (short replication ) {
74+ final StorageType [] types = new StorageType [replication ];
75+ int i = 0 ;
76+ for (; i < types .length && i < storageTypes .length ; i ++) {
77+ types [i ] = storageTypes [i ];
78+ }
79+ final StorageType last = storageTypes [storageTypes .length - 1 ];
80+ for (; i < types .length ; i ++) {
81+ types [i ] = last ;
82+ }
83+ return types ;
84+ }
85+
86+ /** @return the fallback {@link StorageType} for creation. */
87+ StorageType getCreationFallback (EnumSet <StorageType > unavailables ) {
88+ return getFallback (unavailables , creationFallbacks );
89+ }
90+
91+ /** @return the fallback {@link StorageType} for replication. */
92+ StorageType getReplicationFallback (EnumSet <StorageType > unavailables ) {
93+ return getFallback (unavailables , replicationFallbacks );
94+ }
95+
96+ @ Override
97+ public String toString () {
98+ return getClass ().getSimpleName () + "{" + name + ":" + id
99+ + ", storageTypes=" + Arrays .asList (storageTypes )
100+ + ", creationFallbacks=" + Arrays .asList (creationFallbacks )
101+ + ", replicationFallbacks=" + Arrays .asList (replicationFallbacks );
102+ }
103+
104+ private static StorageType getFallback (EnumSet <StorageType > unavailables ,
105+ StorageType [] fallbacks ) {
106+ for (StorageType fb : fallbacks ) {
107+ if (!unavailables .contains (fb )) {
108+ return fb ;
109+ }
110+ }
111+ return null ;
112+ }
113+
114+ private static byte parseID (String string ) {
115+ final byte id = Byte .parseByte (string );
116+ if (id < 1 ) {
117+ throw new IllegalArgumentException (
118+ "Invalid block storage policy ID: id = " + id + " < 1" );
119+ }
120+ if (id > 15 ) {
121+ throw new IllegalArgumentException (
122+ "Invalid block storage policy ID: id = " + id + " > MAX = " + ID_MAX );
123+ }
124+ return id ;
125+ }
126+
127+ private static StorageType [] parseStorageTypes (String [] strings ) {
128+ if (strings == null ) {
129+ return StorageType .EMPTY_ARRAY ;
130+ }
131+ final StorageType [] types = new StorageType [strings .length ];
132+ for (int i = 0 ; i < types .length ; i ++) {
133+ types [i ] = StorageType .valueOf (strings [i ].trim ().toUpperCase ());
134+ }
135+ return types ;
136+ }
137+
138+ private static StorageType [] readStorageTypes (byte id , String keyPrefix ,
139+ Configuration conf ) {
140+ final String [] values = conf .getStrings (keyPrefix + id );
141+ return parseStorageTypes (values );
142+ }
143+
144+ public static BlockStoragePolicy readBlockStoragePolicy (byte id , String name ,
145+ Configuration conf ) {
146+ final StorageType [] storageTypes = readStorageTypes (id ,
147+ DFS_BLOCK_STORAGE_POLICY_KEY_PREFIX , conf );
148+ final StorageType [] creationFallbacks = readStorageTypes (id ,
149+ DFS_BLOCK_STORAGE_POLICY_CREATION_FALLBACK_KEY_PREFIX , conf );
150+ final StorageType [] replicationFallbacks = readStorageTypes (id ,
151+ DFS_BLOCK_STORAGE_POLICY_REPLICATION_FALLBACK_KEY_PREFIX , conf );
152+ return new BlockStoragePolicy (id , name , storageTypes , creationFallbacks ,
153+ replicationFallbacks );
154+ }
155+
156+ public static BlockStoragePolicy [] readBlockStoragePolicies (
157+ Configuration conf ) {
158+ final BlockStoragePolicy [] policies = new BlockStoragePolicy [ID_MAX + 1 ];
159+ final String [] values = conf .getStrings (DFS_BLOCK_STORAGE_POLICIES_KEY );
160+ for (String v : values ) {
161+ v = v .trim ();
162+ final int i = v .indexOf (':' );
163+ final String name = v .substring (0 , i );
164+ final byte id = parseID (v .substring (i + 1 ));
165+ if (policies [id ] != null ) {
166+ throw new IllegalArgumentException (
167+ "Policy duplication: ID " + id + " appears more than once in "
168+ + DFS_BLOCK_STORAGE_POLICIES_KEY );
169+ }
170+ policies [id ] = readBlockStoragePolicy (id , name , conf );
171+ LOG .info (policies [id ]);
172+ }
173+ return policies ;
174+ }
175+ }
0 commit comments