1 /**
2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
9 *
10 * http://www.apache.org/licenses/LICENSE-2.0
11 *
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS,
14 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 * See the License for the specific language governing permissions and
16 * limitations under the License.
17 */
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import java.io.IOException;
22 import java.util.HashSet;
23 import java.util.LinkedList;
24 import java.util.List;
25 import java.util.Map;
26 import java.util.Set;
27 import java.util.TreeMap;
28
29 import org.apache.hadoop.classification.InterfaceAudience;
30 import org.apache.hadoop.fs.FileStatus;
31 import org.apache.hadoop.fs.FileSystem;
32 import org.apache.hadoop.fs.Path;
33 import org.apache.hadoop.hbase.io.HFileLink;
34 import org.apache.hadoop.hbase.regionserver.wal.HLogUtil;
35 import org.apache.hadoop.hbase.util.FSUtils;
36 import org.apache.hadoop.hbase.util.FSVisitor;
37
38 /**
39 * Utility methods for interacting with the snapshot referenced files.
40 */
41 @InterfaceAudience.Private
42 public final class SnapshotReferenceUtil {
43 public interface FileVisitor extends FSVisitor.StoreFileVisitor,
44 FSVisitor.RecoveredEditsVisitor, FSVisitor.LogFileVisitor {
45 }
46
47 private SnapshotReferenceUtil() {
48 // private constructor for utility class
49 }
50
51 /**
52 * Get log directory for a server in a snapshot.
53 *
54 * @param snapshotDir directory where the specific snapshot is stored
55 * @param serverName name of the parent regionserver for the log files
56 * @return path to the log home directory for the archive files.
57 */
58 public static Path getLogsDir(Path snapshotDir, String serverName) {
59 return new Path(snapshotDir, HLogUtil.getHLogDirectoryName(serverName));
60 }
61
62 /**
63 * Get the snapshotted recovered.edits dir for the specified region.
64 *
65 * @param snapshotDir directory where the specific snapshot is stored
66 * @param regionName name of the region
67 * @return path to the recovered.edits directory for the specified region files.
68 */
69 public static Path getRecoveredEditsDir(Path snapshotDir, String regionName) {
70 return HLogUtil.getRegionDirRecoveredEditsDir(new Path(snapshotDir, regionName));
71 }
72
73 /**
74 * Get the snapshot recovered.edits file
75 *
76 * @param snapshotDir directory where the specific snapshot is stored
77 * @param regionName name of the region
78 * @param logfile name of the edit file
79 * @return full path of the log file for the specified region files.
80 */
81 public static Path getRecoveredEdits(Path snapshotDir, String regionName, String logfile) {
82 return new Path(getRecoveredEditsDir(snapshotDir, regionName), logfile);
83 }
84
85 /**
86 * Iterate over the snapshot store files, restored.edits and logs
87 *
88 * @param fs {@link FileSystem}
89 * @param snapshotDir {@link Path} to the Snapshot directory
90 * @param visitor callback object to get the referenced files
91 * @throws IOException if an error occurred while scanning the directory
92 */
93 public static void visitReferencedFiles(final FileSystem fs, final Path snapshotDir,
94 final FileVisitor visitor) throws IOException {
95 visitTableStoreFiles(fs, snapshotDir, visitor);
96 visitRecoveredEdits(fs, snapshotDir, visitor);
97 visitLogFiles(fs, snapshotDir, visitor);
98 }
99
100 /**
101 * Iterate over the snapshot store files
102 *
103 * @param fs {@link FileSystem}
104 * @param snapshotDir {@link Path} to the Snapshot directory
105 * @param visitor callback object to get the store files
106 * @throws IOException if an error occurred while scanning the directory
107 */
108 public static void visitTableStoreFiles(final FileSystem fs, final Path snapshotDir,
109 final FSVisitor.StoreFileVisitor visitor) throws IOException {
110 FSVisitor.visitTableStoreFiles(fs, snapshotDir, visitor);
111 }
112
113 /**
114 * Iterate over the snapshot store files in the specified region
115 *
116 * @param fs {@link FileSystem}
117 * @param regionDir {@link Path} to the Snapshot region directory
118 * @param visitor callback object to get the store files
119 * @throws IOException if an error occurred while scanning the directory
120 */
121 public static void visitRegionStoreFiles(final FileSystem fs, final Path regionDir,
122 final FSVisitor.StoreFileVisitor visitor) throws IOException {
123 FSVisitor.visitRegionStoreFiles(fs, regionDir, visitor);
124 }
125
126 /**
127 * Iterate over the snapshot recovered.edits
128 *
129 * @param fs {@link FileSystem}
130 * @param snapshotDir {@link Path} to the Snapshot directory
131 * @param visitor callback object to get the recovered.edits files
132 * @throws IOException if an error occurred while scanning the directory
133 */
134 public static void visitRecoveredEdits(final FileSystem fs, final Path snapshotDir,
135 final FSVisitor.RecoveredEditsVisitor visitor) throws IOException {
136 FSVisitor.visitTableRecoveredEdits(fs, snapshotDir, visitor);
137 }
138
139 /**
140 * Iterate over the snapshot log files
141 *
142 * @param fs {@link FileSystem}
143 * @param snapshotDir {@link Path} to the Snapshot directory
144 * @param visitor callback object to get the log files
145 * @throws IOException if an error occurred while scanning the directory
146 */
147 public static void visitLogFiles(final FileSystem fs, final Path snapshotDir,
148 final FSVisitor.LogFileVisitor visitor) throws IOException {
149 FSVisitor.visitLogFiles(fs, snapshotDir, visitor);
150 }
151
152 /**
153 * Returns the set of region names available in the snapshot.
154 *
155 * @param fs {@link FileSystem}
156 * @param snapshotDir {@link Path} to the Snapshot directory
157 * @throws IOException if an error occurred while scanning the directory
158 * @return the set of the regions contained in the snapshot
159 */
160 public static Set<String> getSnapshotRegionNames(final FileSystem fs, final Path snapshotDir)
161 throws IOException {
162 FileStatus[] regionDirs = FSUtils.listStatus(fs, snapshotDir, new FSUtils.RegionDirFilter(fs));
163 if (regionDirs == null) return null;
164
165 Set<String> regions = new HashSet<String>();
166 for (FileStatus regionDir: regionDirs) {
167 regions.add(regionDir.getPath().getName());
168 }
169 return regions;
170 }
171
172 /**
173 * Get the list of hfiles for the specified snapshot region.
174 * NOTE: The current implementation keeps one empty file per HFile in the region.
175 * The file name matches the one in the original table, and by reconstructing
176 * the path you can quickly jump to the referenced file.
177 *
178 * @param fs {@link FileSystem}
179 * @param snapshotRegionDir {@link Path} to the Snapshot region directory
180 * @return Map of hfiles per family, the key is the family name and values are hfile names
181 * @throws IOException if an error occurred while scanning the directory
182 */
183 public static Map<String, List<String>> getRegionHFileReferences(final FileSystem fs,
184 final Path snapshotRegionDir) throws IOException {
185 final Map<String, List<String>> familyFiles = new TreeMap<String, List<String>>();
186
187 visitRegionStoreFiles(fs, snapshotRegionDir,
188 new FSVisitor.StoreFileVisitor() {
189 public void storeFile (final String region, final String family, final String hfile)
190 throws IOException {
191 List<String> hfiles = familyFiles.get(family);
192 if (hfiles == null) {
193 hfiles = new LinkedList<String>();
194 familyFiles.put(family, hfiles);
195 }
196 hfiles.add(hfile);
197 }
198 });
199
200 return familyFiles;
201 }
202
203 /**
204 * Returns the store file names in the snapshot.
205 *
206 * @param fs {@link FileSystem}
207 * @param snapshotDir {@link Path} to the Snapshot directory
208 * @throws IOException if an error occurred while scanning the directory
209 * @return the names of hfiles in the specified snaphot
210 */
211 public static Set<String> getHFileNames(final FileSystem fs, final Path snapshotDir)
212 throws IOException {
213 final Set<String> names = new HashSet<String>();
214 visitTableStoreFiles(fs, snapshotDir, new FSVisitor.StoreFileVisitor() {
215 public void storeFile (final String region, final String family, final String hfile)
216 throws IOException {
217 if (HFileLink.isHFileLink(hfile)) {
218 names.add(HFileLink.getReferencedHFileName(hfile));
219 } else {
220 names.add(hfile);
221 }
222 }
223 });
224 return names;
225 }
226
227 /**
228 * Returns the log file names available in the snapshot.
229 *
230 * @param fs {@link FileSystem}
231 * @param snapshotDir {@link Path} to the Snapshot directory
232 * @throws IOException if an error occurred while scanning the directory
233 * @return the names of hlogs in the specified snaphot
234 */
235 public static Set<String> getHLogNames(final FileSystem fs, final Path snapshotDir)
236 throws IOException {
237 final Set<String> names = new HashSet<String>();
238 visitLogFiles(fs, snapshotDir, new FSVisitor.LogFileVisitor() {
239 public void logFile (final String server, final String logfile) throws IOException {
240 names.add(logfile);
241 }
242 });
243 return names;
244 }
245 }