1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19 package org.apache.hadoop.hbase.snapshot;
20
21 import static org.junit.Assert.assertEquals;
22 import static org.junit.Assert.assertFalse;
23 import static org.junit.Assert.assertTrue;
24
25 import java.io.IOException;
26 import java.net.URI;
27 import java.util.ArrayList;
28 import java.util.Arrays;
29 import java.util.List;
30 import java.util.HashSet;
31 import java.util.Set;
32
33 import org.apache.commons.logging.Log;
34 import org.apache.commons.logging.LogFactory;
35
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.FileStatus;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.hbase.HBaseTestingUtility;
41 import org.apache.hadoop.hbase.HColumnDescriptor;
42 import org.apache.hadoop.hbase.HTableDescriptor;
43 import org.apache.hadoop.hbase.KeyValue;
44 import org.apache.hadoop.hbase.MediumTests;
45 import org.apache.hadoop.hbase.MiniHBaseCluster;
46 import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
47 import org.apache.hadoop.hbase.client.HBaseAdmin;
48 import org.apache.hadoop.hbase.client.HTable;
49 import org.apache.hadoop.hbase.util.Bytes;
50 import org.apache.hadoop.hbase.util.FSUtils;
51 import org.apache.hadoop.hbase.util.Pair;
52 import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
53 import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
54 import org.apache.hadoop.mapreduce.Job;
55 import org.junit.After;
56 import org.junit.AfterClass;
57 import org.junit.Before;
58 import org.junit.BeforeClass;
59 import org.junit.Test;
60 import org.junit.experimental.categories.Category;
61
62
63
64
65 @Category(MediumTests.class)
66 public class TestExportSnapshot {
67 private final Log LOG = LogFactory.getLog(getClass());
68
69 private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
70
71 private final static byte[] FAMILY = Bytes.toBytes("cf");
72
73 private byte[] snapshotName;
74 private byte[] tableName;
75 private HBaseAdmin admin;
76
77 @BeforeClass
78 public static void setUpBeforeClass() throws Exception {
79 TEST_UTIL.getConfiguration().setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
80 TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100);
81 TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250);
82 TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6);
83 TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true);
84 TEST_UTIL.startMiniCluster(3);
85 }
86
87 @AfterClass
88 public static void tearDownAfterClass() throws Exception {
89 TEST_UTIL.shutdownMiniCluster();
90 }
91
92
93
94
95 @Before
96 public void setUp() throws Exception {
97 this.admin = TEST_UTIL.getHBaseAdmin();
98
99 long tid = System.currentTimeMillis();
100 tableName = Bytes.toBytes("testtb-" + tid);
101 snapshotName = Bytes.toBytes("snaptb0-" + tid);
102
103
104 HTableDescriptor htd = new HTableDescriptor(tableName);
105 htd.addFamily(new HColumnDescriptor(FAMILY));
106 admin.createTable(htd, null);
107 HTable table = new HTable(TEST_UTIL.getConfiguration(), tableName);
108 TEST_UTIL.loadTable(table, FAMILY);
109
110
111 admin.disableTable(tableName);
112 admin.snapshot(snapshotName, tableName);
113 admin.enableTable(tableName);
114 }
115
116 @After
117 public void tearDown() throws Exception {
118 this.admin.close();
119 }
120
121
122
123
124
125
126
127
128
129
130 @Test
131 public void testBalanceSplit() throws Exception {
132
133 List<Pair<Path, Long>> files = new ArrayList<Pair<Path, Long>>();
134 for (long i = 0; i <= 20; i++) {
135 files.add(new Pair<Path, Long>(new Path("file-" + i), i));
136 }
137
138
139
140
141
142
143
144 List<List<Path>> splits = ExportSnapshot.getBalancedSplits(files, 5);
145 assertEquals(5, splits.size());
146 assertEquals(Arrays.asList(new Path("file-20"), new Path("file-11"),
147 new Path("file-10"), new Path("file-1"), new Path("file-0")), splits.get(0));
148 assertEquals(Arrays.asList(new Path("file-19"), new Path("file-12"),
149 new Path("file-9"), new Path("file-2")), splits.get(1));
150 assertEquals(Arrays.asList(new Path("file-18"), new Path("file-13"),
151 new Path("file-8"), new Path("file-3")), splits.get(2));
152 assertEquals(Arrays.asList(new Path("file-17"), new Path("file-14"),
153 new Path("file-7"), new Path("file-4")), splits.get(3));
154 assertEquals(Arrays.asList(new Path("file-16"), new Path("file-15"),
155 new Path("file-6"), new Path("file-5")), splits.get(4));
156 }
157
158
159
160
161 @Test
162 public void testExportFileSystemState() throws Exception {
163 Path copyDir = TEST_UTIL.getDataTestDir("export-" + System.currentTimeMillis());
164 URI hdfsUri = FileSystem.get(TEST_UTIL.getConfiguration()).getUri();
165 FileSystem fs = FileSystem.get(copyDir.toUri(), new Configuration());
166 copyDir = copyDir.makeQualified(fs);
167
168
169 int res = ExportSnapshot.innerMain(TEST_UTIL.getConfiguration(), new String[] {
170 "-snapshot", Bytes.toString(snapshotName),
171 "-copy-to", copyDir.toString()
172 });
173 assertEquals(0, res);
174
175
176 FileStatus[] rootFiles = fs.listStatus(copyDir);
177 assertEquals(2, rootFiles.length);
178 for (FileStatus fileStatus: rootFiles) {
179 String name = fileStatus.getPath().getName();
180 assertTrue(fileStatus.isDir());
181 assertTrue(name.equals(".snapshot") || name.equals(".archive"));
182 }
183
184
185 final FileSystem hdfs = FileSystem.get(hdfsUri, TEST_UTIL.getConfiguration());
186 final Path snapshotDir = new Path(".snapshot", Bytes.toString(snapshotName));
187 verifySnapshot(hdfs, new Path(TEST_UTIL.getDefaultRootDirPath(), snapshotDir),
188 fs, new Path(copyDir, snapshotDir));
189 verifyArchive(fs, copyDir, Bytes.toString(snapshotName));
190
191
192 fs.delete(copyDir, true);
193 }
194
195
196
197
198 private void verifySnapshot(final FileSystem fs1, final Path root1,
199 final FileSystem fs2, final Path root2) throws IOException {
200 Set<String> s = new HashSet<String>();
201 assertEquals(listFiles(fs1, root1, root1), listFiles(fs2, root2, root2));
202 }
203
204
205
206
207 private void verifyArchive(final FileSystem fs, final Path rootDir, final String snapshotName)
208 throws IOException {
209 final Path exportedSnapshot = new Path(rootDir, new Path(".snapshot", snapshotName));
210 final Path exportedArchive = new Path(rootDir, ".archive");
211 LOG.debug(listFiles(fs, exportedArchive, exportedArchive));
212 SnapshotReferenceUtil.visitReferencedFiles(fs, exportedSnapshot,
213 new SnapshotReferenceUtil.FileVisitor() {
214 public void storeFile (final String region, final String family, final String hfile)
215 throws IOException {
216 verifyNonEmptyFile(new Path(exportedArchive,
217 new Path(Bytes.toString(tableName), new Path(region, new Path(family, hfile)))));
218 }
219
220 public void recoveredEdits (final String region, final String logfile)
221 throws IOException {
222 verifyNonEmptyFile(new Path(exportedSnapshot,
223 new Path(Bytes.toString(tableName), new Path(region, logfile))));
224 }
225
226 public void logFile (final String server, final String logfile)
227 throws IOException {
228 verifyNonEmptyFile(new Path(exportedSnapshot, new Path(server, logfile)));
229 }
230
231 private void verifyNonEmptyFile(final Path path) throws IOException {
232 LOG.debug(path);
233 assertTrue(fs.exists(path));
234 assertTrue(fs.getFileStatus(path).getLen() > 0);
235 }
236 });
237 }
238
239 private Set<String> listFiles(final FileSystem fs, final Path root, final Path dir)
240 throws IOException {
241 Set<String> files = new HashSet<String>();
242 int rootPrefix = root.toString().length();
243 FileStatus[] list = FSUtils.listStatus(fs, dir);
244 if (list != null) {
245 for (FileStatus fstat: list) {
246 LOG.debug(fstat.getPath());
247 if (fstat.isDir()) {
248 files.addAll(listFiles(fs, root, fstat.getPath()));
249 } else {
250 files.add(fstat.getPath().toString().substring(rootPrefix));
251 }
252 }
253 }
254 return files;
255 }
256 }
257