1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.master.snapshot;
19  
20  import static org.junit.Assert.assertFalse;
21  import static org.junit.Assert.assertTrue;
22  
23  import java.io.IOException;
24  import java.util.Collection;
25  import java.util.HashSet;
26  
27  import org.apache.commons.logging.Log;
28  import org.apache.commons.logging.LogFactory;
29  import org.apache.hadoop.fs.FileSystem;
30  import org.apache.hadoop.fs.Path;
31  import org.apache.hadoop.hbase.HBaseTestingUtility;
32  import org.apache.hadoop.hbase.HConstants;
33  import org.apache.hadoop.hbase.MediumTests;
34  import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
35  import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
36  import org.apache.hadoop.hbase.snapshot.SnapshotReferenceUtil;
37  import org.apache.hadoop.hbase.snapshot.TakeSnapshotUtils;
38  import org.apache.hadoop.hbase.util.FSUtils;
39  import org.junit.After;
40  import org.junit.AfterClass;
41  import org.junit.BeforeClass;
42  import org.junit.Test;
43  import org.junit.experimental.categories.Category;
44  
45  /**
46   * Test that we correctly reload the cache, filter directories, etc.
47   */
48  @Category(MediumTests.class)
49  public class TestSnapshotFileCache {
50  
51    private static final Log LOG = LogFactory.getLog(TestSnapshotFileCache.class);
52    private static final HBaseTestingUtility UTIL = new HBaseTestingUtility();
53    private static FileSystem fs;
54    private static Path rootDir;
55  
56    @BeforeClass
57    public static void startCluster() throws Exception {
58      UTIL.startMiniDFSCluster(1);
59      fs = UTIL.getDFSCluster().getFileSystem();
60      rootDir = UTIL.getDefaultRootDirPath();
61    }
62  
63    @AfterClass
64    public static void stopCluster() throws Exception {
65      UTIL.shutdownMiniDFSCluster();
66    }
67  
68    @After
69    public void cleanupFiles() throws Exception {
70      // cleanup the snapshot directory
71      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
72      fs.delete(snapshotDir, true);
73    }
74  
75    @Test(timeout = 10000000)
76    public void testLoadAndDelete() throws Exception {
77      // don't refresh the cache unless we tell it to
78      long period = Long.MAX_VALUE;
79      Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
80      SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
81          "test-snapshot-file-cache-refresh", new SnapshotFiles());
82  
83      Path snapshot = new Path(snapshotDir, "snapshot");
84      Path region = new Path(snapshot, "7e91021");
85      Path family = new Path(region, "fam");
86      Path file1 = new Path(family, "file1");
87      Path file2 = new Path(family, "file2");
88  
89      // create two hfiles under the snapshot
90      fs.create(file1);
91      fs.create(file2);
92  
93      FSUtils.logFileSystemState(fs, rootDir, LOG);
94  
95      // then make sure the cache finds them
96      assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
97      assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
98      String not = "file-shouldn't-be-found";
99      assertFalse("Cache found '" + not + "', but it shouldn't have.", cache.contains(not));
100 
101     // make sure we get a little bit of separation in the modification times
102     // its okay if we sleep a little longer (b/c of GC pause), as long as we sleep a little
103     Thread.sleep(10);
104 
105     LOG.debug("Deleting snapshot.");
106     // then delete the snapshot and make sure that we can still find the files
107     if (!fs.delete(snapshot, true)) {
108       throw new IOException("Couldn't delete " + snapshot + " for an unknown reason.");
109     }
110     FSUtils.logFileSystemState(fs, rootDir, LOG);
111 
112 
113     LOG.debug("Checking to see if file is deleted.");
114     assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
115     assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
116 
117     // then trigger a refresh
118     cache.triggerCacheRefreshForTesting();
119     // and not it shouldn't find those files
120     assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
121       cache.contains(file1.getName()));
122     assertFalse("Cache found '" + file2 + "', but it shouldn't have.",
123       cache.contains(file2.getName()));
124 
125     fs.delete(snapshotDir, true);
126   }
127 
128   @Test
129   public void testLoadsTmpDir() throws Exception {
130     // don't refresh the cache unless we tell it to
131     long period = Long.MAX_VALUE;
132     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
133     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
134         "test-snapshot-file-cache-refresh", new SnapshotFiles());
135 
136     // create a file in a 'completed' snapshot
137     Path snapshot = new Path(snapshotDir, "snapshot");
138     Path region = new Path(snapshot, "7e91021");
139     Path family = new Path(region, "fam");
140     Path file1 = new Path(family, "file1");
141     fs.create(file1);
142 
143     // create an 'in progress' snapshot
144     SnapshotDescription desc = SnapshotDescription.newBuilder().setName("working").build();
145     snapshot = SnapshotDescriptionUtils.getWorkingSnapshotDir(desc, rootDir);
146     region = new Path(snapshot, "7e91021");
147     family = new Path(region, "fam");
148     Path file2 = new Path(family, "file2");
149     fs.create(file2);
150 
151     FSUtils.logFileSystemState(fs, rootDir, LOG);
152 
153     // then make sure the cache finds both files
154     assertTrue("Cache didn't find:" + file1, cache.contains(file1.getName()));
155     assertTrue("Cache didn't find:" + file2, cache.contains(file2.getName()));
156   }
157 
158   @Test
159   public void testJustFindLogsDirectory() throws Exception {
160     // don't refresh the cache unless we tell it to
161     long period = Long.MAX_VALUE;
162     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
163     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
164         "test-snapshot-file-cache-refresh", new SnapshotFileCache.SnapshotFileInspector() {
165             public Collection<String> filesUnderSnapshot(final Path snapshotDir)
166                 throws IOException {
167               return SnapshotReferenceUtil.getHLogNames(fs, snapshotDir);
168             }
169         });
170 
171     // create a file in a 'completed' snapshot
172     Path snapshot = new Path(snapshotDir, "snapshot");
173     Path region = new Path(snapshot, "7e91021");
174     Path family = new Path(region, "fam");
175     Path file1 = new Path(family, "file1");
176     fs.create(file1);
177 
178     // and another file in the logs directory
179     Path logs = TakeSnapshotUtils.getSnapshotHLogsDir(snapshot, "server");
180     Path log = new Path(logs, "me.hbase.com%2C58939%2C1350424310315.1350424315552");
181     fs.create(log);
182 
183     FSUtils.logFileSystemState(fs, rootDir, LOG);
184 
185     // then make sure the cache only finds the log files
186     assertFalse("Cache found '" + file1 + "', but it shouldn't have.",
187       cache.contains(file1.getName()));
188     assertTrue("Cache didn't find:" + log, cache.contains(log.getName()));
189   }
190 
191   @Test
192   public void testReloadModifiedDirectory() throws IOException {
193     // don't refresh the cache unless we tell it to
194     long period = Long.MAX_VALUE;
195     Path snapshotDir = SnapshotDescriptionUtils.getSnapshotsDir(rootDir);
196     SnapshotFileCache cache = new SnapshotFileCache(fs, rootDir, period, 10000000,
197         "test-snapshot-file-cache-refresh", new SnapshotFiles());
198 
199     Path snapshot = new Path(snapshotDir, "snapshot");
200     Path region = new Path(snapshot, "7e91021");
201     Path family = new Path(region, "fam");
202     Path file1 = new Path(family, "file1");
203     Path file2 = new Path(family, "file2");
204 
205     // create two hfiles under the snapshot
206     fs.create(file1);
207     fs.create(file2);
208 
209     FSUtils.logFileSystemState(fs, rootDir, LOG);
210 
211     assertTrue("Cache didn't find " + file1, cache.contains(file1.getName()));
212 
213     // now delete the snapshot and add a file with a different name
214     fs.delete(snapshot, true);
215     Path file3 = new Path(family, "new_file");
216     fs.create(file3);
217 
218     FSUtils.logFileSystemState(fs, rootDir, LOG);
219     assertTrue("Cache didn't find new file:" + file3, cache.contains(file3.getName()));
220   }
221 
222   class SnapshotFiles implements SnapshotFileCache.SnapshotFileInspector {
223     public Collection<String> filesUnderSnapshot(final Path snapshotDir) throws IOException {
224       Collection<String> files =  new HashSet<String>();
225       files.addAll(SnapshotReferenceUtil.getHLogNames(fs, snapshotDir));
226       files.addAll(SnapshotReferenceUtil.getHFileNames(fs, snapshotDir));
227       return files;
228     }
229   };
230 }