1   /*
2    * Copyright 2009 The Apache Software Foundation
3    *
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *     http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing, software
15   * distributed under the License is distributed on an "AS IS" BASIS,
16   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17   * See the License for the specific language governing permissions and
18   * limitations under the License.
19   */
20  
21  package org.apache.hadoop.hbase.io;
22  
23  import java.io.IOException;
24  import java.nio.ByteBuffer;
25  import java.util.ArrayList;
26  import java.util.TreeMap;
27  import java.util.concurrent.ConcurrentHashMap;
28  import java.util.concurrent.ConcurrentSkipListMap;
29  import java.util.concurrent.CopyOnWriteArrayList;
30  import java.util.concurrent.CopyOnWriteArraySet;
31  import java.util.concurrent.atomic.AtomicBoolean;
32  import java.util.concurrent.atomic.AtomicInteger;
33  import java.util.concurrent.atomic.AtomicLong;
34  import java.util.concurrent.locks.ReentrantReadWriteLock;
35  
36  import junit.framework.TestCase;
37  
38  import org.apache.commons.logging.Log;
39  import org.apache.commons.logging.LogFactory;
40  import org.apache.hadoop.hbase.KeyValue;
41  import org.apache.hadoop.hbase.SmallTests;
42  import org.apache.hadoop.hbase.client.Put;
43  import org.apache.hadoop.hbase.io.hfile.BlockCacheKey;
44  import org.apache.hadoop.hbase.io.hfile.CachedBlock;
45  import org.apache.hadoop.hbase.io.hfile.LruBlockCache;
46  import org.apache.hadoop.hbase.regionserver.HRegion;
47  import org.apache.hadoop.hbase.regionserver.MemStore;
48  import org.apache.hadoop.hbase.regionserver.Store;
49  import org.apache.hadoop.hbase.regionserver.metrics.SchemaConfigured;
50  import org.apache.hadoop.hbase.util.Bytes;
51  import org.apache.hadoop.hbase.util.ClassSize;
52  import org.junit.experimental.categories.Category;
53  
54  /**
55   * Testing the sizing that HeapSize offers and compares to the size given by
56   * ClassSize.
57   */
58  @Category(SmallTests.class)
59  public class TestHeapSize extends TestCase {
60    static final Log LOG = LogFactory.getLog(TestHeapSize.class);
61    // List of classes implementing HeapSize
62    // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry<K,V>, HStoreKey
63    // KeyValue, LruBlockCache, LruHashMap<K,V>, Put, HLogKey
64  
65    /**
66     * Test our hard-coded sizing of native java objects
67     */
68    public void testNativeSizes() throws IOException {
69      @SuppressWarnings("rawtypes")
70      Class cl = null;
71      long expected = 0L;
72      long actual = 0L;
73  
74      // ArrayList
75      cl = ArrayList.class;
76      expected = ClassSize.estimateBase(cl, false);
77      actual = ClassSize.ARRAYLIST;
78      if(expected != actual) {
79        ClassSize.estimateBase(cl, true);
80        assertEquals(expected, actual);
81      }
82  
83      // ByteBuffer
84      cl = ByteBuffer.class;
85      expected = ClassSize.estimateBase(cl, false);
86      actual = ClassSize.BYTE_BUFFER;
87      if(expected != actual) {
88        ClassSize.estimateBase(cl, true);
89        assertEquals(expected, actual);
90      }
91  
92      // Integer
93      cl = Integer.class;
94      expected = ClassSize.estimateBase(cl, false);
95      actual = ClassSize.INTEGER;
96      if(expected != actual) {
97        ClassSize.estimateBase(cl, true);
98        assertEquals(expected, actual);
99      }
100 
101     // Map.Entry
102     // Interface is public, all others are not.  Hard to size via ClassSize
103 //    cl = Map.Entry.class;
104 //    expected = ClassSize.estimateBase(cl, false);
105 //    actual = ClassSize.MAP_ENTRY;
106 //    if(expected != actual) {
107 //      ClassSize.estimateBase(cl, true);
108 //      assertEquals(expected, actual);
109 //    }
110 
111     // Object
112     cl = Object.class;
113     expected = ClassSize.estimateBase(cl, false);
114     actual = ClassSize.OBJECT;
115     if(expected != actual) {
116       ClassSize.estimateBase(cl, true);
117       assertEquals(expected, actual);
118     }
119 
120     // TreeMap
121     cl = TreeMap.class;
122     expected = ClassSize.estimateBase(cl, false);
123     actual = ClassSize.TREEMAP;
124     if(expected != actual) {
125       ClassSize.estimateBase(cl, true);
126       assertEquals(expected, actual);
127     }
128 
129     // String
130     cl = String.class;
131     expected = ClassSize.estimateBase(cl, false);
132     actual = ClassSize.STRING;
133     if(expected != actual) {
134       ClassSize.estimateBase(cl, true);
135       assertEquals(expected, actual);
136     }
137 
138     // ConcurrentHashMap
139     cl = ConcurrentHashMap.class;
140     expected = ClassSize.estimateBase(cl, false);
141     actual = ClassSize.CONCURRENT_HASHMAP;
142     if(expected != actual) {
143       ClassSize.estimateBase(cl, true);
144       assertEquals(expected, actual);
145     }
146 
147     // ConcurrentSkipListMap
148     cl = ConcurrentSkipListMap.class;
149     expected = ClassSize.estimateBase(cl, false);
150     actual = ClassSize.CONCURRENT_SKIPLISTMAP;
151     if(expected != actual) {
152       ClassSize.estimateBase(cl, true);
153       assertEquals(expected, actual);
154     }
155 
156     // ReentrantReadWriteLock
157     cl = ReentrantReadWriteLock.class;
158     expected = ClassSize.estimateBase(cl, false);
159     actual = ClassSize.REENTRANT_LOCK;
160     if(expected != actual) {
161       ClassSize.estimateBase(cl, true);
162       assertEquals(expected, actual);
163     }
164 
165     // AtomicLong
166     cl = AtomicLong.class;
167     expected = ClassSize.estimateBase(cl, false);
168     actual = ClassSize.ATOMIC_LONG;
169     if(expected != actual) {
170       ClassSize.estimateBase(cl, true);
171       assertEquals(expected, actual);
172     }
173 
174     // AtomicInteger
175     cl = AtomicInteger.class;
176     expected = ClassSize.estimateBase(cl, false);
177     actual = ClassSize.ATOMIC_INTEGER;
178     if(expected != actual) {
179       ClassSize.estimateBase(cl, true);
180       assertEquals(expected, actual);
181     }
182 
183     // AtomicBoolean
184     cl = AtomicBoolean.class;
185     expected = ClassSize.estimateBase(cl, false);
186     actual = ClassSize.ATOMIC_BOOLEAN;
187     if(expected != actual) {
188       ClassSize.estimateBase(cl, true);
189       assertEquals(expected, actual);
190     }
191 
192     // CopyOnWriteArraySet
193     cl = CopyOnWriteArraySet.class;
194     expected = ClassSize.estimateBase(cl, false);
195     actual = ClassSize.COPYONWRITE_ARRAYSET;
196     if(expected != actual) {
197       ClassSize.estimateBase(cl, true);
198       assertEquals(expected, actual);
199     }
200 
201     // CopyOnWriteArrayList
202     cl = CopyOnWriteArrayList.class;
203     expected = ClassSize.estimateBase(cl, false);
204     actual = ClassSize.COPYONWRITE_ARRAYLIST;
205     if(expected != actual) {
206       ClassSize.estimateBase(cl, true);
207       assertEquals(expected, actual);
208     }
209 
210 
211   }
212 
213   /**
214    * Testing the classes that implements HeapSize and are a part of 0.20.
215    * Some are not tested here for example BlockIndex which is tested in
216    * TestHFile since it is a non public class
217    * @throws IOException
218    */
219   public void testSizes() throws IOException {
220     @SuppressWarnings("rawtypes")
221     Class cl = null;
222     long expected = 0L;
223     long actual = 0L;
224 
225     //KeyValue
226     cl = KeyValue.class;
227     expected = ClassSize.estimateBase(cl, false);
228     KeyValue kv = new KeyValue();
229     actual = kv.heapSize();
230     if(expected != actual) {
231       ClassSize.estimateBase(cl, true);
232       assertEquals(expected, actual);
233     }
234 
235     //Put
236     cl = Put.class;
237     expected = ClassSize.estimateBase(cl, false);
238     //The actual TreeMap is not included in the above calculation
239     expected += ClassSize.TREEMAP;
240     Put put = new Put(Bytes.toBytes(""));
241     actual = put.heapSize();
242     if(expected != actual) {
243       ClassSize.estimateBase(cl, true);
244       assertEquals(expected, actual);
245     }
246 
247     //LruBlockCache Overhead
248     cl = LruBlockCache.class;
249     actual = LruBlockCache.CACHE_FIXED_OVERHEAD;
250     expected = ClassSize.estimateBase(cl, false);
251     if(expected != actual) {
252       ClassSize.estimateBase(cl, true);
253       assertEquals(expected, actual);
254     }
255 
256     // CachedBlock Fixed Overhead
257     // We really need "deep" sizing but ClassSize does not do this.
258     // Perhaps we should do all these more in this style....
259     cl = CachedBlock.class;
260     actual = CachedBlock.PER_BLOCK_OVERHEAD;
261     expected = ClassSize.estimateBase(cl, false);
262     expected += ClassSize.estimateBase(String.class, false);
263     expected += ClassSize.estimateBase(ByteBuffer.class, false);
264     if(expected != actual) {
265       ClassSize.estimateBase(cl, true);
266       ClassSize.estimateBase(String.class, true);
267       ClassSize.estimateBase(ByteBuffer.class, true);
268       assertEquals(expected, actual);
269     }
270 
271     // MemStore Overhead
272     cl = MemStore.class;
273     actual = MemStore.FIXED_OVERHEAD;
274     expected = ClassSize.estimateBase(cl, false);
275     if(expected != actual) {
276       ClassSize.estimateBase(cl, true);
277       assertEquals(expected, actual);
278     }
279 
280     // MemStore Deep Overhead
281     actual = MemStore.DEEP_OVERHEAD;
282     expected = ClassSize.estimateBase(cl, false);
283     expected += ClassSize.estimateBase(ReentrantReadWriteLock.class, false);
284     expected += ClassSize.estimateBase(AtomicLong.class, false);
285     expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
286     expected += ClassSize.estimateBase(ConcurrentSkipListMap.class, false);
287     expected += ClassSize.estimateBase(CopyOnWriteArraySet.class, false);
288     expected += ClassSize.estimateBase(CopyOnWriteArrayList.class, false);
289     if(expected != actual) {
290       ClassSize.estimateBase(cl, true);
291       ClassSize.estimateBase(ReentrantReadWriteLock.class, true);
292       ClassSize.estimateBase(AtomicLong.class, true);
293       ClassSize.estimateBase(ConcurrentSkipListMap.class, true);
294       ClassSize.estimateBase(CopyOnWriteArraySet.class, true);
295       ClassSize.estimateBase(CopyOnWriteArrayList.class, true);
296       assertEquals(expected, actual);
297     }
298 
299     // SchemaConfigured
300     LOG.debug("Heap size for: " + SchemaConfigured.class.getName());
301     SchemaConfigured sc = new SchemaConfigured(null, "myTable", "myCF");
302     assertEquals(ClassSize.estimateBase(SchemaConfigured.class, true),
303         sc.heapSize());
304 
305     // Store Overhead
306     cl = Store.class;
307     actual = Store.FIXED_OVERHEAD;
308     expected = ClassSize.estimateBase(cl, false);
309     if(expected != actual) {
310       ClassSize.estimateBase(cl, true);
311       assertEquals(expected, actual);
312     }
313 
314     // Region Overhead
315     cl = HRegion.class;
316     actual = HRegion.FIXED_OVERHEAD;
317     expected = ClassSize.estimateBase(cl, false);
318     if (expected != actual) {
319       ClassSize.estimateBase(cl, true);
320       assertEquals(expected, actual);
321     }
322 
323     // Block cache key overhead
324     cl = BlockCacheKey.class;
325     // Passing zero length file name, because estimateBase does not handle
326     // deep overhead.
327     actual = new BlockCacheKey("", 0).heapSize();
328     expected  = ClassSize.estimateBase(cl, false);
329     if (expected != actual) {
330       ClassSize.estimateBase(cl, true);
331       assertEquals(expected, actual);
332     }
333 
334     // Currently NOT testing Deep Overheads of many of these classes.
335     // Deep overheads cover a vast majority of stuff, but will not be 100%
336     // accurate because it's unclear when we're referencing stuff that's already
337     // accounted for.  But we have satisfied our two core requirements.
338     // Sizing is quite accurate now, and our tests will throw errors if
339     // any of these classes are modified without updating overhead sizes.
340   }
341 
342   @org.junit.Rule
343   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
344     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
345 }
346