1   /*
2    * Licensed to the Apache Software Foundation (ASF) under one or more
3    * contributor license agreements. See the NOTICE file distributed with this
4    * work for additional information regarding copyright ownership. The ASF
5    * licenses this file to you under the Apache License, Version 2.0 (the
6    * "License"); you may not use this file except in compliance with the License.
7    * You may obtain a copy of the License at
8    *
9    * http://www.apache.org/licenses/LICENSE-2.0
10   *
11   * Unless required by applicable law or agreed to in writing, software
12   * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14   * License for the specific language governing permissions and limitations
15   * under the License.
16   */
17  package org.apache.hadoop.hbase.util;
18  
19  import static org.junit.Assert.assertEquals;
20  
21  import java.io.IOException;
22  import java.util.ArrayList;
23  import java.util.Collection;
24  import java.util.List;
25  
26  import org.apache.commons.logging.Log;
27  import org.apache.commons.logging.LogFactory;
28  import org.apache.hadoop.conf.Configuration;
29  import org.apache.hadoop.hbase.HBaseTestingUtility;
30  import org.apache.hadoop.hbase.HColumnDescriptor;
31  import org.apache.hadoop.hbase.HConstants;
32  import org.apache.hadoop.hbase.LargeTests;
33  import org.apache.hadoop.hbase.TableNotFoundException;
34  import org.apache.hadoop.hbase.client.HBaseAdmin;
35  import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
36  import org.apache.hadoop.hbase.io.hfile.Compression;
37  import org.junit.After;
38  import org.junit.Before;
39  import org.junit.Test;
40  import org.junit.experimental.categories.Category;
41  import org.junit.runner.RunWith;
42  import org.junit.runners.Parameterized;
43  import org.junit.runners.Parameterized.Parameters;
44  
45  /**
46   * A write/read/verify load test on a mini HBase cluster. Tests reading
47   * and then writing.
48   */
49  @Category(LargeTests.class)
50  @RunWith(Parameterized.class)
51  public class TestMiniClusterLoadSequential {
52  
53    private static final Log LOG = LogFactory.getLog(
54        TestMiniClusterLoadSequential.class);
55  
56    protected static final byte[] TABLE = Bytes.toBytes("load_test_tbl");
57    protected static final byte[] CF = Bytes.toBytes("load_test_cf");
58    protected static final int NUM_THREADS = 8;
59    protected static final int NUM_RS = 2;
60    protected static final int TIMEOUT_MS = 180000;
61    protected static final HBaseTestingUtility TEST_UTIL =
62        new HBaseTestingUtility();
63  
64    protected final Configuration conf = TEST_UTIL.getConfiguration();
65    protected final boolean isMultiPut;
66    protected final DataBlockEncoding dataBlockEncoding;
67  
68    protected MultiThreadedWriter writerThreads;
69    protected MultiThreadedReader readerThreads;
70    protected int numKeys;
71  
72    protected Compression.Algorithm compression = Compression.Algorithm.NONE;
73  
74    public TestMiniClusterLoadSequential(boolean isMultiPut,
75        DataBlockEncoding dataBlockEncoding) {
76      this.isMultiPut = isMultiPut;
77      this.dataBlockEncoding = dataBlockEncoding;
78      conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024 * 1024);
79    }
80  
81    @Parameters
82    public static Collection<Object[]> parameters() {
83      List<Object[]> parameters = new ArrayList<Object[]>();
84      for (boolean multiPut : new boolean[]{false, true}) {
85        for (DataBlockEncoding dataBlockEncoding : new DataBlockEncoding[] {
86            DataBlockEncoding.NONE, DataBlockEncoding.PREFIX }) {
87          parameters.add(new Object[]{multiPut, dataBlockEncoding});
88        }
89      }
90      return parameters;
91    }
92  
93    @Before
94    public void setUp() throws Exception {
95      LOG.debug("Test setup: isMultiPut=" + isMultiPut);
96      TEST_UTIL.startMiniCluster(1, NUM_RS);
97    }
98  
99    @After
100   public void tearDown() throws Exception {
101     LOG.debug("Test teardown: isMultiPut=" + isMultiPut);
102     TEST_UTIL.shutdownMiniCluster();
103   }
104 
105   @Test(timeout=TIMEOUT_MS)
106   public void loadTest() throws Exception {
107     prepareForLoadTest();
108     runLoadTestOnExistingTable();
109   }
110 
111   protected void runLoadTestOnExistingTable() throws IOException {
112     writerThreads.start(0, numKeys, NUM_THREADS);
113     writerThreads.waitForFinish();
114     assertEquals(0, writerThreads.getNumWriteFailures());
115 
116     readerThreads.start(0, numKeys, NUM_THREADS);
117     readerThreads.waitForFinish();
118     assertEquals(0, readerThreads.getNumReadFailures());
119     assertEquals(0, readerThreads.getNumReadErrors());
120     assertEquals(numKeys, readerThreads.getNumKeysVerified());
121   }
122 
123   protected void prepareForLoadTest() throws IOException {
124     LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding +
125         ", isMultiPut=" + isMultiPut);
126     numKeys = numKeys();
127     HBaseAdmin admin = new HBaseAdmin(conf);
128     while (admin.getClusterStatus().getServers().size() < NUM_RS) {
129       LOG.info("Sleeping until " + NUM_RS + " RSs are online");
130       Threads.sleepWithoutInterrupt(1000);
131     }
132     admin.close();
133 
134     int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf,
135         TABLE, CF, compression, dataBlockEncoding);
136 
137     TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
138 
139     writerThreads = new MultiThreadedWriter(conf, TABLE, CF);
140     writerThreads.setMultiPut(isMultiPut);
141     readerThreads = new MultiThreadedReader(conf, TABLE, CF, 100);
142   }
143 
144   protected int numKeys() {
145     return 10000;
146   }
147 
148   protected HColumnDescriptor getColumnDesc(HBaseAdmin admin)
149       throws TableNotFoundException, IOException {
150     return admin.getTableDescriptor(TABLE).getFamily(CF);
151   }
152 
153 }