1   package org.apache.hadoop.hbase.regionserver;
2   
3   import static org.junit.Assert.assertEquals;
4   
5   import java.io.IOException;
6   import java.util.ArrayList;
7   import java.util.List;
8   import java.util.concurrent.CountDownLatch;
9   
10  import org.apache.hadoop.conf.Configuration;
11  import org.apache.hadoop.fs.FileSystem;
12  import org.apache.hadoop.fs.Path;
13  import org.apache.hadoop.hbase.HBaseConfiguration;
14  import org.apache.hadoop.hbase.HConstants;
15  import org.apache.hadoop.hbase.HRegionInfo;
16  import org.apache.hadoop.hbase.HTableDescriptor;
17  import org.apache.hadoop.hbase.KeyValue;
18  import org.apache.hadoop.hbase.SmallTests;
19  import org.apache.hadoop.hbase.MultithreadedTestUtil;
20  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestContext;
21  import org.apache.hadoop.hbase.MultithreadedTestUtil.TestThread;
22  import org.apache.hadoop.hbase.client.Mutation;
23  import org.apache.hadoop.hbase.client.Put;
24  import org.apache.hadoop.hbase.client.Scan;
25  import org.apache.hadoop.hbase.filter.BinaryComparator;
26  import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
27  import org.apache.hadoop.hbase.io.HeapSize;
28  import org.apache.hadoop.hbase.regionserver.wal.HLog;
29  import org.apache.hadoop.hbase.util.Bytes;
30  import org.apache.hadoop.hbase.util.Pair;
31  import org.junit.Test;
32  import org.junit.experimental.categories.Category;
33  
34  import com.google.common.collect.Lists;
35  
36  /**
37   * Test of HBASE-7051; that checkAndPuts and puts behave atomically with respect to each other.
38   * Rather than perform a bunch of trials to verify atomicity, this test recreates a race condition
39   * that causes the test to fail if checkAndPut doesn't wait for outstanding put transactions
40   * to complete.  It does this by invasively overriding HRegion function to affect the timing of
41   * the operations.
42   */
43  @Category(SmallTests.class)
44  public class TestHBase7051 {
45  
46    private static CountDownLatch latch = new CountDownLatch(1);
47    private enum TestStep {
48      INIT,                  // initial put of 10 to set value of the cell
49      PUT_STARTED,           // began doing a put of 50 to cell
50      PUT_COMPLETED,         // put complete (released RowLock, but may not have advanced MVCC).
51      CHECKANDPUT_STARTED,   // began checkAndPut: if 10 -> 11
52      CHECKANDPUT_COMPLETED  // completed checkAndPut
53      // NOTE: at the end of these steps, the value of the cell should be 50, not 11!
54    }
55    private static volatile TestStep testStep = TestStep.INIT;
56    private final String family = "f1";
57    	 
58    @Test
59    public void testPutAndCheckAndPutInParallel() throws Exception {
60  
61      final String tableName = "testPutAndCheckAndPut";
62      Configuration conf = HBaseConfiguration.create();
63      conf.setClass(HConstants.REGION_IMPL, MockHRegion.class, HeapSize.class);
64      final MockHRegion region = (MockHRegion) TestHRegion.initHRegion(Bytes.toBytes(tableName),
65          tableName, conf, Bytes.toBytes(family));
66  
67      List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
68      Put[] puts = new Put[1];
69      Put put = new Put(Bytes.toBytes("r1"));
70      put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("10"));
71      puts[0] = put;
72      Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
73  
74      putsAndLocks.add(pair);
75  
76      region.batchMutate(putsAndLocks.toArray(new Pair[0]));
77      MultithreadedTestUtil.TestContext ctx =
78        new MultithreadedTestUtil.TestContext(conf);
79      ctx.addThread(new PutThread(ctx, region));
80      ctx.addThread(new CheckAndPutThread(ctx, region));
81      ctx.startThreads();
82      while (testStep != TestStep.CHECKANDPUT_COMPLETED) {
83        Thread.sleep(100);
84      }
85      ctx.stop();
86      Scan s = new Scan();
87      RegionScanner scanner = region.getScanner(s);
88      List<KeyValue> results = new ArrayList<KeyValue>();
89      scanner.next(results, 2);
90      for (KeyValue keyValue : results) {
91        assertEquals("50",Bytes.toString(keyValue.getValue()));
92      }
93  
94    }
95  
96    private class PutThread extends TestThread {
97      private MockHRegion region;
98      PutThread(TestContext ctx, MockHRegion region) {
99        super(ctx);
100       this.region = region;
101     }
102 
103     public void doWork() throws Exception {
104       List<Pair<Mutation, Integer>> putsAndLocks = Lists.newArrayList();
105       Put[] puts = new Put[1];
106       Put put = new Put(Bytes.toBytes("r1"));
107       put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("50"));
108       puts[0] = put;
109       Pair<Mutation, Integer> pair = new Pair<Mutation, Integer>(puts[0], null);
110       putsAndLocks.add(pair);
111       testStep = TestStep.PUT_STARTED;
112       region.batchMutate(putsAndLocks.toArray(new Pair[0]));
113     }
114   }
115 
116   private class CheckAndPutThread extends TestThread {
117     private MockHRegion region;
118     CheckAndPutThread(TestContext ctx, MockHRegion region) {
119       super(ctx);
120       this.region = region;
121    }
122 
123     public void doWork() throws Exception {
124       Put[] puts = new Put[1];
125       Put put = new Put(Bytes.toBytes("r1"));
126       put.add(Bytes.toBytes(family), Bytes.toBytes("q1"), Bytes.toBytes("11"));
127       puts[0] = put;
128       while (testStep != TestStep.PUT_COMPLETED) {
129         Thread.sleep(100);
130       }
131       testStep = TestStep.CHECKANDPUT_STARTED;
132       region.checkAndMutate(Bytes.toBytes("r1"), Bytes.toBytes(family), Bytes.toBytes("q1"),
133         CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("10")), put, null, true);
134       testStep = TestStep.CHECKANDPUT_COMPLETED;
135     }
136   }
137 
138   public static class MockHRegion extends HRegion {
139 
140     public MockHRegion(Path tableDir, HLog log, FileSystem fs, Configuration conf,
141         final HRegionInfo regionInfo, final HTableDescriptor htd, RegionServerServices rsServices) {
142       super(tableDir, log, fs, conf, regionInfo, htd, rsServices);
143     }
144 
145     @Override
146     public void releaseRowLock(Integer lockId) {
147       if (testStep == TestStep.INIT) {
148         super.releaseRowLock(lockId);
149         return;
150       }
151 
152       if (testStep == TestStep.PUT_STARTED) {
153         try {
154           testStep = TestStep.PUT_COMPLETED;
155           super.releaseRowLock(lockId);
156           // put has been written to the memstore and the row lock has been released, but the
157           // MVCC has not been advanced.  Prior to fixing HBASE-7051, the following order of
158           // operations would cause the non-atomicity to show up:
159           // 1) Put releases row lock (where we are now)
160           // 2) CheckAndPut grabs row lock and reads the value prior to the put (10)
161           //    because the MVCC has not advanced
162           // 3) Put advances MVCC
163           // So, in order to recreate this order, we wait for the checkAndPut to grab the rowLock
164           // (see below), and then wait some more to give the checkAndPut time to read the old
165           // value.
166           latch.await();
167           Thread.sleep(1000);
168         } catch (InterruptedException e) {
169           Thread.currentThread().interrupt();
170         }
171       }
172       else if (testStep == TestStep.CHECKANDPUT_STARTED) {
173         super.releaseRowLock(lockId);
174       }
175     }
176 
177     @Override
178     public Integer getLock(Integer lockid, byte[] row, boolean waitForLock) throws IOException {
179       if (testStep == TestStep.CHECKANDPUT_STARTED) {
180         latch.countDown();
181       }
182       return super.getLock(lockid, row, waitForLock);
183     }
184 
185   }
186 
187 }