1   /**
2    * Licensed to the Apache Software Foundation (ASF) under one
3    * or more contributor license agreements.  See the NOTICE file
4    * distributed with this work for additional information
5    * regarding copyright ownership.  The ASF licenses this file
6    * to you under the Apache License, Version 2.0 (the
7    * "License"); you may not use this file except in compliance
8    * with the License.  You may obtain a copy of the License at
9    *
10   *     http://www.apache.org/licenses/LICENSE-2.0
11   *
12   * Unless required by applicable law or agreed to in writing, software
13   * distributed under the License is distributed on an "AS IS" BASIS,
14   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15   * See the License for the specific language governing permissions and
16   * limitations under the License.
17   */
18  package org.apache.hadoop.hbase.regionserver;
19  
20  import java.io.IOException;
21  import java.util.ArrayList;
22  import java.util.List;
23  
24  import org.apache.hadoop.hbase.*;
25  import org.apache.hadoop.hbase.client.Delete;
26  import org.apache.hadoop.hbase.client.Get;
27  import org.apache.hadoop.hbase.client.Put;
28  import org.apache.hadoop.hbase.client.Result;
29  import org.apache.hadoop.hbase.client.Scan;
30  import org.apache.hadoop.hbase.util.Bytes;
31  import org.junit.experimental.categories.Category;
32  
33  @Category(SmallTests.class)
34  public class TestKeepDeletes extends HBaseTestCase {
35    private final byte[] T0 = Bytes.toBytes("0");
36    private final byte[] T1 = Bytes.toBytes("1");
37    private final byte[] T2 = Bytes.toBytes("2");
38    private final byte[] T3 = Bytes.toBytes("3");
39    private final byte[] T4 = Bytes.toBytes("4");
40    private final byte[] T5 = Bytes.toBytes("5");
41    private final byte[] T6 = Bytes.toBytes("6");
42  
43    private final byte[] c0 = COLUMNS[0];
44    private final byte[] c1 = COLUMNS[1];
45  
46    /**
47     * Make sure that deleted rows are retained.
48     * Family delete markers are deleted.
49     * Column Delete markers are versioned
50     * Time range scan of deleted rows are possible
51     */
52    public void testBasicScenario() throws Exception {
53      // keep 3 versions, rows do not expire
54      HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
55          HConstants.FOREVER, true);
56      HRegion region = createNewHRegion(htd, null, null);
57  
58      long ts = System.currentTimeMillis();
59      Put p = new Put(T1, ts);
60      p.add(c0, c0, T1);
61      region.put(p);
62      p = new Put(T1, ts+1);
63      p.add(c0, c0, T2);
64      region.put(p);
65      p = new Put(T1, ts+2);
66      p.add(c0, c0, T3);
67      region.put(p);
68      p = new Put(T1, ts+4);
69      p.add(c0, c0, T4);
70      region.put(p);
71  
72      // now place a delete marker at ts+2
73      Delete d = new Delete(T1, ts+2, null);
74      region.delete(d, null, true);
75  
76      // a raw scan can see the delete markers
77      // (one for each column family)
78      assertEquals(3, countDeleteMarkers(region));
79  
80      // get something *before* the delete marker
81      Get g = new Get(T1);
82      g.setMaxVersions();
83      g.setTimeRange(0L, ts+2);
84      Result r = region.get(g, null);
85      checkResult(r, c0, c0, T2,T1);
86  
87      // flush
88      region.flushcache();
89  
90      // yep, T2 still there, T1 gone
91      r = region.get(g, null);
92      checkResult(r, c0, c0, T2);
93  
94      // major compact
95      region.compactStores(true);
96      region.compactStores(true);
97  
98      // one delete marker left (the others did not
99      // have older puts)
100     assertEquals(1, countDeleteMarkers(region));
101 
102     // still there (even after multiple compactions)
103     r = region.get(g, null);
104     checkResult(r, c0, c0, T2);
105 
106     // a timerange that includes the delete marker won't see past rows
107     g.setTimeRange(0L, ts+4);
108     r = region.get(g, null);
109     assertTrue(r.isEmpty());
110 
111     // two more puts, this will expire the older puts.
112     p = new Put(T1, ts+5);
113     p.add(c0, c0, T5);
114     region.put(p);
115     p = new Put(T1, ts+6);
116     p.add(c0, c0, T6);
117     region.put(p);
118 
119     // also add an old put again
120     // (which is past the max versions)
121     p = new Put(T1, ts);
122     p.add(c0, c0, T1);
123     region.put(p);
124     r = region.get(g, null);
125     assertTrue(r.isEmpty());
126 
127     region.flushcache();
128     region.compactStores(true);
129     region.compactStores(true);
130 
131     // verify that the delete marker itself was collected
132     region.put(p);
133     r = region.get(g, null);
134     checkResult(r, c0, c0, T1);
135     assertEquals(0, countDeleteMarkers(region));
136 
137     region.close();
138     region.getLog().closeAndDelete();
139   }
140 
141   /**
142    * Even when the store does not keep deletes a "raw" scan will 
143    * return everything it can find (unless discarding cells is guaranteed
144    * to have no effect).
145    * Assuming this the desired behavior. Could also disallow "raw" scanning
146    * if the store does not have KEEP_DELETED_CELLS enabled.
147    * (can be changed easily)
148    */
149   public void testRawScanWithoutKeepingDeletes() throws Exception {
150     // KEEP_DELETED_CELLS is NOT enabled
151     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
152         HConstants.FOREVER, false);
153     HRegion region = createNewHRegion(htd, null, null);
154 
155     long ts = System.currentTimeMillis();
156     Put p = new Put(T1, ts);
157     p.add(c0, c0, T1);
158     region.put(p);
159 
160     Delete d = new Delete(T1, ts, null);
161     d.deleteColumn(c0, c0, ts);
162     region.delete(d, null, true);
163 
164     // scan still returns delete markers and deletes rows
165     Scan s = new Scan();
166     s.setRaw(true);
167     s.setMaxVersions();
168     InternalScanner scan = region.getScanner(s);
169     List<KeyValue> kvs = new ArrayList<KeyValue>();
170     scan.next(kvs);
171     assertEquals(2, kvs.size());
172 
173     region.flushcache();
174     region.compactStores(true);
175 
176     // after compaction they are gone
177     // (note that this a test with a Store without
178     //  KEEP_DELETED_CELLS)
179     s = new Scan();
180     s.setRaw(true);
181     s.setMaxVersions();
182     scan = region.getScanner(s);
183     kvs = new ArrayList<KeyValue>();
184     scan.next(kvs);
185     assertTrue(kvs.isEmpty());
186 
187     region.close();
188     region.getLog().closeAndDelete();
189   }
190 
191   /**
192    * basic verification of existing behavior
193    */
194   public void testWithoutKeepingDeletes() throws Exception {
195     // KEEP_DELETED_CELLS is NOT enabled
196     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
197         HConstants.FOREVER, false);
198     HRegion region = createNewHRegion(htd, null, null);
199 
200     long ts = System.currentTimeMillis();  
201     Put p = new Put(T1, ts);
202     p.add(c0, c0, T1);
203     region.put(p);
204     Delete d = new Delete(T1, ts+2, null);
205     d.deleteColumn(c0, c0, ts);
206     region.delete(d, null, true);
207 
208     // "past" get does not see rows behind delete marker
209     Get g = new Get(T1);
210     g.setMaxVersions();
211     g.setTimeRange(0L, ts+1);
212     Result r = region.get(g, null);
213     assertTrue(r.isEmpty());
214 
215     // "past" scan does not see rows behind delete marker
216     Scan s = new Scan();
217     s.setMaxVersions();
218     s.setTimeRange(0L, ts+1);
219     InternalScanner scanner = region.getScanner(s);
220     List<KeyValue> kvs = new ArrayList<KeyValue>();
221     while(scanner.next(kvs));
222     assertTrue(kvs.isEmpty());
223 
224     // flushing and minor compaction keep delete markers
225     region.flushcache();
226     region.compactStores();
227     assertEquals(1, countDeleteMarkers(region));
228     region.compactStores(true);
229     // major compaction deleted it
230     assertEquals(0, countDeleteMarkers(region));
231 
232     region.close();
233     region.getLog().closeAndDelete();
234   }
235 
236   /**
237    * The ExplicitColumnTracker does not support "raw" scanning.
238    */
239   public void testRawScanWithColumns() throws Exception {
240     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
241         HConstants.FOREVER, true);
242     HRegion region = createNewHRegion(htd, null, null);
243 
244     Scan s = new Scan();
245     s.setRaw(true);
246     s.setMaxVersions();
247     s.addColumn(c0, c0);
248     
249     try {
250       InternalScanner scan = region.getScanner(s);
251       fail("raw scanner with columns should have failed");
252     } catch (DoNotRetryIOException dnre) {
253       // ok!
254     }
255 
256     region.close();
257     region.getLog().closeAndDelete();
258   }
259 
260   /**
261    * Verify that "raw" scanning mode return delete markers and deletes rows.
262    */
263   public void testRawScan() throws Exception {
264     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
265         HConstants.FOREVER, true);
266     HRegion region = createNewHRegion(htd, null, null);
267 
268     long ts = System.currentTimeMillis();
269     Put p = new Put(T1, ts);
270     p.add(c0, c0, T1);
271     region.put(p);
272     p = new Put(T1, ts+2);
273     p.add(c0, c0, T2);
274     region.put(p);
275     p = new Put(T1, ts+4);
276     p.add(c0, c0, T3);
277     region.put(p);
278 
279     Delete d = new Delete(T1, ts+1);
280     region.delete(d, true);
281 
282     d = new Delete(T1, ts+2);
283     d.deleteColumn(c0, c0, ts+2);
284     region.delete(d, true);
285 
286     d = new Delete(T1, ts+3);
287     d.deleteColumns(c0, c0, ts+3);
288     region.delete(d, true);
289 
290     Scan s = new Scan();
291     s.setRaw(true);
292     s.setMaxVersions();
293     InternalScanner scan = region.getScanner(s);
294     List<KeyValue> kvs = new ArrayList<KeyValue>();
295     scan.next(kvs);
296     assertEquals(8, kvs.size());
297     assertTrue(kvs.get(0).isDeleteFamily());
298     assertEquals(kvs.get(1).getValue(), T3);
299     assertTrue(kvs.get(2).isDelete());
300     assertTrue(kvs.get(3).isDeleteType());
301     assertEquals(kvs.get(4).getValue(), T2);
302     assertEquals(kvs.get(5).getValue(), T1);
303     // we have 3 CFs, so there are two more delete markers
304     assertTrue(kvs.get(6).isDeleteFamily());
305     assertTrue(kvs.get(7).isDeleteFamily());
306 
307     // verify that raw scans honor the passed timerange
308     s = new Scan();
309     s.setRaw(true);
310     s.setMaxVersions();
311     s.setTimeRange(0, 1);
312     scan = region.getScanner(s);
313     kvs = new ArrayList<KeyValue>();
314     scan.next(kvs);
315     // nothing in this interval, not even delete markers
316     assertTrue(kvs.isEmpty());
317 
318     // filter new delete markers
319     s = new Scan();
320     s.setRaw(true);
321     s.setMaxVersions();
322     s.setTimeRange(0, ts+2);
323     scan = region.getScanner(s);
324     kvs = new ArrayList<KeyValue>();
325     scan.next(kvs);
326     assertEquals(4, kvs.size());
327     assertTrue(kvs.get(0).isDeleteFamily());
328     assertEquals(kvs.get(1).getValue(), T1);
329     // we have 3 CFs
330     assertTrue(kvs.get(2).isDeleteFamily());
331     assertTrue(kvs.get(3).isDeleteFamily());
332 
333     // filter old delete markers
334     s = new Scan();
335     s.setRaw(true);
336     s.setMaxVersions();
337     s.setTimeRange(ts+3, ts+5);
338     scan = region.getScanner(s);
339     kvs = new ArrayList<KeyValue>();
340     scan.next(kvs);
341     assertEquals(2, kvs.size());
342     assertEquals(kvs.get(0).getValue(), T3);
343     assertTrue(kvs.get(1).isDelete());
344 
345     region.close();
346     region.getLog().closeAndDelete();
347   }
348 
349   /**
350    * Verify that delete markers are removed from an otherwise empty store.
351    */
352   public void testDeleteMarkerExpirationEmptyStore() throws Exception {
353     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
354         HConstants.FOREVER, true);
355     HRegion region = createNewHRegion(htd, null, null);
356 
357     long ts = System.currentTimeMillis();
358 
359     Delete d = new Delete(T1, ts, null);
360     d.deleteColumns(c0, c0, ts);
361     region.delete(d, null, true);
362 
363     d = new Delete(T1, ts, null);
364     d.deleteFamily(c0);
365     region.delete(d, null, true);
366 
367     d = new Delete(T1, ts, null);
368     d.deleteColumn(c0, c0, ts+1);
369     region.delete(d, null, true);
370     
371     d = new Delete(T1, ts, null);
372     d.deleteColumn(c0, c0, ts+2);
373     region.delete(d, null, true);
374 
375     // 1 family marker, 1 column marker, 2 version markers
376     assertEquals(4, countDeleteMarkers(region));
377 
378     // neither flush nor minor compaction removes any marker
379     region.flushcache();
380     assertEquals(4, countDeleteMarkers(region));
381     region.compactStores(false);
382     assertEquals(4, countDeleteMarkers(region));
383 
384     // major compaction removes all, since there are no puts they affect
385     region.compactStores(true);
386     assertEquals(0, countDeleteMarkers(region));
387 
388     region.close();
389     region.getLog().closeAndDelete();
390   }
391 
392   /**
393    * Test delete marker removal from store files.
394    */
395   public void testDeleteMarkerExpiration() throws Exception {
396     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
397         HConstants.FOREVER, true);
398     HRegion region = createNewHRegion(htd, null, null);
399 
400     long ts = System.currentTimeMillis();
401 
402     Put p = new Put(T1, ts);
403     p.add(c0, c0, T1);
404     region.put(p);
405 
406     // a put into another store (CF) should have no effect
407     p = new Put(T1, ts-10);
408     p.add(c1, c0, T1);
409     region.put(p);
410 
411     // all the following deletes affect the put
412     Delete d = new Delete(T1, ts, null);
413     d.deleteColumns(c0, c0, ts);
414     region.delete(d, null, true);
415 
416     d = new Delete(T1, ts, null);
417     d.deleteFamily(c0, ts);
418     region.delete(d, null, true);
419 
420     d = new Delete(T1, ts, null);
421     d.deleteColumn(c0, c0, ts+1);
422     region.delete(d, null, true);
423     
424     d = new Delete(T1, ts, null);
425     d.deleteColumn(c0, c0, ts+2);
426     region.delete(d, null, true);
427 
428     // 1 family marker, 1 column marker, 2 version markers
429     assertEquals(4, countDeleteMarkers(region));
430 
431     region.flushcache();
432     assertEquals(4, countDeleteMarkers(region));
433     region.compactStores(false);
434     assertEquals(4, countDeleteMarkers(region));
435 
436     // another put will push out the earlier put...
437     p = new Put(T1, ts+3);
438     p.add(c0, c0, T1);
439     region.put(p);
440 
441     region.flushcache();
442     // no markers are collected, since there is an affected put
443     region.compactStores(true);
444     assertEquals(4, countDeleteMarkers(region));
445 
446     // the last collections collected the earlier put
447     // so after this collection all markers
448     region.compactStores(true);
449     assertEquals(0, countDeleteMarkers(region));
450 
451     region.close();
452     region.getLog().closeAndDelete();
453   }
454 
455   /**
456    * Verify correct range demarcation
457    */
458   public void testRanges() throws Exception {
459     HTableDescriptor htd = createTableDescriptor(getName(), 0, 3,
460         HConstants.FOREVER, true);
461     HRegion region = createNewHRegion(htd, null, null);
462 
463     long ts = System.currentTimeMillis();
464     Put p = new Put(T1, ts);
465     p.add(c0, c0, T1);
466     p.add(c0, c1, T1);
467     p.add(c1, c0, T1);
468     p.add(c1, c1, T1);
469     region.put(p);
470 
471     p = new Put(T2, ts);
472     p.add(c0, c0, T1);
473     p.add(c0, c1, T1);
474     p.add(c1, c0, T1);
475     p.add(c1, c1, T1);
476     region.put(p);
477 
478     p = new Put(T1, ts+1);
479     p.add(c0, c0, T2);
480     p.add(c0, c1, T2);
481     p.add(c1, c0, T2);
482     p.add(c1, c1, T2);
483     region.put(p);
484 
485     p = new Put(T2, ts+1);
486     p.add(c0, c0, T2);
487     p.add(c0, c1, T2);
488     p.add(c1, c0, T2);
489     p.add(c1, c1, T2);
490     region.put(p);
491 
492     Delete d = new Delete(T1, ts+2, null);
493     d.deleteColumns(c0, c0, ts+2);
494     region.delete(d, null, true);
495 
496     d = new Delete(T1, ts+2, null);
497     d.deleteFamily(c1, ts+2);
498     region.delete(d, null, true);
499 
500     d = new Delete(T2, ts+2, null);
501     d.deleteFamily(c0, ts+2);
502     region.delete(d, null, true);
503 
504     // add an older delete, to make sure it is filtered
505     d = new Delete(T1, ts-10, null);
506     d.deleteFamily(c1, ts-10);
507     region.delete(d, null, true);
508 
509     // ts + 2 does NOT include the delete at ts+2
510     checkGet(region, T1, c0, c0, ts+2, T2, T1);
511     checkGet(region, T1, c0, c1, ts+2, T2, T1);
512     checkGet(region, T1, c1, c0, ts+2, T2, T1);
513     checkGet(region, T1, c1, c1, ts+2, T2, T1);
514 
515     checkGet(region, T2, c0, c0, ts+2, T2, T1);
516     checkGet(region, T2, c0, c1, ts+2, T2, T1);
517     checkGet(region, T2, c1, c0, ts+2, T2, T1);
518     checkGet(region, T2, c1, c1, ts+2, T2, T1);
519 
520     // ts + 3 does
521     checkGet(region, T1, c0, c0, ts+3);
522     checkGet(region, T1, c0, c1, ts+3, T2, T1);
523     checkGet(region, T1, c1, c0, ts+3);
524     checkGet(region, T1, c1, c1, ts+3);
525 
526     checkGet(region, T2, c0, c0, ts+3);
527     checkGet(region, T2, c0, c1, ts+3);
528     checkGet(region, T2, c1, c0, ts+3, T2, T1);
529     checkGet(region, T2, c1, c1, ts+3, T2, T1);
530 
531     region.close();
532     region.getLog().closeAndDelete();
533   }
534 
535   /**
536    * Verify that column/version delete makers are sorted
537    * with their respective puts and removed correctly by
538    * versioning (i.e. not relying on the store earliestPutTS).
539    */
540   public void testDeleteMarkerVersioning() throws Exception {
541     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
542         HConstants.FOREVER, true);
543     HRegion region = createNewHRegion(htd, null, null);
544 
545     long ts = System.currentTimeMillis();
546     Put p = new Put(T1, ts);
547     p.add(c0, c0, T1);
548     region.put(p);
549 
550     // this prevents marker collection based on earliestPut
551     // (cannot keep earliest put per column in the store file)
552     p = new Put(T1, ts-10);
553     p.add(c0, c1, T1);
554     region.put(p);
555     
556     Delete d = new Delete(T1, ts, null);
557     // test corner case (Put and Delete have same TS)
558     d.deleteColumns(c0, c0, ts);
559     region.delete(d, null, true);
560 
561     d = new Delete(T1, ts+1, null);
562     d.deleteColumn(c0, c0, ts+1);
563     region.delete(d, null, true);
564     
565     d = new Delete(T1, ts+3, null);
566     d.deleteColumn(c0, c0, ts+3);
567     region.delete(d, null, true);
568 
569     region.flushcache();
570     region.compactStores(true);
571     region.compactStores(true);
572     assertEquals(3, countDeleteMarkers(region));
573 
574     // add two more puts, since max version is 1
575     // the 2nd put (and all delete markers following)
576     // will be removed.
577     p = new Put(T1, ts+2);
578     p.add(c0, c0, T2);
579     region.put(p);
580     
581     // delete, put, delete, delete, put
582     assertEquals(3, countDeleteMarkers(region));
583 
584     p = new Put(T1, ts+3);
585     p.add(c0, c0, T3);
586     region.put(p);
587 
588     // This is potentially questionable behavior.
589     // This could be changed by not letting the ScanQueryMatcher
590     // return SEEK_NEXT_COL if a put is past VERSIONS, but instead
591     // return SKIP if the store has KEEP_DELETED_CELLS set.
592     //
593     // As it stands, the 1 here is correct here.
594     // There are two puts, VERSIONS is one, so after the 1st put the scanner
595     // knows that there can be no more KVs (put or delete) that have any effect.
596     //
597     // delete, put, put | delete, delete
598     assertEquals(1, countDeleteMarkers(region));
599 
600     // flush cache only sees what is in the memstore
601     region.flushcache();
602 
603     // Here we have the three markers again, because the flush above
604     // removed the 2nd put before the file is written.
605     // So there's only one put, and hence the deletes already in the store
606     // files cannot be removed safely.
607     // delete, put, delete, delete
608     assertEquals(3, countDeleteMarkers(region));
609 
610     region.compactStores(true);
611     assertEquals(3, countDeleteMarkers(region));
612 
613     // add one more put
614     p = new Put(T1, ts+4);
615     p.add(c0, c0, T4);
616     region.put(p);
617 
618     region.flushcache();
619     // one trailing delete marker remains (but only one)
620     // because delete markers do not increase the version count
621     assertEquals(1, countDeleteMarkers(region));
622     region.compactStores(true);
623     region.compactStores(true);
624     assertEquals(1, countDeleteMarkers(region));
625 
626     region.close();
627     region.getLog().closeAndDelete();
628   }
629 
630   /**
631    * Verify scenarios with multiple CFs and columns
632    */
633   public void testWithMixedCFs() throws Exception {
634     HTableDescriptor htd = createTableDescriptor(getName(), 0, 1,
635         HConstants.FOREVER, true);
636     HRegion region = createNewHRegion(htd, null, null);
637 
638     long ts = System.currentTimeMillis();
639 
640     Put p = new Put(T1, ts);
641     p.add(c0, c0, T1);
642     p.add(c0, c1, T1);
643     p.add(c1, c0, T1);
644     p.add(c1, c1, T1);
645     region.put(p);
646 
647     p = new Put(T2, ts+1);
648     p.add(c0, c0, T2);
649     p.add(c0, c1, T2);
650     p.add(c1, c0, T2);
651     p.add(c1, c1, T2);
652     region.put(p);
653 
654     // family markers are each family
655     Delete d = new Delete(T1, ts+1, null);
656     region.delete(d, null, true);
657 
658     d = new Delete(T2, ts+2, null);
659     region.delete(d, null, true);
660 
661     Scan s = new Scan(T1);
662     s.setTimeRange(0, ts+1);
663     InternalScanner scanner = region.getScanner(s);
664     List<KeyValue> kvs = new ArrayList<KeyValue>();
665     scanner.next(kvs);
666     assertEquals(4, kvs.size());
667     scanner.close();
668 
669     s = new Scan(T2);
670     s.setTimeRange(0, ts+2);
671     scanner = region.getScanner(s);
672     kvs = new ArrayList<KeyValue>();
673     scanner.next(kvs);
674     assertEquals(4, kvs.size());
675     scanner.close();
676 
677     region.close();
678     region.getLog().closeAndDelete();
679   }
680 
681   /**
682    * Test keeping deleted rows together with min versions set
683    * @throws Exception
684    */
685   public void testWithMinVersions() throws Exception {
686     HTableDescriptor htd = createTableDescriptor(getName(), 3, 1000, 1, true);
687     HRegion region = createNewHRegion(htd, null, null);
688 
689     long ts = System.currentTimeMillis() - 2000; // 2s in the past
690 
691     Put p = new Put(T1, ts);
692     p.add(c0, c0, T3);
693     region.put(p);
694     p = new Put(T1, ts-1);
695     p.add(c0, c0, T2);
696     region.put(p);
697     p = new Put(T1, ts-3);
698     p.add(c0, c0, T1);
699     region.put(p);
700     p = new Put(T1, ts-4);
701     p.add(c0, c0, T0);
702     region.put(p);
703 
704     // all puts now are just retained because of min versions = 3
705 
706     // place a family delete marker
707     Delete d = new Delete(T1, ts-1, null);
708     region.delete(d, null, true);
709     // and a column delete marker
710     d = new Delete(T1, ts-2, null);
711     d.deleteColumns(c0, c0, ts-1);
712     region.delete(d, null, true);
713 
714     Get g = new Get(T1);
715     g.setMaxVersions();
716     g.setTimeRange(0L, ts-2);
717     Result r = region.get(g, null);
718     checkResult(r, c0, c0, T1,T0);
719 
720     // 3 families, one column delete marker
721     assertEquals(4, countDeleteMarkers(region));
722 
723     region.flushcache();
724     // no delete marker removes by the flush
725     assertEquals(4, countDeleteMarkers(region));
726 
727     r = region.get(g, null);
728     checkResult(r, c0, c0, T1);
729     p = new Put(T1, ts+1);
730     p.add(c0, c0, T4);
731     region.put(p);
732     region.flushcache();
733 
734     assertEquals(4, countDeleteMarkers(region));
735 
736     r = region.get(g, null);
737     checkResult(r, c0, c0, T1);
738 
739     // this will push out the last put before
740     // family delete marker
741     p = new Put(T1, ts+2);
742     p.add(c0, c0, T5);
743     region.put(p);
744 
745     region.flushcache();
746     region.compactStores(true);
747     // the two family markers without puts are gone
748     assertEquals(2, countDeleteMarkers(region));
749 
750     // the last compactStores updated the earliestPutTs,
751     // so after the next compaction the last family delete marker is also gone
752     region.compactStores(true);
753     assertEquals(0, countDeleteMarkers(region));
754 
755     region.close();
756     region.getLog().closeAndDelete();
757   }
758 
759   private void checkGet(HRegion region, byte[] row, byte[] fam, byte[] col,
760       long time, byte[]... vals) throws IOException {
761     Get g = new Get(row);
762     g.addColumn(fam, col);
763     g.setMaxVersions();
764     g.setTimeRange(0L, time);
765     Result r = region.get(g, null);
766     checkResult(r, fam, col, vals);
767 
768   }
769 
770   private int countDeleteMarkers(HRegion region) throws IOException {
771     Scan s = new Scan();
772     s.setRaw(true);
773     // use max versions from the store(s)
774     s.setMaxVersions(region.getStores().values().iterator().next().getScanInfo().getMaxVersions());
775     InternalScanner scan = region.getScanner(s);
776     List<KeyValue> kvs = new ArrayList<KeyValue>();
777     int res = 0;
778     boolean hasMore;
779     do {
780       hasMore = scan.next(kvs);
781       for (KeyValue kv : kvs) {
782         if(kv.isDelete()) res++;
783       }
784       kvs.clear();
785     } while (hasMore);
786     scan.close();
787     return res;
788   }
789 
790   private void checkResult(Result r, byte[] fam, byte[] col, byte[] ... vals) {
791     assertEquals(r.size(), vals.length);
792     List<KeyValue> kvs = r.getColumn(fam, col);
793     assertEquals(kvs.size(), vals.length);
794     for (int i=0;i<vals.length;i++) {
795       assertEquals(kvs.get(i).getValue(), vals[i]);
796     }
797   }
798 
799 
800   @org.junit.Rule
801   public org.apache.hadoop.hbase.ResourceCheckerJUnitRule cu =
802     new org.apache.hadoop.hbase.ResourceCheckerJUnitRule();
803 }
804