1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18 package org.apache.hadoop.hbase.procedure;
19
20 import static org.junit.Assert.assertNull;
21 import static org.junit.Assert.assertTrue;
22 import static org.mockito.Matchers.any;
23 import static org.mockito.Matchers.anyListOf;
24 import static org.mockito.Matchers.anyString;
25 import static org.mockito.Matchers.eq;
26 import static org.mockito.Mockito.atLeastOnce;
27 import static org.mockito.Mockito.doAnswer;
28 import static org.mockito.Mockito.doThrow;
29 import static org.mockito.Mockito.inOrder;
30 import static org.mockito.Mockito.mock;
31 import static org.mockito.Mockito.never;
32 import static org.mockito.Mockito.reset;
33 import static org.mockito.Mockito.spy;
34 import static org.mockito.Mockito.times;
35 import static org.mockito.Mockito.verify;
36 import static org.mockito.Mockito.when;
37
38 import java.io.IOException;
39 import java.util.Arrays;
40 import java.util.List;
41 import java.util.concurrent.ThreadPoolExecutor;
42
43 import org.apache.hadoop.hbase.SmallTests;
44 import org.apache.hadoop.hbase.errorhandling.ForeignException;
45 import org.apache.hadoop.hbase.errorhandling.ForeignExceptionDispatcher;
46 import org.junit.After;
47 import org.junit.Test;
48 import org.junit.experimental.categories.Category;
49 import org.mockito.InOrder;
50 import org.mockito.invocation.InvocationOnMock;
51 import org.mockito.stubbing.Answer;
52
53 import com.google.common.collect.Lists;
54
55
56
57
58
59
60
61 @Category(SmallTests.class)
62 public class TestProcedureCoordinator {
63
64 private static final long WAKE_FREQUENCY = 1000;
65 private static final long TIMEOUT = 100000;
66 private static final long POOL_KEEP_ALIVE = 1;
67 private static final String nodeName = "node";
68 private static final String procName = "some op";
69 private static final byte[] procData = new byte[0];
70 private static final List<String> expected = Lists.newArrayList("remote1", "remote2");
71
72
73 private final ProcedureCoordinatorRpcs controller = mock(ProcedureCoordinatorRpcs.class);
74 private final Procedure task = mock(Procedure.class);
75 private final ForeignExceptionDispatcher monitor = mock(ForeignExceptionDispatcher.class);
76
77
78 private ProcedureCoordinator coordinator;
79
80 @After
81 public void resetTest() throws IOException {
82
83 reset(controller, task, monitor);
84
85 if (coordinator != null) coordinator.close();
86 }
87
88 private ProcedureCoordinator buildNewCoordinator() {
89 ThreadPoolExecutor pool = ProcedureCoordinator.defaultPool(nodeName, POOL_KEEP_ALIVE, 1, WAKE_FREQUENCY);
90 return spy(new ProcedureCoordinator(controller, pool));
91 }
92
93
94
95
96
97 @Test
98 public void testThreadPoolSize() throws Exception {
99 ProcedureCoordinator coordinator = buildNewCoordinator();
100 Procedure proc = new Procedure(coordinator, monitor,
101 WAKE_FREQUENCY, TIMEOUT, procName, procData, expected);
102 Procedure procSpy = spy(proc);
103
104 Procedure proc2 = new Procedure(coordinator, monitor,
105 WAKE_FREQUENCY, TIMEOUT, procName +"2", procData, expected);
106 Procedure procSpy2 = spy(proc2);
107 when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
108 .thenReturn(procSpy, procSpy2);
109
110 coordinator.startProcedure(procSpy.getErrorMonitor(), procName, procData, expected);
111
112 assertNull("Coordinator successfully ran two tasks at once with a single thread pool.",
113 coordinator.startProcedure(proc2.getErrorMonitor(), "another op", procData, expected));
114 }
115
116
117
118
119 @Test(timeout = 5000)
120 public void testUnreachableControllerDuringPrepare() throws Exception {
121 coordinator = buildNewCoordinator();
122
123 List<String> expected = Arrays.asList("cohort");
124 Procedure proc = new Procedure(coordinator, WAKE_FREQUENCY,
125 TIMEOUT, procName, procData, expected);
126 final Procedure procSpy = spy(proc);
127
128 when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
129 .thenReturn(procSpy);
130
131
132 IOException cause = new IOException("Failed to reach comms during acquire");
133 doThrow(cause).when(controller)
134 .sendGlobalBarrierAcquire(eq(procSpy), eq(procData), anyListOf(String.class));
135
136
137 proc = coordinator.startProcedure(proc.getErrorMonitor(), procName, procData, expected);
138
139 proc.waitForCompleted();
140 verify(procSpy, atLeastOnce()).receive(any(ForeignException.class));
141 verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
142 verify(controller, times(1)).sendGlobalBarrierAcquire(procSpy, procData, expected);
143 verify(controller, never()).sendGlobalBarrierReached(any(Procedure.class),
144 anyListOf(String.class));
145 }
146
147
148
149
150 @Test(timeout = 5000)
151 public void testUnreachableControllerDuringCommit() throws Exception {
152 coordinator = buildNewCoordinator();
153
154
155 List<String> expected = Arrays.asList("cohort");
156 final Procedure spy = spy(new Procedure(coordinator,
157 WAKE_FREQUENCY, TIMEOUT, procName, procData, expected));
158
159 when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
160 .thenReturn(spy);
161
162
163 IOException cause = new IOException("Failed to reach controller during prepare");
164 doAnswer(new AcquireBarrierAnswer(procName, new String[] { "cohort" }))
165 .when(controller).sendGlobalBarrierAcquire(eq(spy), eq(procData), anyListOf(String.class));
166 doThrow(cause).when(controller).sendGlobalBarrierReached(eq(spy), anyListOf(String.class));
167
168
169 Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
170
171 task.waitForCompleted();
172 verify(spy, atLeastOnce()).receive(any(ForeignException.class));
173 verify(coordinator, times(1)).rpcConnectionFailure(anyString(), eq(cause));
174 verify(controller, times(1)).sendGlobalBarrierAcquire(eq(spy),
175 eq(procData), anyListOf(String.class));
176 verify(controller, times(1)).sendGlobalBarrierReached(any(Procedure.class),
177 anyListOf(String.class));
178 }
179
180 @Test(timeout = 1000)
181 public void testNoCohort() throws Exception {
182 runSimpleProcedure();
183 }
184
185 @Test(timeout = 1000)
186 public void testSingleCohortOrchestration() throws Exception {
187 runSimpleProcedure("one");
188 }
189
190 @Test(timeout = 1000)
191 public void testMultipleCohortOrchestration() throws Exception {
192 runSimpleProcedure("one", "two", "three", "four");
193 }
194
195 public void runSimpleProcedure(String... members) throws Exception {
196 coordinator = buildNewCoordinator();
197 Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY,
198 TIMEOUT, procName, procData, Arrays.asList(members));
199 final Procedure spy = spy(task);
200 runCoordinatedProcedure(spy, members);
201 }
202
203
204
205
206 @Test(timeout = 1000)
207 public void testEarlyJoiningBarrier() throws Exception {
208 final String[] cohort = new String[] { "one", "two", "three", "four" };
209 coordinator = buildNewCoordinator();
210 final ProcedureCoordinator ref = coordinator;
211 Procedure task = new Procedure(coordinator, monitor, WAKE_FREQUENCY,
212 TIMEOUT, procName, procData, Arrays.asList(cohort));
213 final Procedure spy = spy(task);
214
215 AcquireBarrierAnswer prepare = new AcquireBarrierAnswer(procName, cohort) {
216 public void doWork() {
217
218
219 ref.memberAcquiredBarrier(this.opName, this.cohort[0]);
220 ref.memberFinishedBarrier(this.opName, this.cohort[0]);
221
222 ref.memberAcquiredBarrier(this.opName, this.cohort[1]);
223
224 ref.memberAcquiredBarrier(this.opName, this.cohort[2]);
225 ref.memberFinishedBarrier(this.opName, this.cohort[2]);
226
227 ref.memberAcquiredBarrier(this.opName, this.cohort[3]);
228 }
229 };
230
231 BarrierAnswer commit = new BarrierAnswer(procName, cohort) {
232 @Override
233 public void doWork() {
234 ref.memberFinishedBarrier(opName, this.cohort[1]);
235 ref.memberFinishedBarrier(opName, this.cohort[3]);
236 }
237 };
238 runCoordinatedOperation(spy, prepare, commit, cohort);
239 }
240
241
242
243
244
245
246
247
248
249
250 public void runCoordinatedProcedure(Procedure spy, String... cohort) throws Exception {
251 runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort),
252 new BarrierAnswer(procName, cohort), cohort);
253 }
254
255 public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepare,
256 String... cohort) throws Exception {
257 runCoordinatedOperation(spy, prepare, new BarrierAnswer(procName, cohort), cohort);
258 }
259
260 public void runCoordinatedOperation(Procedure spy, BarrierAnswer commit,
261 String... cohort) throws Exception {
262 runCoordinatedOperation(spy, new AcquireBarrierAnswer(procName, cohort), commit, cohort);
263 }
264
265 public void runCoordinatedOperation(Procedure spy, AcquireBarrierAnswer prepareOperation,
266 BarrierAnswer commitOperation, String... cohort) throws Exception {
267 List<String> expected = Arrays.asList(cohort);
268 when(coordinator.createProcedure(any(ForeignExceptionDispatcher.class), eq(procName), eq(procData), anyListOf(String.class)))
269 .thenReturn(spy);
270
271
272 doAnswer(prepareOperation).when(controller).sendGlobalBarrierAcquire(spy, procData, expected);
273 doAnswer(commitOperation).when(controller)
274 .sendGlobalBarrierReached(eq(spy), anyListOf(String.class));
275
276
277 Procedure task = coordinator.startProcedure(spy.getErrorMonitor(), procName, procData, expected);
278
279 task.waitForCompleted();
280
281
282 prepareOperation.ensureRan();
283
284 InOrder inorder = inOrder(spy, controller);
285 inorder.verify(spy).sendGlobalBarrierStart();
286 inorder.verify(controller).sendGlobalBarrierAcquire(task, procData, expected);
287 inorder.verify(spy).sendGlobalBarrierReached();
288 inorder.verify(controller).sendGlobalBarrierReached(eq(task), anyListOf(String.class));
289 }
290
291 private abstract class OperationAnswer implements Answer<Void> {
292 private boolean ran = false;
293
294 public void ensureRan() {
295 assertTrue("Prepare mocking didn't actually run!", ran);
296 }
297
298 @Override
299 public final Void answer(InvocationOnMock invocation) throws Throwable {
300 this.ran = true;
301 doWork();
302 return null;
303 }
304
305 protected abstract void doWork() throws Throwable;
306 }
307
308
309
310
311 private class AcquireBarrierAnswer extends OperationAnswer {
312 protected final String[] cohort;
313 protected final String opName;
314
315 public AcquireBarrierAnswer(String opName, String... cohort) {
316 this.cohort = cohort;
317 this.opName = opName;
318 }
319
320 @Override
321 public void doWork() {
322 if (cohort == null) return;
323 for (String member : cohort) {
324 TestProcedureCoordinator.this.coordinator.memberAcquiredBarrier(opName, member);
325 }
326 }
327 }
328
329
330
331
332 private class BarrierAnswer extends OperationAnswer {
333 protected final String[] cohort;
334 protected final String opName;
335
336 public BarrierAnswer(String opName, String... cohort) {
337 this.cohort = cohort;
338 this.opName = opName;
339 }
340
341 @Override
342 public void doWork() {
343 if (cohort == null) return;
344 for (String member : cohort) {
345 TestProcedureCoordinator.this.coordinator.memberFinishedBarrier(opName, member);
346 }
347 }
348 }
349 }