From e6f4168813f441b2f3c340ed30093721e99aa7a5 Mon Sep 17 00:00:00 2001
From: Eli Collins <eli@cloudera.com>
Date: Mon, 23 Aug 2010 12:00:42 -0700
Subject: [PATCH] HADOOP-6762. Interrupting a thread performing an RPC should not hang that thread.

Description: Moves the sending of parameters for RPC calls to a separate
thread, such that interrupting a thread that is making an RPC call does not
negatively affect the shared RPC channel.

Reason: Fixes occasional hangs of HBase under heavy load during failure testing.
Author: Sam Rash
Ref: CDH-659, CDH-1084
---
 src/core/org/apache/hadoop/ipc/Client.java  |  121 +++++++++++++++++++++-----
 src/test/org/apache/hadoop/ipc/TestRPC.java |   67 +++++++++++++++
 2 files changed, 164 insertions(+), 24 deletions(-)

diff --git a/src/core/org/apache/hadoop/ipc/Client.java b/src/core/org/apache/hadoop/ipc/Client.java
index 79cd142..a053d25 100644
--- a/src/core/org/apache/hadoop/ipc/Client.java
+++ b/src/core/org/apache/hadoop/ipc/Client.java
@@ -35,7 +35,14 @@ import java.io.InputStream;
 import java.util.Hashtable;
 import java.util.Iterator;
 import java.util.Map.Entry;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 
 import javax.net.SocketFactory;
@@ -81,6 +88,25 @@ public class Client {
   final static int DEFAULT_PING_INTERVAL = 60000; // 1 min
   final static int PING_CALL_ID = -1;
   
+  private static final ThreadFactory DAEMON_THREAD_FACTORY = new ThreadFactory() {
+    private final ThreadFactory defaultThreadFactory = 
+      Executors.defaultThreadFactory();
+    private final AtomicInteger counter = new AtomicInteger(0);
+    @Override
+    public Thread newThread(Runnable r) {
+      Thread thread = defaultThreadFactory.newThread(r);
+        
+      thread.setDaemon(true);
+      thread.setName("sendParams-" + counter.getAndIncrement());
+        
+      return thread;
+    }
+  };
+  
+  private static final ExecutorService SEND_PARAMS_EXECUTOR = 
+    Executors.newCachedThreadPool(DAEMON_THREAD_FACTORY);
+  
+  
   /**
    * set the ping interval value in configuration
    * 
@@ -187,6 +213,8 @@ public class Client {
     private AtomicLong lastActivity = new AtomicLong();// last I/O activity time
     private AtomicBoolean shouldCloseConnection = new AtomicBoolean();  // indicate if the connection is closed
     private IOException closeException; // close reason
+    
+    private final Object sendParamsLock = new Object();
 
     public Connection(ConnectionId remoteId) throws IOException {
       this.remoteId = remoteId;
@@ -467,34 +495,64 @@ public class Client {
      * Note: this is not called from the Connection thread, but by other
      * threads.
      */
-    public void sendParam(Call call) {
+    public void sendParam(final Call call) throws InterruptedException {
       if (shouldCloseConnection.get()) {
         return;
       }
+      
+      // lock the connection for the period of submission and waiting
+      // in order to bound the # of threads in the executor by the number
+      // of connections
+      synchronized (sendParamsLock) {
+        Future senderFuture = SEND_PARAMS_EXECUTOR.submit(new Runnable() {
+          @Override
+          public void run() {
+            DataOutputBuffer d = null;
+
+            synchronized (Connection.this.out) {
+              try {
+                if (shouldCloseConnection.get()) {
+                  return;
+                }
+                if (LOG.isDebugEnabled()) {
+                  LOG.debug(getName() + " sending #" + call.id);
+                }
+
+                //for serializing the
+                //data to be written
+                d = new DataOutputBuffer();
+                d.writeInt(call.id);
+                call.param.write(d);
+                byte[] data = d.getData();
+                int dataLength = d.getLength();
+                out.writeInt(dataLength);      //first put the data length
+                out.write(data, 0, dataLength);//write the data
+                out.flush();
+
+              } catch (IOException e) {
+                markClosed(e);
+              } finally {
+                //the buffer is just an in-memory buffer, but it is still polite to
+                // close early
+                IOUtils.closeStream(d);
+              }
+            }
+          }
+        });
 
-      DataOutputBuffer d=null;
-      try {
-        synchronized (this.out) {
-          if (LOG.isDebugEnabled())
-            LOG.debug(getName() + " sending #" + call.id);
-          
-          //for serializing the
-          //data to be written
-          d = new DataOutputBuffer();
-          d.writeInt(call.id);
-          call.param.write(d);
-          byte[] data = d.getData();
-          int dataLength = d.getLength();
-          out.writeInt(dataLength);      //first put the data length
-          out.write(data, 0, dataLength);//write the data
-          out.flush();
+        try {
+          senderFuture.get();
+        } catch (ExecutionException e) {
+          Throwable cause = e.getCause();
+
+          // cause should only be a RuntimeException as the Runnable above
+          // catches IOException
+          if (cause instanceof RuntimeException) {
+            throw (RuntimeException) cause;
+          } else {
+            throw new RuntimeException("checked exception made it here", cause);
+          }
         }
-      } catch(IOException e) {
-        markClosed(e);
-      } finally {
-        //the buffer is just an in-memory buffer, but it is still polite to
-        // close early
-        IOUtils.closeStream(d);
       }
     }  
 
@@ -728,7 +786,16 @@ public class Client {
                        throws InterruptedException, IOException {
     Call call = new Call(param);
     Connection connection = getConnection(addr, protocol, ticket, call);
-    connection.sendParam(call);                 // send the parameter
+    try {
+      connection.sendParam(call);                 // send the parameter
+    } catch (RejectedExecutionException e) {
+      throw new IOException("connection has been closed", e);
+    } catch (InterruptedException e) {
+      Thread.currentThread().interrupt();
+      LOG.warn("interrupted waiting to send params to server", e);
+      throw new IOException(e);
+    }
+
     boolean interrupted = false;
     synchronized (call) {
       while (!call.done) {
@@ -819,11 +886,17 @@ public class Client {
           Connection connection = 
             getConnection(addresses[i], protocol, ticket, call);
           connection.sendParam(call);             // send each parameter
+        } catch (RejectedExecutionException e) {
+          throw new IOException("connection has been closed", e);
         } catch (IOException e) {
           // log errors
           LOG.info("Calling "+addresses[i]+" caught: " + 
                    e.getMessage(),e);
           results.size--;                         //  wait for one fewer result
+        } catch (InterruptedException e) {
+          Thread.currentThread().interrupt();
+          LOG.warn("interrupted waiting to send params to server", e);
+          throw new IOException(e);
         }
       }
       while (results.count != results.size) {
diff --git a/src/test/org/apache/hadoop/ipc/TestRPC.java b/src/test/org/apache/hadoop/ipc/TestRPC.java
index d0db263..730b52d 100644
--- a/src/test/org/apache/hadoop/ipc/TestRPC.java
+++ b/src/test/org/apache/hadoop/ipc/TestRPC.java
@@ -26,6 +26,10 @@ import java.lang.reflect.Method;
 import junit.framework.TestCase;
 
 import java.util.Arrays;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.apache.commons.logging.*;
 
@@ -68,6 +72,7 @@ public class TestRPC extends TestCase {
     int error() throws IOException;
     void testServerGet() throws IOException;
     int[] exchange(int[] values) throws IOException;
+    String toString();
   }
 
   public class TestImpl implements TestProtocol {
@@ -382,6 +387,68 @@ public class TestRPC extends TestCase {
     conf.set(ACL_CONFIG, "invalid invalid");
     doRPCs(conf, true);
   }
+
+  public void testRPCInterrupted3() throws IOException, InterruptedException {
+    final Configuration conf = new Configuration();
+    Server server = RPC.getServer(
+      new TestImpl(), ADDRESS, 0, 5, true, conf);
+
+    server.start();
+
+    int numConcurrentRPC = 200;
+    InetSocketAddress addr = NetUtils.getConnectAddress(server);
+    final CyclicBarrier barrier = new CyclicBarrier(numConcurrentRPC);
+    final CountDownLatch latch = new CountDownLatch(numConcurrentRPC);
+    final AtomicBoolean leaderRunning = new AtomicBoolean(true);
+    final AtomicReference<Throwable> error = new AtomicReference<Throwable>(null);
+    Thread leaderThread = null;
+    
+    for (int i = 0; i < numConcurrentRPC; i++) {
+      final int num = i;
+      final TestProtocol proxy = (TestProtocol) RPC.getProxy(
+      TestProtocol.class, TestProtocol.versionID, addr, conf);
+      Thread rpcThread = new Thread(new Runnable() {
+        @Override
+        public void run() {
+          try {
+            barrier.await();
+            while (num == 0 || leaderRunning.get()) {
+              proxy.slowPing(false);
+            }
+            
+            proxy.slowPing(false);
+          } catch (Exception e) {
+            if (num == 0) {
+              leaderRunning.set(false);
+            } else {
+              error.set(e);
+            }
+
+            LOG.error(e);
+          } finally {
+            latch.countDown();
+          }
+        }
+      });
+      rpcThread.start();
+
+      if (leaderThread == null) {
+       leaderThread = rpcThread;
+      }
+    }
+    // let threads get past the barrier
+    Thread.sleep(1000);
+
+    // stop a single thread
+    while (leaderRunning.get()) {
+      leaderThread.interrupt();
+    }
+
+    latch.await();
+    
+    // should not cause any other thread to get an error
+    assertNull("rpc got ClosedChannelException", error.get());
+  }
   
   public static void main(String[] args) throws Exception {
 
-- 
1.6.0.4

