From 06ceb079693292a41085af795c5b2bbc3fd10af2 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Fri, 12 Mar 2010 17:34:24 -0800
Subject: [PATCH] HADOOP-6454. Create setup.py for EC2 cloud scripts

Description: This would make it easier to install the scripts.
Reason: Installation improvement
Author: Tom White
Ref: UNKNOWN
---
 src/contrib/cloud/src/py/VERSION                   |    1 -
 src/contrib/cloud/src/py/hadoop-ec2-init-remote.sh |  548 --------------------
 src/contrib/cloud/src/py/hadoop/cloud/__init__.py  |    3 +-
 src/contrib/cloud/src/py/hadoop/cloud/cli.py       |    4 +-
 .../py/hadoop/cloud/data/hadoop-ec2-init-remote.sh |  548 ++++++++++++++++++++
 src/contrib/cloud/src/py/hadoop/cloud/service.py   |    3 +-
 src/contrib/cloud/src/py/setup.py                  |   30 ++
 7 files changed, 583 insertions(+), 554 deletions(-)
 delete mode 100644 src/contrib/cloud/src/py/VERSION
 delete mode 100644 src/contrib/cloud/src/py/hadoop-ec2-init-remote.sh
 create mode 100644 src/contrib/cloud/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh
 create mode 100644 src/contrib/cloud/src/py/setup.py

diff --git a/src/contrib/cloud/src/py/VERSION b/src/contrib/cloud/src/py/VERSION
deleted file mode 100644
index a7f3fc2..0000000
--- a/src/contrib/cloud/src/py/VERSION
+++ /dev/null
@@ -1 +0,0 @@
-0.22.0
\ No newline at end of file
diff --git a/src/contrib/cloud/src/py/hadoop-ec2-init-remote.sh b/src/contrib/cloud/src/py/hadoop-ec2-init-remote.sh
deleted file mode 100644
index 2c1049d..0000000
--- a/src/contrib/cloud/src/py/hadoop-ec2-init-remote.sh
+++ /dev/null
@@ -1,548 +0,0 @@
-#!/bin/bash -x
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-################################################################################
-# Script that is run on each EC2 instance on boot. It is passed in the EC2 user
-# data, so should not exceed 16K in size after gzip compression.
-#
-# This script is executed by /etc/init.d/ec2-run-user-data, and output is
-# logged to /var/log/messages.
-################################################################################
-
-################################################################################
-# Initialize variables
-################################################################################
-
-# Substitute environment variables passed by the client
-export %ENV%
-
-HADOOP_VERSION=${HADOOP_VERSION:-0.20.1}
-HADOOP_HOME=/usr/local/hadoop-$HADOOP_VERSION
-HADOOP_CONF_DIR=$HADOOP_HOME/conf
-SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname`
-for role in $(echo "$ROLES" | tr "," "\n"); do
-  case $role in
-  nn)
-    NN_HOST=$SELF_HOST
-    ;;
-  jt)
-    JT_HOST=$SELF_HOST
-    ;;
-  esac
-done
-
-function register_auto_shutdown() {
-  if [ ! -z "$AUTO_SHUTDOWN" ]; then
-    shutdown -h +$AUTO_SHUTDOWN >/dev/null &
-  fi
-}
-
-# Install a list of packages on debian or redhat as appropriate
-function install_packages() {
-  if which dpkg &> /dev/null; then
-    apt-get update
-    apt-get -y install $@
-  elif which rpm &> /dev/null; then
-    yum install -y $@
-  else
-    echo "No package manager found."
-  fi
-}
-
-# Install any user packages specified in the USER_PACKAGES environment variable
-function install_user_packages() {
-  if [ ! -z "$USER_PACKAGES" ]; then
-    install_packages $USER_PACKAGES
-  fi
-}
-
-function install_hadoop() {
-  useradd hadoop
-
-  hadoop_tar_url=http://s3.amazonaws.com/hadoop-releases/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
-  hadoop_tar_file=`basename $hadoop_tar_url`
-  hadoop_tar_md5_file=`basename $hadoop_tar_url.md5`
-
-  curl="curl --retry 3 --silent --show-error --fail"
-  for i in `seq 1 3`;
-  do
-    $curl -O $hadoop_tar_url
-    $curl -O $hadoop_tar_url.md5
-    if md5sum -c $hadoop_tar_md5_file; then
-      break;
-    else
-      rm -f $hadoop_tar_file $hadoop_tar_md5_file
-    fi
-  done
-
-  if [ ! -e $hadoop_tar_file ]; then
-    echo "Failed to download $hadoop_tar_url. Aborting."
-    exit 1
-  fi
-
-  tar zxf $hadoop_tar_file -C /usr/local
-  rm -f $hadoop_tar_file $hadoop_tar_md5_file
-
-  echo "export HADOOP_HOME=$HADOOP_HOME" >> ~root/.bashrc
-  echo 'export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH' >> ~root/.bashrc
-}
-
-function prep_disk() {
-  mount=$1
-  device=$2
-  automount=${3:-false}
-
-  echo "warning: ERASING CONTENTS OF $device"
-  mkfs.xfs -f $device
-  if [ ! -e $mount ]; then
-    mkdir $mount
-  fi
-  mount -o defaults,noatime $device $mount
-  if $automount ; then
-    echo "$device $mount xfs defaults,noatime 0 0" >> /etc/fstab
-  fi
-}
-
-function wait_for_mount {
-  mount=$1
-  device=$2
-
-  mkdir $mount
-
-  i=1
-  echo "Attempting to mount $device"
-  while true ; do
-    sleep 10
-    echo -n "$i "
-    i=$[$i+1]
-    mount -o defaults,noatime $device $mount || continue
-    echo " Mounted."
-    break;
-  done
-}
-
-function make_hadoop_dirs {
-  for mount in "$@"; do
-    if [ ! -e $mount/hadoop ]; then
-      mkdir -p $mount/hadoop
-      chown hadoop:hadoop $mount/hadoop
-    fi
-  done
-}
-
-# Configure Hadoop by setting up disks and site file
-function configure_hadoop() {
-
-  install_packages xfsprogs # needed for XFS
-
-  INSTANCE_TYPE=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-type`
-
-  if [ -n "$EBS_MAPPINGS" ]; then
-    # EBS_MAPPINGS is like "/ebs1,/dev/sdj;/ebs2,/dev/sdk"
-    DFS_NAME_DIR=''
-    FS_CHECKPOINT_DIR=''
-    DFS_DATA_DIR=''
-    for mapping in $(echo "$EBS_MAPPINGS" | tr ";" "\n"); do
-      # Split on the comma (see "Parameter Expansion" in the bash man page)
-      mount=${mapping%,*}
-      device=${mapping#*,}
-      wait_for_mount $mount $device
-      DFS_NAME_DIR=${DFS_NAME_DIR},"$mount/hadoop/hdfs/name"
-      FS_CHECKPOINT_DIR=${FS_CHECKPOINT_DIR},"$mount/hadoop/hdfs/secondary"
-      DFS_DATA_DIR=${DFS_DATA_DIR},"$mount/hadoop/hdfs/data"
-      FIRST_MOUNT=${FIRST_MOUNT-$mount}
-      make_hadoop_dirs $mount
-    done
-    # Remove leading commas
-    DFS_NAME_DIR=${DFS_NAME_DIR#?}
-    FS_CHECKPOINT_DIR=${FS_CHECKPOINT_DIR#?}
-    DFS_DATA_DIR=${DFS_DATA_DIR#?}
-
-    DFS_REPLICATION=3 # EBS is internally replicated, but we also use HDFS replication for safety
-  else
-    case $INSTANCE_TYPE in
-    m1.xlarge|c1.xlarge)
-      DFS_NAME_DIR=/mnt/hadoop/hdfs/name,/mnt2/hadoop/hdfs/name
-      FS_CHECKPOINT_DIR=/mnt/hadoop/hdfs/secondary,/mnt2/hadoop/hdfs/secondary
-      DFS_DATA_DIR=/mnt/hadoop/hdfs/data,/mnt2/hadoop/hdfs/data,/mnt3/hadoop/hdfs/data,/mnt4/hadoop/hdfs/data
-      ;;
-    m1.large)
-      DFS_NAME_DIR=/mnt/hadoop/hdfs/name,/mnt2/hadoop/hdfs/name
-      FS_CHECKPOINT_DIR=/mnt/hadoop/hdfs/secondary,/mnt2/hadoop/hdfs/secondary
-      DFS_DATA_DIR=/mnt/hadoop/hdfs/data,/mnt2/hadoop/hdfs/data
-      ;;
-    *)
-      # "m1.small" or "c1.medium"
-      DFS_NAME_DIR=/mnt/hadoop/hdfs/name
-      FS_CHECKPOINT_DIR=/mnt/hadoop/hdfs/secondary
-      DFS_DATA_DIR=/mnt/hadoop/hdfs/data
-      ;;
-    esac
-    FIRST_MOUNT=/mnt
-    DFS_REPLICATION=3
-  fi
-
-  case $INSTANCE_TYPE in
-  m1.xlarge|c1.xlarge)
-    prep_disk /mnt2 /dev/sdc true &
-    disk2_pid=$!
-    prep_disk /mnt3 /dev/sdd true &
-    disk3_pid=$!
-    prep_disk /mnt4 /dev/sde true &
-    disk4_pid=$!
-    wait $disk2_pid $disk3_pid $disk4_pid
-    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local,/mnt2/hadoop/mapred/local,/mnt3/hadoop/mapred/local,/mnt4/hadoop/mapred/local
-    MAX_MAP_TASKS=8
-    MAX_REDUCE_TASKS=4
-    CHILD_OPTS=-Xmx680m
-    CHILD_ULIMIT=1392640
-    ;;
-  m1.large)
-    prep_disk /mnt2 /dev/sdc true
-    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local,/mnt2/hadoop/mapred/local
-    MAX_MAP_TASKS=4
-    MAX_REDUCE_TASKS=2
-    CHILD_OPTS=-Xmx1024m
-    CHILD_ULIMIT=2097152
-    ;;
-  c1.medium)
-    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local
-    MAX_MAP_TASKS=4
-    MAX_REDUCE_TASKS=2
-    CHILD_OPTS=-Xmx550m
-    CHILD_ULIMIT=1126400
-    ;;
-  *)
-    # "m1.small"
-    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local
-    MAX_MAP_TASKS=2
-    MAX_REDUCE_TASKS=1
-    CHILD_OPTS=-Xmx550m
-    CHILD_ULIMIT=1126400
-    ;;
-  esac
-
-  make_hadoop_dirs `ls -d /mnt*`
-
-  # Create tmp directory
-  mkdir /mnt/tmp
-  chmod a+rwxt /mnt/tmp
-  
-  mkdir /etc/hadoop
-  ln -s $HADOOP_CONF_DIR /etc/hadoop/conf
-
-  ##############################################################################
-  # Modify this section to customize your Hadoop cluster.
-  ##############################################################################
-  cat > $HADOOP_CONF_DIR/hadoop-site.xml <<EOF
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<configuration>
-<property>
-  <name>dfs.block.size</name>
-  <value>134217728</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.data.dir</name>
-  <value>$DFS_DATA_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.datanode.du.reserved</name>
-  <value>1073741824</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.datanode.handler.count</name>
-  <value>3</value>
-  <final>true</final>
-</property>
-<!--property>
-  <name>dfs.hosts</name>
-  <value>$HADOOP_CONF_DIR/dfs.hosts</value>
-  <final>true</final>
-</property-->
-<!--property>
-  <name>dfs.hosts.exclude</name>
-  <value>$HADOOP_CONF_DIR/dfs.hosts.exclude</value>
-  <final>true</final>
-</property-->
-<property>
-  <name>dfs.name.dir</name>
-  <value>$DFS_NAME_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.namenode.handler.count</name>
-  <value>5</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.permissions</name>
-  <value>true</value>
-  <final>true</final>
-</property>
-<property>
-  <name>dfs.replication</name>
-  <value>$DFS_REPLICATION</value>
-</property>
-<property>
-  <name>fs.checkpoint.dir</name>
-  <value>$FS_CHECKPOINT_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>fs.default.name</name>
-  <value>hdfs://$NN_HOST:8020/</value>
-</property>
-<property>
-  <name>fs.trash.interval</name>
-  <value>1440</value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.tmp.dir</name>
-  <value>/mnt/tmp/hadoop-\${user.name}</value>
-  <final>true</final>
-</property>
-<property>
-  <name>io.file.buffer.size</name>
-  <value>65536</value>
-</property>
-<property>
-  <name>mapred.child.java.opts</name>
-  <value>$CHILD_OPTS</value>
-</property>
-<property>
-  <name>mapred.child.ulimit</name>
-  <value>$CHILD_ULIMIT</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.job.tracker</name>
-  <value>$JT_HOST:8021</value>
-</property>
-<property>
-  <name>mapred.job.tracker.handler.count</name>
-  <value>5</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.local.dir</name>
-  <value>$MAPRED_LOCAL_DIR</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.map.tasks.speculative.execution</name>
-  <value>true</value>
-</property>
-<property>
-  <name>mapred.reduce.parallel.copies</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.reduce.tasks</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.reduce.tasks.speculative.execution</name>
-  <value>false</value>
-</property>
-<property>
-  <name>mapred.submit.replication</name>
-  <value>10</value>
-</property>
-<property>
-  <name>mapred.system.dir</name>
-  <value>/hadoop/system/mapred</value>
-</property>
-<property>
-  <name>mapred.tasktracker.map.tasks.maximum</name>
-  <value>$MAX_MAP_TASKS</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.tasktracker.reduce.tasks.maximum</name>
-  <value>$MAX_REDUCE_TASKS</value>
-  <final>true</final>
-</property>
-<property>
-  <name>tasktracker.http.threads</name>
-  <value>46</value>
-  <final>true</final>
-</property>
-<property>
-  <name>mapred.compress.map.output</name>
-  <value>true</value>
-</property>
-<property>
-  <name>mapred.output.compression.type</name>
-  <value>BLOCK</value>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.default</name>
-  <value>org.apache.hadoop.net.StandardSocketFactory</value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
-  <value></value>
-  <final>true</final>
-</property>
-<property>
-  <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
-  <value></value>
-  <final>true</final>
-</property>
-<property>
-  <name>io.compression.codecs</name>
-  <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec</value>
-</property>
-<property>
-  <name>fs.s3.awsAccessKeyId</name>
-  <value>$AWS_ACCESS_KEY_ID</value>
-</property>
-<property>
-  <name>fs.s3.awsSecretAccessKey</name>
-  <value>$AWS_SECRET_ACCESS_KEY</value>
-</property>
-<property>
-  <name>fs.s3n.awsAccessKeyId</name>
-  <value>$AWS_ACCESS_KEY_ID</value>
-</property>
-<property>
-  <name>fs.s3n.awsSecretAccessKey</name>
-  <value>$AWS_SECRET_ACCESS_KEY</value>
-</property>
-</configuration>
-EOF
-
-  # Keep PID files in a non-temporary directory
-  sed -i -e "s|# export HADOOP_PID_DIR=.*|export HADOOP_PID_DIR=/var/run/hadoop|" \
-    $HADOOP_CONF_DIR/hadoop-env.sh
-  mkdir -p /var/run/hadoop
-  chown -R hadoop:hadoop /var/run/hadoop
-
-  # Set SSH options within the cluster
-  sed -i -e 's|# export HADOOP_SSH_OPTS=.*|export HADOOP_SSH_OPTS="-o StrictHostKeyChecking=no"|' \
-    $HADOOP_CONF_DIR/hadoop-env.sh
-
-  # Hadoop logs should be on the /mnt partition
-  sed -i -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/var/log/hadoop/logs|' \
-    $HADOOP_CONF_DIR/hadoop-env.sh
-  rm -rf /var/log/hadoop
-  mkdir /mnt/hadoop/logs
-  chown hadoop:hadoop /mnt/hadoop/logs
-  ln -s /mnt/hadoop/logs /var/log/hadoop
-  chown -R hadoop:hadoop /var/log/hadoop
-
-}
-
-# Sets up small website on cluster.
-function setup_web() {
-
-  if which dpkg &> /dev/null; then
-    apt-get -y install thttpd
-    WWW_BASE=/var/www
-  elif which rpm &> /dev/null; then
-    yum install -y thttpd
-    chkconfig --add thttpd
-    WWW_BASE=/var/www/thttpd/html
-  fi
-
-  cat > $WWW_BASE/index.html << END
-<html>
-<head>
-<title>Hadoop EC2 Cluster</title>
-</head>
-<body>
-<h1>Hadoop EC2 Cluster</h1>
-To browse the cluster you need to have a proxy configured.
-Start the proxy with <tt>hadoop-ec2 proxy &lt;cluster_name&gt;</tt>,
-and point your browser to
-<a href="http://apache-hadoop-ec2.s3.amazonaws.com/proxy.pac">this Proxy
-Auto-Configuration (PAC)</a> file.  To manage multiple proxy configurations,
-you may wish to use
-<a href="https://addons.mozilla.org/en-US/firefox/addon/2464">FoxyProxy</a>.
-<ul>
-<li><a href="http://$NN_HOST:50070/">NameNode</a>
-<li><a href="http://$JT_HOST:50030/">JobTracker</a>
-</ul>
-</body>
-</html>
-END
-
-  service thttpd start
-
-}
-
-function start_namenode() {
-  if which dpkg &> /dev/null; then
-    AS_HADOOP="su -s /bin/bash - hadoop -c"
-  elif which rpm &> /dev/null; then
-    AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
-  fi
-
-  # Format HDFS
-  [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
-
-  $AS_HADOOP "$HADOOP_HOME/bin/hadoop-daemon.sh start namenode"
-
-  $AS_HADOOP "$HADOOP_HOME/bin/hadoop dfsadmin -safemode wait"
-  $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /user"
-  # The following is questionable, as it allows a user to delete another user
-  # It's needed to allow users to create their own user directories
-  $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /user"
-
-}
-
-function start_daemon() {
-  if which dpkg &> /dev/null; then
-    AS_HADOOP="su -s /bin/bash - hadoop -c"
-  elif which rpm &> /dev/null; then
-    AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
-  fi
-  $AS_HADOOP "$HADOOP_HOME/bin/hadoop-daemon.sh start $1"
-}
-
-register_auto_shutdown
-install_user_packages
-install_hadoop
-configure_hadoop
-
-for role in $(echo "$ROLES" | tr "," "\n"); do
-  case $role in
-  nn)
-    setup_web
-    start_namenode
-    ;;
-  snn)
-    start_daemon secondarynamenode
-    ;;
-  jt)
-    start_daemon jobtracker
-    ;;
-  dn)
-    start_daemon datanode
-    ;;
-  tt)
-    start_daemon tasktracker
-    ;;
-  esac
-done
-
diff --git a/src/contrib/cloud/src/py/hadoop/cloud/__init__.py b/src/contrib/cloud/src/py/hadoop/cloud/__init__.py
index 13878a1..7025626 100644
--- a/src/contrib/cloud/src/py/hadoop/cloud/__init__.py
+++ b/src/contrib/cloud/src/py/hadoop/cloud/__init__.py
@@ -11,4 +11,5 @@
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
-# limitations under the License.
\ No newline at end of file
+# limitations under the License.
+VERSION="cdh-0.21.0"
diff --git a/src/contrib/cloud/src/py/hadoop/cloud/cli.py b/src/contrib/cloud/src/py/hadoop/cloud/cli.py
index 2403e59..3827ba4 100644
--- a/src/contrib/cloud/src/py/hadoop/cloud/cli.py
+++ b/src/contrib/cloud/src/py/hadoop/cloud/cli.py
@@ -16,6 +16,7 @@
 from __future__ import with_statement
 
 import ConfigParser
+from hadoop.cloud import VERSION
 from hadoop.cloud.cluster import get_cluster
 from hadoop.cloud.service import InstanceTemplate
 from hadoop.cloud.service import HadoopService
@@ -32,9 +33,6 @@ from optparse import make_option
 import os
 import sys
 
-version_file = os.path.join(sys.path[0], "VERSION")
-VERSION = open(version_file, "r").read().strip()
-
 DEFAULT_CLOUD_PROVIDER = 'ec2'
 
 DEFAULT_CONFIG_DIR_NAME = '.hadoop-cloud'
diff --git a/src/contrib/cloud/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh b/src/contrib/cloud/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh
new file mode 100644
index 0000000..2c1049d
--- /dev/null
+++ b/src/contrib/cloud/src/py/hadoop/cloud/data/hadoop-ec2-init-remote.sh
@@ -0,0 +1,548 @@
+#!/bin/bash -x
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+################################################################################
+# Script that is run on each EC2 instance on boot. It is passed in the EC2 user
+# data, so should not exceed 16K in size after gzip compression.
+#
+# This script is executed by /etc/init.d/ec2-run-user-data, and output is
+# logged to /var/log/messages.
+################################################################################
+
+################################################################################
+# Initialize variables
+################################################################################
+
+# Substitute environment variables passed by the client
+export %ENV%
+
+HADOOP_VERSION=${HADOOP_VERSION:-0.20.1}
+HADOOP_HOME=/usr/local/hadoop-$HADOOP_VERSION
+HADOOP_CONF_DIR=$HADOOP_HOME/conf
+SELF_HOST=`wget -q -O - http://169.254.169.254/latest/meta-data/public-hostname`
+for role in $(echo "$ROLES" | tr "," "\n"); do
+  case $role in
+  nn)
+    NN_HOST=$SELF_HOST
+    ;;
+  jt)
+    JT_HOST=$SELF_HOST
+    ;;
+  esac
+done
+
+function register_auto_shutdown() {
+  if [ ! -z "$AUTO_SHUTDOWN" ]; then
+    shutdown -h +$AUTO_SHUTDOWN >/dev/null &
+  fi
+}
+
+# Install a list of packages on debian or redhat as appropriate
+function install_packages() {
+  if which dpkg &> /dev/null; then
+    apt-get update
+    apt-get -y install $@
+  elif which rpm &> /dev/null; then
+    yum install -y $@
+  else
+    echo "No package manager found."
+  fi
+}
+
+# Install any user packages specified in the USER_PACKAGES environment variable
+function install_user_packages() {
+  if [ ! -z "$USER_PACKAGES" ]; then
+    install_packages $USER_PACKAGES
+  fi
+}
+
+function install_hadoop() {
+  useradd hadoop
+
+  hadoop_tar_url=http://s3.amazonaws.com/hadoop-releases/core/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz
+  hadoop_tar_file=`basename $hadoop_tar_url`
+  hadoop_tar_md5_file=`basename $hadoop_tar_url.md5`
+
+  curl="curl --retry 3 --silent --show-error --fail"
+  for i in `seq 1 3`;
+  do
+    $curl -O $hadoop_tar_url
+    $curl -O $hadoop_tar_url.md5
+    if md5sum -c $hadoop_tar_md5_file; then
+      break;
+    else
+      rm -f $hadoop_tar_file $hadoop_tar_md5_file
+    fi
+  done
+
+  if [ ! -e $hadoop_tar_file ]; then
+    echo "Failed to download $hadoop_tar_url. Aborting."
+    exit 1
+  fi
+
+  tar zxf $hadoop_tar_file -C /usr/local
+  rm -f $hadoop_tar_file $hadoop_tar_md5_file
+
+  echo "export HADOOP_HOME=$HADOOP_HOME" >> ~root/.bashrc
+  echo 'export PATH=$JAVA_HOME/bin:$HADOOP_HOME/bin:$PATH' >> ~root/.bashrc
+}
+
+function prep_disk() {
+  mount=$1
+  device=$2
+  automount=${3:-false}
+
+  echo "warning: ERASING CONTENTS OF $device"
+  mkfs.xfs -f $device
+  if [ ! -e $mount ]; then
+    mkdir $mount
+  fi
+  mount -o defaults,noatime $device $mount
+  if $automount ; then
+    echo "$device $mount xfs defaults,noatime 0 0" >> /etc/fstab
+  fi
+}
+
+function wait_for_mount {
+  mount=$1
+  device=$2
+
+  mkdir $mount
+
+  i=1
+  echo "Attempting to mount $device"
+  while true ; do
+    sleep 10
+    echo -n "$i "
+    i=$[$i+1]
+    mount -o defaults,noatime $device $mount || continue
+    echo " Mounted."
+    break;
+  done
+}
+
+function make_hadoop_dirs {
+  for mount in "$@"; do
+    if [ ! -e $mount/hadoop ]; then
+      mkdir -p $mount/hadoop
+      chown hadoop:hadoop $mount/hadoop
+    fi
+  done
+}
+
+# Configure Hadoop by setting up disks and site file
+function configure_hadoop() {
+
+  install_packages xfsprogs # needed for XFS
+
+  INSTANCE_TYPE=`wget -q -O - http://169.254.169.254/latest/meta-data/instance-type`
+
+  if [ -n "$EBS_MAPPINGS" ]; then
+    # EBS_MAPPINGS is like "/ebs1,/dev/sdj;/ebs2,/dev/sdk"
+    DFS_NAME_DIR=''
+    FS_CHECKPOINT_DIR=''
+    DFS_DATA_DIR=''
+    for mapping in $(echo "$EBS_MAPPINGS" | tr ";" "\n"); do
+      # Split on the comma (see "Parameter Expansion" in the bash man page)
+      mount=${mapping%,*}
+      device=${mapping#*,}
+      wait_for_mount $mount $device
+      DFS_NAME_DIR=${DFS_NAME_DIR},"$mount/hadoop/hdfs/name"
+      FS_CHECKPOINT_DIR=${FS_CHECKPOINT_DIR},"$mount/hadoop/hdfs/secondary"
+      DFS_DATA_DIR=${DFS_DATA_DIR},"$mount/hadoop/hdfs/data"
+      FIRST_MOUNT=${FIRST_MOUNT-$mount}
+      make_hadoop_dirs $mount
+    done
+    # Remove leading commas
+    DFS_NAME_DIR=${DFS_NAME_DIR#?}
+    FS_CHECKPOINT_DIR=${FS_CHECKPOINT_DIR#?}
+    DFS_DATA_DIR=${DFS_DATA_DIR#?}
+
+    DFS_REPLICATION=3 # EBS is internally replicated, but we also use HDFS replication for safety
+  else
+    case $INSTANCE_TYPE in
+    m1.xlarge|c1.xlarge)
+      DFS_NAME_DIR=/mnt/hadoop/hdfs/name,/mnt2/hadoop/hdfs/name
+      FS_CHECKPOINT_DIR=/mnt/hadoop/hdfs/secondary,/mnt2/hadoop/hdfs/secondary
+      DFS_DATA_DIR=/mnt/hadoop/hdfs/data,/mnt2/hadoop/hdfs/data,/mnt3/hadoop/hdfs/data,/mnt4/hadoop/hdfs/data
+      ;;
+    m1.large)
+      DFS_NAME_DIR=/mnt/hadoop/hdfs/name,/mnt2/hadoop/hdfs/name
+      FS_CHECKPOINT_DIR=/mnt/hadoop/hdfs/secondary,/mnt2/hadoop/hdfs/secondary
+      DFS_DATA_DIR=/mnt/hadoop/hdfs/data,/mnt2/hadoop/hdfs/data
+      ;;
+    *)
+      # "m1.small" or "c1.medium"
+      DFS_NAME_DIR=/mnt/hadoop/hdfs/name
+      FS_CHECKPOINT_DIR=/mnt/hadoop/hdfs/secondary
+      DFS_DATA_DIR=/mnt/hadoop/hdfs/data
+      ;;
+    esac
+    FIRST_MOUNT=/mnt
+    DFS_REPLICATION=3
+  fi
+
+  case $INSTANCE_TYPE in
+  m1.xlarge|c1.xlarge)
+    prep_disk /mnt2 /dev/sdc true &
+    disk2_pid=$!
+    prep_disk /mnt3 /dev/sdd true &
+    disk3_pid=$!
+    prep_disk /mnt4 /dev/sde true &
+    disk4_pid=$!
+    wait $disk2_pid $disk3_pid $disk4_pid
+    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local,/mnt2/hadoop/mapred/local,/mnt3/hadoop/mapred/local,/mnt4/hadoop/mapred/local
+    MAX_MAP_TASKS=8
+    MAX_REDUCE_TASKS=4
+    CHILD_OPTS=-Xmx680m
+    CHILD_ULIMIT=1392640
+    ;;
+  m1.large)
+    prep_disk /mnt2 /dev/sdc true
+    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local,/mnt2/hadoop/mapred/local
+    MAX_MAP_TASKS=4
+    MAX_REDUCE_TASKS=2
+    CHILD_OPTS=-Xmx1024m
+    CHILD_ULIMIT=2097152
+    ;;
+  c1.medium)
+    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local
+    MAX_MAP_TASKS=4
+    MAX_REDUCE_TASKS=2
+    CHILD_OPTS=-Xmx550m
+    CHILD_ULIMIT=1126400
+    ;;
+  *)
+    # "m1.small"
+    MAPRED_LOCAL_DIR=/mnt/hadoop/mapred/local
+    MAX_MAP_TASKS=2
+    MAX_REDUCE_TASKS=1
+    CHILD_OPTS=-Xmx550m
+    CHILD_ULIMIT=1126400
+    ;;
+  esac
+
+  make_hadoop_dirs `ls -d /mnt*`
+
+  # Create tmp directory
+  mkdir /mnt/tmp
+  chmod a+rwxt /mnt/tmp
+  
+  mkdir /etc/hadoop
+  ln -s $HADOOP_CONF_DIR /etc/hadoop/conf
+
+  ##############################################################################
+  # Modify this section to customize your Hadoop cluster.
+  ##############################################################################
+  cat > $HADOOP_CONF_DIR/hadoop-site.xml <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+<property>
+  <name>dfs.block.size</name>
+  <value>134217728</value>
+  <final>true</final>
+</property>
+<property>
+  <name>dfs.data.dir</name>
+  <value>$DFS_DATA_DIR</value>
+  <final>true</final>
+</property>
+<property>
+  <name>dfs.datanode.du.reserved</name>
+  <value>1073741824</value>
+  <final>true</final>
+</property>
+<property>
+  <name>dfs.datanode.handler.count</name>
+  <value>3</value>
+  <final>true</final>
+</property>
+<!--property>
+  <name>dfs.hosts</name>
+  <value>$HADOOP_CONF_DIR/dfs.hosts</value>
+  <final>true</final>
+</property-->
+<!--property>
+  <name>dfs.hosts.exclude</name>
+  <value>$HADOOP_CONF_DIR/dfs.hosts.exclude</value>
+  <final>true</final>
+</property-->
+<property>
+  <name>dfs.name.dir</name>
+  <value>$DFS_NAME_DIR</value>
+  <final>true</final>
+</property>
+<property>
+  <name>dfs.namenode.handler.count</name>
+  <value>5</value>
+  <final>true</final>
+</property>
+<property>
+  <name>dfs.permissions</name>
+  <value>true</value>
+  <final>true</final>
+</property>
+<property>
+  <name>dfs.replication</name>
+  <value>$DFS_REPLICATION</value>
+</property>
+<property>
+  <name>fs.checkpoint.dir</name>
+  <value>$FS_CHECKPOINT_DIR</value>
+  <final>true</final>
+</property>
+<property>
+  <name>fs.default.name</name>
+  <value>hdfs://$NN_HOST:8020/</value>
+</property>
+<property>
+  <name>fs.trash.interval</name>
+  <value>1440</value>
+  <final>true</final>
+</property>
+<property>
+  <name>hadoop.tmp.dir</name>
+  <value>/mnt/tmp/hadoop-\${user.name}</value>
+  <final>true</final>
+</property>
+<property>
+  <name>io.file.buffer.size</name>
+  <value>65536</value>
+</property>
+<property>
+  <name>mapred.child.java.opts</name>
+  <value>$CHILD_OPTS</value>
+</property>
+<property>
+  <name>mapred.child.ulimit</name>
+  <value>$CHILD_ULIMIT</value>
+  <final>true</final>
+</property>
+<property>
+  <name>mapred.job.tracker</name>
+  <value>$JT_HOST:8021</value>
+</property>
+<property>
+  <name>mapred.job.tracker.handler.count</name>
+  <value>5</value>
+  <final>true</final>
+</property>
+<property>
+  <name>mapred.local.dir</name>
+  <value>$MAPRED_LOCAL_DIR</value>
+  <final>true</final>
+</property>
+<property>
+  <name>mapred.map.tasks.speculative.execution</name>
+  <value>true</value>
+</property>
+<property>
+  <name>mapred.reduce.parallel.copies</name>
+  <value>10</value>
+</property>
+<property>
+  <name>mapred.reduce.tasks</name>
+  <value>10</value>
+</property>
+<property>
+  <name>mapred.reduce.tasks.speculative.execution</name>
+  <value>false</value>
+</property>
+<property>
+  <name>mapred.submit.replication</name>
+  <value>10</value>
+</property>
+<property>
+  <name>mapred.system.dir</name>
+  <value>/hadoop/system/mapred</value>
+</property>
+<property>
+  <name>mapred.tasktracker.map.tasks.maximum</name>
+  <value>$MAX_MAP_TASKS</value>
+  <final>true</final>
+</property>
+<property>
+  <name>mapred.tasktracker.reduce.tasks.maximum</name>
+  <value>$MAX_REDUCE_TASKS</value>
+  <final>true</final>
+</property>
+<property>
+  <name>tasktracker.http.threads</name>
+  <value>46</value>
+  <final>true</final>
+</property>
+<property>
+  <name>mapred.compress.map.output</name>
+  <value>true</value>
+</property>
+<property>
+  <name>mapred.output.compression.type</name>
+  <value>BLOCK</value>
+</property>
+<property>
+  <name>hadoop.rpc.socket.factory.class.default</name>
+  <value>org.apache.hadoop.net.StandardSocketFactory</value>
+  <final>true</final>
+</property>
+<property>
+  <name>hadoop.rpc.socket.factory.class.ClientProtocol</name>
+  <value></value>
+  <final>true</final>
+</property>
+<property>
+  <name>hadoop.rpc.socket.factory.class.JobSubmissionProtocol</name>
+  <value></value>
+  <final>true</final>
+</property>
+<property>
+  <name>io.compression.codecs</name>
+  <value>org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.GzipCodec</value>
+</property>
+<property>
+  <name>fs.s3.awsAccessKeyId</name>
+  <value>$AWS_ACCESS_KEY_ID</value>
+</property>
+<property>
+  <name>fs.s3.awsSecretAccessKey</name>
+  <value>$AWS_SECRET_ACCESS_KEY</value>
+</property>
+<property>
+  <name>fs.s3n.awsAccessKeyId</name>
+  <value>$AWS_ACCESS_KEY_ID</value>
+</property>
+<property>
+  <name>fs.s3n.awsSecretAccessKey</name>
+  <value>$AWS_SECRET_ACCESS_KEY</value>
+</property>
+</configuration>
+EOF
+
+  # Keep PID files in a non-temporary directory
+  sed -i -e "s|# export HADOOP_PID_DIR=.*|export HADOOP_PID_DIR=/var/run/hadoop|" \
+    $HADOOP_CONF_DIR/hadoop-env.sh
+  mkdir -p /var/run/hadoop
+  chown -R hadoop:hadoop /var/run/hadoop
+
+  # Set SSH options within the cluster
+  sed -i -e 's|# export HADOOP_SSH_OPTS=.*|export HADOOP_SSH_OPTS="-o StrictHostKeyChecking=no"|' \
+    $HADOOP_CONF_DIR/hadoop-env.sh
+
+  # Hadoop logs should be on the /mnt partition
+  sed -i -e 's|# export HADOOP_LOG_DIR=.*|export HADOOP_LOG_DIR=/var/log/hadoop/logs|' \
+    $HADOOP_CONF_DIR/hadoop-env.sh
+  rm -rf /var/log/hadoop
+  mkdir /mnt/hadoop/logs
+  chown hadoop:hadoop /mnt/hadoop/logs
+  ln -s /mnt/hadoop/logs /var/log/hadoop
+  chown -R hadoop:hadoop /var/log/hadoop
+
+}
+
+# Sets up small website on cluster.
+function setup_web() {
+
+  if which dpkg &> /dev/null; then
+    apt-get -y install thttpd
+    WWW_BASE=/var/www
+  elif which rpm &> /dev/null; then
+    yum install -y thttpd
+    chkconfig --add thttpd
+    WWW_BASE=/var/www/thttpd/html
+  fi
+
+  cat > $WWW_BASE/index.html << END
+<html>
+<head>
+<title>Hadoop EC2 Cluster</title>
+</head>
+<body>
+<h1>Hadoop EC2 Cluster</h1>
+To browse the cluster you need to have a proxy configured.
+Start the proxy with <tt>hadoop-ec2 proxy &lt;cluster_name&gt;</tt>,
+and point your browser to
+<a href="http://apache-hadoop-ec2.s3.amazonaws.com/proxy.pac">this Proxy
+Auto-Configuration (PAC)</a> file.  To manage multiple proxy configurations,
+you may wish to use
+<a href="https://addons.mozilla.org/en-US/firefox/addon/2464">FoxyProxy</a>.
+<ul>
+<li><a href="http://$NN_HOST:50070/">NameNode</a>
+<li><a href="http://$JT_HOST:50030/">JobTracker</a>
+</ul>
+</body>
+</html>
+END
+
+  service thttpd start
+
+}
+
+function start_namenode() {
+  if which dpkg &> /dev/null; then
+    AS_HADOOP="su -s /bin/bash - hadoop -c"
+  elif which rpm &> /dev/null; then
+    AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
+  fi
+
+  # Format HDFS
+  [ ! -e $FIRST_MOUNT/hadoop/hdfs ] && $AS_HADOOP "$HADOOP_HOME/bin/hadoop namenode -format"
+
+  $AS_HADOOP "$HADOOP_HOME/bin/hadoop-daemon.sh start namenode"
+
+  $AS_HADOOP "$HADOOP_HOME/bin/hadoop dfsadmin -safemode wait"
+  $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /user"
+  # The following is questionable, as it allows a user to delete another user
+  # It's needed to allow users to create their own user directories
+  $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /user"
+
+}
+
+function start_daemon() {
+  if which dpkg &> /dev/null; then
+    AS_HADOOP="su -s /bin/bash - hadoop -c"
+  elif which rpm &> /dev/null; then
+    AS_HADOOP="/sbin/runuser -s /bin/bash - hadoop -c"
+  fi
+  $AS_HADOOP "$HADOOP_HOME/bin/hadoop-daemon.sh start $1"
+}
+
+register_auto_shutdown
+install_user_packages
+install_hadoop
+configure_hadoop
+
+for role in $(echo "$ROLES" | tr "," "\n"); do
+  case $role in
+  nn)
+    setup_web
+    start_namenode
+    ;;
+  snn)
+    start_daemon secondarynamenode
+    ;;
+  jt)
+    start_daemon jobtracker
+    ;;
+  dn)
+    start_daemon datanode
+    ;;
+  tt)
+    start_daemon tasktracker
+    ;;
+  esac
+done
+
diff --git a/src/contrib/cloud/src/py/hadoop/cloud/service.py b/src/contrib/cloud/src/py/hadoop/cloud/service.py
index 84327fd..5be74d5 100644
--- a/src/contrib/cloud/src/py/hadoop/cloud/service.py
+++ b/src/contrib/cloud/src/py/hadoop/cloud/service.py
@@ -247,7 +247,8 @@ echo Proxy pid %s;""" % (process.pid, process.pid)
   
 
   def _get_default_user_data_file_template(self):
-    return os.path.join(sys.path[0], 'hadoop-%s-init-remote.sh' %
+    data_path = os.path.join(os.path.dirname(__file__), 'data')
+    return os.path.join(data_path, 'hadoop-%s-init-remote.sh' %
                  self.cluster.get_provider_code())
   
   def _get_master(self):
diff --git a/src/contrib/cloud/src/py/setup.py b/src/contrib/cloud/src/py/setup.py
new file mode 100644
index 0000000..238b4a9
--- /dev/null
+++ b/src/contrib/cloud/src/py/setup.py
@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distutils.core import setup
+
+version = __import__('hadoop.cloud').cloud.VERSION
+
+setup(name='hadoop-cloud',
+      version=version,
+      description='Scripts for running Hadoop on cloud providers',
+      license = 'Apache License (2.0)',
+      url = 'http://hadoop.apache.org/common/',
+      packages=['hadoop', 'hadoop.cloud','hadoop.cloud.providers'],
+      package_data={'hadoop.cloud': ['data/*.sh']},
+      scripts=['hadoop-ec2'],
+      author = 'Apache Hadoop Contributors',
+      author_email = 'common-dev@hadoop.apache.org',
+)
-- 
1.6.0.4

