From 5359a3bbd2b09644825be99fdd354ff3276a5d59 Mon Sep 17 00:00:00 2001
From: Aaron Kimball <aaron@cloudera.com>
Date: Fri, 12 Mar 2010 14:21:36 -0800
Subject: [PATCH] CLOUDERA-BUILD. New versions of cloudera packaging scripts

---
 bin/hadoop-config.sh       |   72 +++++++++++++++++++++++++++++++++++++-------
 cloudera/README.cloudera   |    9 +++++
 cloudera/do-release-build  |   72 ++++++++++++++++++++++++++++++++++++++++++++
 cloudera/install_hadoop.sh |   45 +++++++++++----------------
 4 files changed, 160 insertions(+), 38 deletions(-)
 create mode 100644 cloudera/README.cloudera
 create mode 100755 cloudera/do-release-build

diff --git a/bin/hadoop-config.sh b/bin/hadoop-config.sh
index 8c4b8ad..9c0e799 100644
--- a/bin/hadoop-config.sh
+++ b/bin/hadoop-config.sh
@@ -17,28 +17,79 @@
 # should not be executable directly
 # also should not be passed any arguments, since we need original $*
 
-# Honor the JAVA_HOME variable if set, otherwise try to find Java
-if [ -z $JAVA_HOME ]; then
-  if [ -e @JAVA_HOME@ ]; then
-	export JAVA_HOME="@JAVA_HOME@"
+# resolve links - $0 may be a softlink
+
+this="$0"
+while [ -h "$this" ]; do
+  ls=`ls -ld "$this"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '.*/.*' > /dev/null; then
+    this="$link"
   else
-	cat <<MSG
+    this=`dirname "$this"`/"$link"
+  fi
+done
+
+# convert relative path to absolute path
+bin=`dirname "$this"`
+script=`basename "$this"`
+bin=`cd "$bin"; pwd`
+this="$bin/$script"
+
+# the root of the Hadoop installation
+if [ -z "$HADOOP_HOME" ]; then
+  export HADOOP_HOME=`dirname "$this"`/..
+fi
+
+# double check that our HADOOP_HOME looks reasonable
+# cding to / here verifies that we have an absolute path, which is
+# necessary for the daemons to function properly
+if [ -z $(cd / && ls $HADOOP_HOME/hadoop-*-core.jar 2>/dev/null) ]; then
+  cat 1>&2 <<EOF
++======================================================================+
+|      Error: HADOOP_HOME is not set and could not be guessed          |
++----------------------------------------------------------------------+
+| This script attempts to find HADOOP_HOME based on its location, but  |
+| could not locate it. If you have a nonstandard install, please set   |
+| your HADOOP_HOME variable to the absolute path of the directory that |
+| contains hadoop-VERSION-core.jar                                     |
++======================================================================+
+EOF
+  exit 1
+fi
+
+# attempt to find java
+if [ -z "$JAVA_HOME" ]; then
+  for candidate in \
+    /usr/lib/jvm/java-6-sun \
+    /usr/lib/j2sdk1.6-sun \
+    /usr/java/jdk1.6* \
+    /usr/java/jre1.6* \
+    /Library/Java/Home ; do
+    if [ -e $candidate/bin/java ]; then
+      export JAVA_HOME=$candidate
+      break
+    fi
+  done
+  # if we didn't set it
+  if [ -z "$JAVA_HOME" ]; then
+    cat 1>&2 <<EOF
 +======================================================================+
 |      Error: JAVA_HOME is not set and Java could not be found         |
 +----------------------------------------------------------------------+
 | Please download the latest Sun JDK from the Sun Java web site        |
 |       > http://java.sun.com/javase/downloads/ <                      |
 |                                                                      |
+| Hadoop requires Java 1.6 or later.                                   |
 | NOTE: This script will find Sun Java whether you install using the   |
 |       binary or the RPM based installer.                             |
 +======================================================================+
-MSG
-	exit 1
+EOF
+    exit 1
   fi
 fi
 
-# the root of the Hadoop installation
-export HADOOP_HOME="${HADOOP_HOME:-/usr/lib/hadoop-0.20}"
+
 
 #check to see if the conf dir is given as an optional argument
 if [ $# -gt 1 ]
@@ -53,8 +104,7 @@ then
 fi
  
 # Allow alternate conf dir location.
-HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-/etc/hadoop-0.20/conf}"
-HADOOP_LOG_DIR="${HADOOP_LOG_DIR:-/var/log/hadoop-0.20}"
+HADOOP_CONF_DIR="${HADOOP_CONF_DIR:-$HADOOP_HOME/conf}"
 
 #check to see it is specified whether to use the slaves or the
 # masters file
diff --git a/cloudera/README.cloudera b/cloudera/README.cloudera
new file mode 100644
index 0000000..394d47c
--- /dev/null
+++ b/cloudera/README.cloudera
@@ -0,0 +1,9 @@
+This build was generated by Cloudera's build system in the following manner:
+
+1) The pristine open-source release tarball was unpacked
+
+2) The patches contained within the patches/ directory next to this README
+were applied using the apply-patches script. A complete log of these changes
+is also included in CHANGES.cloudera.txt.
+
+3) The project was built by running the do-release-build script in this directory.
\ No newline at end of file
diff --git a/cloudera/do-release-build b/cloudera/do-release-build
new file mode 100755
index 0000000..6555e36
--- /dev/null
+++ b/cloudera/do-release-build
@@ -0,0 +1,72 @@
+#!/bin/bash
+# Copyright (c) 2009 Cloudera, inc
+#
+# Performs a release build
+
+set -e
+
+if [ $(uname -m) != "x86_64" ]; then
+  echo Release build should be done on a 64-bit box to generate 1>&2
+  echo both 64 and 32 bit native libraries. 1>&2
+  exit 1
+fi
+
+JAVA32_HOME=${JAVA32_HOME:-$JAVA_HOME}
+JAVA64_HOME=${JAVA64_HOME:-$JAVA_HOME}
+
+# Check that JVMs support the right architectures
+# Passing -d with an architecture that's not supported results
+# in a non-zero exit code even with -version
+if ! $JAVA32_HOME/bin/java -d32 -version ; then
+  echo Your jvm in $JAVA32_HOME seems like it doesnt support 32-bit mode 1>&2
+  echo Please set JAVA32_HOME to point to a 32-bit JDK
+  exit 1
+fi
+
+if ! $JAVA64_HOME/bin/java -d64 -version ; then
+  echo Your jvm in $JAVA64_HOME seems like it doesnt support 64-bit mode 1>&2
+  echo Please set JAVA64_HOME to point to a 64-bit JDK
+  exit 1
+fi
+
+if ! [ -e $JAVA5_HOME/bin/java ]; then
+  echo No JAVA5_HOME set. Forrest documentation requires Java 5 to be 1>&2
+  echo installed. 1>&2
+  exit 1
+fi
+
+if ! [ -e $FORREST_HOME/bin/forrest ]; then
+  echo No FORREST_HOME set. Forrest documentation requires Forrest to be 1>&2
+  echo installed. 1>&2
+  exit 1
+fi
+
+# Do the build
+BIN_DIR=$(readlink -f $(dirname $0))
+RELEASE_DIR=$BIN_DIR/..
+
+cd $RELEASE_DIR
+
+JAVA_HOME=$JAVA32_HOME \
+  CFLAGS=-m32 \
+  CXXFLAGS=-m32 \
+  ant \
+  -Dlibhdfs=true \
+  -Dcompile.native=true \
+  -Dcompile.c++=true \
+  -Djava5.home=$JAVA5_HOME \
+  -Dforrest.home=$FORREST_HOME \
+  -propertyfile cloudera/build.properties \
+  clean tar
+
+JAVA_HOME=$JAVA64_HOME \
+  CFLAGS=-m64 \
+  CXXFLAGS=-m64 \
+  ant \
+  -Dlibhdfs=true \
+  -Dcompile.native=true \
+  -Dcompile.c++=true \
+  -Djava5.home=$JAVA5_HOME \
+  -Dforrest.home=$FORREST_HOME \
+  -propertyfile cloudera/build.properties \
+  compile-core-native compile-c++ tar
\ No newline at end of file
diff --git a/cloudera/install_hadoop.sh b/cloudera/install_hadoop.sh
index 93bb879..d225958 100755
--- a/cloudera/install_hadoop.sh
+++ b/cloudera/install_hadoop.sh
@@ -13,8 +13,6 @@ usage: $0 <options>
 
   Optional options:
      --native-build-string       eg Linux-amd-64 (optional - no native installed if not set)
-     --hadoop-config-location    eg /usr/bin/hadoop-config.sh or /etc/default/hadoop
-     --java-home                 eg /usr/lib/jvm/java-6-sun/
      ... [ see source for more similar options ]
   "
   exit 1
@@ -27,13 +25,11 @@ OPTS=$(getopt \
   -l 'prefix:' \
   -l 'build-dir:' \
   -l 'native-build-string:' \
-  -l 'hadoop-config-location:' \
   -l 'installed-lib-dir:' \
   -l 'lib-dir:' \
   -l 'src-dir:' \
   -l 'etc-dir:' \
   -l 'doc-dir:' \
-  -l 'java-home:' \
   -l 'man-dir:' \
   -l 'example-dir:' \
   -- "$@")
@@ -66,15 +62,9 @@ while true ; do
         --etc-dir)
         ETC_DIR=$2 ; shift 2
         ;;
-        --hadoop-config-location)
-        HADOOP_CONFIG_INSTALL_LOCATION=$2 ; shift 2
-        ;;
         --installed-lib-dir)
         INSTALLED_LIB_DIR=$2 ; shift 2
         ;;
-        --java-home)
-        JAVA_HOME=$2 ; shift 2
-        ;;
         --man-dir)
         MAN_DIR=$2 ; shift 2
         ;;
@@ -109,8 +99,6 @@ MAN_DIR=${MAN_DIR:-$PREFIX/usr/man}
 EXAMPLE_DIR=${EXAMPLE_DIR:-$DOC_DIR/examples}
 SRC_DIR=${SRC_DIR:-$PREFIX/usr/src/hadoop}
 ETC_DIR=${ETC_DIR:-$PREFIX/etc/hadoop}
-HADOOP_CONFIG_INSTALL_LOCATION=${HADOOP_CONFIG_INSTALL_LOCATION:-/etc/default/hadoop}
-JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-6-sun/}
 
 INSTALLED_LIB_DIR=${INSTALLED_LIB_DIR:-/usr/lib/hadoop}
 
@@ -127,19 +115,18 @@ for x in docs lib/native c++ src conf ; do
 done
 
 
-# First, lets point all the bin scripts to the correct hadoop-config.sh file
-for file in $LIB_DIR/bin/* ; do
-    sed -i -e "s|^.*hadoop-config.sh.*\$|. $HADOOP_CONFIG_INSTALL_LOCATION|" $file
-    sed -i -e 's|"$HADOOP_HOME"/bin/hadoop|/usr/bin/hadoop|' $file
-done
+# Make bin wrappers
+mkdir -p $BIN_DIR
 
-# Point the sqoop script at the correct target for its jar.
-sed -i -e "s|SQOOP_PREFIX|$INSTALLED_LIB_DIR|" "$LIB_DIR/bin/sqoop"
+for bin_wrapper in hadoop sqoop ; do
+  cat > $BIN_DIR/$bin_wrapper <<EOF
+#!/bin/sh
 
-# Mv bin elsewhere
-mkdir -p $BIN_DIR
-ln -sf $INSTALLED_LIB_DIR/bin/hadoop $BIN_DIR
-ln -sf $INSTALLED_LIB_DIR/bin/sqoop $BIN_DIR
+export HADOOP_HOME=$INSTALLED_LIB_DIR
+exec $INSTALLED_LIB_DIR/bin/$bin_wrapper \$*
+EOF
+  chmod 755 $BIN_DIR/$bin_wrapper
+done
 
 # Fix some bad permissions in HOD
 chmod 755 $LIB_DIR/contrib/hod/support/checklimits.sh
@@ -167,6 +154,14 @@ cp -a ${HADOOP_SRC_DIR}/* $SRC_DIR/
 install -d -m 0755 $ETC_DIR/conf.empty
 (cd ${BUILD_DIR}/conf && tar cf - .) | (cd $ETC_DIR/conf.empty && tar xf -)
 
+# Link the HADOOP_HOME conf dir and log dir to installed locations
+rm -rf $LIB_DIR/conf
+ln -s ${ETC_DIR#$PREFIX}/conf $LIB_DIR/conf
+rm -rf $LIB_DIR/logs
+ln -s /var/log/hadoop $LIB_DIR/logs
+
+
+
 # Make the pseudo-distributed config
 for conf in conf.pseudo ; do
   install -d -m 0755 $ETC_DIR/$conf
@@ -176,10 +171,6 @@ for conf in conf.pseudo ; do
   (cd ${BUILD_DIR}/../../example-confs/$conf && tar -cf - .) | (cd $ETC_DIR/$conf && tar -xf -)
 done
 
-mkdir -p `dirname $PREFIX$HADOOP_CONFIG_INSTALL_LOCATION`
-mv $LIB_DIR/bin/hadoop-config.sh $PREFIX$HADOOP_CONFIG_INSTALL_LOCATION
-sed -i -e "s|@JAVA_HOME@|$JAVA_HOME|" $PREFIX$HADOOP_CONFIG_INSTALL_LOCATION
-
 # man page
 mkdir -p $MAN_DIR/man1
 cp ${CLOUDERA_SOURCE_DIR}/hadoop.1.gz $MAN_DIR/man1/
-- 
1.6.0.4

