Commit 25f12d8c authored by David Johnson's avatar David Johnson

A simple port of Unni's Hadoop expect scripts to sh.

Also some Hadoop utility scripts, and wrappers to run jobs we care about
for testing or whatever.

These don't have the password login Unni had done in expect.  Right now
we don't need that.  If we need that, we can add it as the "frontend"
script and then use these for the rest.

The other part of these scripts is the part that bakes the hadoop image,
in ../setup-capnet-basic.sh .
parent 51100be1
Pipeline #1308 skipped
#!/bin/sh
set -x
HADOOPDIR=/home/ubuntu/hadoop-2.7.1
$HADOOPDIR/sbin/stop-yarn.sh
$HADOOPDIR/sbin/stop-dfs.sh
$HADOOPDIR/sbin/start-dfs.sh
$HADOOPDIR/sbin/start-yarn.sh
exit 0
#!/bin/sh
set -x
arg1=30
arg2=30
if [ $# -gt 0 ]; then
arg1=$1
fi
if [ $# -gt 1 ]; then
arg2=$2
fi
HADOOPDIR=/home/ubuntu/hadoop-2.7.1
HDFS="$HADOOPDIR/bin/hdfs dfs"
$HADOOPDIR/bin/hadoop jar \
$HADOOPDIR/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar \
pi $arg1 $arg2
exit 0
#!/bin/sh
set -x
if [ $# -gt 0 ]; then
SIZE=$1
else
SIZE=`expr 1024*1024*10`
fi
HADOOPDIR=/home/ubuntu/hadoop-2.7.1
HDFS="$HADOOPDIR/bin/hdfs dfs"
$HDFS -mkdir /tests
$HDFS -ls /tests/words-input-$SIZE.txt
if [ ! $? -eq 0 ]; then
# Create the input file
rm -f words-input.txt
touch words-input.txt
cursize=`stat --printf "%s" words-input.txt`
while [ $cursize -lt $SIZE ]; do
cat words.txt >> words-input.txt
cursize=`stat --printf "%s" words-input.txt`
done
# Upload into HDFS
$HDFS -copyFromLocal words-input.txt /tests/words-input-$SIZE.txt
fi
OUTDIRNUM=0
$HDFS -ls /tests/output.$OUTDIRNUM
while [ $? -eq 0 ]; do
OUTDIRNUM=`expr $OUTDIRNUM + 1`
$HDFS -ls /tests/output.$OUTDIRNUM
done
$HADOOPDIR/bin/hadoop jar \
$HADOOPDIR/share/hadoop/mapreduce/hadoop-mapreduce-examples-2.7.1.jar \
wordcount /tests/words-input-$SIZE.txt /tests/output.$OUTDIRNUM
exit 0
#!/bin/sh
PSCP='/usr/bin/parallel-scp -O StrictHostKeyChecking=no '
PSSH='/usr/bin/parallel-ssh -O StrictHostKeyChecking=no '
SSH='ssh -o StrictHostKeyChecking=no '
HOMEDIR=/home/ubuntu
TARBALLDIR="$HOMEDIR"
HADOOPTARBALL="${TARBALLDIR}/hadoop-2.7.1.tar.gz"
HADOOPDIR="$HOMEDIR/hadoop-2.7.1"
MCTARBALL="${TARBALLDIR}/hadoop_master_conf.tar.gz"
RMCTARBALL="${TARBALLDIR}/hadoop_resourcemanager_conf.tar.gz"
SCTARBALL="${TARBALLDIR}/hadoop_slaves_conf.tar.gz"
#!/bin/sh
set -x
DIRNAME=`dirname $0`
. "$DIRNAME/setup-lib.sh"
#
# Get the IPs of all our nodes and build parallel ssh hostfiles.
#
i=0
slaves=""
touch all.hostfile
touch slaves.namefile
touch slaves.hostfile
while [ $# -gt 0 ]; do
if [ $i -eq 0 ]; then
master="$1"
echo "$master" > master.hostfile
echo "$master" > all.hostfile
echo "master" > master.namefile
elif [ $i -eq 1 ]; then
resourcemanager="$1"
echo "$resourcemanager" > resourcemanager.hostfile
echo "$resourcemanager" >> all.hostfile
echo "resourcemanager" > resourcemanager.namefile
else
slavenum=`expr $i - 2`
slaves="$slaves $1"
echo "$1" >> all.hostfile
echo "$1" >> slaves.hostfile
echo "slave${slavenum}" >> slaves.namefile
fi
i=`expr $i + 1`
shift
done
#
# Build an /etc/hosts file and append it to all VMs /etc/hosts.
#
echo "$master master-link-1 master-0 master" > etc.hosts
echo "$resourcemanager resourcemanager-link-1 resourcemanager-0 resourcemanager" \
>> etc.hosts
i=0
for slave in $slaves ; do
echo "${slave} slave$i-link-1 slave$i-0 slave$i" >> etc.hosts
i=`expr $i + 1`
done
#cat etc.hosts | sudo tee -a /etc/hosts
cat etc.hosts | $PSSH --send-input -h all.hostfile sudo tee -a /etc/hosts
#
# Setup an ssh config file with StrictHostKeyChecking no so that all
# the hadoop scripts don't get hung up on that.
#
touch $HOMEDIR/.ssh/config
chmod 600 $HOMEDIR/.ssh/config
chown ubuntu:ubuntu $HOMEDIR/.ssh/config
echo "StrictHostKeyChecking no" >> $HOMEDIR/.ssh/config
$PSCP -h all.hostfile $HOMEDIR/.ssh/config $HOMEDIR/.ssh/config
#
# parallel-scp the slaves file to all hosts.
#
$PSCP -h all.hostfile $HOMEDIR/slaves.namefile $HOMEDIR/slaves.namefile
#
# Setup the master.
#
$SSH $master $HOMEDIR/setup_master.sh
#
# Setup the resourcemanager.
#
$SSH $resourcemanager $HOMEDIR/setup_resourcemanager.sh
#
# Setup all the slaves.
#
$PSSH -h slaves.hostfile $HOMEDIR/setup_slave.sh
#
# Let a parallelized hadoop startup script actually start everything up
# on the master, resourcemanager, and slave nodes.
#
$HOMEDIR/hadoop-2.7.1/bin/hadoop namenode -format
$HOMEDIR/hadoop-2.7.1/sbin/start-dfs.sh
$HOMEDIR/hadoop-2.7.1/sbin/start-yarn.sh
exit 0
#!/bin/sh
set -x
DIRNAME=`dirname $0`
. "$DIRNAME/setup-lib.sh"
#
# Unpack our tarballs.
#
tar -xzvf $HADOOPTARBALL -C $HOMEDIR
cp -pR $HADOOPDIR/etc/hadoop $HADOOPDIR/etc/hadoop.bak
rm -rf $HADOOPDIR/etc/hadoop
tar -xzvf $MCTARBALL -C $HADOOPDIR/etc/
#sudo rsync -av hadoop_master_conf/hadoop/ /etc/hadoop/
#
# Copy the slaves file into place
#
cp $HOMEDIR/slaves.namefile $HADOOPDIR/etc/hadoop/slaves
echo master >> $HADOOPDIR/etc/hadoop/slaves
mkdir -p $HADOOPDIR/logs
chmod 777 $HADOOPDIR/logs
sudo mkdir -p /mnt
sudo chown -R ubuntu /mnt
mkdir -p /mnt/hadoop
mkdir -p /mnt/datanode
# Give hadoop more tmp space to work with.
sudo mount -o remount,size=2G /tmp
exit 0
#!/bin/sh
set -x
DIRNAME=`dirname $0`
. "$DIRNAME/setup-lib.sh"
#
# Unpack our tarballs.
#
tar -xzvf $HADOOPTARBALL -C $HOMEDIR
cp -pR $HADOOPDIR/etc/hadoop $HADOOPDIR/etc/hadoop.bak
rm -rf $HADOOPDIR/etc/hadoop
tar -xzvf $RMCTARBALL -C $HADOOPDIR/etc/
#sudo rsync -av hadoop_resourcemanager_conf/hadoop/ /etc/hadoop/
#
# Copy the slaves file into place
#
cp slaves.namefile $HADOOPDIR/etc/hadoop/slaves
mkdir -p $HADOOPDIR/logs
chmod 777 $HADOOPDIR/logs
sudo mkdir -p /mnt
sudo chown -R ubuntu /mnt
mkdir -p /mnt/hadoop
mkdir -p /mnt/datanode
exit 0
#!/bin/sh
set -x
DIRNAME=`dirname $0`
. "$DIRNAME/setup-lib.sh"
#
# Unpack our tarballs.
#
tar -xzvf $HADOOPTARBALL -C $HOMEDIR
cp -pR $HADOOPDIR/etc/hadoop $HADOOPDIR/etc/hadoop.bak
rm -rf $HADOOPDIR/etc/hadoop
tar -xzvf $SCTARBALL -C $HADOOPDIR/etc/
#sudo rsync -av hadoop_resourcemanager_conf/hadoop/ /etc/hadoop/
#
# Copy the slaves file into place
#
cp slaves.namefile $HADOOPDIR/etc/hadoop/slaves
mkdir -p $HADOOPDIR/logs
chmod 777 $HADOOPDIR/logs
sudo mkdir -p /mnt
sudo chown -R ubuntu /mnt
mkdir -p /mnt/hadoop
mkdir -p /mnt/datanode
exit 0
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment