编写自动安装Hadoop的shell脚本

完整流程: 大数据—Hadoop安装教程（二）

文章目录

• 编写自动安装Hadoop的shell脚本
• 上传压缩包
• 编写shell脚本
• vim hadoopautoinstall.sh
• 运行

上传压缩包

在opt目录下创建连个目录install和soft
将压缩包上传到install目录下

编写shell脚本

创建目录来放shell脚本
[root@hadoop shellfile]# mkdir shellfile
修改hadoopautoinstall.sh的文件权限
chmod 777 ./hadoopautoinstall.sh

vim hadoopautoinstall.sh

#! /bin/bash
echo 'auto install hadoop begining....'#global var
hadoop=trueif [ "$hadoop" = true ];thenecho 'hadoop install set true'echo 'setup hadoop 3'tar -zxf /opt/install/hadoop-3.1.3.tar.gz -C /opt/soft/mv /opt/soft/hadoop-3.1.3 /opt/soft/hadoop313sed -i '78a\export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop' /etc/profilesed -i '78a\export HADOOP_LIBEXEC_DIR=$HADOOP_HOME/libexec' /etc/profilesed -i '78a\export HADOOP_COMMON_LIB_NATIVE_DIR=$HADOOP_HOME/lib/native' /etc/profilesed -i '78a\export HADOOP_INSTALL=$HADOOP_HOME' /etc/profilesed -i '78a\export HADOOP_YARN_HOME=$HADOOP_HOME' /etc/profilesed -i '78a\export HADOOP_HDFS_HOME=$HADOOP_HOME' /etc/profilesed -i '78a\export HADOOP_COMMON_HOME=$HADOOP_HOME' /etc/profilesed -i '78a\export HADOOP_MAPRED_HOME=$HADOOP_HOME' /etc/profilesed -i '78a\export YARN_NODEMANAGER_USER=root' /etc/profilesed -i '78a\export YARN_RESOURCEMANAGER_USER=root' /etc/profilesed -i '78a\export HDFS_ZKFC_USER=root' /etc/profilesed -i '78a\export HDFS_JOURNALNODE_USER=root' /etc/profilesed -i '78a\export HDFS_SECONDARYNAMENODE_USER=root' /etc/profilesed -i '78a\export HDFS_DATANODE_USER=root' /etc/profilesed -i '78a\export HDFS_NAMENODE_USER=root' /etc/profilesed -i '78a\export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/lib' /etc/profilesed -i '78a\export HADOOP_HOME=/opt/soft/hadoop313' /etc/profilesed -i '78a\#HADOOP_HOME' /etc/profile#core-sitesed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    *' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    hadoop.proxyuser.root.groups' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    *' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    hadoop.proxyuser.root.hosts' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    读写队列缓存：128k' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    131072' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    io.file.buffer.size' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    root' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    hadoop.http.staticuser.user' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    namenode上本地的hadoop临时文件夹' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    /opt/soft/hadoop313/data' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    hadoop.tmp.dir' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    hdfs://hadoop:9000' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\    fs.defaultFS' /opt/soft/hadoop313/etc/hadoop/core-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/core-site.xml#hadoop-env.shsed -i '53a\export JAVA_HOME=/opt/soft/jdk180'  /opt/soft/hadoop313/etc/hadoop/hadoop-env.sh#hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    关闭权限验证' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    false' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    dfs.permissions.enabled' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    datanode上数据块的物理存储位置目录' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    /opt/soft/hadoop313/data/dfs/data' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    dfs.datanode.data.dir' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    namenode上存储hdfsq名字空间元数据的目录' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    /opt/soft/hadoop313/data/dfs/name' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    dfs.namenode.name.dir' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    hadoop中每一个block文件的备份数量' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    1' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\    dfs.replication' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/hdfs-site.xml#mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    1024' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    mapreduce.reduce.memory.mb' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    1024' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    mapreduce.map.memory.mb' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    hadoop:19888' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    mapreduce.jobhistory.webapp.address' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    hadoop:10020' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    mapreduce.jobhistory.address' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    /opt/soft/hadoop313/etc/hadoop:/opt/soft/hadoop313/share/hadoop/common/lib/*:/opt/soft/hadoop313/share/hadoop/common/*:/opt/soft/hadoop313/share/hadoop/hdfs/*:/opt/soft/hadoop313/share/hadoop/hdfs/lib/*:/opt/soft/hadoop313/share/hadoop/mapreduce/*:/opt/soft/hadoop313/share/hadoop/mapreduce/lib/*:/opt/soft/hadoop313/share/hadoop/yarn/*:/opt/soft/hadoop313/share/hadoop/yarn/lib/*' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\   mapreduce.application.classpath' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    true' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    job执行框架：local,classic or yarn' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    yarn' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\    mapreduce.framework.name' /opt/soft/hadoop313/etc/hadoop/mapred-site.xmlsed -i '19a\  ' /opt/soft/hadoop313/etc/hadoop/mapred-site.xml#yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    /opt/soft/hadoop313/yarndata/log' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.nodemanager.log-dirs' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    /opt/soft/hadoop313/yarndata/yarn' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.nodemanager.local-dirs' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    mapreduce_shuffle' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.nodemanager.aux-services' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    hadoop:8042' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.nodemanager.webapp.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    hadoop:8050' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.nodemanager.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    hadoop:8040' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.nodemanager.localizer.address' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.resourcemanager.scheduler.class' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    20000' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\    yarn.resourcemanager.connect.retry-interval.ms' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlsed -i '17a\  ' /opt/soft/hadoop313/etc/hadoop/yarn-site.xmlecho 'setup hadoop 3 success!!!'
fi

运行

刷新资源
[root@hadoop shellfile]# source /etc/profile
运行文件
[root@hadoop shellfile]# ./hadoopautoinstall.sh

格式化
[root@hadoop hadoop313]# hdfs namenode -format

开始
[root@hadoop hadoop313]# start-all.sh
[root@hadoop hadoop313]# jps