准备

下载包

cloudera-manager-centos7-cm5.9.0_x86_64.tar.gz

CDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcelCDH-5.9.0-1.cdh5.9.0.p0.23-el7.parcel.sha1manifest.json

依赖包

yum install -y python-lxml  httpd mod_ssl

yum -y install cyrus-sasl-plain cyrus-sasl-devel cyrus-sasl-gssapi

rpm jdk包

rpm -vih jdk-6u31-linux-amd64.rpm rpm -vih oracle-j2sdk1.7-1.7.0+update67-1.x86_64.rpm

挂载nfs

mkdir /opt/nfsmount -t nfs 10.0.1.19:/opt/software /opt/nfs

hosts文件

10.0.1.120 cdh1.ops.net10.0.1.121 cdh2.ops.net10.0.1.122 cdh3.ops.net10.0.1.123 cdh4.ops.net

推送ssh key

## 生成mkdir /opt/nfs/software/hadoop/key -pcd  /opt/nfs/software/hadoop/key && ssh-keygen -t rsa -P '' -f hadoop #生成key## 所有主机的操作mkdir /root/.sshcp /opt/nfs/software/hadoop/key/hadoop.pub /root/.ssh/authorized_keyschmod 600 /root/.ssh/authorized_keyscp /opt/nfs/software/hadoop/key/hadoop /root/.ssh/id_rsa

master1 ntp(确保时间一致性)

which ntpd|xargs rpm -qf ||yum install ntp -y

ntp配置文件

restrict 0.0.0.0 mask 0.0.0.0 nomodify notrap noqueryrestrict 127.0.0.1restrict default nomodify notrapserver ntp1.aliyun.com iburstserver ntp2.aliyun.com iburst# 阿里云ntpserver 127.127.1.0fudge 127.127.1.0 stratum 10# 定义的server都不可用时，将使用local时间作为ntp服务提供给ntp客户端includefile /etc/ntp/crypto/pwdriftfile /var/lib/ntp/driftkeys /etc/ntp/keys

启用服务

systemctl enable ntpd && systemctl start ntpd

测试命令

ntpstat

synchronised to NTP server (120.25.115.20) at stratum 3 time correct to within 27 ms polling server every 64 s

其他主机同步时间

grep 'cdh-1' /var/spool/cron/root &>/dev/null ||echo '*/5 * * * * ntpdate cdh-1'  > /var/spool/cron/root

禁用Transparent Hugepage

echo never > /sys/kernel/mm/transparent_hugepage/defragecho never > /sys/kernel/mm/transparent_hugepage/enabledcat /sys/kernel/mm/transparent_hugepage/enabled #查看状态

调整vm.swappiness Linux内核参数

grep vm.swappiness /etc/sysctl.conf || echo "vm.swappiness = 1" >> /etc/sysctl.confsysctl -p

安装mysql

master1 节点安装mysql

安装jdk

java -version

java version "1.8.0_102" Java(TM) SE Runtime Environment (build 1.8.0_102-b14) Java HotSpot(TM) 64-Bit Server VM (build 25.102-b14, mixed mode)

安装MySQL JDBC驱动程序

wget https://cdn.mysql.com//Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gztar zxvf mysql-connector-java-5.1.46.tar.gzmkdir -p /usr/share/java/cd mysql-connector-java-5.1.46cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar#http://download.softagency.net/MySQL/Downloads/Connector-J/mysql-connector-java-5.1.42.tar.gz#JDBC目前已经有6.x，但与CDH 5.x存在不兼容

Cloudera Manager 安装与集群配置

...

安装CDH5

...

清理

mysql

drop database oozie;drop database hive;drop database hue;drop database scm;

删除文件

rm -Rf /var/lib/flume-ng/ /var/lib/hadoop* /var/lib/navigator /var/lib/hue /var/lib/oozie /var/lib/solr /var/lib/sqoop* /var/lib/zookeeper /etc/hadoop/ /etc/hbase /etc/hive /dfs /var/lib/hive /usr/lib/hue /var/lib/cloudera* /var/cache/yum/cloudera* /var/log/cloudera* /var/run/cloudera*rm /opt/cm -rf #软连接rm -rf /tmp/.scm_prepare_node.lockrm /opt/cm-5.15.1/lib/cloudera-scm-agent/cm_guid -frm /opt/cm/lib/cloudera-scm-agent/cm_guid -f