03_5_HDP_Installation_AWS

0. Objectives

This document provides step-by-step instruction for installing each component of the HDP & HDF platform in AWS Centos 7.

1. Pre-Installation

(All nodes)Config hostname and hosts

# (all servers)/etc/hosts
vim /etc/hosts
172.31.2.160 hdp01.hadoop.com hdp01
172.31.5.171 hdp02.hadoop.com hdp02
172.31.4.113 hdp03.hadoop.com hdp03

# (all servers)hostname
vim /etc/hostname

(All nodes)Disable SELinux

sudo sed -i -e "s/^SELINUX=enforcing$/SELINUX=disabled/g" /etc/selinux/config
cat /etc/selinux/config
getenforce
setenforce 0

(All nodes)Install and Start ntpd Service

1
2
3

yum install -y ntp
systemctl enable ntpd
systemctl start ntpd

(On ambari server only)Setting Login without password

ssh-keygen -t rsa
cat .ssh/id_rsa.pub
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPODX0nRWzqkpzyT3SZtBIhOSWPVJB8dOQ3fI6HxHNVRBTaDXYQCavA3+9C/esrysQkAMd2o/97VpAHtMmjK4+wSikNmZv3kxuNkL9ue0UKu3Z/HulrzN1LSVUg/Qdm9BnFAvnU0xrnnP7PPqi155d6uplYBkU43cKaA1suSsXRM6wipOQ2JXxo1o7BzdEpEib7iHJ+3vGv9BGQ2O2DDZlf3/OY7HHYqprZGxkLB/qxvlj/6uUoVEtI+QR8glpFxX2aecS5gytK8uNY2oBTGM5/1xVgtx33Surj+wTulEBS7IV2RdCAEK86ymJWmT52xIddNjjgum3HrHmfjAaXI6/ root@hdp01.hadoop.com

vim .ssh/authorized_keys
chmod 700 .ssh
chmod 400 .ssh/authorized_keys

# allow root to ssh
vi /etc/ssh/sshd_config
PermitRootLogin yes

service sshd reload

(All nodes)install java

yum search jdk
yum install -y java-1.8.0-openjdk.x86_64
echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile
echo "export PATH=$JAVA_HOME/bin:$PATH" >> /etc/profile
source /etc/profile
java -version

Install and config database(mariadb)

(On hdp01 server)

yum install -y mariadb-server
yum install -y mysql-connector-java*
systemctl start mariadb.service
systemctl enable mariadb.service
/usr/bin/mysql_secure_installation

mysql -u root -proot
-- hive
CREATE USER 'hive'@'localhost' IDENTIFIED BY 'bigdata';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'localhost';
CREATE USER 'hive'@'%' IDENTIFIED BY 'bigdata';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'%';
CREATE USER 'hive'@'hdp03.hadoop.com' IDENTIFIED BY 'bigdata';
GRANT ALL PRIVILEGES ON *.* TO 'hive'@'hdp03.hadoop.com';
CREATE DATABASE metastore;

-- ambari
CREATE USER 'ambari'@'%' IDENTIFIED BY 'bigdata';
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'%';
CREATE USER 'ambari'@'localhost' IDENTIFIED BY 'bigdata';
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'localhost';
CREATE USER 'ambari'@'hdp01.hadoop.com' IDENTIFIED BY 'bigdata';
GRANT ALL PRIVILEGES ON *.* TO 'ambari'@'hdp01.hadoop.com';
CREATE DATABASE ambari;

-- oozie
CREATE DATABASE oozie;
USE oozie;
CREATE USER 'oozie'@'%' IDENTIFIED BY 'oozie';
GRANT ALL ON *.* TO 'oozie'@'%';

-- knox
CREATE DATABASE knox;
USE knox;
CREATE USER 'knox'@'%' IDENTIFIED BY 'knox';
GRANT ALL ON *.* TO 'knox'@'%';

-- ranger
CREATE USER 'rangerdba'@'localhost' IDENTIFIED BY 'rangerdba';
GRANT ALL PRIVILEGES ON *.* TO 'rangerdba'@'localhost';
CREATE USER 'rangerdba'@'%' IDENTIFIED BY 'rangerdba';
CREATE USER 'rangerdba'@'hdp02.hadoop.com' IDENTIFIED BY 'rangerdba';
GRANT ALL PRIVILEGES ON *.* TO 'rangerdba'@'hdp02.hadoop.com' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'rangerdba'@'localhost' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'rangerdba'@'%' WITH GRANT OPTION;
create database ranger;
CREATE USER 'rangerdba'@'hdp01.hadoop.com' IDENTIFIED BY 'rangerdba';
GRANT ALL PRIVILEGES ON *.* TO 'rangerdba'@'hdp01.hadoop.com' WITH GRANT OPTION;

-- Hue
CREATE USER 'hue'@'%' IDENTIFIED BY 'hue';
GRANT ALL ON *.* TO 'hue'@'%';
CREATE USER 'hue'@'localhost' IDENTIFIED BY 'hue';
GRANT ALL PRIVILEGES on *.* to 'hue'@'localhost' WITH GRANT OPTION;
GRANT ALL on hue.* to 'hue'@'localhost';
CREATE DATABASE hue;

--SAM
create database registry;
CREATE USER 'registry'@'%' IDENTIFIED BY 'registry';
GRANT ALL ON *.* TO 'registry'@'%';
create database streamline;
CREATE USER 'streamline'@'%' IDENTIFIED BY 'streamline';
GRANT ALL ON *.* TO 'streamline'@'%';

--Druid
CREATE DATABASE druid DEFAULT CHARACTER SET utf8;
CREATE DATABASE superset DEFAULT CHARACTER SET utf8;
CREATE USER 'druid'@'%' IDENTIFIED BY 'druid';
CREATE USER 'superset'@'%' IDENTIFIED BY 'superset';
GRANT ALL PRIVILEGES ON *.* TO 'druid'@'%' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'superset'@'%' WITH GRANT OPTION;
CREATE USER 'druid'@'localhost' IDENTIFIED BY 'druid';
CREATE USER 'superset'@'localhost' IDENTIFIED BY 'superset';
GRANT ALL PRIVILEGES ON *.* TO 'druid'@'localhost' WITH GRANT OPTION;
GRANT ALL PRIVILEGES ON *.* TO 'superset'@'localhost' WITH GRANT OPTION;
FLUSH PRIVILEGES;
exit;

Config Ambari Repo

(On hdp01 server)

cd /etc/yum.repos.d/
wget -nv http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.7.3.0/ambari.repo -O /etc/yum.repos.d/ambari.repo
yum clean all
yum makecache

ambari installation and Config

(On hdp01 server) ambari installation

1	yum install -y ambari-server

(On hdp01 server)ambari setup

1	ambari-server setup

(On hdp01 server)ambari start

1
2
3

ambari-server start
systemctl disable firewalld
systemctl stop firewalld

HDF mpack install

(On hdp01 server)HDF mpack install

wget http://public-repo-1.hortonworks.com/HDF/centos7/3.x/updates/3.4.0.0/tars/hdf_ambari_mp/hdf-ambari-mpack-3.4.0.0-155.tar.gz

mv hdf-ambari-mpack-3.4.0.0-155.tar.gz /tmp/
ambari-server install-mpack \
--mpack=/tmp/hdf-ambari-mpack-3.4.0.0-155.tar.gz \
--verbose

ambari-server restart

Back up mysql db

(On hdp01 server)Back up mysql db

1	mysqldump -uroot -proot --all-databases > alldb.sql;

HDP/HDF deploy

(On Web)

ambari-server setup --jdbc-db=mysql --jdbc-driver=/usr/share/java/mysql-connector-java.jar

http://ec2-18-188-54-141.us-east-2.compute.amazonaws.com:8080
admin/admin

###