526互联

Hadoop3.x和Hive3.x环境搭建

发布时间 2023-11-03 14:08:39作者: 骑白马走三关

Hadoop3.3.6

创建hadoop用户

useradd hadoop
su - hadoop
ssh-keygen -t rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys

配置hdfs

core-site.xml

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop91:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/software/hadoop/tmp</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/opt/software/hadoop/tmp</value>
    </property>
    <!--
    “*”表示可通过超级代理“xxx”操作hadoop的用户、用户组和主机
    -->
    <property>
        <name>hadoop.proxyuser.hadoop.groups</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.hadoop.hosts</name>
        <value>*</value>
    </property>

</configuration>

hdfs-site.xml

<configuration>
    <!--设置名称节点的目录-->
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/opt/software/hadoop/tmp/namenode</value>
    </property>
    <!--设置数据节点的目录-->
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/opt/software/hadoop/tmp/datanode</value>
    </property>
    <!--设置辅助名称节点-->
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>hadoop91:50090</value>
    </property>
    <!--hdfs web的地址，默认为9870，可不配置-->
    <!--注意如果使用hadoop2，默认为50070-->
    <property>
        <name>dfs.namenode.http-address</name>
        <value>0.0.0.0:9870</value>
    </property>
    <!--副本数，默认为3，可不配置-->
    <property>
        <name>dfs.replication</name>
        <value>1</value>
    </property>
    <!--是否启用hdfs权限，当值为false时，代表关闭-->
    <property>
        <name>dfs.permissions.enabled</name>
        <value>false</value>
    </property>
</configuration>

yarn-site.xml

<configuration>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
    <property>
        <name>yarn.nodemanager.env-whitelist</name>
        <value>JAVA_HOME,HADOOP_COMMON_HOME,HADOOP_HDFS_HOME,HADOOP_CONF_DIR,CLASSPATH_PREPEND_DISTCACHE,HADOOP_YARN_HOME,HADOOP_HOME,PATH,LANG,TZ,HADOOP_MAPRED_HOME</value>
    </property>
</configuration>

mapred-site.xml

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.application.classpath</name>
        <value>$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/*:$HADOOP_MAPRED_HOME/share/hadoop/mapreduce/lib/*</value>
    </property>
</configuration>

hadoop-env.sh

export JAVA_HOME=/opt/software/java

yarn-env.sh

export JAVA_HOME=/opt/software/java
export HADOOP_HOME=/opt/software/hadoop
export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

格式化hdfs

hdfs namenode -format

# 启动 访问http://hadoop91:9870/测试
start-dfs.sh

hive3.1.3

前置配置

# 删除自带guava,防止和hadoop guava冲突
rm -f $HIVE_HOME/lib/guava-19.0.ja
cp $HADOOP_HOME/share/hadoop/common/lib/guava-27.0-jre.jar $HIVE_HOME/lib
cp mysql-connector-java-5.1.44.jar $HIVE_HOME/lib

配置hive

cd $HIVE_HOME/conf
cp hive-env.sh.template hive-env.sh
cp hive-default.xml.template hive-site.xml

hive-site.xml

<configuration>
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://hadoop91:12306/hive?createDatabaseIfNotExist=true&amp;useSSL=false&amp;useUnicode=true&amp;characterEncoding=UTF-8</value>
    </property>

    <!-- 这里注意MySQL 版本，我的是 5.X , 如果是 8 需要换成 8 的驱动 -->
    <property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>root</value>
    </property>

    <property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>Admin@2023!</value>
    </property>

    <!-- 关闭元数据存储授权  -->
    <property>
        <name>hive.metastore.event.db.notification.api.auth</name>
        <value>false</value>
    </property>

    <!-- 关闭元数据存储版本的验证 -->
    <property>
        <name>hive.metastore.schema.verification</name>
        <value>false</value>
    </property>
</configuration>

hive-env.sh

export HADOOP_HOME=/opt/software/hadoop
export HIVE_CONF_DIR=/opt/software/hive/conf
export HIVE_AUX_JARS_PATH=/opt/software/hive/lib

启动hive

#metastore
nohup /opt/software/hive/bin/hive --service metastore > nohup.out 2>&1 &
#hiveserver2
nohup /opt/software/hive/bin/hive --service hiveserver2 nohup.out 2>&1 &

#beeline测试
beeline

# 连接
!connect jdbc:hive2://hadoop91:10000