526互联

Hadoop-3.3.5单机版安装步骤

发布时间 2023-05-17 16:26:34作者: SpringCore

1.下载JDK和Hadoop[略]

2.解压[略]

3.创建hadoop数据存储的目录

mkdir -p /home/hadoop/tmp /home/hadoop/hdfs/data /home/hadoop/hdfs/name

4.配置JAVA环境和HADOOP_HOME

vim /etc/profile

添加如下内容

JAVA_HOME=/home/fanqi/jdk1.8.0_202
HADOOP_HOME=/home/hadoop/hadoop-3.3.5
PATH=$PATH:$JAVA_HOME/bin:$HADOOP_HOME/sbin:$HADOOP_HOME/bin
CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar
export JAVA_HOME PATH CLASSPATH HADOOP_HOME

5.刷新环境变量

source /etc/profile

6.hadoop-env.sh添加相关配置

vim /home/hadoop/hadoop-3.3.5/etc/hadoop/hadoop-env.sh

export JAVA_HOME=/home/fanqi/jdk1.8.0_202
export HADOOP_HOME=/home/hadoop/hadoop-3.3.5
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root

7.修改core-site.xml

vim /home/hadoop/hadoop-3.3.5/etc/hadoop/core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://hadoop:9000</value>
    </property>
    <property>
        <name>io.file.buffer.size</name>
        <value>131072</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/home/hadoop/tmp</value>
    </property>
</configuration>

8.修改hdfs-site.xml

vim /home/hadoop/hadoop-3.3.5/etc/hadoop/hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>/home/hadoop/hdfs/name</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/home/hadoop/hdfs/data</value>
    </property>

    <property>
        <name>dfs.replication</name>
        <value>1</value>
        <description>副本个数，配置默认是3,应小于datanode机器数量</description>
    </property>

    <property>
        <name>dfs.namenode.secondary.http.address</name>
        <value>master:9870</value>
    </property>

    <property>
        <name>dfs.permissions</name>
        <value>false</value>
    </property>

    <property>
        <name>dfs.webhdfs.enabled</name>
        <value>true</value>
    </property>

    <property>
        <name>dfs.http.address</name>
        <value>0.0.0.0:50070</value>
        <description>将绑定IP改为0.0.0.0，而不是本地回环IP，这样，就能够实现外网访问本机的50070端口了</description>
    </property>
</configuration>

9.修改mapre-site.xml

vim /home/hadoop/hadoop-3.3.5/etc/hadoop/mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>

<configuration>
    <property>
        <name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>yarn.app.mapreduce.am.env</name>
        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
    </property>
    <property>
        <name>mapreduce.map.env</name>
        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
    </property>
    <property>
        <name>mapreduce.reduce.env</name>
        <value>HADOOP_MAPRED_HOME=${HADOOP_HOME}</value>
    </property>

    <property>
        <name>mapred.job.tracker</name>
        <value>master:9001</value>
    </property>

    <property>
        <name>mapreduce.map.memory.mb</name>
        <value>1536</value>
    </property>
    <property>
        <name>mapreduce.map.java.opts</name>
        <value>-Xmx1024M</value>
    </property>
    <property>
        <name>mapreduce.reduce.memory.mb</name>
        <value>3072</value>
    </property>
    <property>
        <name>mapreduce.reduce.java.opts</name>
        <value>-Xmx2560M</value>
    </property>

    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>master:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>master:19888</value>
    </property>
</configuration>

10.修改yarn-site.xml

vim /home/hadoop/hadoop-3.3.5/etc/hadoop/yarn-site.xml

<?xml version="1.0"?>

<configuration>
<!-- Site specific YARN configuration properties -->

    <property>
        <name>yarn.resourcemanager.hostname</name>
        <value>master</value>
    </property>
    <property>
        <name>yarn.nodemanager.aux-services</name>
        <value>mapreduce_shuffle</value>
    </property>
</configuration>