526互联

Hadoop-3.3.5 自动安装

发布时间 2023-10-14 13:29:21作者: Y&Qter

Hadoop-3.3.5 自动安装脚本

为了方便数据科学与大数据技术的 hadoop-3.3.5 安装与配置

目前适用于新安装的纯净虚拟机，未配置冲突检测..

所以安装的话请使用全新创建的机器以避免错误...

如何使用

  1. ssh localhost 输入你目前用户的密码
  2. exit 退出当前ssh终端（此举是为自动建立.ssh文件夹并记录known_hosts文件）
  3. 然后将下列代码复制进 hadoop_create.sh中 （vim hadoop_create.sh）
  4. chmod +x hadoop_create.sh （为该文件添加可执行权限）
  5. 在你保存该文件的文件夹里输入./hadoop_create.sh，启动该脚本。
#!/bin/bash
# Set up Java下载链接、目录和Java安装目录
download_url_java="https://aka.ms/download-jdk/microsoft-jdk-21.0.0-linux-x64.tar.gz"
download_dir="$HOME/Download"
java_install_dir="$HOME/JavaRuntime"

# 设置SSH密钥和目录
ssh_key_dir="$HOME/.ssh"
ssh_key_file="$ssh_key_dir/id_rsa"

# 设置Hadoop下载链接、目录和Hadoop安装目录
hadoop_download_url="https://archive.apache.org/dist/hadoop/common/hadoop-3.3.5/hadoop-3.3.5.tar.gz"
hadoop_download_dir="$download_dir"
hadoop_install_dir="$HOME/HadoopRuntime"

# 创建Hadoop数据目录结构
hadoop_tmp_dir="$hadoop_install_dir/data/tmp"
hadoop_datanode_dir="$hadoop_install_dir/data/datanode"
hadoop_namenode_dir="$hadoop_install_dir/data/namenode"
mkdir -p "$hadoop_tmp_dir"
mkdir -p "$hadoop_datanode_dir"
mkdir -p "$hadoop_namenode_dir"

# 生成SSH密钥并添加到authorized_keys
echo '========================================='
echo '=                                       ='
echo '=                                       ='
echo '=   Now is Configure the SSH            ='
echo '=                                       ='
echo '=                                       ='
echo '=                       --- Utree       ='
echo '=                                       ='
echo '========================================='

if [ ! -f "$ssh_key_file" ]; then
    mkdir -p "$ssh_key_dir"
    ssh-keygen -t rsa -f "$ssh_key_file" -N ""
    cat "$ssh_key_file.pub" >> "$ssh_key_dir/authorized_keys"
    echo "SSH密钥对已生成并添加到authorized_keys."
else
    echo "SSH密钥对已存在，未执行生成步骤."
fi

# 确保Download目录存在
mkdir -p "$download_dir"

echo '========================================='
echo '=                                       ='
echo '=                                       ='
echo '=   Now is Configure the Java Runtime   ='
echo '=                                       ='
echo '=                                       ='
echo '=                       --- Utree       ='
echo '=                                       ='
echo '========================================='

# 使用wget下载Java文件
wget "$download_url_java" -P "$download_dir" -O "$download_dir/jdk-21_linux-x64_bin.tar.gz"

# 检查Java下载是否成功
if [ $? -eq 0 ]; then
    echo "Java 21下载成功，已保存在$download_dir 目录下."

    # 创建Java安装目录
    mkdir -p "$java_install_dir"

    # 解压Java文件
    tar -xzvf "$download_dir/jdk-21_linux-x64_bin.tar.gz" -C "$java_install_dir"

    # 配置Java环境变量
    echo "export JAVA_HOME=$java_install_dir/jdk-21+35" >> "$HOME/.bashrc"
    echo 'export PATH=$JAVA_HOME/bin:$PATH' >> "$HOME/.bashrc"

    # 使配置生效
    source "$HOME/.bashrc"

    echo "Java 21已安装到$java_install_dir 目录，并已配置到~/.bashrc中."
else
    echo "Java 21下载失败."
fi

# 下载和安装Hadoop
# 使用wget下载Hadoop文件
echo '========================================='
echo '=                                       ='
echo '=                                       ='
echo '=   Now is Configure the Hadoop.        ='
echo '=                                       ='
echo '=                                       ='
echo '=                       --- Utree       ='
echo '=                                       ='
echo '========================================='

wget "$hadoop_download_url" -P "$hadoop_download_dir"

# 检查Hadoop下载是否成功
if [ $? -eq 0 ]; then
    echo "Hadoop 3.3.5下载成功，已保存在$download_dir 目录下."

    # 创建Hadoop安装目录
    mkdir -p "$hadoop_install_dir"

    # 解压Hadoop文件
    tar -xzvf "$hadoop_download_dir/hadoop-3.3.5.tar.gz" -C "$hadoop_install_dir" --strip-components=1

    echo "Hadoop 3.3.5已安装到$hadoop_install_dir 目录."

    # 配置core-site.xml
    core_site_file="$hadoop_install_dir/etc/hadoop/core-site.xml"
    cat <<EOL > "$core_site_file"
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>$hadoop_tmp_dir</value>
        <description>A base for other temporary directories.</description>
    </property>
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://localhost:9000</value>
    </property>
</configuration>
EOL

    # 配置hdfs-site.xml
    hdfs_site_file="$hadoop_install_dir/etc/hadoop/hdfs-site.xml"
    cat <<EOL > "$hdfs_site_file"
<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
    <property>
        <name>dfs.namenode.name.dir</name>
        <value>file:$hadoop_namenode_dir</value>
    </property>
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>file:$hadoop_datanode_dir</value>
    </property>
</configuration>
EOL

    echo '========================================='
    echo '=                                       ='
    echo '=                                       ='
    echo '=   Now is Configure namenode and       ='
    echo '=   datanode....                        ='
    echo '=                                       ='
    echo '=                       --- Utree       ='
    echo '=                                       ='
    echo '========================================='
    # 配置hadoop-env.sh
    hadoop_env_file="$hadoop_install_dir/etc/hadoop/hadoop-env.sh"
    echo "export JAVA_HOME=$java_install_dir/jdk-21+35" >> "$hadoop_env_file"

    echo "Hadoop配置已完成."
    # 格式化HDFS名称节点
    $hadoop_install_dir/bin/hdfs namenode -format

echo "Hadoop配置已完成，HDFS名称节点已格式化."
else
    echo "Hadoop 3.3.5下载失败."
fi