DataSphere Studio & Linkis 单机部署-526互联

一、环境要求

参考文档：DataSphere Studio & Linkis 单机一键部署文档

1.系统要求

CentOS为6或者7

2.安装依赖命令

命令依赖：
telnet; tar; sed; dos2unix; mysql; yum; java; unzip; zip; expect

yum provides */telnet  #如果命令不存在通过此命令查看命令所属安装包安装即可,如telnet

3.JDK

tar xf jdk-8u201-linux-x64.tar.gz -C /usr/local/
cd /usr/local/
ln -sv jdk1.8.0_201/ java
vim /etc/profile.d/java.sh
export JAVA_HOME=/usr/local/java
export CLASSPATH=$JAVA_HOME/lib/tools.jar
export PATH=$JAVA_HOME/bin:$PATH

. /etc/profile.d/java.sh
java -version

4.调大用户文件描述符

vim /etc/security/limits.conf
* soft nofile 65536
* hard nofile 131072
* soft nproc 2048
* hard nproc 4096
* soft memlock unlimited
* hard memlock unlimited

5.本机免密登录

ssh-keygen -t rsa
ssh-copy-id -i 192.168.1.134

6.安装 python matplotlib模块

python -m pip install matplotlib

7.主机名解析

192.168.1.134 mq002

二、安装依赖服务

1.Nginx

yum install nginx -y      #应用会自动添加Nginx配置和重启Nginx，自定义Nginx目前未验证是否有问题
systemctl start nginx

2.MySQL

参考：https://www.cnblogs.com/panwenbin-logs/p/11425652.html
确保tmpdir目录MySQL用户有写权限 show variables like '%tmpdir%';

创建 dss和hive库
create database dss default character set utf8mb4 COLLATE utf8mb4_general_ci;
CREATE USER 'dss'@'%' IDENTIFIED BY 'Qwer@123';
GRANT ALL PRIVILEGES ON dss.* TO 'dss'@'%';


create database hive default character set utf8mb4 COLLATE utf8mb4_general_ci;
CREATE USER 'hive'@'%' IDENTIFIED BY 'Qwer@123';
GRANT ALL PRIVILEGES ON hive.* TO 'hive'@'%';

FLUSH PRIVILEGES;
exit;

3.Hadoop

tar xf hadoop-2.7.2.tar.gz -C /usr/local/
ln -sv hadoop-2.7.2/ hadoop
cd hadoop/etc/hadoop/

cp hadoop-env.sh hadoop-env.sh-bak
vim hadoop-env.sh
export HDFS_NAMENODE_USER=root
export HDFS_DATANODE_USER=root
export HDFS_SECONDARYNAMENODE_USER=root
export YARN_RESOURCEMANAGER_USER=root
export YARN_NODEMANAGER_USER=root
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin


cp core-site.xml core-site.xml-bak
vim core-site.xml
<configuration>
   <!-- 指定HDFS老大（namenode）的通信地址 -->
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://192.168.1.134:9000</value>
    </property>
    <!-- 指定hadoop运行时产生文件的存储路径 -->
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/Hadoop/tmp</value>
    </property>
    <!-- 设置HDFS web UI访问用户 -->
    <property>
        <name>hadoop.http.staticuser.user</name>
        <value>root</value>
    </property>
</configuration>


cp hdfs-site.xml hdfs-site.xml-bak
vim hdfs-site.xml
<configuration>
    <!-- 设置namenode的http通讯地址 -->
    <property>
       <name>dfs.namenode.http-address</name>
       <value>192.168.1.134:50070</value>
    </property>
     <!-- 设置namenode存放的路径 -->
    <property>
       <name>dfs.namenode.name.dir</name>
       <value>/Hadoop/name</value>
    </property>
    <!-- 设置datanode存放的路径 -->
    <property>
        <name>dfs.datanode.data.dir</name>
        <value>/Hadoop/data</value>
    </property>


cp mapred-site.xml.template mapred-site.xml
vim mapred-site.xml
<configuration>
    <!-- 通知框架MR使用YARN -->
    <property>
       <name>mapreduce.framework.name</name>
       <value>yarn</value>
    </property>
</configuration>



vim yarn-site.xml
<configuration>

<!-- Site specific YARN configuration properties -->
<property>
    <name>yarn.resourcemanager.hostname</name>
    <value>192.168.1.134</value>
</property>
 
<property>
    <name>yarn.nodemanager.aux-services</name>
    <value>mapreduce_shuffle</value>
</property>
 

<!--检查每个任务正使用的物理内存量，如果任务超出分配值，则将其杀掉，默认是true -->
 <property>
     <name>yarn.nodemanager.pmem-check-enabled</name>
     <value>false</value>
 </property>

 <!--检查每个任务正使用的虚拟内存量，如果任务超出分配值，则将其杀掉，默认是true -->
 <property>
     <name>yarn.nodemanager.vmem-check-enabled</name>
     <value>false</value>
 </property>
</configuration>

vim slaves
192.168.1.134

mkdir -pv /Hadoop/{data,name,tmp}
vim /etc/profile.d/hadoop.sh
export HADOOP_HOME=/usr/local/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
. /etc/profile.d/hadoop.sh

hdfs namenode -format
start-dfs.sh
jps

访问地址：192.168.1.134:8088

4.hive

vim /etc/profile.d/hive.sh
export HIVE_HOME=/usr/local/hive
export PATH=$JAVA_HOME/bin:$PATH:$HADOOP_HOME/bin:$HIVE_HOME/bin
. /etc/profile.d/hive.sh


tar xf apache-hive-2.3.3-bin.tar.gz -C /usr/local/
cd /usr/local/
ln -sv apache-hive-2.3.3-bin/ hive
cd hive/conf

cp hive-env.sh.template hive-env.sh
vim hive-env.sh
export JAVA_HOME=/usr/local/java
export HADOOP_HOME=/usr/local/hadoop

vim hive-site.xml
<configuration>
    <property>
        <name>javax.jdo.option.ConnectionURL</name>
        <value>jdbc:mysql://192.168.1.134:3306/hive?useSSL=false&amp;createDatabaseIfNotExist=true&amp;characterEncoding=UTF-8</value>
    </property>
<property>
        <name>javax.jdo.option.ConnectionDriverName</name>
        <value>com.mysql.jdbc.Driver</value>
    </property>
    <property>
        <name>javax.jdo.option.ConnectionUserName</name>
        <value>hive</value>         # mysql用户
    </property>
<property>
        <name>javax.jdo.option.ConnectionPassword</name>
        <value>Qwer@123</value>    # mysql密码
    </property>

    <property>
        <name>datanucleus.schema.autoCreateAll</name>
        <value>true</value>
    </property>
</configuration>



mv mysql-connector-java-5.1.27-bin.jar /usr/local/hive/lib/
schematool -initSchema -dbType mysql
hive -e "show databases"

5.spark(on yarn)

tar xf spark-2.1.1-bin-hadoop2.7.tgz -C /usr/local/
cd /usr/local/
ln -sv spark-2.1.1-bin-hadoop2.7/ spark
cd spark/conf/

cp spark-env.sh.template spark-env.sh
vim spark-env.sh
export JAVA_HOME=/usr/local/java
export HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/
export YARN_CONF_DIR=/usr/local/hadoop/etc/hadoop/
export SPARK_DIST_CLASSPATH=$(/usr/local/hadoop/bin/hadoop classpath)

vim /root/.bash_profile
export PATH=/usr/local/spark/bin:$PATH
. /root/.bash_profile

spark-submit --class org.apache.spark.examples.SparkPi --master yarn --deploy-mode cluster --driver-memory 4g --executor-memory 2g --executor-cores 1 --queue default examples/jars/spark-examples*.jar 10


spark-sql -e "show databases"

三、安装DataSphere Studio & Linkis

mv dss_linkis_one-click_install_20221201.zip /mnt
cd /mnt
unzip -q dss_linkis_one-click_install_20221201.zip
cd dss_linkis_one-click_install_20221201/
cd conf/

cp config.sh config.sh-bak
vim config.sh
deployUser=root
LINKIS_VERSION=1.1.1
DSS_NGINX_IP=192.168.1.134
DSS_WEB_PORT=8085
DSS_VERSION=1.1.1
WORKSPACE_USER_ROOT_PATH=file:///root/linkis/ 
HDFS_USER_ROOT_PATH=hdfs:///tmp/linkis 
RESULT_SET_ROOT_PATH=hdfs:///tmp/linkis 
ENGINECONN_ROOT_PATH=/tmp
HADOOP_CONF_DIR=/usr/local/hadoop/etc/hadoop/
HIVE_CONF_DIR=/usr/local/hive/conf/
SPARK_CONF_DIR=/usr/local/spark/conf/
LINKIS_PUBLIC_MODULE=lib/linkis-commons/public-module
YARN_RESTFUL_URL=http://192.168.1.134:8088
SPARK_VERSION=2.1.1
HIVE_VERSION=2.3.3
PYTHON_VERSION=python2
LINKIS_EUREKA_INSTALL_IP=192.168.1.134
LINKIS_EUREKA_PORT=9600
LINKIS_GATEWAY_PORT=9001
LINKIS_MANAGER_PORT=9101
LINKIS_ENGINECONNMANAGER_PORT=9102
LINKIS_ENGINECONN_PLUGIN_SERVER_PORT=9103
LINKIS_ENTRANCE_PORT=9104
LINKIS_PUBLICSERVICE_PORT=9105
LINKIS_CS_PORT=9108
export SERVER_HEAP_SIZE="512M"
EMAIL_HOST=smtp.163.com
EMAIL_PORT=25
EMAIL_USERNAME=xxx@163.com
EMAIL_PASSWORD=xxxxx
EMAIL_PROTOCOL=smtp
ORCHESTRATOR_FILE_PATH=/tmp/dss
EXECUTION_LOG_PATH=/tmp/dss



cp db.sh db.sh-bak
vim db.sh
MYSQL_HOST=192.168.1.134
MYSQL_PORT=3306
MYSQL_DB=dss
MYSQL_USER=dss
MYSQL_PASSWORD=Qwer@123
HIVE_HOST=192.168.1.134
HIVE_PORT=3306
HIVE_DB=hive
HIVE_USER=hive
HIVE_PASSWORD=Qwer@123

执行安装脚本

cd /mnt/dss/dss_linkis_one-click_install_20221201/bin

sh install.sh # 安装时，脚本会询问您是否需要初始化数据库并导入元数据，Linkis 和 DSS 均会询问，第一次安装必须选是,其他按需

启动

sh start-all.sh

http://192.168.1.134:9600/  #Eureka地址查看微服务是否正常
http://192.168.1.134:8085/  #登录界面  用户：安装用户名称 密码：hadoop

datasphere单机studio linkis

脚本单机ansible linkis

linkis

datasphere

datasphere appconn studio