526互联

Flink安装

发布时间 2023-04-03 11:12:54作者: edclol

Flink安装

  1. 下载并解压

    f_v=1.14.4
mkdir -p /opt/software && cd /opt/software
wget https://mirrors.bfsu.edu.cn/apache/flink/flink-${f_v}/flink-${f_v}-bin-scala_2.11.tgz --no-check-certificate
tar -zxvf flink-${f_v}-bin-scala_2.11.tgz

  2. 安装jar包连接器

    f_v=1.14.4

cd flink-${f_v}/lib

#mysql连接驱动
wget https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.19/mysql-connector-java-8.0.19.jar

#Flink sql 连接kafka 连接器
wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-kafka_2.11/${f_v}/flink-sql-connector-kafka_2.11-${f_v}.jar

#Flink jdbc连接器
wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-jdbc_2.11/${f_v}/flink-connector-jdbc_2.11-${f_v}.jar

wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-hive_2.11/${f_v}/flink-connector-hive_2.11-${f_v}.jar

#高版本自带 不用下载

wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-csv/${f_v}/flink-csv-${f_v}.jar

wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-json/${f_v}/flink-json-${f_v}.jar

#CDC
wget https://repo.maven.apache.org/maven2/com/ververica/flink-connector-mysql-cdc/2.2.0/flink-connector-mysql-cdc-2.2.0.jar

whttps://repo1.maven.org/maven2/com/ververica/flink-sql-connector-mysql-cdc/2.2.0/flink-sql-connector-mysql-cdc-2.2.0.jar

#KAFKA
wget https://repo.maven.apache.org/maven2/org/apache/flink/flink-connector-kafka_2.11/1.14.4/flink-connector-kafka_2.11-1.14.4.jar

  3. 启动集群

    ./bin/start-cluster.sh

  4. 启动范例

    ./bin/flink run examples/streaming/WordCount.jar

  5. 停止集群

    ./bin/stop-cluster.sh

  6. 启动sqlclient

    cd /opt/software/flink-${f_v}/

bin/sql-client.sh embedded --jar lib/flink-sql-connector-kafka_2.11-${f_v}.jar

  7. 配置目录

    vi ./bin/yarn-session.sh

export HADOOP_CONF_DIR=/etc/hadoop/conf
export HBASE_CONF_DIR=/etc/hbase/conf
export HADOOP_CLASSPATH=`hadoop classpath`

vi conf/flink-conf.yaml
classloader.check-leaked-classloader: false
classloader.resolve-order: parent-first
env.java.opts: "-Dfile.encoding=UTF-8"

    参考https://nightlies.apache.org/flink/flink-docs-release-1.10/ops/deployment/hadoop.html

  8. 停止yarn-per-job

    echo "stop" | ./bin/yarn-session.sh -id application_1651050121613_0094