526互联

Geoserver+Geomesa+HBase环境搭建

发布时间 2024-01-04 17:11:47作者: 粒子先生

版本

zookeeper-3.5.10
hadoop-2.8.5
hbase-2.2.4
geomesa-hbase_2.11-3.1.1
geoserver-2.17.1

安装HBase

zoo.cfg

tickTime=2000
initLimit=10
syncLimit=5

clientPort=2181
dataLogDir=/data/soft/apache-zookeeper-3.5.10-bin/logs
dataDir=/data/soft/apache-zookeeper-3.5.10-bin/data

autopurge.snapRetainCount=500
autopurge.purgeInterval=24

server.1=jcmap01:2888:3888
server.2=jcmap02:2888:3888
server.3=jcmap03:2888:3888

core-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
            <name>fs.default.name</name>
            <value>hdfs://jcmap01:9000</value>
    </property>
    <property>
            <name>hadoop.tmp.dir</name>
            <value>file://data/soft/hadoop-2.8.5/tmp</value>
            <description>
                    Abase for other temporary directories.
            </description>
    </property>
    <property>
            <name>hadoop.proxyuser.root.hosts</name>
            <value>jcmap01</value>
    </property>
    <property>
            <name>hadoop.proxyuser.root.groups</name>
            <value>*</value>
    </property>
</configuration>

hdfs-site.xml

<?xml version="1.0" encoding="UTF-8"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property>
            <name>dfs.datanode.data.dir</name>
            <value>/data/soft/hadoop-2.8.5/hdf/data</value>
            <final>true</final>
    </property>
    <property>
            <name>dfs.namenode.name.dir</name>
            <value>/data/soft/hadoop-2.8.5/hdf/name</value>
            <final>true</final>
    </property>
</configuration>

yarn-site.xml

<?xml version="1.0"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->
<configuration>

<!-- Site specific YARN configuration properties -->
    <property>
            <name>yarn.nodemanager.aux-services.mapreduce.shuffle.class</name>
            <value>org.apache.mapred.ShuffleHandler</value>
    </property>
    <property>
            <name>yarn.resourcemanager.address</name>
            <value>jcmap01:8032</value>
    </property>
    <property>
            <name>yarn.resourcemanager.scheduler.address</name>
            <value>jcmap01:8030</value>
    </property>
    <property>
            <name>yarn.resourcemanager.resource-tracker.address</name>
            <value>jcmap01:8031</value>
    </property>
    <property>
            <name>yarn.resourcemanager.admin.address</name>
            <value>jcmap01:8033</value>
    </property>
    <property>
            <name>yarn.resourcemanager.webapp.address</name>
            <value>jcmap01:8088</value>
    </property>
</configuration>

 mapred-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  See the License for the specific language governing permissions and
  limitations under the License. See accompanying LICENSE file.
-->

<!-- Put site-specific property overrides in this file. -->

<configuration>
    <property><name>mapreduce.framework.name</name>
        <value>yarn</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.address</name>
        <value>jcmap01:10020</value>
    </property>
    <property>
        <name>mapreduce.jobhistory.webapp.address</name>
        <value>jcmap01:19888</value>
    </property>
</configuration>

hadoop-env.sh

# The java implementation to use.
export JAVA_HOME=/usr/lib/jvm/java-1.11.0-openjdk-amd64

slaves

jcmap02
jcmap03

hbase-site.xml

<?xml version="1.0"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<!--
/**
 *
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
-->
<configuration>
        <property>
            <name>hbase.rootdir</name>
            <value>hdfs://jcmap01:9000/hbase</value>
    </property>
    <property>
            <name>hbase.cluster.distributed</name>
          <value>true</value>
    </property>
    <property>
            <name>hbase.zookeeper.quorum</name>
            <value>jcmap01,jcmap02,jcmap03</value>
    </property>
    <property>
            <name>hbase.master.maxclockskew</name>
            <value>180000</value>
    </property>
    <property>
            <name>hbase.zookeeper.property.dataDir</name>
            <value>/data/soft/hbase-2.2.4/data</value>
    </property>
    <property>
            <name>hbase.tmp.dir</name>
            <value>/data/soft/hbase-2.2.4/temp</value>
    </property>
    <property>
            <name>hbase.master</name>
            <value>hdfs://jcmap01:60000</value>
    </property>
    <property>
            <name>hbase.master.info.port</name>
            <value>60010</value>
   </property>
    <property>
            <name>hbase.regionserver.info.port</name>
            <value>60030</value>
    </property>
    <property>
            <name>hbase.client.scanner.caching</name>
            <value>200</value>
    </property>
    <property>
            <name>hbase.balancer.period</name>
            <value>300000</value>
    </property>
    <property>
            <name>hbase.client.write.buffer</name>
            <value>10485760</value>
    </property>
    <property>
            <name>hbase.hregion.majorcompaction</name>
            <value>7200000</value>
    </property>
    <property>
            <name>hbase.hregion.max.filesize</name>
            <value>67108864</value>
   </property>
    <property>
            <name>hbase.hregion.memstore.flush.size</name>
            <value>1048576</value>
    </property>
    <property>
            <name>hbase.server.thread.wakefrequency</name>
            <value>30000</value>
    </property>
    <property>
            <name>hbase.server.keyvalue.maxsize</name>
            <value>52428800</value>
    </property>
    <property>
            <name>hbase.client.keyvalue.maxsize</name>
            <value>52428800</value>
    </property>
    <property>
            <name>hbase.coprocessor.user.region.classes</name>
            <value>org.locationtech.geomesa.hbase.server.coprocessor.GeoMesaCoprocessor</value>
    </property>
</configuration>

regionservers

jcmap02
jcmap03

环境变量

# hbase2.2.4
export ZOOKEEPER_HOME=/data/soft/apache-zookeeper-3.5.10-bin
export PATH=$PATH:$ZOOKEEPER_HOME/bin
export HBASE_HOME=/data/soft/hbase-2.2.4
export PATH=$PATH:$HBASE_HOME/bin
export HADOOP_HOME=/data/soft/hadoop-2.8.5
export PATH=$PATH:$HADOOP_HOME/bin

安装Geomesa

下载安装包并拷贝至hbase安装目录

https://github.com/locationtech/geomesa/releases/download/geomesa_2.11-3.1.1/geomesa-hbase_2.11-3.1.1-bin.tar.gz
tar xvf geomesa-hbase_2.11-3.1.1-bin.tar.gz
cd geomesa-hbase_2.11-3.1.1

${GEOMESA_HBASE_HOME}/dist/hbase/geomesa-hbase-distributed-runtime-hbase1_2.11-3.1.1.jar拷贝到${HBASE_HOME}/lib目录下
Hbase2请使用geomesa-hbase-distributed-runtime-hbase2_2.11-3.1.1.jar
Hbase1请使用geomesa-hbase-distributed-runtime-hbase1_2.11-3.1.1.jar

初始化geomesa-hbase

bin/geomesa-hbase configure

注册coprocessor

Geomesa使用HBase提供的coprocessor工具将处理过程移动到服务器端运行来提高查询效率，最简单的注册方式就是直接修改hbase-site.xml，增加以下内容

    <property>
            <name>hbase.coprocessor.user.region.classes</name>
            <value>org.locationtech.geomesa.hbase.server.coprocessor.GeoMesaCoprocessor</value>
    </property>

环境变量

export GEOMESA_HBASE_HOME=/data/soft/geomesa-hbase_2.11-3.1.1
export GEOMESA_LIB=$GEOMESA_HBASE_HOME/lib
export GEOMESA_LOG_DIR=$GEOMESA_HBASE_HOME/logs
export PATH=$PATH:$GEOMESA_HBASE_HOME/bin

重启hbase 

导入数据

geomesa-hbase ingest --catalog osm_pois --feature-name osm_pois --input-format shp /data/data/vector/china-latest-free/gis_osm_pois_free_1.shp

安装Geoserver

复制插件

拷贝./geomesa-hbase_2.11-3.1.0/dist/gs-plugins目录下的geomesa-hbase-gs-plugin_2.11-3.1.0-install.tar.gz，复制到geoserver的lib目录下，并解压。

a.将 $GEOMESA_HBASE_HOME/dist/gs-plugins/geomesa-hbase-gs-plugin_2.11-$VERSION-install.tar.gz解压到WBE-INF/lib路径下；

b. 由于geomesa-hbase-gs-plugin并不包含有Hadoop、HBase有关的依赖jar文件，所以需要手动将以下jar文件复制到WBE-INF/lib路径下（仔细观察geoserver相关包是否已存在，小心jar冲突）

commons-cli-1.2.jar
commons-configuration-1.6.jar
commons-io-2.5.jar
commons-logging-1.1.3.jar
netty-all-4.1.48.Final.jar
hadoop-auth-2.8.5.jar
hadoop-client-2.8.5.jar
hadoop-common-2.8.5.jar
hadoop-hdfs-2.8.5.jar
hadoop-hdfs-client-2.8.5.jar
hadoop-mapreduce-client-core-2.8.5.jar
hbase-client-2.2.6.jar
hbase-common-2.2.6.jar
hbase-hadoop-compat-2.2.6.jar
hbase-mapreduce-2.2.6.jar
hbase-protocol-2.2.6.jar
hbase-protocol-shaded-2.2.6.jar
hbase-shaded-miscellaneous-2.2.1.jar
hbase-shaded-netty-2.2.1.jar
hbase-shaded-protobuf-2.2.1.jar
htrace-core-3.1.0-incubating.jar
htrace-core4-4.1.0-incubating.jar
protobuf-java-2.5.0.jar
zookeeper-3.4.14.jar

geoserver中配置hbase
将hbase-site.xml文件复制到geoserver/WEB-INF/classes下。

 cp /etc/hbase/conf.cloudera.hbase/hbase-site.xml /usr/local/tomcat8/webapps/geoserver/WEB-INF/classes/

环境变量

export GEOSERVER_HOME=/data/soft/geoserver-2.17.1

启动geoserver

cd /data/soft/geoserver-2.17.1/bin/
./startup.sh

登录 admin geoserver

创建数据存储

 

 

选择数据发布即可