Spark Streaming 开发环境搭建

说明

主要是基于之前的操作在 pom.xml 中添加对应的依赖

前提

1. 安装配置好 IDEA
2. 在 Windows10 中安装配置好 Maven3.3.9

添加依赖

在pom.xml 中添加依赖如下:

<properties>
   <scala.version>2.11.8</scala.version>
   <kafka.version>0.9.0.0</kafka.version>
   <spark.version>2.2.0</spark.version>
   <hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
   <hbase.version>1.2.0-cdh5.7.0</hbase.version>
 </properties>

 <!-- 添加cloudera的repository -->
 <repositories>
   <repository>
     <id>cloudera</id>
     <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
   </repository>
 </repositories>

 <dependencies>
   <!-- Scala 依赖 -->
   <dependency>
     <groupId>org.scala-lang</groupId>
     <artifactId>scala-library</artifactId>
     <version>${scala.version}</version>
   </dependency>

   <!-- Kafka 依赖 -->
   <dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka_2.11</artifactId>
     <version>${kafka.version}</version>
   </dependency>

   <dependency>
     <groupId>org.apache.kafka</groupId>
     <artifactId>kafka-clients</artifactId>
     <version>${kafka.version}</version>
   </dependency>

   <!-- Hadoop 依赖 -->
   <dependency>
     <groupId>org.apache.hadoop</groupId>
     <artifactId>hadoop-client</artifactId>
     <version>${hadoop.version}</version>
   </dependency>

   <!-- HBase 依赖 -->
   <dependency>
     <groupId>org.apache.hbase</groupId>
     <artifactId>hbase-client</artifactId>
     <version>${hbase.version}</version>
   </dependency>

   <dependency>
     <groupId>org.apache.hbase</groupId>
     <artifactId>hbase-server</artifactId>
     <version>${hbase.version}</version>
   </dependency>

   <!-- Spark Streaming 依赖 -->
   <dependency>
     <groupId>org.apache.spark</groupId>
     <artifactId>spark-streaming_2.11</artifactId>
     <version>${spark.version}</version>
   </dependency>

 </dependencies>

以上完成了使用 IDEA 整合 Maven 搭建 Spark Streaming 开发环境