大数据调试环境配置(1):外部链接HBase调试环境配置
一、目的
在开发过程中,不可避免需要在集群外部调试程序,以便快速开发,测试。
二、环境
1.IDEA
2.CDH 5.7.0(Hbase、HADOOP)
三、步骤
1.创建Maven项目
2.加载对应CDH需求依赖
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>CDH</groupId>
<artifactId>hbase</artifactId>
<version>1.0</version>
<repositories>
<repository>
<id>cloudera</id>
<url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
</repository>
</repositories>
<properties>
<hadoop.version>2.6.0-cdh5.7.0</hadoop.version>
<hbase.version>1.2.0-cdh5.7.0</hbase.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<!-- <dependency>-->
<!-- <groupId>org.apache.hadoop</groupId>-->
<!-- <artifactId>hadoop-common</artifactId>-->
<!-- <version>2.5.0-cdh5.2.0</version>-->
<!-- </dependency>-->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<version>${hadoop.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
<version>${hbase.version}</version>
</dependency>
<!-- https://mvnrepository.com/artifact/com.google.protobuf/protobuf-java -->
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>2.5.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.10</version>
<scope>test</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/commons-logging/commons-logging -->
<!-- <dependency>-->
<!-- <groupId>commons-logging</groupId>-->
<!-- <artifactId>commons-logging</artifactId>-->
<!-- <version>1.2</version>-->
<!-- </dependency>-->
</dependencies>
</project>
3.加载hbase-site.xml、hdfs-site.xml、core-site.xml到resources目录下
4.将hadoop-2.6.0-cdh5.7.0的tar包直接解压缩到本地,并且配置Hadoop环境变量
5.添加winutils.exe到hadoop-2.6.0-cdh5.7.0\bin目录下
6.开启hdfs和zookeeper以及hbase
四、链接HBase测试代码
1.创建namespace并且创建表
import org.apache.commons.math.linear.NonSquareMatrixException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.util.Bytes;
import java.io.IOException;
import java.util.NavigableMap;
public class HbaseUtils {
public static Configuration getHBaseConf(){
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.rootdir","hdfs://hadoop01:8020/hbase");
conf.set("hbase.zookeeper.quorum","hadoop01");
conf.set("hbase.zookeeper.property.clientPort","2181");
return conf;
}
/**
* createTable:创建namespace,表名,列簇。
* @throws Exception
*/
@SuppressWarnings("resource")
// @Test
public static void createTable() throws Exception {
String nameSpace = "ns2";
String tbName = "actor";
String colFamily = "info";
// 1.读取配置文件
//的hbase-default.xml,也可以把hbase-site.xml放到resources中
Configuration conf=HbaseUtils.getHBaseConf();
Connection connection;
connection = ConnectionFactory.createConnection(conf);
// 2.创建一个管理员,管理表格的创建和删除
// HBaseAdmin admin = new HBaseAdmin(conf);
Admin admin=connection.getAdmin();
// 3.创建命名空间描述器
NamespaceDescriptor ns = NamespaceDescriptor.create(nameSpace).build();
// 4.创建表格,表格对象
TableName tableName = TableName.valueOf(nameSpace + ":" + tbName);
// 创建表的描述器
HTableDescriptor tableDesc = new HTableDescriptor(tableName);
// 5.创建列簇的描述器
HColumnDescriptor columnDesc = new HColumnDescriptor(colFamily);
// 6.将列簇添加到表的描述器
tableDesc.addFamily(columnDesc);
// 7.创建命名空间,创建表
admin.createNamespace(ns);
admin.createTable(tableDesc);
System.out.println("创建表 " + nameSpace + ":" + tbName + " 成功");
}
}
2.结果
创建成功!