Hadoop入门之azkaban的安装和使用
1.azkaban的安装
2.azkaban的几种使用
2.1 使用小Demo
#command
type=command
command=sh /home/hadoop/shell/say_hello.sh
shell脚本
#!/bin/sh
format="+%Y%m%d_%H:%M:%S"
now_date=`date $format`
echo $now_date hello world! >> /home/hadoop/logs/say_hello.log
2.2 依赖任务的Demo
运行的Job
#command
type=command
command=sh /home/hadoop/shell/say_hello.sh A
#command
type=command
command=sh /home/hadoop/shell/say_hello.sh B
#command
type=command
command=sh /home/hadoop/shell/say_hello.sh C
dependencies=A,B
执行的脚本
#!/bin/sh
user_name=$1
format="+%Y%m%d_%H:%M:%S"
now_date=`date $format`
echo $user_name $now_date hello world! >> /home/hadoop/logs/say_hello.log
2.3 操作HDFS的Demo
#command
type=command
command=hadoop fs -mkdir /azkaban_demo
2.4 操作Hive
在HDFS中创建目录/aztest/hiveinput
上传一个用户名文件如下:
1,xiaoming
2,xiaohong
3,xiaofang
5,xiaozhang
azkaban执行Job文件
#command
type=command
command=/home/hadoop/app/hive/bin/hive -f '/home/hadoop/sql/create_demo_table.sql'
create_demo_table.sql 文件内容
use default;
drop table aztest;
create table aztest(id int,name string) row format delimited fields terminated by ',';
load data inpath '/aztest/hiveinput' into table aztest;
create table azres as select * from aztest;
insert overwrite directory '/aztest/hiveoutput' select count(1) from aztest;
查看执行结果: