大数据全系列 教程
1869个小节阅读:464.8k
JAVA全系列 教程
面向对象的程序设计语言
Python全系列 教程
Python3.x版本,未来主流的版本
人工智能 教程
顺势而为,AI创新未来
大厂算法 教程
算法,程序员自我提升必经之路
C++ 教程
一门通用计算机编程语言
微服务 教程
目前业界流行的框架组合
web前端全系列 教程
通向WEB技术世界的钥匙
大数据全系列 教程
站在云端操控万千数据
AIGC全能工具班
A A
White Night
修改pom.xml文件
xxxxxxxxxx
<version>1.0-SNAPSHOT</version>
改为:
xxxxxxxxxx
<version>1.0</version>
打包loganalyser-1.0.jar
将该jar包从window系统下拷贝到node2上
将loganalyser-1.0.jar上传到hdfs的/itbaizhan/udf文件夹中
xxxxxxxxxx
[root@node2 ~]# hdfs dfs -mkdir -p /itbaizhan/udf
[root@node2 ~]# hdfs dfs -put loganalyser-1.0.jar /itbaizhan/udf
创建hive的function
xxxxxxxxxx
create function dateFunc as 'cn.itbaizhan.udf.DateDimensionUDF' using jar 'hdfs://mycluster/itbaizhan/udf/loganalyser-1.0.jar';
create function platformFunc as 'cn.itbaizhan.udf.PlatformDimensionUDF' using jar 'hdfs://mycluster/itbaizhan/udf/loganalyser-1.0.jar';
设置本地模式运行
xxxxxxxxxx
hive> set hive.exec.mode.local.auto=true;
清空hive中的stats_view_depth表中的数据:
xxxxxxxxxx
hive> truncate table stats_view_depth;
修改hql语句
xxxxxxxxxx
with tmp as
(select pl,`day`,ct as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv1' union all
select pl,`day`,0 as pv1,ct as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv2' union all
select pl,`day`,0 as pv1,0 as pv2,ct as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv3' union all
select pl,`day`,0 as pv1,0 as pv2,0 as pv3,ct as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv4' union all
select pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,ct as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv5_10' union all
select pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,ct as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv10_30' union all
select pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,ct as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv30_60' union all
select pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,ct as pv60_plus
from stats_view_depth_tmp where col='pv60_plus' union all
select 'all' as pl,`day`,ct as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv1' union all
select 'all' as pl,`day`,0 as pv1,ct as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv2' union all
select 'all' as pl,`day`,0 as pv1,0 as pv2,ct as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv3' union all
select 'all' as pl,`day`,0 as pv1,0 as pv2,0 as pv3,ct as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus
from stats_view_depth_tmp where col='pv4' union all
select 'all' as pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,ct as pv5_10,0 as pv10_30,0 as pv30_60,0 as pv60_plus from stats_view_depth_tmp where col='pv5_10' union all
select 'all' as pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,ct as pv10_30,0 as pv30_60,0 as pv60_plus from stats_view_depth_tmp where col='pv10_30' union all
select 'all' as pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,ct as pv30_60,0 as pv60_plus from stats_view_depth_tmp where col='pv30_60' union all
select 'all' as pl,`day`,0 as pv1,0 as pv2,0 as pv3,0 as pv4,0 as pv5_10,0 as pv10_30,0 as pv30_60,ct as pv60_plus from stats_view_depth_tmp where col='pv60_plus')
from tmp
insert overwrite table stats_view_depth
select platformFunc(pl),dateFunc(day),1,sum(pv1),sum(pv2),sum(pv3),sum(pv4),sum(pv5_10),sum(pv10_30),sum(pv30_60),sum(pv60_plus),day
group by pl,day;
执行hql语句
执行后查询stats_view_depth表中数据
xxxxxxxxxx
hive> select * from stats_view_depth;
OK
platform_dimension_id date_dimension_id kpi_dimension_id pv1 stats_view_depth.pv2 pv3 pv4 pv5_10 pv10_30 pv30_60 pv60_plus created
1 1 1 233 82 18 7 0 0 0 0 2022-01-20
2 1 1 233 82 18 7 0 0 0 0 2022-01-20