2.2 DWS公共匯總粒度事實(shí)層
Hive數(shù)據(jù)庫建庫建表:
創(chuàng)建Hive庫并進(jìn)入:
create database if not exists dws_nshop;
use dws_nshop;
2.2.1 用戶主題
2.2.2.1 用戶啟動(dòng)【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_ulog_launch (
user_id string COMMENT '用戶id',
device_num string COMMENT '設(shè)備號(hào)',
device_type string COMMENT '設(shè)備類型',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
launch_count int COMMENT '啟動(dòng)次數(shù)'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_ulog_launch/';
通過對(duì)dwd層的用戶啟動(dòng)日志表做一個(gè)count聚合即可,sql如下:
insert overwrite table dws_nshop.dws_nshop_ulog_launch partition(bdp_day='20200618')
select
user_id,
device_num,
device_type,
os,
os_version,
manufacturer,
carrier,
network_type,
area_code,
count(device_num) over(partition by device_num) as launch_count
from
dwd_nshop.dwd_nshop_actlog_launch
where
bdp_day="20200618"
用戶啟動(dòng)7days【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_ulog_launch_7d (
user_id string COMMENT '用戶id',
device_num string COMMENT '設(shè)備號(hào)',
device_type string COMMENT '設(shè)備類型',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
launch_count int COMMENT '啟動(dòng)次數(shù)'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_ulog_launch_7d/';
入庫:和上面邏輯一樣胯究,只是加一周時(shí)間條件限制:
insert overwrite table dws_nshop.dws_nshop_ulog_launch_7d partition(bdp_day='20200618')
select
user_id,
device_num,
device_type,
os,
os_version,
manufacturer,
carrier,
network_type,
area_code,
count(device_num) over(partition by device_num) as launch_count
from
dwd_nshop.dwd_nshop_actlog_launch
where
bdp_day between '20200611' and '20200618'
2.2.2.2 用戶瀏覽表
建表:
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_ulog_view (
user_id string COMMENT '用戶id',
device_num string COMMENT '設(shè)備號(hào)',
device_type string COMMENT '設(shè)備類型',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
view_count INT COMMENT '瀏覽次數(shù)'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_ulog_view/';
數(shù)據(jù)從DWD層的用戶產(chǎn)品瀏覽表聚合得來:
insert overwrite table dws_nshop.dws_nshop_ulog_view partition(bdp_day='20200618')
select
user_id ,
device_num ,
device_type,
os ,
os_version ,
manufacturer,
carrier ,
network_type,
area_code,
count(device_num) over(partition by device_num)as view_count
from dwd_nshop.dwd_nshop_actlog_pdtview
where
bdp_day='20200618'
用戶查詢【DWS】
建表:
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_ulog_search (
user_id string COMMENT '用戶id',
device_num string COMMENT '設(shè)備號(hào)',
device_type string COMMENT '設(shè)備類型',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
search_count int COMMENT '查詢次數(shù)'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_ulog_search/'
對(duì)查詢主題表進(jìn)行count聚合即可,如下:
insert overwrite table dws_nshop.dws_nshop_ulog_search partition(bdp_day='20200618')
select
user_id,
device_num,
device_type,
os,
os_version,
manufacturer,
carrier,
network_type,
area_code,
count(device_num) over(partition by device_num) as search_count
from
dwd_nshop.dwd_nshop_actlog_pdtsearch
where
bdp_day="20200618"
2.2.2.3 用戶關(guān)注【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_ulog_comment (
user_id string COMMENT '用戶id',
device_num string COMMENT '設(shè)備號(hào)',
device_type string COMMENT '設(shè)備類型',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
comment_count int COMMENT '評(píng)論次數(shù)',
comment_target_count int COMMENT '類別評(píng)論次數(shù)',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_ulog_comment/';
聚合用戶對(duì)產(chǎn)品的關(guān)注,使用count聚合出關(guān)注次數(shù)
insert overwrite table dws_nshop.dws_nshop_ulog_comment partition(bdp_day='20200618')
select
user_id,
device_num,
device_type,
os,
os_version,
manufacturer,
carrier,
network_type,
area_code,
count(target_id) over(partition by comment_count) as comment_count,
count(distinct target_id) over(partition by target_id) as comment_target_count,
bdp_day
from
dwd_nshop.dwd_actlog_product_comment
where
bdp_day="20200618"
用戶交易寬表【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_user_orders (
user_id string COMMENT '用戶id',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
orders_count INT COMMENT '訂單數(shù)量',
orders_pay DECIMAL (10, 1) COMMENT '訂單金額',
orders_shipping DECIMAL (10, 1) COMMENT '訂單運(yùn)費(fèi)金額',
orders_district DECIMAL (10, 1) COMMENT '訂單優(yōu)惠金額',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_user_orders/'
用戶交易寬表的數(shù)據(jù)主要是從DWD層交易訂單明細(xì)流水表中抽取出來蜕琴,再結(jié)合用戶表查出用戶相關(guān)信息:
with tborder as(
select
o.order_id,
o.district_money,
o.shipping_money,
o.payment_money,
c.customer_id,
c.customer_natives
from dwd_nshop.dwd_nshop_orders_details o
join ods_nshop.ods_02_customer c
on o.customer_id=c.customer_id
where
bdp_day='20200618'
)
insert overwrite table dws_nshop.dws_nshop_user_orders partition(bdp_day='20200618')
select
customer_id,
customer_natives,
count(order_id) over(partition by customer_id) as orders_count,
sum(payment_money) over(partition by customer_id)as orders_pay,
sum(shipping_money) over(partition by customer_id)as orders_shipping,
sum(district_money) over(partition by customer_id)as orders_district,
current_timestamp() as ct
from tborder
用戶交易寬表7day【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_user_orders_7d (
user_id string COMMENT '用戶id',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
orders_count INT COMMENT '訂單數(shù)量',
orders_pay DECIMAL (10, 1) COMMENT '訂單金額',
orders_shipping DECIMAL (10, 1) COMMENT '訂單運(yùn)費(fèi)金額',
orders_district DECIMAL (10, 1) COMMENT '訂單優(yōu)惠金額',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_week string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_user_orders_7d/'
用戶投訴訂單寬表【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_user_complainant (
user_id string COMMENT '用戶id',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
compl_orders_count INT COMMENT '訂單數(shù)量',
compl_orders_pay DECIMAL (10, 1) COMMENT '訂單金額',
compl_supplier_count INT COMMENT '商家數(shù)量',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_user_complainant/'
用戶投訴訂單寬表7day【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_user_complainant_7d (
user_id string COMMENT '用戶id',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
compl_orders_count INT COMMENT '訂單數(shù)量',
compl_orders_pay DECIMAL (10, 1) COMMENT '訂單金額',
compl_supplier_count INT COMMENT '商家數(shù)量',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_week string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_user_complainant_7d/'
用戶營銷活動(dòng)寬表【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_user_release (
user_id string COMMENT '用戶id',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
source_count INT COMMENT '投放來源數(shù)量',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_user_release/'
用戶營銷活動(dòng)寬表7day【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_user_release_7d (
user_id string COMMENT '用戶id',
os string COMMENT '手機(jī)系統(tǒng)',
os_version string COMMENT '手機(jī)系統(tǒng)版本',
manufacturer string COMMENT '手機(jī)制造商',
carrier string COMMENT '電信運(yùn)營商',
network_type string COMMENT '網(wǎng)絡(luò)類型',
area_code string COMMENT '地區(qū)編碼',
source_count INT COMMENT '投放來源數(shù)量',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_week string) stored AS parquet location '/data/nshop/dws/user/dws_nshop_user_release_7d/'
商家用戶交互記錄寬表【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_supplier_user (
supplier_id string COMMENT '商家id',
supplier_type INT COMMENT '供應(yīng)商類型:1.自營,2.官方 3其他',
view_count INT COMMENT '瀏覽次數(shù)',
comment_users INT COMMENT '關(guān)注人數(shù)',
comment_area_code INT COMMENT '關(guān)注地區(qū)數(shù)量',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/supplier/dws_nshop_supplier_user/'
商家用戶交互記錄的數(shù)據(jù)主要統(tǒng)計(jì):
1.商家維度下用戶的瀏覽次數(shù)
所以要結(jié)合DWD層的用戶產(chǎn)品瀏覽表dwd_nshop_actlog_pdtview結(jié)合頁面布局維度表宵溅、產(chǎn)品維度表和店鋪維度表查出相關(guān)信息
2.商家維度下關(guān)注人數(shù)凌简、關(guān)注地區(qū)數(shù)
所以要結(jié)合DWD層的用戶產(chǎn)品關(guān)注表dwd_actlog_product_comment 結(jié)合頁面布局維度表,產(chǎn)品維度表和店鋪維度表查出相關(guān)信息
整合sql如下:
-- 統(tǒng)計(jì)商家維度下用戶的瀏覽次數(shù)
with pgview as(
select
su.supplier_code,
su.supplier_type,
count(*) as view_count
from dwd_nshop.dwd_nshop_actlog_pdtview pv
join
ods_nshop.dim_pub_page pp
on
pp.page_type='4'
and pp.page_code=pv.target_id
join
ods_nshop.dim_pub_product pr
on
pr.product_code=pp.page_code
join
ods_nshop.dim_pub_supplier su
on
su.supplier_code=pr.supplier_code
where
pv.bdp_day='20200618'
group by
su.supplier_code,
su.supplier_type
),
-- 統(tǒng)計(jì)商家維度下關(guān)注人數(shù)恃逻、關(guān)注地區(qū)數(shù)
prcomment as(
select
su.supplier_code,
su.supplier_type,
count(distinct pc.user_id) as comment_users,
count(distinct pc.area_code)as comment_area_code
from
dwd_nshop.dwd_actlog_product_comment pc
join
ods_nshop.dim_pub_page pp
on
pp.page_type='4'
and pp.page_code=pc.target_id
join
ods_nshop.dim_pub_product pr
on
pr.product_code=pp.page_code
join
ods_nshop.dim_pub_supplier su
on
su.supplier_code=pr.supplier_code
where
pc.bdp_day='20200618'
group by
su.supplier_code,
su.supplier_type
)
-- 整合指標(biāo)到DWS表
insert overwrite table dws_nshop.dws_nshop_supplier_user partition(bdp_day='20200618')
select
pgview.supplier_code,
pgview.supplier_type,
pgview.view_count,
prcomment.comment_users,
prcomment.comment_area_code,
current_timestamp() as ct
from pgview
join
prcomment
on
pgview.supplier_code=prcomment.supplier_code
and
pgview.supplier_type=prcomment.supplier_type
商家用戶交互記錄寬表7day【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_supplier_user_7d (
supplier_id string COMMENT '商家id',
supplier_type INT COMMENT '供應(yīng)商類型:1.自營雏搂,2.官方 3其他',
view_count INT COMMENT '瀏覽次數(shù)',
comment_users INT COMMENT '關(guān)注人數(shù)',
comment_area_code INT COMMENT '關(guān)注地區(qū)數(shù)量',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/supplier/dws_nshop_supplier_user_7d/'
商家日流水寬表【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_supplier_sales (
supplier_id string COMMENT '商家id',
supplier_type INT COMMENT '供應(yīng)商類型:1.自營,2.官方 3其他',
sales_users INT COMMENT '購物人數(shù)',
sales_users_area INT COMMENT '購物地區(qū)數(shù)量',
sales_orders INT COMMENT '購物訂單數(shù)',
salaes_orders_pay DECIMAL (10, 1) COMMENT '訂單金額',
salaes_orders_district DECIMAL (10, 1) COMMENT '訂單優(yōu)惠金額',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/supplier/dws_nshop_supplier_sales/'
商家日流水寬表的數(shù)據(jù)主要是從訂單明細(xì)流水表里抽取,再結(jié)合用戶信息表和產(chǎn)品信息表寇损、店鋪表查出相關(guān)信息:
insert overwrite table dws_nshop.dws_nshop_supplier_sales partition(bdp_day='20200321')
select
od.supplier_code,
su.supplier_type,
count(distinct od.customer_id) sales_users,
count(distinct oc.customer_natives) sales_users_area,
count(distinct od.order_id) sales_orders,
sum(pr.product_price * od.product_cnt) salaes_orders_pay,
sum(od.district_money) salaes_orders_district,
current_timestamp() as ct
from dwd_nshop.dwd_nshop_orders_details od
join
ods_nshop.ods_02_customer oc
on
od.customer_id=oc.custxomer_id
join
ods_nshop.dim_pub_product pr
on
od.supplier_code=pr.supplier_code
join
ods_nshop.dim_pub_supplier su
on
su.supplier_code=od.supplier_code
where
od.bdp_day='20200321'
group by
od.supplier_code,
su.supplier_type
商家日流水寬表7day【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_supplier_sales_7d (
supplier_id string COMMENT '商家id',
supplier_type INT COMMENT '供應(yīng)商類型:1.自營凸郑,2.官方 3其他',
sales_users INT COMMENT '購物人數(shù)',
sales_users_area INT COMMENT '購物地區(qū)數(shù)量',
sales_orders INT COMMENT '購物訂單數(shù)',
salaes_orders_pay DECIMAL (10, 1) COMMENT '訂單金額',
salaes_orders_district DECIMAL (10, 1) COMMENT '訂單優(yōu)惠金額',
ct BIGINT COMMENT '產(chǎn)生時(shí)間'
) partitioned BY (bdp_week string) stored AS parquet location '/data/nshop/dws/supplier/dws_nshop_supplier_sales_7d/'
2.3 營銷活動(dòng)主題
廣告投放用戶寬表【DWS】
CREATE external TABLE
IF NOT EXISTS dws_nshop.dws_nshop_release_user (
release_sources string COMMENT '投放渠道',
release_category string COMMENT '投放瀏覽產(chǎn)品分類',
release_users INT COMMENT '投放瀏覽用戶數(shù)',
release_product_page INT COMMENT '投放瀏覽產(chǎn)品頁面數(shù)',
ct BIGINT COMMENT '創(chuàng)建時(shí)間'
) partitioned BY (bdp_day string) stored AS parquet location '/data/nshop/dws/release/dws_nshop_release_user/'
對(duì)用戶數(shù)和產(chǎn)品數(shù)進(jìn)行聚合即可:
insert overwrite table dws_nshop.dws_nshop_release_user partition(bdp_day='20200321')
select
release_sources,
release_category,
count(distinct customer_id) release_users,
count(1) release_product_page,
current_timestamp() ct
from
dwd_nshop.dwd_nshop_releasedatas
where
bdp_day='20200321'
group by
release_sources,
release_category