源碼閱讀(數(shù)據(jù)寫入)
HoodieSparkSqlWriter.write(){
// 數(shù)據(jù)寫入
checkWriteStatus(){
// 提交數(shù)據(jù)
client.commit(commitTime, writeStatuses)
AbstractHoodieWriteClient.commit(){
// 提交狀態(tài)
commitStats(instantTime, stats, Option<Map<String, String>> ){
// 保存完成
activeTimeline.saveAsComplete()
// 提交
postCommit(metadata, commitTime, extraMetadata){
// 壓縮操作
forceCompact(extraMetadata);
// 歸檔
archiveLog.archiveIfRequired(jsc);
// clean操作
clean(instantTime);
}
// 提交Metrics
emitCommitMetrics(instantTime, metadata, actionType);
}
}
}
}
源碼閱讀(數(shù)據(jù)清理)
CleanActionExecutor.execute(){
// get clean plan
Option<HoodieCleanerPlan> cleanerPlanOpt = requestClean(instantTime);
// 清理
runClean(table, HoodieTimeline.getCleanRequestedInstant(instantTime), cleanerPlan);
}
關(guān)于大數(shù)據(jù)方面技術(shù)問題可以咨詢,替你解決你的苦惱垃僚。參考WX:hainanzhongjian