根據(jù)LightLDA的輸出文件得到文檔-主題分布和主題-詞分布以及表示某篇文檔的topN關鍵詞乞娄。
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.util.List;
import java.util.PriorityQueue;
/**
* Created by yangxin on 2017/8/11.
*/
public class LDAResult {
private double alpha; //主題分布Dirichlet分布參數(shù)
private double beta; //詞分布Dirichlet分布參數(shù)
private int topic_num; //主題數(shù)目
private int vocab_num; //詞數(shù)目
private int doc_num; //文檔數(shù)目
private double[][] doc_topic_mat = null; //文檔_主題概率矩陣
private double[][] topic_vocab_mat = null; //主題_詞概率矩陣
private Item[][] doc_word_info = null; //文檔_top詞的信息矩陣
/**
* lda每個doc對應的前n個詞Id
*/
public static class Item implements Comparable<Item>{
public double prob;
public int word_id;
public Item(double prob, int word_id) {
this.prob = prob;
this.word_id = word_id;
}
@Override
public String toString() {
return "Item{" +
"prob=" + prob +
", word_id=" + word_id +
'}';
}
@Override
public int compareTo(Item o) {
return prob - o.prob > 0 ? 1 : -1;
}
}
public LDAResult(double alpha, double beta, int topic_num, int vocab_num, int doc_num) {
this.alpha = alpha;
this.beta = beta;
this.topic_num = topic_num;
this.vocab_num = vocab_num;
this.doc_num = doc_num;
doc_topic_mat = new double[topic_num][doc_num];
topic_vocab_mat = new double[vocab_num][topic_num];
}
/**
* 得到每個文檔前n個關鍵詞
* @param n
* @return
*/
public Item[][] getDocTopWordInfo(int n){
doc_word_info = new Item[doc_num][n];
for(int i = 0; i < doc_num; ++i){ //每篇文檔
PriorityQueue<Item> queue = new PriorityQueue<>();
for(int j = 0; j < vocab_num; ++j){ //每個詞
double prob = 0;
for(int k = 0; k < topic_num; ++k){ //每個主題
prob += doc_topic_mat[k][i] * topic_vocab_mat[j][k];
}
Item item = new Item(prob, j);
queue.offer(item);
if(queue.size() > n){
queue.poll();
}
}
int q = queue.size();
while(!queue.isEmpty()){
doc_word_info[i][--q] = queue.poll();
}
}
return doc_word_info;
}
/**
* 寫每個文檔的前n個關鍵詞到文件中
* @param n
* @param output 輸出文件
* @param titles doc標題列表
* @param words 詞列表
* @throws Exception
*/
public void dumpTopResult(int n, String output, final List<String> titles, final List<String> words) throws Exception{
if(n <= 0) return;
BufferedWriter bw = new BufferedWriter(new FileWriter(output));
if(doc_word_info == null){
doc_word_info = getDocTopWordInfo(n);
}
for(int i = 0; i < doc_num; ++i){ //doc_id
bw.write(titles.get(i) + " : ");
for(Item item : doc_word_info[i]){
bw.write(words.get(item.word_id) + "/" + item.prob + " ");
}
bw.newLine();
bw.flush();
}
bw.close();
}
/**
* 加載文檔_主題模型
* @param model_path
* @throws Exception
*/
public void loadDocTopicModel(String model_path) throws Exception{
//將計數(shù)寫入到矩陣中
BufferedReader br = new BufferedReader(new FileReader(model_path));
String line = null;
while((line = br.readLine()) != null){
String[] doc_info = line.split("[\t ]+");
int doc_id = Integer.parseInt(doc_info[0]); //文檔號禀酱，從0開始
for(int i = 1; i < doc_info.length; ++i){
String[] topic_info = doc_info[i].split(":"); //對應的主題信息
int topic_id = Integer.parseInt(topic_info[0]); //主題id
int topic_cnt = Integer.parseInt(topic_info[1]); //主題次數(shù)
doc_topic_mat[topic_id][doc_id] = topic_cnt;
}
}
br.close();
//計數(shù)
int[] doc_cnts = new int[doc_num]; //每個文檔對應的主題數(shù)量和幔欧，即包含詞的數(shù)目
for(int i = 0; i < doc_num; ++i){ //對每個文檔
for(int j = 0; j < topic_num; ++j){ //對每個主題
doc_cnts[i] += doc_topic_mat[j][i];
}
}
//計算概率
double factor = topic_num * alpha;
for(int i = 0; i < doc_num; ++i){ //對每個文檔
for(int j = 0; j < topic_num; ++j){ //對每個主題
doc_topic_mat[j][i] = (doc_topic_mat[j][i] + alpha) / (doc_cnts[i] + factor);
}
}
}
/**
* 加載主題_詞模型
* @param model_path 主題_詞模型位置,對應文件 server_model_0
* @param model_summary_path 主題數(shù)目統(tǒng)計谋减，對應文件 server_model_1
* @throws Exception
*/
public void loadTopicWordModel(String model_path, String model_summary_path) throws Exception{
//將計數(shù)寫入到矩陣中
BufferedReader br = new BufferedReader(new FileReader(model_path));
String line = null;
while((line = br.readLine()) != null){
String[] info = line.split(" ");
int word_id = Integer.parseInt(info[0]); //詞id
for(int i = 1; i < info.length; ++i){
String[] topic_info = info[i].split(":"); //對應的每個topic信息
int topic_id = Integer.parseInt(topic_info[0]); //topic id
int topic_cnt = Integer.parseInt(topic_info[1]); //topic計數(shù)
topic_vocab_mat[word_id][topic_id] = topic_cnt;
}
}
br.close();
//寫每個主題出現(xiàn)的次數(shù)
int[] topic_cnts = new int[topic_num]; //主題出現(xiàn)的次數(shù)
br = new BufferedReader(new FileReader(model_summary_path));
String[] cnts = br.readLine().split(" ");
for(int i = 1; i < cnts.length; ++i){
String[] cnt_info = cnts[i].split(":");
int topic_id = Integer.parseInt(cnt_info[0]);
int topic_cnt = Integer.parseInt(cnt_info[1]);
topic_cnts[topic_id] = topic_cnt;
}
br.close();
//寫概率
double factor = vocab_num * beta; //歸一化因子
for(int i = 0; i < vocab_num; ++i){ //每個詞
for(int j = 0; j < topic_num; ++j){ //每個主題
topic_vocab_mat[i][j] = (topic_vocab_mat[i][j] + beta) / (topic_cnts[j] + factor);
}
}
}
}
調(diào)用
public static void main(String[] args) throws Exception{
String doc_topic_path = "doc_topic.0";
String topic_word_path = "server_0_table_0.model";
String topic_summary = "server_0_table_1.model";
String ori_doc_path = "merge_texts";
String ori_word_path = "vocab";
String output = "result";
LDAResult result = new LDAResult(0.22, 0.1, 220, 1539967, 146119);
result.loadTopicWordModel(topic_word_path, topic_summary); //得到主題-詞概率分布
result.loadDocTopicModel(doc_topic_path); //得到文檔-主題概率分布
List<String> titles = Util.getTitles(ori_doc_path); //所有文檔名
List<String> words = Util.getVocabs(ori_word_path); //所有詞
result.dumpTopResult(10, output, titles, words); //每篇文檔的前10個關鍵詞寫入到output中
}
