#!/usr/bin/python
# coding=utf-8
# TF-IDF提取文本關(guān)鍵詞
# http://scikit-learn.org/stable/modules/feature_extraction.html#tfidf-term-weighting

import sys
import os
from config_ch import *
import chardet
import numpy as np
import pandas as pd
import xlrd
import copy
import glob
import jieba.posseg
import jieba.analyse
import io
from sklearn import feature_extraction
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
"""
       TF-IDF權(quán)重：
           1嚎花、CountVectorizer  構(gòu)建詞頻矩陣
           2浆兰、TfidfTransformer 構(gòu)建TF-IDF權(quán)值計(jì)算
           3菱农、文本的關(guān)鍵詞
           4凿蒜、對(duì)應(yīng)的TF-IDF矩陣
"""
# 數(shù)據(jù)讀取
"""
    輸入數(shù)據(jù)所在文件夾路徑data_path, 輸出data為一字典, 包含'id', 'title', 'abstract'
"""
def dataRead(data_path):
    file_list = os.listdir(data_path)
    idList, titleList, abstractList = range(0, len(file_list)), [], []  # 構(gòu)建3個(gè)list, 用于存放文本編號(hào), 文本標(biāo)題, 文本內(nèi)容
    for file_name in file_list:
        file_path = os.path.join(data_path, file_name)
        if os.path.isfile(file_path):
            f = io.open(file_path, 'rb').read()
            encoding_type = chardet.detect(f)  # 獲取文本的編碼形式
            if not encoding_type['encoding']:
                encoding_type['encoding'] = 'utf-8-sig'  # 一些文本編碼形式為none, 強(qiáng)制轉(zhuǎn)換
            file = f.decode(encoding_type['encoding'])
            titleList.append(file[0:file.find('\n', 1)+1])  # 文本第一行為標(biāo)題
            abstractList.append(file)
    data = {"id": idList, "title": titleList, "abstract": abstractList}
    return data

# 預(yù)處理
"""
    輸入文本text及停用詞表stopword, 輸出分詞結(jié)果text_seg
    預(yù)處理包括jieba分詞, 去停用詞, 篩選詞性
"""
def dataPrepos(text, stopword):
    text_seg = []
    seg = jieba.posseg.cut(text)  # 分詞
    for i in seg:
        if i.word not in stopword and i.flag in pos:  # 去停用詞 + 篩選詞性
            text_seg.append(i.word)
    return text_seg

# 關(guān)鍵詞映射
"""
    輸入關(guān)鍵詞key及映射表mapword, 輸出key_left_mapped,
    包括映射后剩余關(guān)鍵詞"left"及映射得到的關(guān)鍵詞"mapped"
    映射表第1列為atom詞列表, 從第2列起為替換詞列表,
    若key中某詞屬于atom列表, 則將該atom對(duì)應(yīng)的替換詞加入mappedList, 并從leftList中刪除該詞,
    若key中某詞本身屬于替換詞列表, 則將該詞加入mappedList, 并從leftList中刪除
"""
def keysMapping(key, mapword):# key中關(guān)鍵詞若存在于atom中是辕，則加入mappedList，leftList只保留未出現(xiàn)在atom中的關(guān)鍵詞
    leftList, mappedList = copy.deepcopy(key), []  # 初始化leftList, mappedList
    atom = mapword.col_values(0)
    for i in key:
        if i in atom:  # 關(guān)鍵詞為atom列表中的詞, 則用對(duì)應(yīng)的替換詞進(jìn)行替換
            mappedList.extend(mapword.row_values(atom.index(i))[1:])
            mappedList = list(filter(None, mappedList))  # 去除""字符串
            leftList.pop(leftList.index(i))  # 從leftList中刪除
        else:
            for n in range(len(atom)):
                row = mapword.row_values(n)[1:]
                if i in row:  # 關(guān)鍵詞本身為替換詞列表中的詞, 則加入mappedList, 并從leftList中刪除
                    mappedList.extend([i])
                    leftList.pop(leftList.index(i))
                    break

    mappedList = list(set(mappedList))  # 去除重復(fù)詞
    key_left_mapped = {"left": leftList, "mapped": mappedList}
    return key_left_mapped

# TF-IDF提取topK關(guān)鍵詞
"""
    輸入包括數(shù)據(jù)data, 停用詞表stopword, 映射表mapword, 及中間變量mapped和keys_all,
    當(dāng)mode為'tf'時(shí), 每個(gè)文本單獨(dú)調(diào)用getKeyword, 需傳入文本id,
    當(dāng)mode為'tfidf'時(shí), 多個(gè)文本作為整體只調(diào)用一次getKeyword, 不需id, 令id = 0
"""
def getKeywords(data, id, stopword, mapword, mapped, keys_all):
    # 從data中取出id, title, abstract, 構(gòu)建3個(gè)list
    if mode == 'tfidf':
        idList, titleList, abstractList = data['id'], data['title'], data['abstract']
    elif mode == 'tf':  # 取出第id個(gè)文本的信息
        idList, titleList, abstractList = [data['id'][id]], [data['title'][id]], [data['abstract'][id]]

    corpus = []  # 將所有文本到輸出到一個(gè)list中, 每行為一個(gè)文本
    result = pd.DataFrame({"id": [], "title": [], "key": [], "left": [], "mapped": []},
                          columns=['id', 'title', 'key', 'left', 'mapped'])
    # 分別對(duì)每個(gè)文本進(jìn)行預(yù)處理, 將處理后的詞連接成字符串(空格分隔), 輸入到corpus中的一行
    for index in range(len(idList)):
        text = '%s' % abstractList[index]
        text_seg = dataPrepos(text, stopword)
        text_seg = " ".join(text_seg)
        corpus.append(text_seg)
    if corpus == ['']:
        return result  # 空文本
    # 1静浴、構(gòu)建詞頻矩陣勿她，將文本中的詞語轉(zhuǎn)換成詞頻矩陣
    vectorizer = CountVectorizer()
    X = vectorizer.fit_transform(corpus)  # 詞頻矩陣
    # 2、統(tǒng)計(jì)每個(gè)詞的TF-IDF權(quán)值
    transformer = TfidfTransformer()
    tfidf = transformer.fit_transform(X)
    # 3噪生、獲取詞袋模型中的關(guān)鍵詞
    word = vectorizer.get_feature_names()
    # 4裆赵、獲取TF-IDF矩陣
    weight = tfidf.toarray()
    # 5、打印詞語權(quán)重
    # 以下變量分別用于存放文本編號(hào), 標(biāo)題, 提取出的關(guān)鍵詞, 映射得到的關(guān)鍵詞, 映射后剩余的關(guān)鍵詞
    ids, titles, keys, keys_mapped, keys_left = [], [], [], [], []
    for i in range(len(weight)):
        print(u"-------這里輸出第", i+1, u"篇文本的詞語TF-IDF------")
        ids.append(idList[i])  # 添加編號(hào)到ids
        titles.append(titleList[i])  # 添加標(biāo)題到titles
        df_word, df_weight = [], []  # 當(dāng)前文本的所有詞匯列表跺嗽、詞匯對(duì)應(yīng)權(quán)重列表
        for j in range(len(word)):
            print(word[j], weight[i][j])
            if weight[i][j] == 0:
                df_word.append(' ')  # 用空字符串替換權(quán)重為0的詞
            else:
                df_word.append(word[j])
            df_weight.append(weight[i][j])
        # 將df_word和df_weight轉(zhuǎn)換為pandas中的DataFrame形式, 用于排序
        df_word = pd.DataFrame(df_word, columns=['word'])
        df_weight = pd.DataFrame(df_weight, columns=['weight'])
        word_weight = pd.concat([df_word, df_weight], axis=1)  # 拼接詞匯列表和權(quán)重列表
        word_weight = word_weight.sort_values(by="weight", ascending=False)  # 按照權(quán)重值降序排列
        keyword = np.array(word_weight['word'])  # 選擇詞匯列并轉(zhuǎn)成數(shù)組格式
        key = [keyword[x] for x in range(0, min(topK, len(word)))]  # 抽取前topK個(gè)詞匯作為關(guān)鍵詞
        keys_all.extend(key)  # 將當(dāng)前文本提取出的關(guān)鍵詞加入keys_all中, 用于后續(xù)的高頻關(guān)鍵詞提取

        # 關(guān)鍵詞映射
        key_left_mapped = keysMapping(key, mapword)
        # 將list中的詞連接成字符串
        key = " ".join(key)
        key_left_split = " ".join(key_left_mapped["left"])
        key_mapped_split = " ".join(key_left_mapped["mapped"])

        mapped.extend(key_left_mapped["mapped"])  # 將每個(gè)文本映射后的關(guān)鍵詞合并到mapped中, 有重復(fù)

        keys.append(key)
        keys_left.append(key_left_split)
        keys_mapped.append(key_mapped_split)

    result = pd.DataFrame({"id": ids, "title": titles, "key": keys, "left": keys_left, "mapped": keys_mapped}, columns=['id', 'title', 'key', 'left', 'mapped'])
    return result

# 提取topN高頻關(guān)鍵詞
"""
    輸入keys_all為每個(gè)文本提取出的topK關(guān)鍵詞合并后的列表,
    輸出key_most為提取出的topN個(gè)高頻關(guān)鍵詞
"""
def getKeymost(keys_all):
    counts = []
    keys_nodup = list(set(keys_all))  # keys_all去重后結(jié)果
    for item in keys_nodup:
        counts.append(keys_all.count(item))  # 統(tǒng)計(jì)每個(gè)關(guān)鍵詞出現(xiàn)的次數(shù)
    key_word = pd.DataFrame(keys_nodup, columns=['key'])
    count_word = pd.DataFrame(counts, columns=['count'])
    key_count = pd.concat([key_word, count_word], axis=1)
    key_count = key_count.sort_values(by="count", ascending=False)
    key_freq = np.array(key_count['key'])

    key_most = [key_freq[x] for x in range(0, min(topN, len(key_word)))]
    return key_most


def main():

    # 刪除歷史結(jié)果
    for f in glob.glob(os.path.join('result', '*.xls')):
        os.remove(f)

    # 加載停用詞表
    stopword = [w.strip() for w in io.open(stopword_path, 'r', encoding='UTF-8').readlines()]

    # 加載映射表
    mapword = xlrd.open_workbook(map_path).sheet_by_index(0)

    # 加載自定義字典战授，用于jieba分詞
    jieba.load_userdict(dict_path)

    folderList = os.listdir(data_path)

    for folder in folderList:  # 遍歷全部電影文件夾, 每個(gè)文件夾中為1部電影的全部影評(píng)
        folder_path = os.path.join(data_path, folder)

        # 讀取數(shù)據(jù)
        data = dataRead(folder_path)

        keys_all = []  # 用于存放所有文本提取出的關(guān)鍵詞
        mapped = []  # 用于合并所有文本映射后的關(guān)鍵詞

        # 關(guān)鍵詞提取,
        if mode == 'tfidf':
            result = getKeywords(data, 0, stopword, mapword, mapped, keys_all)
            result.to_csv("result/CHkeys_tfidf_" + folder + ".xls", index=False, encoding='utf-8-sig')
        elif mode == 'tf':
            for i in range(len(data['id'])):  # 'tf'模式下, 每個(gè)文本單獨(dú)調(diào)用getKeywords
                result = getKeywords(data, i, stopword, mapword, mapped, keys_all)
                result.to_csv("result/CHkeys_tf_" + folder + ".xls", mode='a', header=False, index=False, encoding='utf-8-sig')

        mapped = list(set(mapped))  # 去除重復(fù)詞
        mapped_result = pd.DataFrame({"mapped": [" ".join(mapped)]}, columns=['mapped'])
        pd.DataFrame({"": [" ".join([])]}).to_csv("result/CHkeys_tf_" + folder + ".xls", mode='a', index=False)  # 增加空行
        mapped_result.to_csv("result/CHkeys_tf_" + folder + ".xls", mode='a', index=False, encoding='utf-8-sig', columns=['', '', 'mapped'])

        # 提取高頻關(guān)鍵詞
        key_most = getKeymost(keys_all)
        key_most = pd.DataFrame({"most mentioned": [" ".join(key_most)]}, columns=['most mentioned'])
        pd.DataFrame({"": [" ".join([])]}).to_csv("result/CHkeys_tf_" + folder + ".xls", mode='a', index=False)  # 增加空行
        key_most.to_csv("result/CHkeys_tf_" + folder + ".xls", mode='a', index=False, encoding='utf-8-sig', columns=['', '', 'most mentioned'])


if __name__ == '__main__':
    main()