原理

FM只能進(jìn)行二階特征交叉，是否可以利用深度神經(jīng)網(wǎng)絡(luò)更強的特征組合能力來改進(jìn)FM模型呢霉囚？2017年新加坡國立大學(xué)基于此提出了NFM模型。
NFM主要思路是采用一個表達(dá)能力更強的函數(shù)代替FM二階隱向量內(nèi)積的部分月劈。

image.png

更強表達(dá)能力的函數(shù)是如何實現(xiàn)的呢越除？
模型在Embedding層和DNN層中間加了BI-Interaction Pooling層。該層的作用是兩兩Embedding向量做元素積操作（兩向量對應(yīng)維度相乘）态辛，在交叉特征向量之后麸澜，對應(yīng)元素求和，最后輸出一個k維的特征向量奏黑。這個k維向量輸入到DNN中進(jìn)一步抽取高階非線性特征炊邦。

模型結(jié)構(gòu)

從模型結(jié)構(gòu)的角度出發(fā)，相比于wide&deep模型熟史，NFM對Deep部分增加了特征交叉池化層馁害，增加了模型特征交叉的能力。

image.png

代碼

import os
import numpy as np
import pandas as pd
from collections import namedtuple

import tensorflow as tf
from tensorflow.keras.layers import *
from tensorflow.keras.models import *
from sklearn.preprocessing import  MinMaxScaler, LabelEncoder

##### 數(shù)據(jù)預(yù)處理
data = pd.read_csv('./data/criteo_sample.txt')

def data_processing(df, dense_features, sparse_features):
    df[dense_features] = df[dense_features].fillna(0.0)
    for f in dense_features:
        df[f] = df[f].apply(lambda x: np.log(x+1) if x > -1 else -1)
    
    df[sparse_features] = df[sparse_features].fillna("-1")
    for f in sparse_features:
        lbe = LabelEncoder()
        df[f] = lbe.fit_transform(df[f])
    return df[dense_features + sparse_features]

dense_features = [i for i in data.columns.values if 'I' in i]
sparse_features = [i for i in data.columns.values if 'C' in i]
df = data_processing(data, dense_features, sparse_features)
df['label'] = data['label']

##### 模型構(gòu)建
# 使用具名元組定義特征標(biāo)記
SparseFeature = namedtuple('SparseFeature', ['name', 'vocabulary_size', 'embedding_size'])
DenseFeature = namedtuple('DenseFeature', ['name', 'dimension'])
VarLenSparseFeature = namedtuple('VarLenSparseFeature', ['name', 'vocabulary_size', 'embedding_size', 'maxlen'])

class BiInteractionPooling(Layer):
    def call(self, inputs):
        """ 0.5 * (和的平方-平方的和) """
        concate_embed_values = inputs # B x n x k
        square_of_sum = tf.square(tf.reduce_sum(concate_embed_values, axis=1, keepdims=True)) # B x k
        sum_of_square = tf.reduce_sum(concate_embed_values * concate_embed_values, axis=1, keepdims=True) # B x k
        output = 0.5 * (square_of_sum - sum_of_square)
        return output
    
    def compute_output_shape(self, input_shape):
        return (None, input_shape[2])

def build_input_layers(feature_columns):
    """ 構(gòu)建輸入層 """
    dense_input_dict, sparse_input_dict = {}, {}
    for f in feature_columns:
        if isinstance(f, DenseFeature):
            dense_input_dict[f.name] = Input(shape=(f.dimension, ), name=f.name)
        elif isinstance(f, SparseFeature):
            sparse_input_dict[f.name] = Input(shape=(1, ), name=f.name)
    return dense_input_dict, sparse_input_dict

def build_embedding_layers(feature_columns, is_linear):
    embedding_layers_dict = {}
    # 篩選出sparse特征列
    sparse_feature_columns = list(filter(lambda x: isinstance(x, SparseFeature), feature_columns)) if feature_columns else []
    if is_linear:
        for f in sparse_feature_columns:
            embedding_layers_dict[f.name] = Embedding(f.vocabulary_size + 1, 1, name='1d_emb_' + f.name)
    else:
        for f in sparse_feature_columns:
            embedding_layers_dict[f.name] = Embedding(f.vocabulary_size + 1, f.embedding_size, name='kd_emb_' + f.name)
    return embedding_layers_dict

def concat_embedding_list(feature_columns, input_layer_dict, embedding_layer_dict, flatten=False):
    """ 拼接embedding特征 """
    sparse_feature_columns = list(filter(lambda x: isinstance(x, SparseFeature), feature_columns)) if feature_columns else []
    embedding_list = []
    for f in sparse_feature_columns:
        _input_layer = input_layer_dict[f.name] 
        _embed = embedding_layer_dict[f.name]
        embed_layer = _embed(_input_layer)
        if flatten:
            embed_layer = Flatten()(embed_layer)
        
        embedding_list.append(embed_layer)
    return embedding_list

def get_linear_logits(dense_input_dict, sparse_input_dict, sparse_feature_columns):
    concat_dense_inputs = Concatenate(axis=1)(list(dense_input_dict.values()))
    dense_logits_output = Dense(1)(concat_dense_inputs)
    
    linear_embedding_layer = build_embedding_layers(sparse_feature_columns, is_linear=True)
    sparse_1d_embed_list = []
    for f in sparse_feature_columns:
        temp_input = sparse_input_dict[f.name]
        temp_embed = Flatten()(linear_embedding_layer[f.name](temp_input))
        sparse_1d_embed_list.append(temp_embed)
    
    sparse_logits_output = Add()(sparse_1d_embed_list)
    linear_logits = Add()([dense_logits_output, sparse_logits_output])
    return linear_logits
    
def get_fm_logits(sparse_input_dict, sparse_feature_columns, dnn_embedding_layers):
    sparse_kd_embed_list = []
    for f in sparse_feature_columns:
        f_input = sparse_input_dict[f.name]
        _embed = dnn_embedding_layers[f.name](f_input)
        sparse_kd_embed_list.append(_embed)
    
    concat_sparse_kd_embed_list = Concatenate(axis=1)(sparse_kd_embed_list) # B x n x k
    fm_logits = BiInteractionPooling()(concat_sparse_kd_embed_list)
    return fm_logits

def get_bi_interaction_pooling_layer(sparse_input_dict, sparse_feature_columns, dnn_embedding_layers):
    """ sparse二階交叉 """
    sparse_kd_embed_list = []
    for f in sparse_feature_columns:
        f_input = sparse_input_dict[f.name]
        _embed = dnn_embedding_layers[f.name](f_input)
        sparse_kd_embed_list.append(_embed)
    
    concat_sparse_kd_embed_list = Concatenate(axis=1)(sparse_kd_embed_list)
    pooling_out = BiInteractionPooling()(concat_sparse_kd_embed_list)
    return pooling_out
    
def get_dnn_logits(pooling_out):    
    # DNN層
    dnn_out = Dropout(0.5)(Dense(1024, activation='relu')(pooling_out))
    dnn_out = Dropout(0.5)(Dense(512, activation='relu')(dnn_out))
    dnn_out = Dropout(0.5)(Dense(256, activation='relu')(dnn_out))
    dnn_logits = Dense(1)(dnn_out)
    return dnn_logits

def NFM(linear_feature_columns, dnn_feature_columns):
    dense_input_dict, sparse_input_dict = build_input_layers(linear_feature_columns + dnn_feature_columns)
    
    # linear
    linear_sparse_feature_columns = list(filter(lambda x: isinstance(x, SparseFeature), linear_feature_columns))
    input_layers = list(dense_input_dict.values()) + list(sparse_input_dict.values())
    linear_logits = get_linear_logits(dense_input_dict, sparse_input_dict, linear_sparse_feature_columns)
    
    # embedding+bi
    dnn_embedding_layers = build_embedding_layers(dnn_feature_columns, is_linear=False)
    dnn_sparse_feature_columns = list(filter(lambda x: isinstance(x, SparseFeature), dnn_feature_columns))
    
    pooling_out = get_bi_interaction_pooling_layer(sparse_input_dict, dnn_sparse_feature_columns, dnn_embedding_layers)
    # BN操作
    pooling_out = BatchNormalization()(pooling_out)
    
    # DNN
    dnn_logits = get_dnn_logits(pooling_out)
    
    output_logits = Add()([linear_logits, dnn_logits])
    output_layer = Activation("sigmoid")(output_logits)
    model = Model(input_layers, output_layer)
    return model

# 定義特征列
linear_feature_columns = [SparseFeature(f, vocabulary_size=df[f].nunique(), embedding_size=4) for f in sparse_features] + \
[DenseFeature(f, 1,) for f in dense_features]

dnn_feature_columns = [SparseFeature(f, vocabulary_size=df[f].nunique(), embedding_size=4) for f in sparse_features] + \
[DenseFeature(f, 1,) for f in dense_features]

model = NFM(linear_feature_columns, dnn_feature_columns)
model.summary()

##### 模型訓(xùn)練
model.compile(optimizer="adam",
             loss="binary_crossentropy",
             metrics=["binary_crossentropy", tf.keras.metrics.AUC(name='auc')])

train_input = {col: df[col] for col in dense_features + sparse_features}
model.fit(train_input, df['label'].values,
         batch_size=64, epochs=5, validation_split=0.2)