一 SENet
SENet 虽然不是特征交叉方法，但是是在FiBiNet这个文章里一起出现的，就说明下，是一种field-wise加权，一共m个field。
1）field维度的平均池化：每一个field做average-pooling，
2）降维压缩：加上一个linear+relu变成m/r 维度
3）还原：再加一个linear+sigmoid还原成m维度
4）加权：生成的m维向量与原始的input进行row-wise mulitply.
def concat_func(inputs, axis=-1, mask=False):if len(inputs) == 1:input = inputs[0]return inputreturn Concat(axis, supports_masking=mask)(inputs)def reduce_mean(input_tensor,axis=None,keep_dims=False,name=None,reduction_indices=None):try:return tf.reduce_mean(input_tensor,axis=axis,keep_dims=keep_dims,name=name,reduction_indices=reduction_indices)except TypeError:return tf.reduce_mean(input_tensor,axis=axis,keepdims=keep_dims,name=name)class SENETLayer(Layer):"""SENETLayer used in FiBiNET.Input shape&Output shape- A list of 3D tensor with shape: ``(batch_size,1,embedding_size)``.'Arguments- **reduction_ratio** : Positive integer, dimensionality of the attention network output space.- **seed** : A Python integer to use as random seed."""def __init__(self, reduction_ratio=3, seed=1024, **kwargs):self.reduction_ratio = reduction_ratioself.seed = seedsuper(SENETLayer, self).__init__(**kwargs)def build(self, input_shape):if not isinstance(input_shape, list) or len(input_shape) < 2:raise ValueError('A `AttentionalFM` layer should be called ''on a list of at least 2 inputs')self.filed_size = len(input_shape) # field的维度mself.embedding_size = input_shape[0][-1] # embedding的维度reduction_size = max(1, self.filed_size // self.reduction_ratio) # 压缩后的维度# 第一层降维，lr参数，glorot_normal等价于xavier_normalself.W_1 = self.add_weight(shape=(self.filed_size, reduction_size), initializer=glorot_normal(seed=self.seed), name="W_1")# 第二层升维度self.W_2 = self.add_weight(shape=(reduction_size, self.filed_size), initializer=glorot_normal(seed=self.seed), name="W_2")# self.tensordot = Lambda(lambda x: tf.tensordot(x[0], x[1], axes=(-1, 0)))# Be sure to call this somewhere!super(SENETLayer, self).build(input_shape)def call(self, inputs, training=None, **kwargs):# inputs=[Tensor(batch_size,1,embedding_size),Tensor(batch_size,1,embedding_size),…]，后面还有concat操作，所以要求每个input维度为3if K.ndim(inputs[0]) != 3:raise ValueError("Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))inputs = concat_func(inputs, axis=1)Z = reduce_mean(inputs, axis=-1, )# 这里利用tf.tensordot手动实现了Dense的功能，也可以直接用Dense，个人感觉无差A_1 = tf.nn.relu(self.tensordot([Z, self.W_1]))A_2 = tf.nn.relu(self.tensordot([A_1, self.W_2]))V = tf.multiply(inputs, tf.expand_dims(A_2, axis=2))return tf.split(V, self.filed_size, axis=1) # 按照filed进行分开，保持输出仍然为一个listdef compute_output_shape(self, input_shape):return input_shapedef get_config(self, ):config = {'reduction_ratio': self.reduction_ratio, 'seed': self.seed}base_config = super(SENETLayer, self).get_config()base_config.update(config)return base_config
二 Bilinear Cross
两个field做交叉得到新的特征，比方把物品和用户地点做embedding，然后做交叉, 如果用inner product和hadamard乘积，必须维度一致，不一致的情况下，Bilinear引入了一个W矩阵，保证两者可以做交叉。
class BilinearInteraction(Layer):"""BilinearInteraction Layer used in FiBiNET.Input shape- A list of 3D tensor with shape: ``(batch_size, 1, embedding_size)``. Its length is ``filed_size``.Output shape- 3D tensor with shape: ``(batch_size, filed_size*(filed_size-1)/2, embedding_size)``.Arguments- **bilinear_type** : String, types of bilinear functions used in this layer.- **seed** : A Python integer to use as random seed."""def __init__(self, bilinear_type="interaction", seed=1024, **kwargs):self.bilinear_type = bilinear_typeself.seed = seedsuper(BilinearInteraction, self).__init__(**kwargs)def build(self, input_shape):if not isinstance(input_shape, list) or len(input_shape) < 2:raise ValueError('A `AttentionalFM` layer should be called ''on a list of at least 2 inputs')embedding_size = int(input_shape[0][-1])# all：所有字段共享同一个权重矩阵 𝑊# each：每个字段有自己独立的权重矩阵。# interaction：针对每对字段交互，使用不同的权重矩阵。if self.bilinear_type == "all":self.W = self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight")elif self.bilinear_type == "each":self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight" + str(i)) for i in range(len(input_shape) - 1)]elif self.bilinear_type == "interaction":self.W_list = [self.add_weight(shape=(embedding_size, embedding_size), initializer=glorot_normal(seed=self.seed), name="bilinear_weight" + str(i) + '_' + str(j)) for i, j in itertools.combinations(range(len(input_shape)), 2)]else:raise NotImplementedErrorsuper(BilinearInteraction, self).build(input_shape) # Be sure to call this somewhere!def call(self, inputs, **kwargs):if K.ndim(inputs[0]) != 3:raise ValueError("Unexpected inputs dimensions %d, expect to be 3 dimensions" % (K.ndim(inputs)))n = len(inputs)if self.bilinear_type == "all":vidots = [tf.tensordot(inputs[i], self.W, axes=(-1, 0)) for i in range(n)]p = [tf.multiply(vidots[i], inputs[j]) for i, j in itertools.combinations(range(n), 2)]elif self.bilinear_type == "each":vidots = [tf.tensordot(inputs[i], self.W_list[i], axes=(-1, 0)) for i in range(n - 1)]p = [tf.multiply(vidots[i], inputs[j]) for i, j in itertools.combinations(range(n), 2)]elif self.bilinear_type == "interaction":p = [tf.multiply(tf.tensordot(v[0], w, axes=(-1, 0)), v[1])for v, w in zip(itertools.combinations(inputs, 2), self.W_list)]else:raise NotImplementedErroroutput = concat_func(p, axis=1)return output
Reference:
1. FiBiNET: Combining Feature Importance and Bilinear feature Interaction for Click-Through Rate Prediction
2. 视频介绍
3. DeepCTR
4. DeepCTR-Torch