11import tensorflow as tf
22from deepctr .layers .activation import activation_layer
33from deepctr .layers .utils import reduce_max , reduce_mean , reduce_sum , concat_func , div , softmax
4- from tensorflow .python .keras .initializers import RandomNormal , Zeros , glorot_normal
4+ from tensorflow .python .keras .initializers import RandomNormal , Zeros , TruncatedNormal
55from tensorflow .python .keras .layers import Layer
66from tensorflow .python .keras .regularizers import l2
77
@@ -103,19 +103,19 @@ def call(self, inputs, training=None, **kwargs):
103103 weight = tf .pow (weight , self .pow_p ) # [x,k_max,1]
104104
105105 if len (inputs ) == 3 :
106- k_user = tf .cast (tf .maximum (
107- 1. ,
108- tf .minimum (
109- tf .cast (self .k_max , dtype = "float32" ), # k_max
110- tf .log1p (tf .cast (inputs [2 ], dtype = "float32" )) / tf .log (2. ) # hist_len
111- )
112- ), dtype = "int64" )
106+ k_user = inputs [2 ]
113107 seq_mask = tf .transpose (tf .sequence_mask (k_user , self .k_max ), [0 , 2 , 1 ])
114108 padding = tf .ones_like (seq_mask , dtype = tf .float32 ) * (- 2 ** 32 + 1 ) # [x,k_max,1]
115109 weight = tf .where (seq_mask , weight , padding )
116110
117- weight = softmax (weight , dim = 1 , name = "weight" )
118- output = reduce_sum (keys * weight , axis = 1 )
111+ if self .pow_p >= 100 :
112+ idx = tf .stack (
113+ [tf .range (tf .shape (keys )[0 ]), tf .squeeze (tf .argmax (weight , axis = 1 , output_type = tf .int32 ), axis = 1 )],
114+ axis = 1 )
115+ output = tf .gather_nd (keys , idx )
116+ else :
117+ weight = softmax (weight , dim = 1 , name = "weight" )
118+ output = tf .reduce_sum (keys * weight , axis = 1 )
119119
120120 return output
121121
@@ -159,6 +159,7 @@ def get_config(self, ):
159159 base_config = super (Similarity , self ).get_config ()
160160 return dict (list (base_config .items ()) + list (config .items ()))
161161
162+
162163class CapsuleLayer (Layer ):
163164 def __init__ (self , input_units , out_units , max_len , k_max , iteration_times = 3 ,
164165 init_std = 1.0 , ** kwargs ):
@@ -171,32 +172,28 @@ def __init__(self, input_units, out_units, max_len, k_max, iteration_times=3,
171172 super (CapsuleLayer , self ).__init__ (** kwargs )
172173
173174 def build (self , input_shape ):
174- self .routing_logits = self .add_weight (shape = [self .max_len , self .k_max , 1 ],
175- initializer = TruncatedNormal (stddev = self .init_std ),
176- trainable = False , name = "B" , dtype = tf .float32 )
177- # N,T,k_max,1
178175 self .bilinear_mapping_matrix = self .add_weight (shape = [self .input_units , self .out_units ],
179176 name = "S" , dtype = tf .float32 )
180177 super (CapsuleLayer , self ).build (input_shape )
181178
182179 def call (self , inputs , ** kwargs ):
183180
184- behavior_embddings = inputs [0 ]
181+ behavior_embedding = inputs [0 ]
185182 seq_len = inputs [1 ]
186- batch_size = tf .shape (behavior_embddings )[0 ]
183+ batch_size = tf .shape (behavior_embedding )[0 ]
187184
188185 mask = tf .reshape (tf .sequence_mask (seq_len , self .max_len , tf .float32 ), [- 1 , self .max_len , 1 , 1 ])
189186
190- behavior_embdding_mapping = tf .matmul ( behavior_embddings , self .bilinear_mapping_matrix )
191- behavior_embdding_mapping = tf .expand_dims (behavior_embdding_mapping , axis = 2 )
187+ behavior_embedding_mapping = tf .tensordot ( behavior_embedding , self .bilinear_mapping_matrix , axes = 1 )
188+ behavior_embedding_mapping = tf .expand_dims (behavior_embedding_mapping , axis = 2 )
192189
193- behavior_embdding_mapping_ = tf .stop_gradient (behavior_embdding_mapping ) # N,max_len,1,E
194- print (behavior_embdding_mapping_ )
190+ behavior_embdding_mapping_ = tf .stop_gradient (behavior_embedding_mapping ) # N,max_len,1,E
195191 try :
196192 routing_logits = tf .truncated_normal ([batch_size , self .max_len , self .k_max , 1 ], stddev = self .init_std )
197193 except AttributeError :
198194 routing_logits = tf .compat .v1 .truncated_normal ([batch_size , self .max_len , self .k_max , 1 ],
199195 stddev = self .init_std )
196+ routing_logits = tf .stop_gradient (routing_logits )
200197
201198 k_user = None
202199 if len (inputs ) == 3 :
@@ -208,32 +205,42 @@ def call(self, inputs, **kwargs):
208205 interest_padding = tf .ones_like (interest_mask ) * - 2 ** 31
209206 interest_mask = tf .cast (interest_mask , tf .bool )
210207
211- routing_logits = tf .stop_gradient (routing_logits )
212- self .routing_logits = routing_logits # N,max_len,k_max,1
213- print (self .routing_logits )
214208 for i in range (self .iteration_times ):
215209 if k_user is not None :
216- self .routing_logits = tf .where (interest_mask , self .routing_logits , interest_padding )
217- weight = tf .nn .softmax (self .routing_logits , 2 ) * mask # N,max_len,k_max,1
210+ routing_logits = tf .where (interest_mask , routing_logits , interest_padding )
211+ try :
212+ weight = softmax (routing_logits , 2 ) * mask
213+ except TypeError :
214+ weight = tf .transpose (softmax (tf .transpose (routing_logits , [0 , 1 , 3 , 2 ])),
215+ [0 , 1 , 3 , 2 ]) * mask # N,max_len,k_max,1
218216 if i < self .iteration_times - 1 :
219217 Z = reduce_sum (tf .matmul (weight , behavior_embdding_mapping_ ), axis = 1 , keep_dims = True ) # N,1,k_max,E
220218 interest_capsules = squash (Z )
221219 delta_routing_logits = reduce_sum (
222220 interest_capsules * behavior_embdding_mapping_ ,
223221 axis = - 1 , keep_dims = True
224222 )
225- self . routing_logits += delta_routing_logits
223+ routing_logits += delta_routing_logits
226224 else :
227- Z = reduce_sum (tf .matmul (weight , behavior_embdding_mapping ), axis = 1 , keep_dims = True )
225+ Z = reduce_sum (tf .matmul (weight , behavior_embedding_mapping ), axis = 1 , keep_dims = True )
228226 interest_capsules = squash (Z )
229227
230228 interest_capsules = tf .reshape (interest_capsules , [- 1 , self .k_max , self .out_units ])
231229 return interest_capsules
232230
231+ def compute_output_shape (self , input_shape ):
232+ return (None , self .k_max , self .out_units )
233+
234+ def get_config (self , ):
235+ config = {'input_units' : self .input_units , 'out_units' : self .out_units , 'max_len' : self .max_len ,
236+ 'k_max' : self .k_max , 'iteration_times' : self .iteration_times , "init_std" : self .init_std }
237+ base_config = super (CapsuleLayer , self ).get_config ()
238+ return dict (list (base_config .items ()) + list (config .items ()))
239+
233240
234241def squash (inputs ):
235242 vec_squared_norm = reduce_sum (tf .square (inputs ), axis = - 1 , keep_dims = True )
236- scalar_factor = vec_squared_norm / (1 + vec_squared_norm ) / tf .sqrt (vec_squared_norm + 1e-8 )
243+ scalar_factor = vec_squared_norm / (1 + vec_squared_norm ) / tf .sqrt (vec_squared_norm + 1e-9 )
237244 vec_squashed = scalar_factor * inputs
238245 return vec_squashed
239246
@@ -255,3 +262,27 @@ def get_config(self, ):
255262 config = {'index' : self .index , }
256263 base_config = super (EmbeddingIndex , self ).get_config ()
257264 return dict (list (base_config .items ()) + list (config .items ()))
265+
266+
267+ class MaskUserEmbedding (Layer ):
268+
269+ def __init__ (self , k_max , ** kwargs ):
270+ self .k_max = k_max
271+ super (MaskUserEmbedding , self ).__init__ (** kwargs )
272+
273+ def build (self , input_shape ):
274+ super (MaskUserEmbedding , self ).build (
275+ input_shape ) # Be sure to call this somewhere!
276+
277+ def call (self , x , training = None , ** kwargs ):
278+ user_embedding , interest_num = x
279+ if not training :
280+ interest_mask = tf .sequence_mask (interest_num , self .k_max , tf .float32 )
281+ interest_mask = tf .reshape (interest_mask , [- 1 , self .k_max , 1 ])
282+ user_embedding *= interest_mask
283+ return user_embedding
284+
285+ def get_config (self , ):
286+ config = {'k_max' : self .k_max , }
287+ base_config = super (MaskUserEmbedding , self ).get_config ()
288+ return dict (list (base_config .items ()) + list (config .items ()))
0 commit comments