Merge pull request #13 from auxten/movielens-din

Test din model on movielens dataset

Author: auxten

.gitignore

@@ -4,6 +4,8 @@
 *.dll
 *.so
 *.dylib
+*.run
+*.rpm
 
 *.zip
 model.txt

README.md

@@ -4,13 +4,29 @@
 
 Deep Learning(Item2vec Embedding + MLP) based Feature-Engineering & Training & Predict all in one Recommendation System that can run on small server or edge device.
 
+# Models implemented
+
+- [x] [Simple 2 layer MLP](./nn/neural_network/multilayer_perceptron.go)
+  - [x] [Simple 2 layer MLP test on MovieLens](./example/movielens/feature_test.go)
+  - [x] Dropout and L2 regularization
+  - [x] Batch Normalization
+- [x] [YouTube DNN](./model/din/simplemlp.go)
+  - [x] [YouTube DNN test on MovieLens](./example/movielens/mlpimpl_test.go)
+  - [x] Dropout and L2 regularization
+  - [ ] Batch Normalization
+- [x] [DeepInterestNetwork](./model/din/din.go)
+  - [x] [DIN test on MovieLens](./example/movielens/dinimpl_test.go)
+  - [x] Euclidean Distance based attention
+  - [x] Dropout and L2 regularization
+  - [ ] Batch Normalization
+
 # Demo
 
 You can run the MovieLens training and predict demo by:
 
 ```shell
 # download and unzip the SQLite DB file
-wget https://github.com/auxten/edgeRec/files/9482338/movielens.db.zip && \
+wget https://github.com/auxten/edgeRec/files/9895974/movielens.db.zip && \
   unzip movielens.db.zip
 # compile the edgeRec and put it in the current directory
 GOBIN=`pwd` go install github.com/auxten/edgeRec@latest && \
@@ -161,6 +177,7 @@ To make this project work, quite a lot of code are copied and modified from the
 
 # Papers related
 
+- [YouTube DNN](https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/45530.pdf)
 - [Deep Interest Network for Click-Through Rate Prediction](https://arxiv.org/abs/1706.06978)
 - [Document Embedding with Paragraph Vectors](https://arxiv.org/abs/1507.07998)
 - [EdgeRec: Recommender System on Edge in Mobile Taobao](https://arxiv.org/abs/2005.08416) // not very identical implementation

example/movielens/dinimpl.go

@@ -0,0 +1,108 @@
+package movielens
+
+import (
+	"fmt"
+
+	"github.com/auxten/edgeRec/model/din"
+	rcmd "github.com/auxten/edgeRec/recommend"
+	log "github.com/sirupsen/logrus"
+	"gonum.org/v1/gonum/mat"
+	"gorgonia.org/tensor"
+)
+
+type dinImpl struct {
+	uProfileDim   int
+	uBehaviorSize int
+	uBehaviorDim  int
+	iFeatureDim   int
+	cFeatureDim   int
+
+	predBatchSize     int
+	batchSize, epochs int
+	sampleInfo        *rcmd.SampleInfo
+
+	// stop training on earlyStop count of no cost improvement
+	// 0 means no early stop
+	earlyStop int
+
+	learner *din.DinNet
+	pred    *din.DinNet
+}
+
+func (d *dinImpl) Predict(X mat.Matrix, Y mat.Mutable) *mat.Dense {
+	numPred, _ := X.Dims()
+	inputTensor := tensor.New(tensor.WithShape(X.Dims()), tensor.WithBacking(X.(*mat.Dense).RawMatrix().Data))
+	y, err := din.Predict(d.pred, numPred, d.predBatchSize, d.sampleInfo, inputTensor)
+	if err != nil {
+		log.Errorf("predict din model failed: %v", err)
+		return nil
+	}
+	yDense := mat.NewDense(numPred, 1, y)
+	if Y != nil {
+		Y.(*mat.Dense).SetRawMatrix(yDense.RawMatrix())
+	}
+
+	return yDense
+}
+
+func (d *dinImpl) Fit(trainSample *rcmd.TrainSample) (pred rcmd.PredictAbstract, err error) {
+	d.uProfileDim = trainSample.Info.UserProfileRange[1] - trainSample.Info.UserProfileRange[0]
+	d.uBehaviorSize = rcmd.UserBehaviorLen
+	d.uBehaviorDim = rcmd.ItemEmbDim
+	d.iFeatureDim = rcmd.ItemEmbDim
+	d.cFeatureDim = trainSample.Info.CtxFeatureRange[1] - trainSample.Info.CtxFeatureRange[0]
+	d.sampleInfo = &trainSample.Info
+
+	sampleLen := len(trainSample.Data)
+	X := mat.NewDense(sampleLen, len(trainSample.Data[0].Input), nil)
+	for i, sample := range trainSample.Data {
+		X.SetRow(i, sample.Input)
+	}
+	Y := mat.NewDense(sampleLen, 1, nil)
+	for i, sample := range trainSample.Data {
+		Y.Set(i, 0, sample.Response[0])
+	}
+
+	d.learner = din.NewDinNet(d.uProfileDim, d.uBehaviorSize, d.uBehaviorDim, d.iFeatureDim, d.cFeatureDim)
+	var (
+		inputs, labels tensor.Tensor
+		numExamples, _ = X.Dims()
+		numLabels, _   = Y.Dims()
+	)
+	if numExamples != numLabels {
+		err = fmt.Errorf("number of examples and labels do not match")
+		return
+	}
+
+	inputs = tensor.New(tensor.WithShape(X.Dims()), tensor.WithBacking(X.RawMatrix().Data))
+	labels = tensor.New(tensor.WithShape(Y.Dims()), tensor.WithBacking(Y.RawMatrix().Data))
+	err = din.Train(d.uProfileDim, d.uBehaviorSize, d.uBehaviorDim, d.iFeatureDim, d.cFeatureDim,
+		numExamples, d.batchSize, d.epochs, d.earlyStop,
+		d.sampleInfo,
+		inputs, labels,
+		d.learner,
+	)
+	if err != nil {
+		log.Errorf("train din model failed: %v", err)
+		return
+	}
+	dinJson, err := d.learner.Marshal()
+	if err != nil {
+		log.Errorf("marshal din model failed: %v", err)
+		return
+	}
+	dinPred, err := din.NewDinNetFromJson(dinJson)
+	if err != nil {
+		log.Errorf("new din model from json failed: %v", err)
+		return
+	}
+	err = din.InitForwardOnlyVm(d.uProfileDim, d.uBehaviorSize, d.uBehaviorDim, d.iFeatureDim, d.cFeatureDim,
+		d.predBatchSize, dinPred)
+	if err != nil {
+		log.Errorf("init forward only vm failed: %v", err)
+		return
+	}
+	d.pred = dinPred
+
+	return d, nil
+}

example/movielens/dinimpl_test.go

@@ -0,0 +1,80 @@
+package movielens
+
+import (
+	"context"
+	"fmt"
+	"math/rand"
+	"testing"
+
+	"github.com/auxten/edgeRec/nn/metrics"
+	rcmd "github.com/auxten/edgeRec/recommend"
+	. "github.com/smartystreets/goconvey/convey"
+	"gonum.org/v1/gonum/mat"
+)
+
+type dinPredictor struct {
+	rcmd.PreRanker
+	rcmd.Predictor
+	rcmd.UserBehavior
+}
+
+func TestDinOnMovielens(t *testing.T) {
+	rand.Seed(42)
+
+	var (
+		movielens = &MovielensRec{
+			DataPath:  "movielens.db",
+			SampleCnt: 79948,
+			//SampleCnt: 10000,
+		}
+		model rcmd.Predictor
+		err   error
+	)
+
+	Convey("Train din model", t, func() {
+		dinModel := &dinImpl{
+			predBatchSize: 100,
+			batchSize:     200,
+			epochs:        100,
+			earlyStop:     20,
+		}
+		trainCtx := context.Background()
+		model, err = rcmd.Train(trainCtx, movielens, dinModel)
+		So(err, ShouldBeNil)
+		So(model, ShouldNotBeNil)
+	})
+
+	Convey("Predict din model", t, func() {
+		testCount := 20600
+		rows, err := db.Query(
+			"SELECT userId, movieId, rating, timestamp FROM ratings_test ORDER BY timestamp, userId ASC LIMIT ?", testCount)
+		So(err, ShouldBeNil)
+		var (
+			userId     int
+			itemId     int
+			rating     float64
+			timestamp  int64
+			yTrue      = mat.NewDense(testCount, 1, nil)
+			sampleKeys = make([]rcmd.Sample, 0, testCount)
+		)
+		for i := 0; rows.Next(); i++ {
+			err = rows.Scan(&userId, &itemId, &rating, &timestamp)
+			if err != nil {
+				t.Errorf("scan error: %v", err)
+			}
+			yTrue.Set(i, 0, BinarizeLabel(rating))
+			sampleKeys = append(sampleKeys, rcmd.Sample{userId, itemId, 0, timestamp})
+		}
+		batchPredictCtx := context.Background()
+		dinPred := &dinPredictor{
+			PreRanker:    movielens,
+			Predictor:    model,
+			UserBehavior: movielens,
+		}
+		yPred, err := rcmd.BatchPredict(batchPredictCtx, dinPred, sampleKeys)
+		So(err, ShouldBeNil)
+		rocAuc := metrics.ROCAUCScore(yTrue, yPred, "", nil)
+		rowCount, _ := yTrue.Dims()
+		fmt.Printf("rocAuc on test set %d: %f\n", rowCount, rocAuc)
+	})
+}