Backend paddle: FNN supports regularization and dropout (#1922)

Author: lijialin03

deepxde/nn/paddle/fnn.py

@@ -3,12 +3,20 @@
 from .nn import NN
 from .. import activations
 from .. import initializers
+from .. import regularizers
 
 
 class FNN(NN):
     """Fully-connected neural network."""
 
-    def __init__(self, layer_sizes, activation, kernel_initializer):
+    def __init__(
+        self,
+        layer_sizes,
+        activation,
+        kernel_initializer,
+        regularization=None,
+        dropout_rate=0,
+    ):
         super().__init__()
         if isinstance(activation, list):
             if not (len(layer_sizes) - 1) == len(activation):
@@ -20,6 +28,13 @@ def __init__(self, layer_sizes, activation, kernel_initializer):
             self.activation = activations.get(activation)
         initializer = initializers.get(kernel_initializer)
         initializer_zero = initializers.get("zeros")
+        self.regularizer = regularizers.get(regularization)
+        self.dropout_rate = dropout_rate
+        if dropout_rate > 0:
+            self.dropouts = [
+                paddle.nn.Dropout(p=dropout_rate)
+                for _ in range(1, len(layer_sizes) - 1)
+            ]
 
         self.linears = paddle.nn.LayerList()
         for i in range(1, len(layer_sizes)):
@@ -37,6 +52,8 @@ def forward(self, inputs):
                 if isinstance(self.activation, list)
                 else self.activation(linear(x))
             )
+            if self.dropout_rate > 0:
+                x = self.dropouts[j](x)
         x = self.linears[-1](x)
         if self._output_transform is not None:
             x = self._output_transform(inputs, x)
@@ -58,11 +75,14 @@ class PFNN(NN):
         kernel_initializer: Initializer for the kernel weights matrix.
     """
 
-    def __init__(self, layer_sizes, activation, kernel_initializer):
+    def __init__(
+        self, layer_sizes, activation, kernel_initializer, regularization=None
+    ):
         super().__init__()
         self.activation = activations.get(activation)
         initializer = initializers.get(kernel_initializer)
         initializer_zero = initializers.get("zeros")
+        self.regularizer = regularizers.get(regularization)
 
         if len(layer_sizes) <= 1:
             raise ValueError("must specify input and output sizes")
@@ -73,7 +93,6 @@ def __init__(self, layer_sizes, activation, kernel_initializer):
 
         n_output = layer_sizes[-1]
 
-
         def make_linear(n_input, n_output):
             linear = paddle.nn.Linear(n_input, n_output)
             initializer(linear.weight)
@@ -92,18 +111,22 @@ def make_linear(n_input, n_output):
                 if isinstance(prev_layer_size, (list, tuple)):
                     # e.g. [8, 8, 8] -> [16, 16, 16]
                     self.layers.append(
-                        paddle.nn.LayerList([
-                            make_linear(prev_layer_size[j], curr_layer_size[j])
-                            for j in range(n_output)
-                        ])
+                        paddle.nn.LayerList(
+                            [
+                                make_linear(prev_layer_size[j], curr_layer_size[j])
+                                for j in range(n_output)
+                            ]
+                        )
                     )
                 else:
                     # e.g. 64 -> [8, 8, 8]
                     self.layers.append(
-                        paddle.nn.LayerList([
-                            make_linear(prev_layer_size, curr_layer_size[j])
-                            for j in range(n_output)
-                        ])
+                        paddle.nn.LayerList(
+                            [
+                                make_linear(prev_layer_size, curr_layer_size[j])
+                                for j in range(n_output)
+                            ]
+                        )
                     )
             else:  # e.g. 64 -> 64
                 if not isinstance(prev_layer_size, int):
@@ -115,10 +138,9 @@ def make_linear(n_input, n_output):
         # output layers
         if isinstance(layer_sizes[-2], (list, tuple)):  # e.g. [3, 3, 3] -> 3
             self.layers.append(
-                paddle.nn.LayerList([
-                    make_linear(layer_sizes[-2][j], 1)
-                    for j in range(n_output)
-                ])
+                paddle.nn.LayerList(
+                    [make_linear(layer_sizes[-2][j], 1) for j in range(n_output)]
+                )
             )
         else:
             self.layers.append(make_linear(layer_sizes[-2], n_output))