No matter what the input is, the prediction is same.

[baichen99]

I am trying to solve NS equation with some points.

reference

After train model many iterations, I try to get some prediction with my model, but no matter what the input is, the output of the network is the same.

Here is my code:

import ipdb
import pandas as pd
import numpy as np
import torch
from glob import glob
import deepxde as dde
from torch import nn
from sklearn.metrics import mean_squared_error as MSE


def pde(x, u):
    # input [x, y, t]
    # output [ux, uy, p]
    # define coefficient
    rho = 5550.0
    mu = 7.22e-4
    up = mu / rho

    du_x = dde.grad.jacobian(u, x, i=0, j=0)
    du_y = dde.grad.jacobian(u, x, i=0, j=1)
    du_t = dde.grad.jacobian(u, x, i=0, j=2)
    
    dv_x = dde.grad.jacobian(u, x, i=1, j=0)
    dv_y = dde.grad.jacobian(u, x, i=1, j=1)
    dv_t = dde.grad.jacobian(u, x, i=1, j=2)
    
    dp_x = dde.grad.jacobian(u, x, i=2, j=0)
    dp_y = dde.grad.jacobian(u, x, i=2, j=1)
    
    du_xx = dde.grad.hessian(u, x, component=0, i=0, j=0)
    du_yy = dde.grad.hessian(u, x, component=0, i=1, j=1)
    dv_xx = dde.grad.hessian(u, x, component=1, i=0, j=0)
    dv_yy = dde.grad.hessian(u, x, component=1, i=1, j=1)
    
    continuity = du_x + dv_y
    eq_1 = du_t + (u[:, 0:1]*du_x + u[:, 1:2] * du_y) + dp_x / rho - up * (du_xx + du_yy)
    eq_2 = dv_t + (u[:, 0:1]*dv_x + u[:, 1:2] * dv_y) + dp_y / rho - up * (dv_xx + dv_yy)
    return [continuity, eq_1, eq_2]

def get_data(size=0.6):
    # np.random.seed(0)
    
    dfs = []
    for file in glob('VTK_∑≈ºÁcsv/*.csv'):
        data = pd.read_csv(file)
        time = int(file.split('/')[-1].replace('s.csv', ''))
        data.insert(0, 'time', time)
        dfs.append(data)
    data = pd.concat(dfs, axis=0)
    data.drop_duplicates(subset=['Points:0', 'Points:1', 'time'])
    msk = np.random.rand(len(data)) < size
    data = data[msk]
    
    d = data[['time', 'Points:0', 'Points:1', 'p', 'U:0', 'U:1']].to_numpy()
    # points:0 is y£¨points:1 is x
    t, y, x, p, u_y, u_x = d[:, 0], d[:, 1], d[:, 2], d[:, 3], d[:, 4], d[:, 5]
    t = np.reshape(t, (-1, 1))
    x = np.reshape(x, (-1, 1))
    y = np.reshape(y, (-1, 1))
    u_x = np.reshape(u_x, (-1, 1))
    u_y = np.reshape(u_y, (-1, 1))
    p = np.reshape(p, (-1, 1))

    X = np.hstack((x, y, t))
    return X, u_x, u_y, p




X, u_x, u_y, p = get_data(size=1)


observe_y1 = dde.icbc.PointSetBC(X, u_x, component=0)
observe_y2 = dde.icbc.PointSetBC(X, u_y, component=1)
x_min, x_max = np.min(X[:, 0]), np.max(X[:, 0])
y_min, y_max = np.min(X[:, 1]), np.max(X[:, 1])

geom = dde.geometry.Rectangle([x_min, y_min], [x_max, y_max])
timedomain = dde.geometry.TimeDomain(0, 7600)
geomtime = dde.geometry.GeometryXTime(geom, timedomain)

data = dde.data.TimePDE(
    geomtime,
    pde,
    [observe_y1, observe_y2],
    num_domain=0,
    num_boundary=0,
    num_initial=0,
    anchors=X,
)

checkpointer = dde.callbacks.ModelCheckpoint(
    filepath="checkpoints/model.ckpt",
    save_better_only=False,
    period=1000
)

net = dde.nn.FNN([3] + [50] * 6 + [3], "tanh", "Glorot uniform")
# net.apply_output_transform(lambda x, y: y * torch.Tensor([1, 1, 1e-5]))
net.apply_feature_transform(lambda x:  (x -torch.min(x))/(torch.max(x) - torch.min(x)))
model = dde.Model(data, net)



model.compile("adam", lr=1e-3)
loss_history, train_state = model.train(
    iterations=10000, callbacks=[checkpointer],
    display_every=1000,
    model_save_path='model'
)

X, ux, uy, p = get_data()
y = model.predict(X)
print(y)
# [x, y, t, ux, uy]
d = np.hstack((X, y))

np.savetxt('predict.csv', d, header=' '.join(['x', 'y', 't', 'Ux', 'Uy', 'p'])

my data looks like this, points:0 is y, points:1 is x

    Points:0  Points:1  time       U:0           U:1         p
0   0.006282  0.065368   500  0.000008 -3.270000e-15  107000.0
1   0.070427  0.166460  1000  0.025690  4.204900e-02  104000.0
2   0.046072 -0.114860  1500  0.010061 -8.458900e-02  105000.0
3   0.087669 -0.240310  6500 -0.017307  1.430800e-02  103000.0
4   0.115720  0.253000  5000  0.000000  0.000000e+00  101000.0
5   0.076694  0.186450  7500  0.076552  9.385900e-02  103000.0
6   0.063388 -0.221220  6500  0.036844 -5.006300e-02  104000.0
7   0.128980 -0.056876  6500  0.001446  1.957200e-01  101000.0
8   0.075587 -0.140030  2000  0.051715 -4.307300e-02  104000.0
9   0.013180  0.055000  7500 -0.003841  8.161700e-02  107000.0
10  0.052229 -0.206460  5000  0.017007 -4.734800e-02  105000.0
11  0.073527 -0.058043   500 -0.007235 -3.706200e-03  104000.0
12  0.145000 -0.084321     0  0.000000  0.000000e+00  100000.0
13  0.122210 -0.161430  5000 -0.031410  1.012300e-02  101000.0
14  0.145000 -0.131220  7000  0.000000  0.000000e+00   99929.0
15  0.065246 -0.215870  7000 -0.008374 -7.772500e-03  104000.0
16  0.100650 -0.041873  7500  0.014483  1.552500e-03  102000.0
17  0.098949  0.061168  4000 -0.118830 -1.086800e-01  102000.0
18  0.038619 -0.136370  4000 -0.033556 -7.648900e-02  105000.0
19  0.021526 -0.122230  4000  0.019074 -1.449000e-02  106000.0
20  0.125270  0.156010  1500  0.020666 -1.147900e-01  101000.0
21  0.094671  0.176740  4000  0.063277 -1.244100e-02  103000.0
22  0.037509 -0.146900     0  0.000000  0.000000e+00  100000.0
23  0.078490  0.056997  1000 -0.036982 -3.200200e-02  103000.0
24  0.142720  0.084349   500 -0.000027 -2.048300e-02  100000.0
25  0.125270 -0.129430  2000  0.013846  1.092900e-01  101000.0
26  0.050762 -0.141540  2500  0.039777 -1.474900e-01  105000.0
27  0.145000  0.055000  5500  0.000000  0.000000e+00   99866.0
28  0.094229  0.119130  5000 -0.013261  1.560100e-02  102000.0
29  0.128960  0.002520  7500 -0.043930  6.878000e-02  101000.0
30  0.143450  0.203840  4000  0.004402 -1.504100e-01   99981.0
31  0.071784 -0.182150  4000  0.043574  7.066200e-02  104000.0
32  0.025485 -0.129340  2500  0.050020 -1.516900e-01  106000.0
33  0.143780 -0.125900  5500  0.002866  4.238500e-03   99947.0
34  0.055016 -0.127710  4000  0.102240 -9.342900e-02  105000.0
35  0.117710 -0.131680  6000 -0.039332  1.813400e-01  101000.0
36  0.063038 -0.096461  7000  0.073312 -5.401900e-02  104000.0
37  0.128650 -0.067410  6500  0.021732  1.903200e-01  101000.0
38  0.138100  0.133460  4000  0.001452 -1.694600e-01  100000.0
39  0.130030  0.216640  1500  0.011389 -4.520100e-02  101000.0
40  0.145000  0.073638     0  0.000000  0.000000e+00  100000.0
41  0.070058 -0.229500     0  0.000000  0.000000e+00  100000.0
42  0.130140 -0.209220  1000  0.015065  4.200900e-02  101000.0
43  0.061086 -0.235380  1500  0.000008 -4.490000e-15  104000.0
44  0.073953  0.111420  4000 -0.057716  3.850300e-02  103000.0
45  0.089352  0.199040  5000  0.076068  3.902000e-02  103000.0
46  0.025403  0.125750     0  0.000000  0.000000e+00  100000.0
47  0.050663 -0.112830  5500 -0.044981 -4.949400e-02  105000.0
48  0.132120  0.062006  6500 -0.013099 -9.563700e-02  101000.0
49  0.128320  0.166630  3000  0.017226 -6.537600e-02  101000.0

loss_history are as follows:

Step      Train loss                                            Test loss                                             Test metric
0         [2.56e-08, 7.70e-10, 3.30e-10, 1.05e-02, 2.21e-03]    [2.56e-08, 7.70e-10, 3.30e-10, 1.05e-02, 2.21e-03]    []  
1000      [5.96e-09, 2.67e-13, 4.12e-12, 7.05e-03, 1.89e-03]    [5.96e-09, 2.67e-13, 4.12e-12, 7.05e-03, 1.89e-03]    []  
2000      [1.23e-09, 1.85e-13, 4.11e-12, 7.05e-03, 1.89e-03]    [1.23e-09, 1.85e-13, 4.11e-12, 7.05e-03, 1.89e-03]    []  
3000      [5.51e-08, 1.15e-12, 3.25e-11, 7.05e-03, 1.89e-03]    [5.51e-08, 1.15e-12, 3.25e-11, 7.05e-03, 1.89e-03]    []  
4000      [4.65e-09, 1.75e-12, 4.42e-12, 7.05e-03, 1.89e-03]    [4.65e-09, 1.75e-12, 4.42e-12, 7.05e-03, 1.89e-03]    []  
5000      [1.46e-09, 2.95e-12, 1.27e-11, 7.05e-03, 1.89e-03]    [1.46e-09, 2.95e-12, 1.27e-11, 7.05e-03, 1.89e-03]    []  
6000      [9.46e-09, 4.12e-12, 2.03e-11, 7.05e-03, 1.89e-03]    [9.46e-09, 4.12e-12, 2.03e-11, 7.05e-03, 1.89e-03]    []  
7000      [1.14e-08, 6.37e-12, 7.90e-12, 7.05e-03, 1.89e-03]    [1.14e-08, 6.37e-12, 7.90e-12, 7.05e-03, 1.89e-03]    []  
8000      [9.44e-09, 1.01e-11, 2.61e-11, 7.05e-03, 1.89e-03]    [9.44e-09, 1.01e-11, 2.61e-11, 7.05e-03, 1.89e-03]    []  
9000      [1.88e-07, 5.06e-12, 1.88e-11, 7.05e-03, 1.89e-03]    [1.88e-07, 5.06e-12, 1.88e-11, 7.05e-03, 1.89e-03]    []  
10000     [3.05e-08, 6.02e-12, 2.20e-11, 7.05e-03, 1.89e-03]    [3.05e-08, 6.02e-12, 2.20e-11, 7.05e-03, 1.89e-03]    []  

I try to predict with train dataset, but got this:

# x y t Ux Uy p
-5.236799999999999788e-02 1.460700000000000054e-01 5.000000000000000000e+03 -1.436957390978932381e-04 1.348556578159332275e-02 -3.312250599265098572e-02
-4.973700000000000343e-02 1.463700000000000001e-01 5.000000000000000000e+03 -1.432035351172089577e-04 1.348813995718955994e-02 -3.312261030077934265e-02
-4.447399999999999964e-02 1.469000000000000028e-01 5.000000000000000000e+03 -1.421803608536720276e-04 1.349332556128501892e-02 -3.312281146645545959e-02
-2.342100000000000085e-02 1.491200000000000025e-01 5.000000000000000000e+03 -1.381508773192763329e-04 1.351399533450603485e-02 -3.312361612915992737e-02
-1.815800000000000053e-02 1.499000000000000055e-01 5.000000000000000000e+03 -1.371987164020538330e-04 1.351907476782798767e-02 -3.312380984425544739e-02
-1.552599999999999987e-02 1.503899999999999959e-01 5.000000000000000000e+03 -1.367337536066770554e-04 1.352158002555370331e-02 -3.312388435006141663e-02
-7.631599999999999669e-03 1.530299999999999994e-01 5.000000000000000000e+03 -1.355925342068076134e-04 1.352861523628234863e-02 -3.312414139509201050e-02
5.000000000000000104e-03 1.639500000000000124e-01 5.000000000000000000e+03 -1.352010294795036316e-04 1.353716664016246796e-02 -3.312451392412185669e-02
7.631599999999999669e-03 1.530299999999999994e-01 5.000000000000000000e+03 -1.323353499174118042e-04 1.354424282908439636e-02 -3.312479332089424133e-02
1.026299999999999955e-02 1.517699999999999882e-01 5.000000000000000000e+03 -1.315146218985319138e-04 1.354744471609592438e-02 -3.312491625547409058e-02
1.289500000000000021e-02 1.509699999999999931e-01 5.000000000000000000e+03 -1.307739876210689545e-04 1.355045475065708160e-02 -3.312503173947334290e-02
1.552599999999999987e-02 1.503899999999999959e-01 5.000000000000000000e+03 -1.300832955166697502e-04 1.355339772999286652e-02 -3.312512114644050598e-02
1.815800000000000053e-02 1.499000000000000055e-01 5.000000000000000000e+03 -1.294186804443597794e-04 1.355627551674842834e-02 -3.312521055340766907e-02
2.078999999999999945e-02 1.494900000000000118e-01 5.000000000000000000e+03 -1.287683844566345215e-04 1.355914957821369171e-02 -3.312535956501960754e-02
2.868400000000000116e-02 1.484800000000000009e-01 5.000000000000000000e+03 -1.268618507310748100e-04 1.356762275099754333e-02 -3.312567993998527527e-02
3.131599999999999662e-02 1.481899999999999884e-01 5.000000000000000000e+03 -1.262443838641047478e-04 1.357044465839862823e-02 -3.312581777572631836e-02
3.657900000000000040e-02 1.476600000000000135e-01 5.000000000000000000e+03 -1.250070054084062576e-04 1.357605867087841034e-02 -3.312602266669273376e-02
3.921000000000000180e-02 1.474000000000000032e-01 5.000000000000000000e+03 -1.243923325091600418e-04 1.357886753976345062e-02 -3.312614560127258301e-02
4.184199999999999725e-02 1.471500000000000030e-01 5.000000000000000000e+03 -1.237812684848904610e-04 1.358165591955184937e-02 -3.312623873353004456e-02
4.447399999999999964e-02 1.469000000000000028e-01 5.000000000000000000e+03 -1.231606584042310715e-04 1.358444616198539734e-02 -3.312633186578750610e-02
4.973700000000000343e-02 1.463700000000000001e-01 5.000000000000000000e+03 -1.219233963638544083e-04 1.359005644917488098e-02 -3.312656655907630920e-02
5.236799999999999788e-02 1.460700000000000054e-01 5.000000000000000000e+03 -1.212942879647016525e-04 1.359288208186626434e-02 -3.312668204307556152e-02
5.500000000000000028e-02 5.310800000000000069e-03 5.000000000000000000e+03 -9.101280011236667633e-05 1.365208439528942108e-02 -3.312870860099792480e-02
5.500000000000000028e-02 6.617900000000000345e-03 5.000000000000000000e+03 -9.128637611865997314e-05 1.365153118968009949e-02 -3.312874585390090942e-02
5.500000000000000028e-02 7.934200000000000627e-03 5.000000000000000000e+03 -9.156251326203346252e-05 1.365101523697376251e-02 -3.312870487570762634e-02
5.500000000000000028e-02 1.097599999999999965e-02 5.000000000000000000e+03 -9.220489300787448883e-05 1.364979334175586700e-02 -3.312869369983673096e-02
5.500000000000000028e-02 1.290000000000000001e-02 5.000000000000000000e+03 -9.261607192456722260e-05 1.364903151988983154e-02 -3.312865644693374634e-02
5.500000000000000028e-02 1.784399999999999875e-02 5.000000000000000000e+03 -9.365985170006752014e-05 1.364704780280590057e-02 -3.312859311699867249e-02
5.500000000000000028e-02 3.897199999999999970e-02 5.000000000000000000e+03 -9.812810458242893219e-05 1.363856345415115356e-02 -3.312826529145240784e-02
5.500000000000000028e-02 4.905600000000000238e-02 5.000000000000000000e+03 -1.002462813630700111e-04 1.363450847566127777e-02 -3.312812373042106628e-02
5.500000000000000028e-02 5.913999999999999813e-02 5.000000000000000000e+03 -1.023794757202267647e-04 1.363045722246170044e-02 -3.312802687287330627e-02
5.500000000000000028e-02 6.418300000000000394e-02 5.000000000000000000e+03 -1.034435117617249489e-04 1.362844556570053101e-02 -3.312790766358375549e-02
5.500000000000000028e-02 6.922499999999999487e-02 5.000000000000000000e+03 -1.045130193233489990e-04 1.362643204629421234e-02 -3.312788531184196472e-02
5.500000000000000028e-02 7.426800000000000068e-02 5.000000000000000000e+03 -1.055739121511578560e-04 1.362438499927520752e-02 -3.312776610255241394e-02
5.500000000000000028e-02 7.930900000000000449e-02 5.000000000000000000e+03 -1.066371332854032516e-04 1.362237334251403809e-02 -3.312773257493972778e-02
5.500000000000000028e-02 9.945600000000000274e-02 5.000000000000000000e+03 -1.108892029151320457e-04 1.361429132521152496e-02 -3.312744200229644775e-02
5.500000000000000028e-02 1.095000000000000001e-01 5.000000000000000000e+03 -1.130100572481751442e-04 1.361024752259254456e-02 -3.312727436423301697e-02
5.500000000000000028e-02 1.145100000000000007e-01 5.000000000000000000e+03 -1.140710664913058281e-04 1.360822096467018127e-02 -3.312719985842704773e-02
5.500000000000000028e-02 1.195100000000000051e-01 5.000000000000000000e+03 -1.151234610006213188e-04 1.360622048377990723e-02 -3.312712162733078003e-02
5.500000000000000028e-02 1.242199999999999971e-01 5.000000000000000000e+03 -1.161211403086781502e-04 1.360434293746948242e-02 -3.312707319855690002e-02
5.500000000000000028e-02 1.317899999999999905e-01 5.000000000000000000e+03 -1.177249941974878311e-04 1.360129751265048981e-02 -3.312698006629943848e-02
5.500000000000000028e-02 1.347200000000000064e-01 5.000000000000000000e+03 -1.183456042781472206e-04 1.360011659562587738e-02 -3.312691673636436462e-02
5.500000000000000028e-02 1.372099999999999986e-01 5.000000000000000000e+03 -1.188664464280009270e-04 1.359912753105163574e-02 -3.312685713171958923e-02
5.500000000000000028e-02 1.411000000000000032e-01 5.000000000000000000e+03 -1.196796074509620667e-04 1.359757408499717712e-02 -3.312686085700988770e-02
5.500000000000000028e-02 1.438099999999999934e-01 5.000000000000000000e+03 -1.202598214149475098e-04 1.359647884964942932e-02 -3.312681987881660461e-02

I don't know why the loss stops dropping soon, and the predicted output is the same no matter what my input is

I have read the issues in Q&A and tried the following methods, but none of them solved my problem.

1. normalize the input
2. set different weights for different losses
3. narrow the time horizon to train and predict only 7000s of data

I don't know what went wrong£¨ can you guys give me some advice, thanks

[lululxvi]

I don't know why the loss stops dropping soon, and the predicted output is the same no matter what my input is

The loss didn't drop too much, always ~1e-2.

[baichen99]

Thank you for your reply !
I would like to ask you if the following reasons could cause the loss not to drop.

1. the data points do not match the pde
2. the time t is taken very sparsely (in my case, only 15 time points of data and the points are taken at 500s intervals)

[lululxvi]

Yes, these could be the issues.