Improving efficiency of learning progress

[saluisto]

Hello community,

wish you all a good start into the new year. Before Christmas I was working on a heat conduction-convection problem [#1598] and I made some modifications to the original code. First of all, now i am using a dimensionless form of the PDE with spatial and temporal domains ranging from [0,1]. I also normalized the temperature in the equation. For learning i use a two step procedure, first the adam algorithm followed by a L-BFGS-B scheme. My problem now is that learning is quite slow and I feel that after 150000 iterations there is still more to achieve. However, I wanted to ask the community if they have some tips for me to speed things up a bit.

My second question is realted to the inverse nature of the problem. I use some observed data as part of the learning process to estimate the unkown convection velocity (q in the code) in time. However, as i set up the problem which is defined in space and time, I get values for q as a function of space and time. What i would like however is that q is only a function of time and not of space, basically in space only a uniform convection velocity should be used. Maybe some of you came across a similar problem and nows how to do this.

Thank you

Here is the code

`import deepxde as dde
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tensorflow.keras.optimizers.schedules import ExponentialDecay

def read_data(filename):
    df = pd.read_csv(filename)
    obs_data = df.dropna()
    obs_data=obs_data.drop(index=range(4))
    return obs_data


df=read_data(filename='complete_new.csv')
df['time'] = pd.to_datetime(df['time'], format="%d.%m.%Y %H:%M:%S")
df = df.drop('depth1', axis=1)
first_time = df['time'].min()
df['time'] = df['time'] - first_time

df['time'] = df['time'].dt.total_seconds()/(60*60*24)
tmaxx=np.float32(df['time'].max())
tmax = dde.Variable(tmaxx)  
df['time'] = df['time']/ df['time'].max()

Temp_max_value = df.max().max()

time = df['time'].values.reshape(-1,1)

df.set_index('time', inplace=True)

observe_x = np.array([0,0.21,0.42,0.62,0.82,1.01,1.22]).reshape(-1,1)
observe_T = df[['depth2','depth3','depth4','depth5','depth6','depth7','depth8']].values
LL=np.float32(observe_x[len(observe_x)-1])
L = dde.Variable(LL)
df.plot()

def pde(x, y):
    ke=3.8*60*60*24
    roh_sed=2650
    c_sed=1078
    roh_water=1024
    c_water=3986    
    D=0.0864
    a=2.06
    T, q = y[:, 0:1], y[:, 1:2] 
    dy_t = dde.grad.jacobian(y, x, i=0, j=1)
    Pe=(a*q*L)/D
    dy_xx = dde.grad.hessian(y, x, component=0, i=0, j=0) 
    dy_dx = dde.grad.jacobian(y, x, i=0, j=0)
    return (
        Pe*dy_t
        - dy_xx
        + Pe *dy_dx
    )

observe_x = observe_x/LL
xx, tt = np.meshgrid(observe_x,time)
X = np.vstack((np.ravel(xx), np.ravel(tt))).T #x,t
observe_T=np.ravel(observe_T).reshape(-1,1)
observe_T=observe_T/Temp_max_value
observe_u = dde.icbc.PointSetBC(X,observe_T,component=0)


geom = dde.geometry.Interval(0,1)
timedomain = dde.geometry.TimeDomain(0, 1)
geomtime = dde.geometry.GeometryXTime(geom, timedomain)

data = dde.data.TimePDE(
    geomtime,
    pde,
    [observe_u],
    num_domain=1000,
    anchors=X,
)

net = dde.nn.PFNN([2, [40, 30], [40, 30], [40, 30], 2], "tanh", "Glorot uniform")


model = dde.Model(data, net)

iterations=150000
initial_learning_rate=0.0035
decay_steps=iterations
decay_rate=0.95

learning_rate_schedule = ExponentialDecay(
    initial_learning_rate,
    decay_steps=decay_steps,
    decay_rate=decay_rate,
    staircase=True  # If True, decay the learning rate at discrete intervals
)

model.compile("adam", lr=initial_learning_rate,loss_weights=[1, 100])
model.train(iterations=iterations)
model.compile("L-BFGS-B",loss_weights=[1, 1])
losshistory, train_state = model.train(iterations=150000)
dde.saveplot(losshistory, train_state, issave=True, isplot=True)


Y = model.predict(X)  
dp = ['depth2','depth3','depth4','depth5','depth6','depth7','depth8']
columns = ['X[m]', 'Time[days]', 'T[C]', 'q[m/day]']
sim=np.concatenate((X,Y),axis=1)
result = pd.DataFrame(sim, columns=columns)
result['X[m]']=result['X[m]']*LL
result['Time[days]']=result['Time[days]']*tmaxx
result['T[C]']=result['T[C]']*Temp_max_value
df.index = df.index *tmaxx

for i in range(len(observe_x)): 
    rslt_d1 = result[result['X[m]'] == float(observe_x[i]*LL)]
    rslt_d1= rslt_d1.drop('X[m]',axis=1)
    rslt_d1['Time[days]'] = rslt_d1['Time[days]']
    
    rslt_d1.plot(x='Time[days]', y='T[C]', kind='scatter', title='Temp_simulated')
    df[dp[i]].plot(kind='line', title=dp[i],color='r')
    rslt_d1.plot(x='Time[days]', y='q[m/day]', kind='line', title='Temp_simulated')`
`
import deepxde as dde
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from tensorflow.keras.optimizers.schedules import ExponentialDecay

def read_data(filename):
    df = pd.read_csv(filename)
    obs_data = df.dropna()
    obs_data=obs_data.drop(index=range(4))
    return obs_data


df=read_data(filename='complete_new.csv')
df['time'] = pd.to_datetime(df['time'], format="%d.%m.%Y %H:%M:%S")
df = df.drop('depth1', axis=1)
first_time = df['time'].min()
df['time'] = df['time'] - first_time

df['time'] = df['time'].dt.total_seconds()/(60*60*24)
tmaxx=np.float32(df['time'].max())
tmax = dde.Variable(tmaxx)  
df['time'] = df['time']/ df['time'].max()

Temp_max_value = df.max().max()

time = df['time'].values.reshape(-1,1)

df.set_index('time', inplace=True)

observe_x = np.array([0,0.21,0.42,0.62,0.82,1.01,1.22]).reshape(-1,1)
observe_T = df[['depth2','depth3','depth4','depth5','depth6','depth7','depth8']].values

LL=np.float32(observe_x[len(observe_x)-1])
L = dde.Variable(LL)


df.plot()

def pde(x, y):
    ke=3.8*60*60*24 #ke is converted from J/(s m K) to J/(day m K)
    roh_sed=2650
    c_sed=1078
    roh_water=1024
    c_water=3986    
    D=0.0864
    a=2.06
    
    T, q = y[:, 0:1], y[:, 1:2] 
    dy_t = dde.grad.jacobian(y, x, i=0, j=1)
    Pe=(a*q*L)/D
    dy_xx = dde.grad.hessian(y, x, component=0, i=0, j=0) 
    dy_dx = dde.grad.jacobian(y, x, i=0, j=0)
    return (
        Pe*dy_t
        - dy_xx
        + Pe *dy_dx
    )

observe_x = observe_x/LL
xx, tt = np.meshgrid(observe_x,time)
X = np.vstack((np.ravel(xx), np.ravel(tt))).T #x,t
observe_T=np.ravel(observe_T).reshape(-1,1)
observe_T=observe_T/Temp_max_value
observe_u = dde.icbc.PointSetBC(X,observe_T,component=0)


geom = dde.geometry.Interval(0,1)
timedomain = dde.geometry.TimeDomain(0, 1)
geomtime = dde.geometry.GeometryXTime(geom, timedomain)

data = dde.data.TimePDE(
    geomtime,
    pde,
    [observe_u],
    num_domain=1000,
    anchors=X,
)

net = dde.nn.PFNN([2, [40, 30], [40, 30], [40, 30], 2], "tanh", "Glorot uniform")


model = dde.Model(data, net)

iterations=150000
initial_learning_rate=0.0035
decay_steps=iterations
decay_rate=0.95

learning_rate_schedule = ExponentialDecay(
    initial_learning_rate,
    decay_steps=decay_steps,
    decay_rate=decay_rate,
    staircase=True  # If True, decay the learning rate at discrete intervals
)

model.compile("adam", lr=initial_learning_rate,loss_weights=[1, 100])
model.train(iterations=iterations)
model.compile("L-BFGS-B",loss_weights=[1, 1])
losshistory, train_state = model.train(iterations=150000)
dde.saveplot(losshistory, train_state, issave=True, isplot=True)


Y = model.predict(X)  
dp = ['depth2','depth3','depth4','depth5','depth6','depth7','depth8']
columns = ['X[m]', 'Time[days]', 'T[C]', 'q[m/day]']
sim=np.concatenate((X,Y),axis=1)
result = pd.DataFrame(sim, columns=columns)
result['X[m]']=result['X[m]']*LL
result['Time[days]']=result['Time[days]']*tmaxx
result['T[C]']=result['T[C]']*Temp_max_value
df.index = df.index *tmaxx

for i in range(len(observe_x)): 
    rslt_d1 = result[result['X[m]'] == float(observe_x[i]*LL)]
    rslt_d1= rslt_d1.drop('X[m]',axis=1)
    rslt_d1['Time[days]'] = rslt_d1['Time[days]']
    
    rslt_d1.plot(x='Time[days]', y='T[C]', kind='scatter', title='Temp_simulated')
    df[dp[i]].plot(kind='line', title=dp[i],color='r')
    rslt_d1.plot(x='Time[days]', y='q[m/day]', kind='line', title='Temp_simulated')

`
Here is the learning progress:

[saluisto]

Small update by switching over to another NN achritecture i could increase accuracy:

layer_size = [2] + [50] * 6 + [2]