@@ -185,27 +185,27 @@ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_
185185
186186```python
187187def fit_polynomial_regression(X_train, y_train, X_test, y_test, degrees):
188- train_errors = []
189- test_errors = []
190-
191- for degree in degrees:
192- poly_features = PolynomialFeatures(degree = degree, include_bias = False )
193- X_train_poly = poly_features.fit_transform(X_train)
194- X_test_poly = poly_features.transform(X_test)
195-
196- model = LinearRegression()
197- model.fit(X_train_poly, y_train)
198-
199- train_pred = model.predict(X_train_poly)
200- test_pred = model.predict(X_test_poly)
201-
202- train_error = mean_squared_error(y_train, train_pred)
203- test_error = mean_squared_error(y_test, test_pred)
204-
205- train_errors.append(train_error)
206- test_errors.append(test_error)
207-
208- return train_errors, test_errors
188+ train_errors = []
189+ test_errors = []
190+
191+ for degree in degrees:
192+ poly_features = PolynomialFeatures(degree = degree, include_bias = False )
193+ X_train_poly = poly_features.fit_transform(X_train)
194+ X_test_poly = poly_features.transform(X_test)
195+
196+ model = LinearRegression()
197+ model.fit(X_train_poly, y_train)
198+
199+ train_pred = model.predict(X_train_poly)
200+ test_pred = model.predict(X_test_poly)
201+
202+ train_error = mean_squared_error(y_train, train_pred)
203+ test_error = mean_squared_error(y_test, test_pred)
204+
205+ train_errors.append(train_error)
206+ test_errors.append(test_error)
207+
208+ return train_errors, test_errors
209209```
210210
211211# ### Example 3
@@ -249,20 +249,20 @@ X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_
249249
250250```python
251251def fit_polynomial_regression(X_train, y_train, X_test, y_test, degree):
252- poly_features = PolynomialFeatures(degree = degree, include_bias = False )
253- X_train_poly = poly_features.fit_transform(X_train)
254- X_test_poly = poly_features.transform(X_test)
255-
256- model = LinearRegression()
257- model.fit(X_train_poly, y_train)
258-
259- train_pred = model.predict(X_train_poly)
260- test_pred = model.predict(X_test_poly)
261-
262- train_mse = mean_squared_error(y_train, train_pred)
263- test_mse = mean_squared_error(y_test, test_pred)
264-
265- return model, poly_features, train_mse, test_mse
252+ poly_features = PolynomialFeatures(degree = degree, include_bias = False )
253+ X_train_poly = poly_features.fit_transform(X_train)
254+ X_test_poly = poly_features.transform(X_test)
255+
256+ model = LinearRegression()
257+ model.fit(X_train_poly, y_train)
258+
259+ train_pred = model.predict(X_train_poly)
260+ test_pred = model.predict(X_test_poly)
261+
262+ train_mse = mean_squared_error(y_train, train_pred)
263+ test_mse = mean_squared_error(y_test, test_pred)
264+
265+ return model, poly_features, train_mse, test_mse
266266```
267267
268268# ### Example 3
@@ -297,42 +297,42 @@ plt.show()
297297from sklearn.linear_model import Ridge
298298
299299def fit_polynomial_regression_with_regularization(X_train, y_train, X_test, y_test, degree, alpha):
300- poly_features = PolynomialFeatures(degree = degree, include_bias = False )
301- X_train_poly = poly_features.fit_transform(X_train)
302- X_test_poly = poly_features.transform(X_test)
303-
304- model = Ridge(alpha = alpha)
305- model.fit(X_train_poly, y_train)
306-
307- train_pred = model.predict(X_train_poly)
308- test_pred = model.predict(X_test_poly)
309-
310- train_mse = mean_squared_error(y_train, train_pred)
311- test_mse = mean_squared_error(y_test, test_pred)
312-
313- return model, poly_features, train_mse, test_mse
314-
300+ poly_features = PolynomialFeatures(degree = degree, include_bias = False )
301+ X_train_poly = poly_features.fit_transform(X_train)
302+ X_test_poly = poly_features.transform(X_test)
303+
304+ model = Ridge(alpha = alpha)
305+ model.fit(X_train_poly, y_train)
306+
307+ train_pred = model.predict(X_train_poly)
308+ test_pred = model.predict(X_test_poly)
309+
310+ train_mse = mean_squared_error(y_train, train_pred)
311+ test_mse = mean_squared_error(y_test, test_pred)
312+
313+ return model, poly_features, train_mse, test_mse
314+
315315# Fit regularized models
316316alphas = [0 , 0.1 , 1 ]
317317degree = 15
318318plt.figure(figsize = (15 , 5 ))
319319
320320for i, alpha in enumerate (alphas):
321- model, poly_features, train_mse, test_mse = fit_polynomial_regression_with_regularization(X_train, y_train, X_test, y_test, degree, alpha)
322-
323- X_plot = np.linspace(0 , 5 , 100 ).reshape(- 1 , 1 )
324- X_plot_poly = poly_features.transform(X_plot)
325- y_plot = model.predict(X_plot_poly)
326-
327- plt.subplot(1 , 3 , i+ 1 )
328- plt.scatter(X_train, y_train, color = ' b' , label = ' Training data' )
329- plt.scatter(X_test, y_test, color = ' r' , label = ' Test data' )
330- plt.plot(X_plot, y_plot, color = ' g' , label = ' Model prediction' )
331- plt.title(f ' Degree { degree} Polynomial, Alpha: { alpha} \n Train MSE: { train_mse:.4f } , Test MSE: { test_mse:.4f } ' )
332- plt.xlabel(' X' )
333- plt.ylabel(' y' )
334- plt.legend()
335-
321+ model, poly_features, train_mse, test_mse = fit_polynomial_regression_with_regularization(X_train, y_train, X_test, y_test, degree, alpha)
322+
323+ X_plot = np.linspace(0 , 5 , 100 ).reshape(- 1 , 1 )
324+ X_plot_poly = poly_features.transform(X_plot)
325+ y_plot = model.predict(X_plot_poly)
326+
327+ plt.subplot(1 , 3 , i+ 1 )
328+ plt.scatter(X_train, y_train, color = ' b' , label = ' Training data' )
329+ plt.scatter(X_test, y_test, color = ' r' , label = ' Test data' )
330+ plt.plot(X_plot, y_plot, color = ' g' , label = ' Model prediction' )
331+ plt.title(f ' Degree { degree} Polynomial, Alpha: { alpha} \n Train MSE: { train_mse:.4f } , Test MSE: { test_mse:.4f } ' )
332+ plt.xlabel(' X' )
333+ plt.ylabel(' y' )
334+ plt.legend()
335+
336336plt.tight_layout()
337337plt.show()
338338```
@@ -2724,4 +2724,4 @@ return amount * 2 + amount.mean()
27242724
27252725# Apply the Pandas UDF to the dataset
27262726result = transactions.withColumn(" calculated_value" , complex_calculation(col(" amount" )))
2727- ```
2727+ ```
0 commit comments