Merge pull request #6 from JoshuaBillson/4-add-support-for-mlj

Add Support for MLJ

Author: JoshuaBillson

.gitignore

@@ -4,3 +4,4 @@
 /Manifest.toml
 /docs/build/
 examples/*.png
+test/Manifest.toml

Project.toml

@@ -7,6 +7,7 @@ version = "0.0.2"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 DocStringExtensions = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
+MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
 Pipe = "b98c9c47-44ae-5843-9183-064241ee97a0"
 ProgressLogging = "33c8b6b6-d38a-422a-b730-caa89a2f386c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
@@ -18,9 +19,3 @@ Flux = "0.13"
 Pipe = "1.3"
 ProgressLogging = "0.1"
 julia = "1.6"
-
-[extras]
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
-
-[targets]
-test = ["Test"]

README.md

@@ -14,41 +14,44 @@ Below is an example of fitting an MDN to the visualized one-to-many distribution
 ![](https://github.com/JoshuaBillson/MixtureDensityNetworks.jl/blob/main/docs/src/figures/PredictedDistribution.png?raw=true)
 
 ```julia
-using MixtureDensityNetworks, Distributions, CairoMakie
+using MixtureDensityNetworks, Distributions, CairoMakie, Logging, TerminalLoggers
 
 const n_samples = 1000
 const epochs = 1000
-const mixtures = 5
+const mixtures = 6
 const layers = [128, 128]
 
+
 function main()
     # Generate Data
-    Y = rand(Uniform(-10.5, 10.5), 1, n_samples)
-    μ = 7sin.(0.75 .* Y) + 0.5 .* Y
-    X = rand.(Normal.(μ, 1.0))
+    X, Y = generate_data(n_samples)
 
     # Create Model
-    model = MDN(epochs=epochs, mixtures=mixtures, layers=layers)
+    machine = MDN(epochs=epochs, mixtures=mixtures, layers=layers) |> Machine
 
     # Fit Model
-    lc = fit!(model, X, Y)
+    report = with_logger(ConsoleLogger()) do 
+        fit!(machine, X, Y)
+    end
 
     # Plot Learning Curve
-    fig, _, _ = lines(1:epochs, lc, axis=(;xlabel="Epochs", ylabel="Loss"))
+    fig, _, _ = lines(1:epochs, report.learning_curve, axis=(;xlabel="Epochs", ylabel="Loss"))
     save("LearningCurve.png", fig)
 
     # Plot Learned Distribution
-    Ŷ = predict(model, X)
+    Ŷ = predict(machine, X)
     fig, ax, plt = scatter(X[1,:], rand.(Ŷ), markersize=4, label="Predicted Distribution")
     scatter!(ax, X[1,:], Y[1,:], markersize=3, label="True Distribution")
     axislegend(ax, position=:lt)
     save("PredictedDistribution.png", fig)
 
     # Plot Conditional Distribution
-    cond = predict(model, reshape([-2.0], (1,1)))[1]
+    cond = predict(machine, reshape([-2.0], (1,1)))[1]
     fig = Figure(resolution=(1000, 500))
     density(fig[1,1], rand(cond, 10000), npoints=10000)
     save("ConditionalDistribution.png", fig)
+
+    return machine
 end
 
 main()

docs/Manifest.toml

@@ -816,6 +816,12 @@ git-tree-sha1 = "2ce8695e1e699b68702c03402672a69f54b8aca9"
 uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
 version = "2022.2.0+0"
 
+[[deps.MLJModelInterface]]
+deps = ["Random", "ScientificTypesBase", "StatisticalTraits"]
+git-tree-sha1 = "c8b7e632d6754a5e36c0d94a4b466a5ba3a30128"
+uuid = "e80e1ace-859a-464e-9ed9-23947d8ae3ea"
+version = "1.8.0"
+
 [[deps.MLStyle]]
 git-tree-sha1 = "bc38dff0548128765760c79eb7388a4b37fae2c8"
 uuid = "d8e11817-5142-5d16-987a-aa16d5891078"
@@ -888,10 +894,10 @@ uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
 version = "1.1.0"
 
 [[deps.MixtureDensityNetworks]]
-deps = ["Distributions", "Flux", "Pipe", "ProgressLogging"]
+deps = ["Distributions", "DocStringExtensions", "Flux", "MLJModelInterface", "Pipe", "ProgressLogging", "Statistics"]
 path = ".."
 uuid = "521d8788-cab4-41cb-a05a-da376f16ad79"
-version = "1.0.0-DEV"
+version = "0.0.2"
 
 [[deps.Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -1214,6 +1220,11 @@ git-tree-sha1 = "2436b15f376005e8790e318329560dcc67188e84"
 uuid = "7b38b023-a4d7-4c5e-8d43-3f3097f304eb"
 version = "0.3.3"
 
+[[deps.ScientificTypesBase]]
+git-tree-sha1 = "a8e18eb383b5ecf1b5e6fc237eb39255044fd92b"
+uuid = "30f210dd-8aff-4c5f-94ba-8e64358c1161"
+version = "3.0.0"
+
 [[deps.Scratch]]
 deps = ["Dates"]
 git-tree-sha1 = "30449ee12237627992a99d5e30ae63e4d78cd24a"
@@ -1316,6 +1327,12 @@ git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a"
 uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c"
 version = "1.4.0"
 
+[[deps.StatisticalTraits]]
+deps = ["ScientificTypesBase"]
+git-tree-sha1 = "30b9236691858e13f167ce829490a68e1a597782"
+uuid = "64bff920-2084-43da-a3e6-9bb72801c0c9"
+version = "3.2.0"
+
 [[deps.Statistics]]
 deps = ["LinearAlgebra", "SparseArrays"]
 uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"

docs/make.jl

@@ -17,7 +17,9 @@ makedocs(;
         assets=String[],
     ),
     pages=[
-        "Home" => "index.md",
+        "Introduction" => "index.md",
+        "MLJ Compatibility" => "mlj.md", 
+        "API (Reference Manual)" => "reference.md",
     ],
 )
 

docs/src/figures/PredictedDistribution.png

@@ -6,7 +6,7 @@ CurrentModule = MixtureDensityNetworks
 
 Mixture Density Networks (MDNs) were first proposed by [Bishop (1994)](https://publications.aston.ac.uk/id/eprint/373/1/NCRG_94_004.pdf). We can think of them as a specialized type of neural network, which are typically employed when our data has a lot of uncertainty or when the relationship between features and labels is one-to-many. Unlike a traditional neural network, which predicts a point-estimate equal to the mode of the learned conditional distribution P(Y|X), an MDN maintains the full condtional distribution by predicting the parameters of a Gaussian Mixture Model (GMM). The multi-modal nature of GMMs are precisely what makes MDNs so well-suited to modeling one-to-many relationships. This package aims to provide a simple interface for defining, training, and deploying MDNs.
 
-## Example
+# Example
 
 First, let's create our dataset. To properly demonstrate the power of MDNs, we'll generate a many-to-one dataset where each x-value can map to more than one y-value.
 ```julia
@@ -68,9 +68,7 @@ const layers = [128, 128]
 
 function main()
     # Generate Data
-    Y = rand(Uniform(-10.5, 10.5), 1, n_samples)
-    μ = 7sin.(0.75 .* Y) + 0.5 .* Y
-    X = rand.(Normal.(μ, 1.0))
+    X, Y = generate_data(n_samples)
 
     # Create Model
     model = MDN(epochs=epochs, mixtures=mixtures, layers=layers)
@@ -97,16 +95,4 @@ function main()
 end
 
 main()
-```
-
-## Index
-
-```@index
-```
-
-## API
-
-```@autodocs
-Modules = [MixtureDensityNetworks]
-Private = false
 ```

docs/src/index.md

@@ -0,0 +1,64 @@
+```@meta
+CurrentModule = MixtureDensityNetworks
+```
+
+# MLJ Compatibility
+
+This package implements the interface specified by [MLJModelInterface](https://github.com/JuliaAI/MLJModelInterface.jl) and is thus fully compatible
+with the MLJ ecosystem. Below is an example demonstrating the use of this package in conjunction with MLJ. 
+
+```julia
+using MixtureDensityNetworks, Distributions, Logging, TerminalLoggers, CairoMakie, MLJ
+
+const n_samples = 1000
+const epochs = 1000
+const mixtures = 6
+const layers = [128, 128]
+
+function main()
+    # Generate Data
+    X, Y = generate_data(n_samples)
+
+    # Create Model
+    mach = MLJ.machine(MDN(epochs=epochs, mixtures=mixtures, layers=layers), MLJ.table(X'), Y[1,:])
+
+    # Evaluate Model
+    with_logger(TerminalLogger()) do 
+        @info "Evaluating..."
+        evaluation = MLJ.evaluate!(
+            mach, 
+            resampling=Holdout(shuffle=true), 
+            measure=[rsq, rmse, mae, mape], 
+            operation=MLJ.predict_mean
+        )
+        names = ["R²", "RMSE", "MAE", "MAPE"]
+        metrics = round.(evaluation.measurement, digits=3)
+        @info "Metrics: " * join(["$name: $metric" for (name, metric) in zip(names, metrics)], ", ")
+    end
+
+    # Fit Model
+    with_logger(TerminalLogger()) do 
+        @info "Training..."
+        MLJ.fit!(mach)
+    end
+
+    # Plot Learning Curve
+    fig, _, _ = lines(1:epochs, MLJ.training_losses(mach), axis=(;xlabel="Epochs", ylabel="Loss"))
+    save("LearningCurve.png", fig)
+
+    # Plot Learned Distribution
+    Ŷ = MLJ.predict(mach) .|> rand
+    fig, ax, plt = scatter(X[1,:], Ŷ, markersize=4, label="Predicted Distribution")
+    scatter!(ax, X[1,:], Y[1,:], markersize=3, label="True Distribution")
+    axislegend(ax, position=:lt)
+    save("PredictedDistribution.png", fig)
+
+    # Plot Conditional Distribution
+    cond = MLJ.predict(mach, MLJ.table(reshape([-2.1], (1,1))))[1]
+    fig = Figure(resolution=(1000, 500))
+    density(fig[1,1], rand(cond, 10000), npoints=10000)
+    save("ConditionalDistribution.png", fig)
+end
+
+main()
+```

docs/src/mlj.md

@@ -0,0 +1,14 @@
+```@meta
+CurrentModule = MixtureDensityNetworks
+```
+# Index
+
+```@index
+```
+
+# API
+
+```@autodocs
+Modules = [MixtureDensityNetworks]
+Private = false
+```

docs/src/reference.md

@@ -0,0 +1,53 @@
+using MixtureDensityNetworks, Distributions, Logging, TerminalLoggers, CairoMakie, MLJ
+
+const n_samples = 1000
+const epochs = 1000
+const mixtures = 6
+const layers = [128, 128]
+
+function main()
+    # Generate Data
+    X, Y = generate_data(n_samples)
+
+    # Create Model
+    mach = MLJ.machine(MDN(epochs=epochs, mixtures=mixtures, layers=layers), MLJ.table(X'), Y[1,:])
+
+    # Evaluate Model
+    with_logger(TerminalLogger()) do 
+        @info "Evaluating..."
+        evaluation = MLJ.evaluate!(
+            mach, 
+            resampling=Holdout(shuffle=true), 
+            measure=[rsq, rmse, mae, mape], 
+            operation=MLJ.predict_mean
+        )
+        names = ["R²", "RMSE", "MAE", "MAPE"]
+        metrics = round.(evaluation.measurement, digits=3)
+        @info "Metrics: " * join(["$name: $metric" for (name, metric) in zip(names, metrics)], ", ")
+    end
+
+    # Fit Model
+    with_logger(TerminalLogger()) do 
+        @info "Training..."
+        MLJ.fit!(mach)
+    end
+
+    # Plot Learning Curve
+    fig, _, _ = lines(1:epochs, MLJ.training_losses(mach), axis=(;xlabel="Epochs", ylabel="Loss"))
+    save("LearningCurve.png", fig)
+
+    # Plot Learned Distribution
+    Ŷ = MLJ.predict(mach) .|> rand
+    fig, ax, plt = scatter(X[1,:], Ŷ, markersize=4, label="Predicted Distribution")
+    scatter!(ax, X[1,:], Y[1,:], markersize=3, label="True Distribution")
+    axislegend(ax, position=:lt)
+    save("PredictedDistribution.png", fig)
+
+    # Plot Conditional Distribution
+    cond = MLJ.predict(mach, MLJ.table(reshape([-2.1], (1,1))))[1]
+    fig = Figure(resolution=(1000, 500))
+    density(fig[1,1], rand(cond, 10000), npoints=10000)
+    save("ConditionalDistribution.png", fig)
+
+    return mach
+end