Skip to content

Commit fbb5e72

Browse files
author
Juan Montesinos
committed
Train MNIST in c++
1 parent 3483283 commit fbb5e72

21 files changed

+925
-1
lines changed

.gitignore

+5
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,8 @@
3030
*.exe
3131
*.out
3232
*.app
33+
34+
*pyc
35+
build/
36+
*.o
37+
*.so

.vscode/c_cpp_properties.json

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"configurations": [
3+
{
4+
"name": "Linux",
5+
"includePath": [
6+
"${workspaceFolder}/**",
7+
"/home/jmt/.pyenv/versions/3.11.2/lib/python3.11/site-packages/pybind11/include",
8+
"/home/jmt/.pyenv/versions/3.11.2/include/python3.11"
9+
10+
],
11+
"defines": [],
12+
"compilerPath": "/usr/bin/gcc",
13+
"cStandard": "c17",
14+
"cppStandard": "gnu++17",
15+
"intelliSenseMode": "linux-gcc-x64"
16+
}
17+
],
18+
"version": 4
19+
}

.vscode/launch.json

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"version": "0.2.0",
3+
"configurations": [
4+
{
5+
"name": "Launch C++ Program",
6+
"type": "cppdbg",
7+
"request": "launch",
8+
"program": "${fileDirname}/${fileBasenameNoExtension}",
9+
"args": [],
10+
"stopAtEntry": false,
11+
"cwd": "${workspaceFolder}",
12+
"environment": [],
13+
"externalConsole": false,
14+
"MIMode": "gdb",
15+
"setupCommands": [
16+
{
17+
"description": "Enable pretty-printing for gdb",
18+
"text": "-enable-pretty-printing",
19+
"ignoreFailures": true
20+
}
21+
],
22+
"preLaunchTask": "compile",
23+
"miDebuggerPath": "/usr/bin/gdb",
24+
"logging": {
25+
"engineLogging": true
26+
}
27+
}
28+
]
29+
}

.vscode/settings.json

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
{
2+
"files.associations": {
3+
".env*": "dotenv",
4+
"*.tcc": "cpp",
5+
"array": "cpp",
6+
"atomic": "cpp",
7+
"bit": "cpp",
8+
"bitset": "cpp",
9+
"cctype": "cpp",
10+
"chrono": "cpp",
11+
"clocale": "cpp",
12+
"cmath": "cpp",
13+
"codecvt": "cpp",
14+
"compare": "cpp",
15+
"concepts": "cpp",
16+
"condition_variable": "cpp",
17+
"cstdarg": "cpp",
18+
"cstddef": "cpp",
19+
"cstdint": "cpp",
20+
"cstdio": "cpp",
21+
"cstdlib": "cpp",
22+
"cstring": "cpp",
23+
"ctime": "cpp",
24+
"cwchar": "cpp",
25+
"cwctype": "cpp",
26+
"deque": "cpp",
27+
"string": "cpp",
28+
"unordered_map": "cpp",
29+
"vector": "cpp",
30+
"exception": "cpp",
31+
"algorithm": "cpp",
32+
"functional": "cpp",
33+
"iterator": "cpp",
34+
"memory": "cpp",
35+
"memory_resource": "cpp",
36+
"numeric": "cpp",
37+
"optional": "cpp",
38+
"random": "cpp",
39+
"ratio": "cpp",
40+
"string_view": "cpp",
41+
"system_error": "cpp",
42+
"tuple": "cpp",
43+
"type_traits": "cpp",
44+
"utility": "cpp",
45+
"fstream": "cpp",
46+
"initializer_list": "cpp",
47+
"iomanip": "cpp",
48+
"iosfwd": "cpp",
49+
"iostream": "cpp",
50+
"istream": "cpp",
51+
"limits": "cpp",
52+
"mutex": "cpp",
53+
"new": "cpp",
54+
"numbers": "cpp",
55+
"ostream": "cpp",
56+
"semaphore": "cpp",
57+
"sstream": "cpp",
58+
"stdexcept": "cpp",
59+
"stop_token": "cpp",
60+
"streambuf": "cpp",
61+
"thread": "cpp",
62+
"typeinfo": "cpp",
63+
"__nullptr": "cpp",
64+
"complex": "cpp",
65+
"forward_list": "cpp",
66+
"list": "cpp",
67+
"map": "cpp",
68+
"set": "cpp",
69+
"unordered_set": "cpp",
70+
"cinttypes": "cpp",
71+
"typeindex": "cpp",
72+
"valarray": "cpp",
73+
"variant": "cpp",
74+
"__functional_base": "cpp",
75+
"__hash_table": "cpp",
76+
"__split_buffer": "cpp",
77+
"__tree": "cpp",
78+
"__memory": "cpp",
79+
"filesystem": "cpp",
80+
"queue": "cpp",
81+
"stack": "cpp"
82+
}
83+
}

.vscode/tasks.json

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"version": "2.0.0",
3+
"tasks": [
4+
{
5+
"label": "compile",
6+
"type": "shell",
7+
"command": "g++",
8+
"args": [
9+
"-ansi",
10+
"-pedantic-errors",
11+
"-std=c++11",
12+
"${file}",
13+
"-o",
14+
"${fileDirname}/${fileBasenameNoExtension}"
15+
],
16+
"group": {
17+
"kind": "build",
18+
"isDefault": true
19+
},
20+
"problemMatcher": ["$gcc"]
21+
}
22+
]
23+
}

Makefile

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Compiler and flags
2+
CXX = g++
3+
PYTHON = $(shell which python3)
4+
PYBIND11_INCLUDE = $(shell $(PYTHON) -m pybind11 --includes)
5+
CXXFLAGS = -O3 -Wall $(PYBIND11_INCLUDE) -fPIC -Wall -Wextra -std=c++17 -fPIC -I include
6+
PYTHON_EXTENSION_SUFFIX = $(shell $(PYTHON) -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX"))')
7+
8+
# Target output
9+
TARGET = libmnist$(PYTHON_EXTENSION_SUFFIX)
10+
BUILD_DIR = build
11+
12+
# Source files
13+
SRC = bindings.cpp \
14+
src/activations.cpp \
15+
src/cross_entropy.cpp \
16+
src/dataloader.cpp \
17+
src/functionals.cpp \
18+
src/linear.cpp
19+
20+
# Object files
21+
OBJ = $(patsubst %.cpp,$(BUILD_DIR)/%.o,$(SRC))
22+
23+
# Default target
24+
all: $(TARGET)
25+
26+
# Create output directories if they don't exist
27+
$(BUILD_DIR):
28+
mkdir -p $(BUILD_DIR)
29+
30+
$(BUILD_DIR)/src:
31+
mkdir -p $(BUILD_DIR)/src
32+
33+
# Compile the shared library
34+
$(TARGET): $(OBJ)
35+
$(CXX) -shared -o $@ $(OBJ)
36+
37+
# Compile object files for root-level source files
38+
$(BUILD_DIR)/%.o: %.cpp | $(BUILD_DIR)
39+
$(CXX) $(CXXFLAGS) -c $< -o $@
40+
41+
# Compile object files for src/ source files
42+
$(BUILD_DIR)/src/%.o: src/%.cpp | $(BUILD_DIR)/src
43+
$(CXX) $(CXXFLAGS) -c $< -o $@
44+
45+
# Show Python interpreter and include paths
46+
python:
47+
@echo $(PYTHON)
48+
@echo $(PYBIND11_INCLUDE)
49+
@echo $(PYTHON_EXTENSION_SUFFIX)
50+
51+
# Clean up build files
52+
clean:
53+
rm -f $(BUILD_DIR)/src/*.o $(BUILD_DIR)/*.o $(TARGET)
54+
rm -rf $(BUILD_DIR)
55+
56+
@if [ -d results/ ]; then rm -rf results/; fi
57+
@mkdir results/
58+
59+
download_mnist:
60+
@if [ -d data/ ]; then rm -rf data/; fi
61+
@mkdir data/
62+
63+
@wget -P data/ https://raw.githubusercontent.com/fgnt/mnist/master/train-images-idx3-ubyte.gz
64+
@wget -P data/ https://raw.githubusercontent.com/fgnt/mnist/master/train-labels-idx1-ubyte.gz
65+
@wget -P data/ https://raw.githubusercontent.com/fgnt/mnist/master/t10k-images-idx3-ubyte.gz
66+
@wget -P data/ https://raw.githubusercontent.com/fgnt/mnist/master/t10k-labels-idx1-ubyte.gz
67+
68+
@gunzip data/*.gz

README.md

+11-1
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,12 @@
11
# MNIST-CPP-and-python
2-
Beginner-friendly repo on how to Code a Simple Neural network with backprop in C++, bind it to python and train MNIST!
2+
Beginner-friendly repo on how to Code a Simple Neural network with backprop in C++, bind it to python and train MNIST!
3+
4+
## Summary
5+
ReLu and Linear layers are implemented in C++ following PyTorch's naming convention. Some functionals like softmax are also implemented. The code is bound to python using pybind11. The model is trained on MNIST dataset using a python script.
6+
7+
## How to run
8+
1. Download the data via `make download_mnist`.
9+
2. Install the python dependencies via `pip install -r requirements.txt`.
10+
3. Compile the C++ code via `make`.
11+
4. Run the python script via `python train.py`.
12+
5. (Optional) Compare against pytorch training via `python train_pytorch.py`.

bindings.cpp

+106
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
// bindings.cpp
2+
#include <pybind11/pybind11.h>
3+
#include <pybind11/stl.h>
4+
#include <pybind11/numpy.h>
5+
6+
#include "include/modules.h"
7+
#include "include/activations.h"
8+
#include "include/functionals.h"
9+
#include "include/cross_entropy.h"
10+
#include "include/dataloader.h"
11+
12+
namespace py = pybind11;
13+
14+
PYBIND11_MODULE(libmnist, m) {
15+
// Bind the ReLu class
16+
py::class_<ReLu>(m, "ReLu")
17+
.def(py::init<>())
18+
.def("forward", [](ReLu& self, const std::vector<float>& input) {
19+
auto output = self.forward(input);
20+
// Return as NumPy array
21+
return py::array_t<float>(output.size(), output.data());
22+
}, "Apply ReLu activation")
23+
.def("backward", [](ReLu& self, const std::vector<float>& grad_output) {
24+
auto grad_input = self.backward(grad_output);
25+
return py::array_t<float>(grad_input.size(), grad_input.data());
26+
}, "Compute the backward pass of ReLu")
27+
.def("update", &ReLu::update, py::arg("lr"), "Update the parameters of ReLu");
28+
29+
// Bind the LinearLayer class
30+
py::class_<LinearLayer>(m, "LinearLayer")
31+
.def(py::init<int, int>())
32+
.def("forward", [](LinearLayer& self, const std::vector<float>& input) {
33+
auto output = self.forward(input);
34+
// Return as NumPy array
35+
return py::array_t<float>(output.size(), output.data());
36+
}, "Perform forward pass with Linear Layer")
37+
.def("backward", [](LinearLayer& self, const std::vector<float>& grad_output) {
38+
auto grad_input = self.backward(grad_output);
39+
return py::array_t<float>(grad_input.size(), grad_input.data());
40+
}, "Compute the backward pass of Linear Layer")
41+
.def("update", &LinearLayer::update, py::arg("lr"), "Update the parameters of LinearLayer")
42+
.def_readwrite("weights", &LinearLayer::weights)
43+
.def_readwrite("bias", &LinearLayer::bias)
44+
.def_readwrite("grad_weights", &LinearLayer::grad_weights)
45+
.def_readwrite("grad_bias", &LinearLayer::grad_bias);
46+
47+
48+
// Bind the SoftmaxndCrossEntropy class
49+
py::class_<SoftmaxndCrossEntropy>(m, "SoftmaxndCrossEntropy")
50+
.def(py::init<int>())
51+
.def("forward", [](SoftmaxndCrossEntropy& self, const std::vector<float>& input, int class_label) {
52+
return self.forward(input, class_label);
53+
}, "Compute the forward pass of Softmax and Cross Entropy")
54+
.def("backward", [](SoftmaxndCrossEntropy& self) {
55+
auto grad = self.backward();
56+
return py::array_t<float>(grad.size(), grad.data());
57+
}, "Compute the backward pass of Softmax and Cross Entropy");
58+
// Bind the DataLoader class
59+
py::class_<DataLoader>(m, "DataLoader")
60+
.def_static("load_images", [](const std::string& filepath) {
61+
auto images = DataLoader::load_images(filepath);
62+
py::ssize_t num_images = static_cast<py::ssize_t>(images.size());
63+
if (num_images == 0) {
64+
throw std::runtime_error("No images loaded");
65+
}
66+
py::ssize_t image_size = static_cast<py::ssize_t>(images[0].size());
67+
68+
// Create a NumPy array of shape (num_images, image_size)
69+
py::array_t<float> result({num_images, image_size});
70+
71+
auto buf = result.mutable_unchecked<2>();
72+
73+
for (py::ssize_t i = 0; i < num_images; ++i) {
74+
if (static_cast<py::ssize_t>(images[i].size()) != image_size) {
75+
throw std::runtime_error("Inconsistent image sizes");
76+
}
77+
for (py::ssize_t j = 0; j < image_size; ++j) {
78+
buf(i, j) = images[i][j];
79+
}
80+
}
81+
return result;
82+
}, "Load images from file")
83+
.def_static("load_labels", [](const std::string& filepath) {
84+
auto labels = DataLoader::load_labels(filepath);
85+
py::ssize_t num_labels = static_cast<py::ssize_t>(labels.size());
86+
87+
py::array_t<int> result({num_labels});
88+
auto buf = result.mutable_unchecked<1>();
89+
for (py::ssize_t i = 0; i < num_labels; ++i) {
90+
buf(i) = labels[i];
91+
}
92+
return result;
93+
}, "Load labels from file");
94+
95+
// Bind the functionals submodule
96+
py::module_ functionals = m.def_submodule("functionals", "Submodule for functional operations");
97+
functionals.def("softmax", [](const std::vector<float>& input) {
98+
auto output = functionals::softmax(input);
99+
return py::array_t<float>(output.size(), output.data());
100+
}, "Compute the softmax of a 1D vector");
101+
functionals.def("flatten2d", [](const std::vector<std::vector<float>>& input) {
102+
auto output = functionals::flatten2d(input);
103+
return py::array_t<float>(output.size(), output.data());
104+
}, "Flatten a 2D vector into a 1D vector");
105+
}
106+

include/activations.h

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#ifndef ACTIVATIONS
2+
#define ACTIVATIONS
3+
4+
#include <vector>
5+
#include "modules.h"
6+
7+
class ReLu : public Module
8+
{
9+
public:
10+
std::vector<float> forward(const std::vector<float> &input) override;
11+
std::vector<float> backward(const std::vector<float> &grad_output) override;
12+
void update(float lr) override;
13+
private:
14+
std::vector<bool> zeroed;
15+
};
16+
17+
#endif

0 commit comments

Comments
 (0)