Skip to content

Commit df5f1c7

Browse files
authored
Merge pull request #75 from patrikhuber/matlab-bindings
Added Matlab bindings for the fitting function
2 parents dc84b9a + 3375cc0 commit df5f1c7

File tree

10 files changed

+631
-0
lines changed

10 files changed

+631
-0
lines changed

.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,6 @@
1010
[submodule "3rdparty/eigen3-nnls"]
1111
path = 3rdparty/eigen3-nnls
1212
url = https://github.com/hmatuschek/eigen3-nnls.git
13+
[submodule "3rdparty/mexplus"]
14+
path = 3rdparty/mexplus
15+
url = https://github.com/kyamagu/mexplus.git

3rdparty/mexplus

Submodule mexplus added at 58b8487

CMakeLists.txt

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,9 @@ option(BUILD_DOCUMENTATION "Build the library documentation." OFF)
6262
message(STATUS "BUILD_DOCUMENTATION: ${BUILD_DOCUMENTATION}")
6363
option(GENERATE_PYTHON_BINDINGS "Build python bindings. Needs BUILD_UTILS enabled too." OFF)
6464
message(STATUS "GENERATE_PYTHON_BINDINGS: ${GENERATE_PYTHON_BINDINGS}")
65+
option(GENERATE_MATLAB_BINDINGS "Build Matlab bindings. Requires Matlab with the compiler installed or the Matlab Compiler Runtime." OFF)
66+
message(STATUS "GENERATE_MATLAB_BINDINGS: ${GENERATE_MATLAB_BINDINGS}")
67+
6568

6669
# Build a CPack driven installer package:
6770
include(InstallRequiredSystemLibraries) # This module will include any runtime libraries that are needed by the project for the current platform
@@ -180,3 +183,7 @@ endif()
180183
if(BUILD_DOCUMENTATION)
181184
add_subdirectory(doc)
182185
endif()
186+
187+
if(GENERATE_MATLAB_BINDINGS)
188+
add_subdirectory(matlab)
189+
endif()

initial_cache.cmake.template

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,15 @@
2424
# ==============================
2525
#set(PYTHON_EXECUTABLE "C:\\Users\\user\\AppData\\Local\\Programs\\Python\\Python35\\python.exe" CACHE PATH "Path to the python interpreter." FORCE)
2626

27+
# Set the path to the Matlab root directory if you want to build the Matlab bindings and it doesn't find Matlab automatically:
28+
# ==============================
29+
#set(Matlab_ROOT_DIR "/opt/matlab" CACHE PATH "Path to the Matlab installation directory." FORCE)
30+
2731
# Configuration options
2832
# ==============================
2933
set(BUILD_EXAMPLES ON CACHE BOOL "Build the example applications." FORCE)
3034
set(BUILD_CERES_EXAMPLE OFF CACHE BOOL "Build the fit-model-ceres example (requires Ceres)." FORCE)
3135
set(BUILD_UTILS OFF CACHE BOOL "Build utility applications." FORCE)
3236
set(BUILD_DOCUMENTATION OFF CACHE BOOL "Build the library documentation." FORCE)
3337
set(GENERATE_PYTHON_BINDINGS OFF CACHE BOOL "Build python bindings. Needs BUILD_UTILS enabled too." FORCE)
38+
set(GENERATE_MATLAB_BINDINGS OFF CACHE BOOL "Build Matlab bindings. Requires Matlab with the compiler installed or the Matlab Compiler Runtime." FORCE)
Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
function [mesh, rendering_parameters] = fit_shape_and_pose(morphable_model, ...
2+
blendshapes, landmarks, landmark_mapper, image_width, image_height, ...
3+
edge_topology, contour_landmarks, model_contour, num_iterations, ...
4+
num_shape_coefficients_to_fit, lambda)
5+
% FIT_SHAPE_AND_POSE Fit a 3DMM shape model to landmarks.
6+
% [ mesh, rendering_parameters ] = FIT_SHAPE_AND_POSE(morphable_model, ...
7+
% blendshapes, landmarks, landmark_mapper, image_width, image_height, ...
8+
% edge_topology, contour_landmarks, model_contour, num_iterations, ...
9+
% num_shape_coefficients_to_fit, lambda)
10+
%
11+
% This function fits a 3D Morphable Model to landmarks in an image.
12+
% It fits the pose (camera), PCA shape model, and expression blendshapes
13+
% in an iterative way.
14+
%
15+
% landmarks must be a 68 x 2 matrix with ibug landmarks, in the order
16+
% from 1 to 68.
17+
%
18+
% Default values for some of the parameters:: num_iterations = 5,
19+
% num_shape_coefficients_to_fit = all (-1), and lambda = 30.0.
20+
%
21+
% Please see the C++ documentation for the description of the parameters:
22+
% http://patrikhuber.github.io/eos/doc/ (TODO: Update to v0.9.1!)
23+
%
24+
% NOTE: In contrast to the C++ function, this Matlab function expects the
25+
% morphable_model, blendshapes, landmark_mapper, edge_topology,
26+
% contour_landmarks and model_contour as *filenames* to the respective
27+
% files in the eos/share/ directory, and not the objects directly.
28+
29+
% We'll use default values to the following arguments, if they're not
30+
% provided:
31+
if (~exist('edge_topology', 'var')), edge_topology = '../share/sfm_3448_edge_topology.json'; end
32+
if (~exist('contour_landmarks', 'var')), contour_landmarks = '../share/ibug2did.txt'; end
33+
if (~exist('model_contour', 'var')), model_contour = '../share/model_contours.json'; end
34+
if (~exist('num_iterations', 'var')), num_iterations = 5; end
35+
if (~exist('num_shape_coefficients_to_fit', 'var')), num_shape_coefficients_to_fit = -1; end
36+
if (~exist('lambda', 'var')), lambda = 30.0; end
37+
38+
[ mesh, rendering_parameters ] = fitting(morphable_model, blendshapes, landmarks, landmark_mapper, image_width, image_height, edge_topology, contour_landmarks, model_contour, num_iterations, num_shape_coefficients_to_fit, lambda);
39+
40+
end
Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
/*
2+
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
3+
*
4+
* File: matlab/+eos/+fitting/private/fitting.cpp
5+
*
6+
* Copyright 2016 Patrik Huber
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
#include "eos/core/LandmarkMapper.hpp"
21+
#include "eos/morphablemodel/MorphableModel.hpp"
22+
#include "eos/morphablemodel/Blendshape.hpp"
23+
#include "eos/morphablemodel/EdgeTopology.hpp"
24+
#include "eos/fitting/contour_correspondence.hpp"
25+
#include "eos/fitting/fitting.hpp"
26+
#include "eos/fitting/RenderingParameters.hpp"
27+
#include "eos/render/Mesh.hpp"
28+
29+
#include "mexplus_eigen.hpp"
30+
#include "mexplus_eos_types.hpp"
31+
32+
#include "mexplus.h"
33+
34+
#include "Eigen/Core"
35+
36+
#include "opencv2/core/core.hpp"
37+
38+
#include "mex.h"
39+
//#include "matrix.h"
40+
41+
#include <string>
42+
43+
using namespace eos;
44+
using namespace mexplus;
45+
46+
void mexFunction(int nlhs, mxArray *plhs[], int nrhs, const mxArray *prhs[])
47+
{
48+
using std::string;
49+
// Check for proper number of input and output arguments:
50+
if (nrhs != 12) {
51+
mexErrMsgIdAndTxt("eos:fitting:nargin", "fit_shape_and_pose requires 12 input arguments.");
52+
}
53+
if (nlhs != 2) {
54+
mexErrMsgIdAndTxt("eos:fitting:nargout", "fit_shape_and_pose returns two output arguments.");
55+
}
56+
57+
InputArguments input(nrhs, prhs, 12);
58+
auto morphablemodel_file = input.get<string>(0);
59+
auto blendshapes_file = input.get<string>(1);
60+
auto landmarks_in = input.get<Eigen::MatrixXd>(2);
61+
auto mapper_file = input.get<string>(3);
62+
auto image_width = input.get<int>(4);
63+
auto image_height = input.get<int>(5);
64+
auto edgetopo_file = input.get<string>(6);
65+
auto contour_lms_file = input.get<string>(7);
66+
auto model_cnt_file = input.get<string>(8);
67+
auto num_iterations = input.get<int>(9);
68+
auto num_shape_coeffs = input.get<int>(10);
69+
auto lambda = input.get<double>(11);
70+
71+
if (landmarks_in.rows() != 68) {
72+
mexErrMsgIdAndTxt("eos:fitting:argin", "Given landmarks must be a 68 x 2 vector with ibug landmarks, in the order from 1 to 68.");
73+
}
74+
// Convert the landmarks (given as matrix in Matlab) to a LandmarkCollection:
75+
core::LandmarkCollection<cv::Vec2f> landmarks;
76+
for (int i = 0; i < 68; ++i)
77+
{
78+
landmarks.push_back(core::Landmark<cv::Vec2f>{ std::to_string(i + 1), cv::Vec2f(landmarks_in(i, 0), landmarks_in(i, 1)) });
79+
}
80+
81+
// Load everything:
82+
const auto morphable_model = morphablemodel::load_model(morphablemodel_file);
83+
auto blendshapes = morphablemodel::load_blendshapes(blendshapes_file);
84+
core::LandmarkMapper landmark_mapper(mapper_file);
85+
auto edge_topology = morphablemodel::load_edge_topology(edgetopo_file);
86+
auto contour_landmarks = fitting::ContourLandmarks::load(contour_lms_file);
87+
auto model_contour = fitting::ModelContour::load(model_cnt_file);
88+
boost::optional<int> num_shape_coefficients_to_fit = num_shape_coeffs == -1 ? boost::none : boost::optional<int>(num_shape_coeffs);
89+
90+
// Now do the actual fitting:
91+
render::Mesh mesh;
92+
fitting::RenderingParameters rendering_parameters;
93+
std::tie(mesh, rendering_parameters) = fitting::fit_shape_and_pose(morphable_model, blendshapes, landmarks, landmark_mapper, image_width, image_height, edge_topology, contour_landmarks, model_contour, num_iterations, num_shape_coefficients_to_fit, lambda);
94+
95+
// C++ counts the vertex indices starting at zero, Matlab starts counting
96+
// at one - therefore, add +1 to all triangle indices:
97+
for (auto&& t : mesh.tvi) {
98+
for (auto&& idx : t) {
99+
idx += 1;
100+
}
101+
}
102+
103+
// Return the mesh and the rendering_parameters:
104+
OutputArguments output(nlhs, plhs, 2);
105+
output.set(0, mesh);
106+
output.set(1, rendering_parameters);
107+
};
108+
109+
void func()
110+
{
111+
int x = 4;
112+
};
113+
114+
int func1()
115+
{
116+
return 5;
117+
};
118+
119+
class MyClass
120+
{
121+
public:
122+
MyClass() = default;
123+
int test() {
124+
return 6;
125+
};
126+
};

matlab/CMakeLists.txt

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
project(matlab-bindings)
2+
cmake_minimum_required(VERSION 3.7.0)
3+
4+
# If Matlab_ROOT_DIR is set, the Matlab at that location is used.
5+
find_package(Matlab COMPONENTS MX_LIBRARY REQUIRED)
6+
7+
# Dependencies of the modules:
8+
find_package(OpenCV 2.4.3 REQUIRED core)
9+
set_target_properties(${OpenCV_LIBS} PROPERTIES MAP_IMPORTED_CONFIG_RELWITHDEBINFO RELEASE)
10+
if(MSVC)
11+
# The standard find_package for boost on Win finds the dynamic libs, so for dynamic linking to boost we need to #define:
12+
add_definitions(-DBOOST_ALL_NO_LIB) # Don't use the automatic library linking by boost with VS (#pragma ...). Instead, we specify everything here in cmake.
13+
add_definitions(-DBOOST_ALL_DYN_LINK) # Link against the dynamic boost lib - needs to match with the version that find_package finds.
14+
endif()
15+
find_package(Boost 1.50.0 COMPONENTS system filesystem REQUIRED) # Why do we need boost for MorphableModel.hpp?
16+
17+
# See: https://cmake.org/cmake/help/v3.7/module/FindMatlab.html?highlight=findmatlab#command:matlab_add_mex
18+
matlab_add_mex(
19+
NAME eos_fitting
20+
#[EXECUTABLE | MODULE | SHARED] # SHARED is the default.
21+
SRC +eos/+fitting/private/fitting.cpp
22+
OUTPUT_NAME fitting #[OUTPUT_NAME output_name]
23+
# DOCUMENTATION +eos/+fitting/fit_shape_and_pose.m # doesn't work - wrong path probably. But it renames the file to fitting.m, so not what we want anyway.
24+
LINK_TO ${OpenCV_LIBS} ${Boost_LIBRARIES} #[LINK_TO target1 target2 ...]
25+
#[...]
26+
)
27+
28+
target_include_directories(eos_fitting PRIVATE ${CMAKE_SOURCE_DIR}/3rdparty/mexplus/include ${CMAKE_SOURCE_DIR}/matlab/include)
29+
30+
install(FILES ${CMAKE_SOURCE_DIR}/matlab/demo.m DESTINATION matlab)
31+
install(DIRECTORY ${CMAKE_SOURCE_DIR}/matlab/include DESTINATION matlab)
32+
install(DIRECTORY ${CMAKE_SOURCE_DIR}/matlab/+eos DESTINATION matlab PATTERN "*.cpp" EXCLUDE)
33+
install(TARGETS eos_fitting DESTINATION matlab/+eos/+fitting/private)

matlab/demo.m

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
%% Demo for running the eos fitting from Matlab
2+
%
3+
%% Set up some required paths to files:
4+
model_file = '../share/sfm_shape_3448.bin';
5+
blendshapes_file = '../share/expression_blendshapes_3448.bin';
6+
landmark_mappings = '../share/ibug2did.txt';
7+
8+
%% Load an image and its landmarks in ibug format:
9+
image = imread('../bin/data/image_0010.png');
10+
landmarks = read_pts_landmarks('../bin/data/image_0010.pts');
11+
image_width = size(image, 2); image_height = size(image, 1);
12+
13+
%% Run the fitting, get back the fitted mesh and pose:
14+
[mesh, render_params] = eos.fitting.fit_shape_and_pose(model_file, blendshapes_file, landmarks, landmark_mappings, image_width, image_height);
15+
% Note: The function actually has a few more arguments to files it
16+
% needs. If you're not running it from within eos/matlab/, you need to
17+
% provide them. See its documentation and .m file.
18+
19+
%% Visualise the fitted mesh using your favourite plot, for example...
20+
figure(1);
21+
plot3(mesh.vertices(:, 1), mesh.vertices(:, 2), mesh.vertices(:, 3), '.');
22+
% or...
23+
FV.vertices = mesh.vertices(:, 1:3);
24+
FV.faces = mesh.tvi;
25+
figure(2);
26+
patch(FV, 'FaceColor', [1 1 1], 'EdgeColor', 'none', 'FaceLighting', 'phong'); light; axis equal; axis off;
27+
28+
%% Visualise the fitting in 2D, on top of the input image:
29+
% Project all vertices to 2D:
30+
points_2d = mesh.vertices * (render_params.viewport*render_params.projection*render_params.modelview)';
31+
% Display the image and plot the projected mesh points on top of it:
32+
figure(3);
33+
imshow(image);
34+
hold on;
35+
plot(points_2d(:, 1), points_2d(:, 2), 'g.');
36+
% We can also plot the landmarks the mesh was fitted to:
37+
plot(landmarks(:, 1), landmarks(:, 2), 'ro');
38+
39+
40+
%% Just a helper function to read ibug .pts landmarks from a file:
41+
function [ landmarks ] = read_pts_landmarks(filename)
42+
43+
file = fopen(filename, 'r');
44+
file_content = textscan(file, '%s');
45+
46+
landmarks = zeros(68, 2);
47+
48+
row_idx = 1;
49+
for i=6:2:141
50+
landmarks(row_idx, 1) = str2double(file_content{1}{i});
51+
landmarks(row_idx, 2) = str2double(file_content{1}{i + 1});
52+
row_idx = row_idx + 1;
53+
end
54+
55+
end

matlab/include/mexplus_eigen.hpp

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* eos - A 3D Morphable Model fitting library written in modern C++11/14.
3+
*
4+
* File: matlab/include/mexplus_eigen.hpp
5+
*
6+
* Copyright 2016 Patrik Huber
7+
*
8+
* Licensed under the Apache License, Version 2.0 (the "License");
9+
* you may not use this file except in compliance with the License.
10+
* You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
#pragma once
21+
22+
#ifndef MEXPLUS_EIGEN_HPP_
23+
#define MEXPLUS_EIGEN_HPP_
24+
25+
#include "mexplus/mxarray.h"
26+
27+
#include "Eigen/Core"
28+
29+
#include "mex.h"
30+
31+
namespace mexplus {
32+
33+
/**
34+
* @brief Define a template specialisation for Eigen::MatrixXd for ... .
35+
*
36+
* Todo: Documentation.
37+
*/
38+
template<>
39+
mxArray* MxArray::from(const Eigen::MatrixXd& eigen_matrix) {
40+
const int num_rows = static_cast<int>(eigen_matrix.rows());
41+
const int num_cols = static_cast<int>(eigen_matrix.cols());
42+
MxArray out_array(MxArray::Numeric<double>(num_rows, num_cols));
43+
44+
// This might not copy the data but it's evil and probably really dangerous!!!:
45+
//mxSetData(const_cast<mxArray*>(matrix.get()), (void*)value.data());
46+
47+
// This copies the data. But I suppose it makes sense that we copy the data when we go
48+
// from C++ to Matlab, since Matlab can unload the C++ mex module at any time I think.
49+
// Loop is column-wise
50+
for (int c = 0; c < num_cols; ++c) {
51+
for (int r = 0; r < num_rows; ++r) {
52+
out_array.set(r, c, eigen_matrix(r, c));
53+
}
54+
}
55+
return out_array.release();
56+
};
57+
58+
/**
59+
* @brief Define a template specialisation for Eigen::MatrixXd for ... .
60+
*
61+
* Todo: Documentation.
62+
*/
63+
template<>
64+
void MxArray::to(const mxArray* in_array, Eigen::MatrixXd* eigen_matrix)
65+
{
66+
MxArray array(in_array);
67+
68+
if (array.dimensionSize() > 2)
69+
{
70+
mexErrMsgIdAndTxt("eos:matlab", "Given array has > 2 dimensions. Can only create 2-dimensional matrices (and vectors).");
71+
}
72+
73+
if (array.dimensionSize() == 1 || array.dimensionSize() == 0)
74+
{
75+
mexErrMsgIdAndTxt("eos:matlab", "Given array has 0 or 1 dimensions but we expected a 2-dimensional matrix (or row/column vector).");
76+
// Even when given a single value dimensionSize() is 2, with n=m=1. When does this happen?
77+
}
78+
79+
if (!array.isDouble())
80+
{
81+
mexErrMsgIdAndTxt("eos:matlab", "Trying to create a Eigen::MatrixXd in C++, but the given data is not of type double.");
82+
}
83+
84+
// We can be sure now that the array is 2-dimensional (or 0, but then we're screwed anyway)
85+
auto nrows = array.dimensions()[0]; // or use array.rows()
86+
auto ncols = array.dimensions()[1];
87+
88+
// I think I can just use Eigen::Matrix, not a Map - the Matrix c'tor that we call creates a Map anyway?
89+
Eigen::Map<Eigen::Matrix<double, Eigen::Dynamic, Eigen::Dynamic, Eigen::ColMajor>> eigen_map(array.getData<double>(), nrows, ncols);
90+
// Not sure that's alright - who owns the data? I think as it is now, everything points to the data in the mxArray owned by Matlab, but I'm not 100% sure.
91+
*eigen_matrix = eigen_map;
92+
};
93+
94+
} /* namespace mexplus */
95+
96+
#endif /* MEXPLUS_EIGEN_HPP_ */

0 commit comments

Comments
 (0)