|
| 1 | +/* |
| 2 | + * eos - A 3D Morphable Model fitting library written in modern C++11/14. |
| 3 | + * |
| 4 | + * File: examples/fit-model-simple.cpp |
| 5 | + * |
| 6 | + * Copyright 2015 Patrik Huber |
| 7 | + * |
| 8 | + * Licensed under the Apache License, Version 2.0 (the "License"); |
| 9 | + * you may not use this file except in compliance with the License. |
| 10 | + * You may obtain a copy of the License at |
| 11 | + * |
| 12 | + * http://www.apache.org/licenses/LICENSE-2.0 |
| 13 | + * |
| 14 | + * Unless required by applicable law or agreed to in writing, software |
| 15 | + * distributed under the License is distributed on an "AS IS" BASIS, |
| 16 | + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 17 | + * See the License for the specific language governing permissions and |
| 18 | + * limitations under the License. |
| 19 | + */ |
| 20 | +#include "eos/core/Landmark.hpp" |
| 21 | +#include "eos/core/LandmarkMapper.hpp" |
| 22 | +#include "eos/fitting/orthographic_camera_estimation_linear.hpp" |
| 23 | +#include "eos/fitting/RenderingParameters.hpp" |
| 24 | +#include "eos/fitting/linear_shape_fitting.hpp" |
| 25 | +#include "eos/render/utils.hpp" |
| 26 | +#include "eos/render/texture_extraction.hpp" |
| 27 | + |
| 28 | +#include "opencv2/core/core.hpp" |
| 29 | +#include "opencv2/highgui/highgui.hpp" |
| 30 | + |
| 31 | +#include "boost/program_options.hpp" |
| 32 | +#include "boost/filesystem.hpp" |
| 33 | + |
| 34 | +#include <vector> |
| 35 | +#include <iostream> |
| 36 | +#include <fstream> |
| 37 | + |
| 38 | +using namespace eos; |
| 39 | +namespace po = boost::program_options; |
| 40 | +namespace fs = boost::filesystem; |
| 41 | +using eos::core::Landmark; |
| 42 | +using eos::core::LandmarkCollection; |
| 43 | +using cv::Mat; |
| 44 | +using cv::Vec2f; |
| 45 | +using cv::Vec3f; |
| 46 | +using cv::Vec4f; |
| 47 | +using std::cout; |
| 48 | +using std::endl; |
| 49 | +using std::vector; |
| 50 | +using std::string; |
| 51 | + |
| 52 | +/** |
| 53 | + * Reads an ibug .pts landmark file and returns an ordered vector with |
| 54 | + * the 68 2D landmark coordinates. |
| 55 | + * |
| 56 | + * @param[in] filename Path to a .pts file. |
| 57 | + * @return An ordered vector with the 68 ibug landmarks. |
| 58 | + */ |
| 59 | +LandmarkCollection<cv::Vec2f> read_pts_landmarks(std::string filename) |
| 60 | +{ |
| 61 | + using std::getline; |
| 62 | + using cv::Vec2f; |
| 63 | + using std::string; |
| 64 | + LandmarkCollection<Vec2f> landmarks; |
| 65 | + landmarks.reserve(68); |
| 66 | + |
| 67 | + std::ifstream file(filename); |
| 68 | + if (!file.is_open()) { |
| 69 | + throw std::runtime_error(string("Could not open landmark file: " + filename)); |
| 70 | + } |
| 71 | + |
| 72 | + string line; |
| 73 | + // Skip the first 3 lines, they're header lines: |
| 74 | + getline(file, line); // 'version: 1' |
| 75 | + getline(file, line); // 'n_points : 68' |
| 76 | + getline(file, line); // '{' |
| 77 | + |
| 78 | + int ibugId = 1; |
| 79 | + while (getline(file, line)) |
| 80 | + { |
| 81 | + if (line == "}") { // end of the file |
| 82 | + break; |
| 83 | + } |
| 84 | + std::stringstream lineStream(line); |
| 85 | + |
| 86 | + Landmark<Vec2f> landmark; |
| 87 | + landmark.name = std::to_string(ibugId); |
| 88 | + if (!(lineStream >> landmark.coordinates[0] >> landmark.coordinates[1])) { |
| 89 | + throw std::runtime_error(string("Landmark format error while parsing the line: " + line)); |
| 90 | + } |
| 91 | + // From the iBug website: |
| 92 | + // "Please note that the re-annotated data for this challenge are saved in the Matlab convention of 1 being |
| 93 | + // the first index, i.e. the coordinates of the top left pixel in an image are x=1, y=1." |
| 94 | + // ==> So we shift every point by 1: |
| 95 | + landmark.coordinates[0] -= 1.0f; |
| 96 | + landmark.coordinates[1] -= 1.0f; |
| 97 | + landmarks.emplace_back(landmark); |
| 98 | + ++ibugId; |
| 99 | + } |
| 100 | + return landmarks; |
| 101 | +}; |
| 102 | + |
| 103 | +/** |
| 104 | + * This app demonstrates estimation of the camera and fitting of the shape |
| 105 | + * model of a 3D Morphable Model from an ibug LFPW image with its landmarks. |
| 106 | + * |
| 107 | + * First, the 68 ibug landmarks are loaded from the .pts file and converted |
| 108 | + * to vertex indices using the LandmarkMapper. Then, an orthographic camera |
| 109 | + * is estimated, and then, using this camera matrix, the shape is fitted |
| 110 | + * to the landmarks. |
| 111 | + */ |
| 112 | +int main(int argc, char *argv[]) |
| 113 | +{ |
| 114 | + fs::path modelfile, isomapfile, imagefile, landmarksfile, mappingsfile, outputfile; |
| 115 | + try { |
| 116 | + po::options_description desc("Allowed options"); |
| 117 | + desc.add_options() |
| 118 | + ("help,h", |
| 119 | + "display the help message") |
| 120 | + ("model,m", po::value<fs::path>(&modelfile)->required()->default_value("../share/sfm_shape_3448.bin"), |
| 121 | + "a Morphable Model stored as cereal BinaryArchive") |
| 122 | + ("image,i", po::value<fs::path>(&imagefile)->required()->default_value("data/image_0010.png"), |
| 123 | + "an input image") |
| 124 | + ("landmarks,l", po::value<fs::path>(&landmarksfile)->required()->default_value("data/image_0010.pts"), |
| 125 | + "2D landmarks for the image, in ibug .pts format") |
| 126 | + ("mapping,p", po::value<fs::path>(&mappingsfile)->required()->default_value("../share/ibug2did.txt"), |
| 127 | + "landmark identifier to model vertex number mapping") |
| 128 | + ("output,o", po::value<fs::path>(&outputfile)->required()->default_value("out"), |
| 129 | + "basename for the output rendering and obj files") |
| 130 | + ; |
| 131 | + po::variables_map vm; |
| 132 | + po::store(po::command_line_parser(argc, argv).options(desc).run(), vm); |
| 133 | + if (vm.count("help")) { |
| 134 | + cout << "Usage: fit-model-simple [options]" << endl; |
| 135 | + cout << desc; |
| 136 | + return EXIT_SUCCESS; |
| 137 | + } |
| 138 | + po::notify(vm); |
| 139 | + } |
| 140 | + catch (const po::error& e) { |
| 141 | + cout << "Error while parsing command-line arguments: " << e.what() << endl; |
| 142 | + cout << "Use --help to display a list of options." << endl; |
| 143 | + return EXIT_SUCCESS; |
| 144 | + } |
| 145 | + |
| 146 | + // Load the image, landmarks, LandmarkMapper and the Morphable Model: |
| 147 | + Mat image = cv::imread(imagefile.string()); |
| 148 | + LandmarkCollection<cv::Vec2f> landmarks; |
| 149 | + try { |
| 150 | + landmarks = read_pts_landmarks(landmarksfile.string()); |
| 151 | + } |
| 152 | + catch (const std::runtime_error& e) { |
| 153 | + cout << "Error reading the landmarks: " << e.what() << endl; |
| 154 | + return EXIT_FAILURE; |
| 155 | + } |
| 156 | + morphablemodel::MorphableModel morphable_model; |
| 157 | + try { |
| 158 | + morphable_model = morphablemodel::load_model(modelfile.string()); |
| 159 | + } |
| 160 | + catch (const std::runtime_error& e) { |
| 161 | + cout << "Error loading the Morphable Model: " << e.what() << endl; |
| 162 | + return EXIT_FAILURE; |
| 163 | + } |
| 164 | + core::LandmarkMapper landmark_mapper = mappingsfile.empty() ? core::LandmarkMapper() : core::LandmarkMapper(mappingsfile); |
| 165 | + |
| 166 | + // Draw the loaded landmarks: |
| 167 | + Mat outimg = image.clone(); |
| 168 | + for (auto&& lm : landmarks) { |
| 169 | + cv::rectangle(outimg, cv::Point2f(lm.coordinates[0] - 2.0f, lm.coordinates[1] - 2.0f), cv::Point2f(lm.coordinates[0] + 2.0f, lm.coordinates[1] + 2.0f), { 255, 0, 0 }); |
| 170 | + } |
| 171 | + |
| 172 | + // These will be the final 2D and 3D points used for the fitting: |
| 173 | + vector<Vec4f> model_points; // the points in the 3D shape model |
| 174 | + vector<int> vertex_indices; // their vertex indices |
| 175 | + vector<Vec2f> image_points; // the corresponding 2D landmark points |
| 176 | + |
| 177 | + // Sub-select all the landmarks which we have a mapping for (i.e. that are defined in the 3DMM): |
| 178 | + for (int i = 0; i < landmarks.size(); ++i) { |
| 179 | + auto converted_name = landmark_mapper.convert(landmarks[i].name); |
| 180 | + if (!converted_name) { // no mapping defined for the current landmark |
| 181 | + continue; |
| 182 | + } |
| 183 | + int vertex_idx = std::stoi(converted_name.get()); |
| 184 | + Vec4f vertex = morphable_model.get_shape_model().get_mean_at_point(vertex_idx); |
| 185 | + model_points.emplace_back(vertex); |
| 186 | + vertex_indices.emplace_back(vertex_idx); |
| 187 | + image_points.emplace_back(landmarks[i].coordinates); |
| 188 | + } |
| 189 | + |
| 190 | + // Estimate the camera (pose) from the 2D - 3D point correspondences |
| 191 | + fitting::ScaledOrthoProjectionParameters pose = fitting::estimate_orthographic_projection_linear(image_points, model_points, true, image.rows); |
| 192 | + fitting::RenderingParameters rendering_params(pose, image.cols, image.rows); |
| 193 | + |
| 194 | + // The 3D head pose can be recovered as follows: |
| 195 | + float yaw_angle = glm::degrees(glm::yaw(rendering_params.get_rotation())); |
| 196 | + // and similarly for pitch and roll. |
| 197 | + |
| 198 | + // Estimate the shape coefficients by fitting the shape to the landmarks: |
| 199 | + Mat affine_from_ortho = fitting::get_3x4_affine_camera_matrix(rendering_params, image.cols, image.rows); |
| 200 | + vector<float> fitted_coeffs = fitting::fit_shape_to_landmarks_linear(morphable_model, affine_from_ortho, image_points, vertex_indices); |
| 201 | + |
| 202 | + // Obtain the full mesh with the estimated coefficients: |
| 203 | + render::Mesh mesh = morphable_model.draw_sample(fitted_coeffs, vector<float>()); |
| 204 | + |
| 205 | + // Extract the texture from the image using given mesh and camera parameters: |
| 206 | + Mat isomap = render::extract_texture(mesh, affine_from_ortho, image); |
| 207 | + |
| 208 | + // Save the mesh as textured obj: |
| 209 | + outputfile += fs::path(".obj"); |
| 210 | + render::write_textured_obj(mesh, outputfile.string()); |
| 211 | + |
| 212 | + // And save the isomap: |
| 213 | + outputfile.replace_extension(".isomap.png"); |
| 214 | + cv::imwrite(outputfile.string(), isomap); |
| 215 | + |
| 216 | + cout << "Finished fitting and wrote result mesh and isomap to files with basename " << outputfile.stem().stem() << "." << endl; |
| 217 | + |
| 218 | + return EXIT_SUCCESS; |
| 219 | +} |
0 commit comments