Skip to content

Commit 243dc80

Browse files
BycobBycob
committed
feat(torch): add inference visualisation at test time
1 parent 0ef1f46 commit 243dc80

File tree

5 files changed

+129
-3
lines changed

5 files changed

+129
-3
lines changed

‎src/backends/torch/torchlib.cc

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2014,6 +2014,7 @@ namespace dd
20142014
{
20152015
APIData ad_res;
20162016
APIData ad_bbox;
2017+
APIData ad_res_bbox;
20172018
APIData ad_out = ad.getobj("parameters").getobj("output");
20182019
int nclasses = _masked_lm ? inputc.vocab_size() : _nclasses;
20192020

@@ -2159,6 +2160,59 @@ namespace dd
21592160
ad_bbox_per_iou[iou_thres].add(std::to_string(entry_id),
21602161
vbad);
21612162
}
2163+
2164+
// Raw results
2165+
APIData bad;
2166+
// predictions
2167+
auto bboxes_acc = bboxes_tensor.accessor<float, 2>();
2168+
auto labels_acc = labels_tensor.accessor<int64_t, 1>();
2169+
auto score_acc = score_tensor.accessor<float, 1>();
2170+
std::vector<APIData> pred_vad;
2171+
2172+
for (int k = 0; k < labels_tensor.size(0); k++)
2173+
{
2174+
APIData pred_ad;
2175+
pred_ad.add("label", labels_acc[k]);
2176+
pred_ad.add("prob", static_cast<double>(score_acc[k]));
2177+
APIData bbox_ad;
2178+
bbox_ad.add("xmin", static_cast<double>(bboxes_acc[k][0]));
2179+
bbox_ad.add("ymin", static_cast<double>(bboxes_acc[k][1]));
2180+
bbox_ad.add("xmax", static_cast<double>(bboxes_acc[k][2]));
2181+
bbox_ad.add("ymax", static_cast<double>(bboxes_acc[k][3]));
2182+
pred_ad.add("bbox", bbox_ad);
2183+
pred_vad.push_back(pred_ad);
2184+
}
2185+
bad.add("predictions", pred_vad);
2186+
// targets
2187+
auto targ_bboxes_acc = targ_bboxes.accessor<float, 2>();
2188+
auto targ_labels_acc = targ_labels.accessor<int64_t, 1>();
2189+
std::vector<APIData> targ_vad;
2190+
2191+
for (int k = start; k < stop; k++)
2192+
{
2193+
APIData targ_ad;
2194+
targ_ad.add("label", targ_labels_acc[k]);
2195+
APIData bbox_ad;
2196+
bbox_ad.add("xmin",
2197+
static_cast<double>(targ_bboxes_acc[k][0]));
2198+
bbox_ad.add("ymin",
2199+
static_cast<double>(targ_bboxes_acc[k][1]));
2200+
bbox_ad.add("xmax",
2201+
static_cast<double>(targ_bboxes_acc[k][2]));
2202+
bbox_ad.add("ymax",
2203+
static_cast<double>(targ_bboxes_acc[k][3]));
2204+
targ_ad.add("bbox", bbox_ad);
2205+
targ_vad.push_back(targ_ad);
2206+
}
2207+
bad.add("targets", targ_vad);
2208+
// pred image
2209+
std::vector<cv::Mat> img_vec;
2210+
img_vec.push_back(torch_utils::tensorToImage(
2211+
batch.data.at(0).index(
2212+
{ torch::indexing::Slice(i, i + 1) }),
2213+
/* rgb = */ true));
2214+
bad.add("image", img_vec);
2215+
ad_res_bbox.add(std::to_string(entry_id), bad);
21622216
++entry_id;
21632217
}
21642218
}
@@ -2340,12 +2394,17 @@ namespace dd
23402394
ad_bbox_per_iou[iou_thres]);
23412395
}
23422396
ad_res.add("0", ad_bbox);
2397+
// raw bbox results
2398+
ad_res.add("raw_bboxes", ad_res_bbox);
23432399
}
23442400
else if (_segmentation)
23452401
ad_res.add("segmentation", true);
23462402
ad_res.add("batch_size",
23472403
entry_id); // here batch_size = tested entries count
23482404
SupervisedOutput::measure(ad_res, ad_out, out, test_id, test_name);
2405+
SupervisedOutput::create_visuals(
2406+
ad_res, ad_out, this->_mlmodel._repo + this->_mlmodel._visuals_dir,
2407+
test_id);
23492408
_module.train();
23502409
return 0;
23512410
}

‎src/backends/torch/torchutils.cc

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ namespace dd
256256
torch_utils::copy_weights(jit_module, module, device, logger, strict);
257257
}
258258

259-
cv::Mat tensorToImage(torch::Tensor tensor)
259+
cv::Mat tensorToImage(torch::Tensor tensor, bool rgb)
260260
{
261261
// 4 channels: batch size, chan, width, height
262262
auto dims = tensor.sizes();
@@ -285,6 +285,13 @@ namespace dd
285285
}
286286
}
287287
}
288+
289+
// convert to bgr
290+
if (rgb)
291+
{
292+
cv::cvtColor(vals_mat, vals_mat, cv::COLOR_RGB2BGR);
293+
}
294+
288295
return vals_mat;
289296
}
290297
}

‎src/backends/torch/torchutils.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -158,8 +158,9 @@ namespace dd
158158

159159
/** Converts a tensor to a CV image that can be saved on the disk.
160160
* XXX(louis) this function is currently debug only, and makes strong
161-
* assumptions on the input tensor format. */
162-
cv::Mat tensorToImage(torch::Tensor tensor);
161+
* assumptions on the input tensor format.
162+
* \param rgb wether the tensor image is rgb */
163+
cv::Mat tensorToImage(torch::Tensor tensor, bool rgb = false);
163164
}
164165
}
165166
#endif

‎src/mlmodel.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,7 @@ namespace dd
181181
std::string _corresp; /**< file name of the class correspondences (e.g.
182182
house / 23) */
183183
std::string _best_model_filename = "/best_model.txt";
184+
std::string _visuals_dir = "/visuals";
184185

185186
#ifdef USE_SIMSEARCH
186187
#ifdef USE_ANNOY

‎src/supervisedoutputconnector.h

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@
2626
#include <sstream>
2727
#include <iomanip>
2828

29+
#include <opencv2/opencv.hpp>
30+
2931
#include "dto/output_connector.hpp"
3032
#include "dto/predict_out.hpp"
3133

@@ -1402,6 +1404,62 @@ namespace dd
14021404
ad_out.add("measure", meas_obj);
14031405
}
14041406

1407+
/** Create visuals from test results and write them in the model
1408+
* repository. */
1409+
static void create_visuals(const APIData &ad_res, APIData &ad_out,
1410+
const std::string &visuals_folder, int test_id)
1411+
{
1412+
(void)ad_out;
1413+
int iteration = static_cast<int>(ad_res.get("iteration").get<double>());
1414+
bool bbox = ad_res.has("bbox") && ad_res.get("bbox").get<bool>();
1415+
std::string targ_dest_folder
1416+
= visuals_folder + "/target/test" + std::to_string(test_id);
1417+
std::string dest_folder = visuals_folder + "/iteration"
1418+
+ std::to_string(iteration) + "/test"
1419+
+ std::to_string(test_id);
1420+
fileops::create_dir(dest_folder, 0755);
1421+
1422+
cv::Scalar colors[]
1423+
= { { 255, 0, 0 }, { 0, 255, 0 }, { 0, 0, 255 },
1424+
{ 0, 255, 255 }, { 255, 0, 255 }, { 255, 255, 0 },
1425+
{ 255, 127, 127 }, { 127, 255, 127 }, { 127, 127, 255 } };
1426+
int ncolors = sizeof(colors) / sizeof(cv::Scalar);
1427+
1428+
if (bbox)
1429+
{
1430+
APIData images_data = ad_res.getobj("raw_bboxes");
1431+
1432+
for (size_t i = 0; i < images_data.size(); ++i)
1433+
{
1434+
APIData bad = images_data.getobj(std::to_string(i));
1435+
cv::Mat img = bad.get("image").get<std::vector<cv::Mat>>().at(0);
1436+
1437+
// pred
1438+
std::vector<APIData> preds = bad.getv("predictions");
1439+
for (size_t k = 0; k < preds.size(); ++k)
1440+
{
1441+
APIData &pred_ad = preds[k];
1442+
// float score = pred_ad.get("prob").get<float>();
1443+
int64_t label = pred_ad.get("label").get<int64_t>();
1444+
APIData bbox = pred_ad.getobj("bbox");
1445+
int xmin = static_cast<int>(bbox.get("xmin").get<double>());
1446+
int ymin = static_cast<int>(bbox.get("ymin").get<double>());
1447+
int xmax = static_cast<int>(bbox.get("xmax").get<double>());
1448+
int ymax = static_cast<int>(bbox.get("ymax").get<double>());
1449+
1450+
auto &color = colors[label % ncolors];
1451+
cv::rectangle(img, cv::Point{ xmin, ymin },
1452+
cv::Point{ xmax, ymax }, color, 3);
1453+
}
1454+
1455+
// write image
1456+
std::string out_img_path
1457+
= dest_folder + "/image" + std::to_string(i) + ".jpg";
1458+
cv::imwrite(out_img_path, img);
1459+
}
1460+
}
1461+
}
1462+
14051463
static void
14061464
timeSeriesMetrics(const APIData &ad, const int timeseries,
14071465
std::vector<double> &mape, std::vector<double> &smape,

0 commit comments

Comments
 (0)