Skip to content
This repository was archived by the owner on Jul 2, 2021. It is now read-only.

Commit ab75bf7

Browse files
yuyu2172yuyu2172
committed
merge to fpn
1 parent 0840f0c commit ab75bf7

File tree

12 files changed

+207
-535
lines changed

12 files changed

+207
-535
lines changed

‎chainercv/links/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
from chainercv.links.model.faster_rcnn.faster_rcnn_vgg import FasterRCNNVGG16 # NOQA
1212
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import FasterRCNNFPNResNet101 # NOQA
1313
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import FasterRCNNFPNResNet50 # NOQA
14+
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import KeypointRCNNFPNResNet101 # NOQA
15+
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import KeypointRCNNFPNResNet50 # NOQA
1416
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import MaskRCNNFPNResNet101 # NOQA
1517
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import MaskRCNNFPNResNet50 # NOQA
1618
from chainercv.links.model.resnet import ResNet101 # NOQA

‎chainercv/links/model/fpn/__init__.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
from chainercv.links.model.fpn.faster_rcnn import FasterRCNN # NOQA
22
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import FasterRCNNFPNResNet101 # NOQA
33
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import FasterRCNNFPNResNet50 # NOQA
4+
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import KeypointRCNNFPNResNet101 # NOQA
5+
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import KeypointRCNNFPNResNet50 # NOQA
46
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import MaskRCNNFPNResNet101 # NOQA
57
from chainercv.links.model.fpn.faster_rcnn_fpn_resnet import MaskRCNNFPNResNet50 # NOQA
68
from chainercv.links.model.fpn.fpn import FPN # NOQA

‎chainercv/links/model/fpn/faster_rcnn.py‎

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -50,10 +50,11 @@ class FasterRCNN(chainer.Chain):
5050
"""
5151

5252
stride = 32
53-
_accepted_return_values = ('rois', 'bboxes', 'labels', 'scores', 'masks')
53+
_accepted_return_values = ('rois', 'bboxes', 'labels', 'scores',
54+
'masks', 'points', 'point_scores')
5455

5556
def __init__(self, extractor, rpn, bbox_head,
56-
mask_head, return_values,
57+
mask_head, keypoint_head, return_values,
5758
min_size=800, max_size=1333):
5859
for value_name in return_values:
5960
if value_name not in self._accepted_return_values:
@@ -64,8 +65,10 @@ def __init__(self, extractor, rpn, bbox_head,
6465

6566
self._store_rpn_outputs = 'rois' in self._return_values
6667
self._run_bbox = any([key in self._return_values
67-
for key in ['bboxes', 'labels', 'scores', 'masks']])
68+
for key in ['bboxes', 'labels', 'scores',
69+
'masks', 'points', 'point_scores']])
6870
self._run_mask = 'masks' in self._return_values
71+
self._run_keypoint = 'points' in self._return_values
6972
super(FasterRCNN, self).__init__()
7073

7174
with self.init_scope():
@@ -75,6 +78,8 @@ def __init__(self, extractor, rpn, bbox_head,
7578
self.bbox_head = bbox_head
7679
if self._run_mask:
7780
self.mask_head = mask_head
81+
if self._run_keypoint:
82+
self.keypoint_head = keypoint_head
7883

7984
self.min_size = min_size
8085
self.max_size = max_size
@@ -174,10 +179,9 @@ def predict(self, imgs):
174179
scores_cpu = [cuda.to_cpu(score) for score in scores]
175180
output.update({'bboxes': bboxes_cpu, 'labels': labels_cpu,
176181
'scores': scores_cpu})
177-
178-
if self._run_mask:
179182
rescaled_bboxes = [bbox * scale
180-
for scale, bbox in zip(scales, bboxes)]
183+
for scale, bbox in zip(scales, bboxes)]
184+
if self._run_mask:
181185
# Change bboxes to RoI and RoI indices format
182186
mask_rois_before_reordering, mask_roi_indices_before_reordering =\
183187
_list_to_flat(rescaled_bboxes)
@@ -200,6 +204,36 @@ def predict(self, imgs):
200204
# Currently MaskHead only supports numpy inputs
201205
masks_cpu = self.mask_head.decode(segms, bboxes_cpu, labels_cpu, sizes)
202206
output.update({'masks': masks_cpu})
207+
208+
if self._run_keypoint:
209+
(point_rois_before_reordering,
210+
point_roi_indices_before_reordering) = _list_to_flat(
211+
rescaled_bboxes)
212+
point_rois, point_roi_indices, order =\
213+
self.keypoint_head.distribute(
214+
point_rois_before_reordering,
215+
point_roi_indices_before_reordering)
216+
with chainer.using_config(
217+
'train', False), chainer.no_backprop_mode():
218+
point_maps = self.keypoint_head(
219+
hs, point_rois, point_roi_indices).data
220+
point_maps = point_maps[order]
221+
point_maps = _flat_to_list(
222+
point_maps, point_roi_indices_before_reordering, len(imgs))
223+
point_maps = [point_map if point_map is not None else
224+
self.xp.zeros(
225+
(0, self.keypoint_head.n_point,
226+
self.keypoint_head.point_map_size,
227+
self.keypoint_head.point_map_size),
228+
dtype=np.float32)
229+
for point_map in point_maps]
230+
point_maps = [
231+
chainer.backends.cuda.to_cpu(point_map)
232+
for point_map in point_maps]
233+
points_cpu, point_scores_cpu = self.keypoint_head.decode(
234+
point_maps, bboxes_cpu)
235+
output.update(
236+
{'points': points_cpu, 'point_scores': point_scores_cpu})
203237
return tuple([output[key] for key in self._return_values])
204238

205239
def prepare(self, imgs):

‎chainercv/links/model/fpn/faster_rcnn_fpn_resnet.py‎

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
from chainercv.links.model.fpn.faster_rcnn import FasterRCNN
88
from chainercv.links.model.fpn.fpn import FPN
99
from chainercv.links.model.fpn.bbox_head import BboxHead
10+
from chainercv.links.model.fpn.keypoint_head import KeypointHead
1011
from chainercv.links.model.fpn.mask_head import MaskHead
1112
from chainercv.links.model.fpn.rpn import RPN
1213
from chainercv.links.model.resnet import ResNet101
@@ -45,10 +46,12 @@ class FasterRCNNFPNResNet(FasterRCNN):
4546
"""
4647

4748
def __init__(self, n_fg_class=None, pretrained_model=None,
49+
n_point=None,
4850
return_values=['bboxes', 'labels', 'scores'],
4951
min_size=800, max_size=1333):
5052
param, path = utils.prepare_pretrained_model(
51-
{'n_fg_class': n_fg_class}, pretrained_model, self._models)
53+
{'n_fg_class': n_fg_class, 'n_point': n_point},
54+
pretrained_model, self._models)
5255

5356
base = self._base(n_class=1, arch='he')
5457
base.pick = ('res2', 'res3', 'res4', 'res5')
@@ -58,11 +61,16 @@ def __init__(self, n_fg_class=None, pretrained_model=None,
5861
extractor = FPN(
5962
base, len(base.pick), (1 / 4, 1 / 8, 1 / 16, 1 / 32, 1 / 64))
6063

64+
if param['n_point'] is not None:
65+
keypoint_head = KeypointHead(param['n_point'], extractor.scales)
66+
else:
67+
keypoint_head = None
6168
super(FasterRCNNFPNResNet, self).__init__(
6269
extractor=extractor,
6370
rpn=RPN(extractor.scales),
6471
bbox_head=BboxHead(param['n_fg_class'] + 1, extractor.scales),
6572
mask_head=MaskHead(param['n_fg_class'] + 1, extractor.scales),
73+
keypoint_head=keypoint_head,
6674
return_values=return_values,
6775
min_size=min_size, max_size=max_size
6876
)
@@ -72,7 +80,7 @@ def __init__(self, n_fg_class=None, pretrained_model=None,
7280
self.extractor.base,
7381
self._base(pretrained_model='imagenet', arch='he'))
7482
elif path:
75-
chainer.serializers.load_npz(path, self)
83+
chainer.serializers.load_npz(path, self, strict=False)
7684

7785

7886
class MaskRCNNFPNResNet(FasterRCNNFPNResNet):
@@ -91,7 +99,30 @@ class MaskRCNNFPNResNet(FasterRCNNFPNResNet):
9199
def __init__(self, n_fg_class=None, pretrained_model=None,
92100
min_size=800, max_size=1333):
93101
super(MaskRCNNFPNResNet, self).__init__(
94-
n_fg_class, pretrained_model, ['masks', 'labels', 'scores'],
102+
n_fg_class, pretrained_model, None,
103+
['masks', 'labels', 'scores'],
104+
min_size, max_size)
105+
106+
107+
class KeypointRCNNFPNResNet(FasterRCNNFPNResNet):
108+
"""Feature Pyramid Networks with ResNet-50.
109+
110+
This is a model of Feature Pyramid Networks [#]_.
111+
This model uses :class:`~chainercv.links.ResNet50` as
112+
its base feature extractor.
113+
114+
.. [#] Tsung-Yi Lin et al.
115+
Feature Pyramid Networks for Object Detection. CVPR 2017
116+
117+
118+
"""
119+
120+
def __init__(self, n_fg_class=None, pretrained_model=None,
121+
n_point=None,
122+
min_size=800, max_size=1333):
123+
super(KeypointRCNNFPNResNet, self).__init__(
124+
n_fg_class, pretrained_model, n_point,
125+
['points', 'labels', 'scores', 'point_scores', 'bboxes'],
95126
min_size, max_size)
96127

97128

@@ -189,6 +220,52 @@ class MaskRCNNFPNResNet101(MaskRCNNFPNResNet):
189220
}
190221

191222

223+
class KeypointRCNNFPNResNet50(KeypointRCNNFPNResNet):
224+
"""Feature Pyramid Networks with ResNet-50.
225+
226+
This is a model of Feature Pyramid Networks [#]_.
227+
This model uses :class:`~chainercv.links.ResNet50` as
228+
its base feature extractor.
229+
230+
.. [#] Tsung-Yi Lin et al.
231+
Feature Pyramid Networks for Object Detection. CVPR 2017
232+
233+
234+
"""
235+
236+
_base = ResNet50
237+
_models = {
238+
'coco': {
239+
'param': {'n_fg_class': 1, 'n_point': 17},
240+
'url': 'https://chainercv-models.preferred.jp/'
241+
'faster_rcnn_fpn_resnet50_keypoint_coco_converted_2019_03_15.npz',
242+
'cv2': True
243+
},
244+
}
245+
246+
247+
class KeypointRCNNFPNResNet101(KeypointRCNNFPNResNet):
248+
"""Feature Pyramid Networks with ResNet-50.
249+
250+
This is a model of Feature Pyramid Networks [#]_.
251+
This model uses :class:`~chainercv.links.ResNet50` as
252+
its base feature extractor.
253+
254+
.. [#] Tsung-Yi Lin et al.
255+
Feature Pyramid Networks for Object Detection. CVPR 2017
256+
257+
258+
"""
259+
260+
_base = ResNet50
261+
_models = {
262+
'coco': {
263+
'param': {'n_fg_class': 80},
264+
'url': '',
265+
'cv2': True
266+
},
267+
}
268+
192269

193270
def _copyparams(dst, src):
194271
if isinstance(dst, chainer.Chain):

chainercv/links/model/mask_rcnn/keypoint_head.py renamed to chainercv/links/model/fpn/keypoint_head.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,8 @@
1515
from chainercv.transforms.image.resize import resize
1616
from chainercv.utils.bbox.bbox_iou import bbox_iou
1717

18-
from chainercv.links.model.mask_rcnn.misc import point_to_roi_points
19-
from chainercv.links.model.mask_rcnn.misc import within_bbox
18+
from chainercv.links.model.fpn.keypoint_utils import point_to_roi_points
19+
from chainercv.links.model.fpn.keypoint_utils import within_bbox
2020

2121

2222
# make a bilinear interpolation kernel

‎chainercv/links/model/fpn/keypoint_utils.py‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
from __future__ import division
2+
3+
import numpy as np
4+
5+
import chainer
6+
7+
8+
def point_to_roi_points(
9+
point, visible, bbox, point_map_size):
10+
xp = chainer.backends.cuda.get_array_module(point)
11+
12+
R, K, _ = point.shape
13+
14+
roi_point = xp.zeros((len(bbox), K, 2))
15+
roi_visible = xp.zeros((len(bbox), K), dtype=np.bool)
16+
17+
offset_y = bbox[:, 0]
18+
offset_x = bbox[:, 1]
19+
scale_y = point_map_size / (bbox[:, 2] - bbox[:, 0])
20+
scale_x = point_map_size / (bbox[:, 3] - bbox[:, 1])
21+
22+
for k in range(K):
23+
y_boundary_index = xp.where(point[:, k, 0] == bbox[:, 2])[0]
24+
x_boundary_index = xp.where(point[:, k, 1] == bbox[:, 3])[0]
25+
26+
ys = (point[:, k, 0] - offset_y) * scale_y
27+
ys = xp.floor(ys)
28+
if len(y_boundary_index) > 0:
29+
ys[y_boundary_index] = point_map_size - 1
30+
xs = (point[:, k, 1] - offset_x) * scale_x
31+
xs = xp.floor(xs)
32+
if len(x_boundary_index) > 0:
33+
xs[x_boundary_index] = point_map_size - 1
34+
35+
valid = xp.logical_and(
36+
xp.logical_and(
37+
xp.logical_and(ys >= 0, xs >= 0),
38+
xp.logical_and(ys < point_map_size, xs < point_map_size)),
39+
visible[:, k])
40+
41+
roi_point[:, k, 0] = ys
42+
roi_point[:, k, 1] = xs
43+
roi_visible[:, k] = valid
44+
return roi_point, roi_visible
45+
46+
47+
def within_bbox(point, bbox):
48+
y_within = (point[:, :, 0] >= bbox[:, 0][:, None]) & (
49+
point[:, :, 0] <= bbox[:, 2][:, None])
50+
x_within = (point[:, :, 1] >= bbox[:, 1][:, None]) & (
51+
point[:, :, 1] <= bbox[:, 3][:, None])
52+
return y_within & x_within

‎chainercv/links/model/fpn/mask_utils.py‎

Lines changed: 0 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -155,50 +155,3 @@ def _expand_boxes(bbox, scale):
155155
expanded_bbox[:, 3] = x_c + w_half
156156

157157
return expanded_bbox
158-
159-
160-
def point_to_roi_points(
161-
point, visible, bbox, point_map_size):
162-
xp = chainer.backends.cuda.get_array_module(point)
163-
164-
R, K, _ = point.shape
165-
166-
roi_point = xp.zeros((len(bbox), K, 2))
167-
roi_visible = xp.zeros((len(bbox), K), dtype=np.bool)
168-
169-
offset_y = bbox[:, 0]
170-
offset_x = bbox[:, 1]
171-
scale_y = point_map_size / (bbox[:, 2] - bbox[:, 0])
172-
scale_x = point_map_size / (bbox[:, 3] - bbox[:, 1])
173-
174-
for k in range(K):
175-
y_boundary_index = xp.where(point[:, k, 0] == bbox[:, 2])[0]
176-
x_boundary_index = xp.where(point[:, k, 1] == bbox[:, 3])[0]
177-
178-
ys = (point[:, k, 0] - offset_y) * scale_y
179-
ys = xp.floor(ys)
180-
if len(y_boundary_index) > 0:
181-
ys[y_boundary_index] = point_map_size - 1
182-
xs = (point[:, k, 1] - offset_x) * scale_x
183-
xs = xp.floor(xs)
184-
if len(x_boundary_index) > 0:
185-
xs[x_boundary_index] = point_map_size - 1
186-
187-
valid = xp.logical_and(
188-
xp.logical_and(
189-
xp.logical_and(ys >= 0, xs >= 0),
190-
xp.logical_and(ys < point_map_size, xs < point_map_size)),
191-
visible[:, k])
192-
193-
roi_point[:, k, 0] = ys
194-
roi_point[:, k, 1] = xs
195-
roi_visible[:, k] = valid
196-
return roi_point, roi_visible
197-
198-
199-
def within_bbox(point, bbox):
200-
y_within = (point[:, :, 0] >= bbox[:, 0][:, None]) & (
201-
point[:, :, 0] <= bbox[:, 2][:, None])
202-
x_within = (point[:, :, 1] >= bbox[:, 1][:, None]) & (
203-
point[:, :, 1] <= bbox[:, 3][:, None])
204-
return y_within & x_within

‎chainercv/links/model/mask_rcnn/__init__.py‎

Lines changed: 0 additions & 11 deletions
This file was deleted.

0 commit comments

Comments
 (0)