Skip to content

Commit 7ea4f4a

Browse files
jaypumejaypume
authored
Merge pull request #91 from qxygxt/main
Optimize the problem of CI test and the structure of OSPP MDIL-SS
2 parents d8fa17a + 89636a3 commit 7ea4f4a

File tree

16 files changed

+242
-185
lines changed

16 files changed

+242
-185
lines changed

‎core/common/constant.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -68,12 +68,12 @@ class SystemMetricType(Enum):
6868
"""
6969
System metric type of ianvs.
7070
"""
71-
# pylint: disable=C0103
7271
SAMPLES_TRANSFER_RATIO = "samples_transfer_ratio"
7372
FWT = "FWT"
7473
BWT = "BWT"
75-
Task_Avg_Acc = "Task_Avg_Acc"
76-
Matrix = "Matrix"
74+
TASK_AVG_ACC = "task_avg_acc"
75+
MATRIX = "MATRIX"
76+
7777

7878
class TestObjectType(Enum):
7979
"""

‎core/testcasecontroller/algorithm/paradigm/lifelong_learning/lifelong_learning.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ def __init__(self, workspace, **kwargs):
6464
self.cloud_task_index = '/tmp/cloud_task/index.pkl'
6565
self.edge_task_index = '/tmp/edge_task/index.pkl'
6666
self.system_metric_info = {SystemMetricType.SAMPLES_TRANSFER_RATIO.value: [],
67-
SystemMetricType.Matrix.value : {},
68-
SystemMetricType.Task_Avg_Acc.value: {}}
67+
SystemMetricType.MATRIX.value : {},
68+
SystemMetricType.TASK_AVG_ACC.value: {}}
6969

7070
def run(self):
7171
# pylint:disable=duplicate-code
@@ -147,7 +147,7 @@ def run(self):
147147
LOGGER.info(f"{entry} scores: {scores}")
148148
task_avg_score['accuracy'] += scores['accuracy']
149149
task_avg_score['accuracy'] = task_avg_score['accuracy']/i
150-
self.system_metric_info[SystemMetricType.Task_Avg_Acc.value] = task_avg_score
150+
self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value] = task_avg_score
151151
LOGGER.info(task_avg_score)
152152
job = self.build_paradigm_job(ParadigmType.LIFELONG_LEARNING.value)
153153
inference_dataset = self.dataset.load_data(self.dataset.test_url, "eval",
@@ -160,7 +160,7 @@ def run(self):
160160
for key in my_dict.keys():
161161
matrix = my_dict[key]
162162
#BWT, FWT = self.compute(key, matrix)
163-
self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix
163+
self.system_metric_info[SystemMetricType.MATRIX.value][key] = matrix
164164

165165
elif mode == 'hard-example-mining':
166166
dataset_files = self._split_dataset(splitting_dataset_times=rounds)
@@ -246,7 +246,7 @@ def run(self):
246246
LOGGER.info(f"{entry} scores: {scores}")
247247
task_avg_score['accuracy'] += scores['accuracy']
248248
task_avg_score['accuracy'] = task_avg_score['accuracy']/i
249-
self.system_metric_info[SystemMetricType.Task_Avg_Acc.value] = task_avg_score
249+
self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value] = task_avg_score
250250
LOGGER.info(task_avg_score)
251251
test_res, unseen_task_train_samples = self._inference(self.edge_task_index,
252252
self.dataset.test_url,
@@ -256,7 +256,7 @@ def run(self):
256256
for key in my_dict.keys():
257257
matrix = my_dict[key]
258258
#BWT, FWT = self.compute(key, matrix)
259-
self.system_metric_info[SystemMetricType.Matrix.value][key] = matrix
259+
self.system_metric_info[SystemMetricType.MATRIX.value][key] = matrix
260260

261261
elif mode != 'multi-inference':
262262
dataset_files = self._split_dataset(splitting_dataset_times=rounds)

‎core/testcasecontroller/metrics/metrics.py‎

Lines changed: 33 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -39,71 +39,56 @@ def samples_transfer_ratio_func(system_metric_info: dict):
3939
4040
"""
4141

42-
info = system_metric_info.get(SystemMetricType.SAMPLES_TRANSFER_RATIO.value)
42+
info = system_metric_info.get(
43+
SystemMetricType.SAMPLES_TRANSFER_RATIO.value)
4344
inference_num = 0
4445
transfer_num = 0
4546
for inference_data, transfer_data in info:
4647
inference_num += len(inference_data)
4748
transfer_num += len(transfer_data)
4849
return round(float(transfer_num) / (inference_num + 1), 4)
4950

51+
5052
def compute(key, matrix):
5153
"""
5254
Compute BWT and FWT scores for a given matrix.
5355
"""
54-
# pylint: disable=C0103
55-
# pylint: disable=C0301
56-
# pylint: disable=C0303
57-
# pylint: disable=R0912
56+
print(
57+
f"compute function: key={key}, matrix={matrix}, type(matrix)={type(matrix)}")
5858

59-
print(f"compute function: key={key}, matrix={matrix}, type(matrix)={type(matrix)}")
60-
6159
length = len(matrix)
6260
accuracy = 0.0
63-
BWT_score = 0.0
64-
FWT_score = 0.0
61+
bwt_score = 0.0
62+
fwt_score = 0.0
6563
flag = True
6664

67-
if key == 'all':
68-
for i in range(length-1, 0, -1):
69-
sum_before_i = sum(item['accuracy'] for item in matrix[i][:i])
70-
sum_after_i = sum(item['accuracy'] for item in matrix[i][-(length - i - 1):])
71-
if i == 0:
72-
seen_class_accuracy = 0.0
73-
else:
74-
seen_class_accuracy = sum_before_i / i
75-
if length - 1 - i == 0:
76-
unseen_class_accuracy = 0.0
77-
else:
78-
unseen_class_accuracy = sum_after_i / (length - 1 - i)
79-
print(f"round {i} : unseen class accuracy is {unseen_class_accuracy}, seen class accuracy is {seen_class_accuracy}")
80-
8165
for row in matrix:
8266
if not isinstance(row, list) or len(row) != length-1:
8367
flag = False
8468
break
8569

8670
if not flag:
87-
BWT_score = np.nan
88-
FWT_score = np.nan
89-
return BWT_score, FWT_score
71+
bwt_score = np.nan
72+
fwt_score = np.nan
73+
return bwt_score, fwt_score
9074

9175
for i in range(length-1):
9276
for j in range(length-1):
9377
if 'accuracy' in matrix[i+1][j] and 'accuracy' in matrix[i][j]:
9478
accuracy += matrix[i+1][j]['accuracy']
95-
BWT_score += matrix[i+1][j]['accuracy'] - matrix[i][j]['accuracy']
96-
79+
bwt_score += matrix[i+1][j]['accuracy'] - \
80+
matrix[i][j]['accuracy']
81+
9782
for i in range(0, length-1):
9883
if 'accuracy' in matrix[i][i] and 'accuracy' in matrix[0][i]:
99-
FWT_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy']
84+
fwt_score += matrix[i][i]['accuracy'] - matrix[0][i]['accuracy']
10085

10186
accuracy = accuracy / ((length-1) * (length-1))
102-
BWT_score = BWT_score / ((length-1) * (length-1))
103-
FWT_score = FWT_score / (length-1)
87+
bwt_score = bwt_score / ((length-1) * (length-1))
88+
fwt_score = fwt_score / (length-1)
10489

105-
print(f"{key} BWT_score: {BWT_score}")
106-
print(f"{key} FWT_score: {FWT_score}")
90+
print(f"{key} BWT_score: {bwt_score}")
91+
print(f"{key} FWT_score: {fwt_score}")
10792

10893
my_matrix = []
10994
for i in range(length-1):
@@ -112,48 +97,53 @@ def compute(key, matrix):
11297
if 'accuracy' in matrix[i+1][j]:
11398
my_matrix[i].append(matrix[i+1][j]['accuracy'])
11499

115-
return my_matrix, BWT_score, FWT_score
100+
return my_matrix, bwt_score, fwt_score
101+
116102

117103
def bwt_func(system_metric_info: dict):
118104
"""
119105
compute BWT
120106
"""
121107
# pylint: disable=C0103
122108
# pylint: disable=W0632
123-
info = system_metric_info.get(SystemMetricType.Matrix.value)
109+
info = system_metric_info.get(SystemMetricType.MATRIX.value)
124110
_, BWT_score, _ = compute("all", info["all"])
125111
return BWT_score
126112

113+
127114
def fwt_func(system_metric_info: dict):
128115
"""
129116
compute FWT
130117
"""
131118
# pylint: disable=C0103
132119
# pylint: disable=W0632
133-
info = system_metric_info.get(SystemMetricType.Matrix.value)
120+
info = system_metric_info.get(SystemMetricType.MATRIX.value)
134121
_, _, FWT_score = compute("all", info["all"])
135122
return FWT_score
136123

124+
137125
def matrix_func(system_metric_info: dict):
138126
"""
139127
compute FWT
140128
"""
141129
# pylint: disable=C0103
142130
# pylint: disable=W0632
143-
info = system_metric_info.get(SystemMetricType.Matrix.value)
131+
info = system_metric_info.get(SystemMetricType.MATRIX.value)
144132
my_dict = {}
145133
for key in info.keys():
146134
my_matrix, _, _ = compute(key, info[key])
147135
my_dict[key] = my_matrix
148136
return my_dict
149137

138+
150139
def task_avg_acc_func(system_metric_info: dict):
151140
"""
152-
compute Task_Avg_Acc
141+
compute task average accuracy
153142
"""
154-
info = system_metric_info.get(SystemMetricType.Task_Avg_Acc.value)
143+
info = system_metric_info.get(SystemMetricType.TASK_AVG_ACC.value)
155144
return info["accuracy"]
156145

146+
157147
def get_metric_func(metric_dict: dict):
158148
"""
159149
get metric func by metric info
@@ -175,9 +165,11 @@ def get_metric_func(metric_dict: dict):
175165
if url:
176166
try:
177167
load_module(url)
178-
metric_func = ClassFactory.get_cls(type_name=ClassType.GENERAL, t_cls_name=name)
168+
metric_func = ClassFactory.get_cls(
169+
type_name=ClassType.GENERAL, t_cls_name=name)
179170
return name, metric_func
180171
except Exception as err:
181-
raise RuntimeError(f"get metric func(url={url}) failed, error: {err}.") from err
172+
raise RuntimeError(
173+
f"get metric func(url={url}) failed, error: {err}.") from err
182174

183175
return name, getattr(sys.modules[__name__], str.lower(name) + "_func")

‎examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/benchmarkingjob.yaml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,3 @@ benchmarkingjob:
6464
# 1> "selected_and_all": save selected and all dataitems;
6565
# 2> "selected_only": save selected dataitems;
6666
save_mode: "selected_and_all"
67-
68-
69-
70-
71-
72-

‎examples/robot-cityscapes-synthia/lifelong_learning_bench/semantic-segmentation/testalgorithms/erfnet/ERFNet/accuracy.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77
__all__ = ('accuracy')
88

9+
910
@ClassFactory.register(ClassType.GENERAL)
1011
def accuracy(y_true, y_pred, **kwargs):
1112
args = val_args()
@@ -35,4 +36,4 @@ def accuracy(y_true, y_pred, **kwargs):
3536
FWIoU = evaluator.Frequency_Weighted_Intersection_over_Union()
3637

3738
print("CPA:{}, mIoU:{}, fwIoU: {}".format(CPA, mIoU, FWIoU))
38-
return CPA
39+
return CPA

0 commit comments

Comments
 (0)