Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 34f4dda

Browse files
KiteSoarKiteSoar
authoredMar 12, 2025··
[ISSUE#13171] Fix the full regular dump for gray configuration. (#13174)
* [ISSUE#13171] Fix the full regular dump for gray configuration. * [ISSUE#13171] Fix CI.
1 parent 368867a commit 34f4dda

File tree

3 files changed

+279
-5
lines changed

3 files changed

+279
-5
lines changed
 

‎config/src/main/java/com/alibaba/nacos/config/server/service/dump/DumpService.java

+13-5
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,7 @@ public DumpService(ConfigInfoPersistService configInfoPersistService,
121121
this.dumpAllTaskMgr.setDefaultTaskProcessor(dumpAllProcessor);
122122

123123
this.dumpAllTaskMgr.addProcessor(DumpAllTask.TASK_ID, dumpAllProcessor);
124+
this.dumpAllTaskMgr.addProcessor(DumpAllGrayTask.TASK_ID, dumpAllGrayProcessor);
124125
DynamicDataSource.getInstance().getDataSource();
125126

126127
NotifyCenter.registerSubscriber(new Subscriber() {
@@ -215,11 +216,7 @@ protected void dumpOperate() throws NacosException {
215216

216217
try {
217218
dumpAllConfigInfoOnStartup(dumpAllProcessor);
218-
219-
LogUtil.DEFAULT_LOG.info("start clear all config-info-gray.");
220-
ConfigDiskServiceFactory.getInstance().clearAllGray();
221-
dumpAllGrayProcessor.process(new DumpAllGrayTask());
222-
219+
dumpAllGrayConfigInfoOnStartup(dumpAllGrayProcessor);
223220
} catch (Exception e) {
224221
LogUtil.FATAL_LOG.error(
225222
"Nacos Server did not start because dumpservice bean construction failure :\n" + e);
@@ -270,6 +267,17 @@ private void dumpAllConfigInfoOnStartup(DumpAllProcessor dumpAllProcessor) {
270267
}
271268
}
272269

270+
private void dumpAllGrayConfigInfoOnStartup(DumpAllGrayProcessor dumpAllGrayProcessor) {
271+
try {
272+
LogUtil.DEFAULT_LOG.info("start to clear all gray-config-info on startup.");
273+
ConfigDiskServiceFactory.getInstance().clearAllGray();
274+
dumpAllGrayProcessor.process(new DumpAllGrayTask());
275+
} catch (Exception e) {
276+
LogUtil.FATAL_LOG.error("failed to dump all gray-config-info on startup." + e.getMessage());
277+
throw e;
278+
}
279+
}
280+
273281
/**
274282
* dump operation.
275283
*

‎config/src/main/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllGrayProcessor.java

+7
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import com.alibaba.nacos.common.task.NacosTaskProcessor;
2121
import com.alibaba.nacos.config.server.model.ConfigInfoGrayWrapper;
2222
import com.alibaba.nacos.config.server.service.ConfigCacheService;
23+
import com.alibaba.nacos.config.server.service.dump.task.DumpAllGrayTask;
2324
import com.alibaba.nacos.config.server.service.repository.ConfigInfoGrayPersistService;
2425
import com.alibaba.nacos.config.server.utils.GroupKey2;
2526
import com.alibaba.nacos.config.server.utils.LogUtil;
@@ -42,6 +43,12 @@ public DumpAllGrayProcessor(ConfigInfoGrayPersistService configInfoGrayPersistSe
4243

4344
@Override
4445
public boolean process(NacosTask task) {
46+
if (!(task instanceof DumpAllGrayTask)) {
47+
DEFAULT_LOG.error(
48+
"[all-dump-gray-error] ,invalid task type {},DumpAllGrayProcessor should process DumpAllGrayTask type.",
49+
task.getClass().getSimpleName());
50+
return false;
51+
}
4552
int rowCount = configInfoGrayPersistService.configInfoGrayCount();
4653
int pageCount = (int) Math.ceil(rowCount * 1.0 / PAGE_SIZE);
4754

‎config/src/test/java/com/alibaba/nacos/config/server/service/dump/processor/DumpAllGrayProcessorTest.java

+259
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,259 @@
1+
/*
2+
* Copyright 1999-$toady.year Alibaba Group Holding Ltd.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.alibaba.nacos.config.server.service.dump.processor;
18+
19+
import com.alibaba.nacos.common.task.NacosTask;
20+
import com.alibaba.nacos.common.utils.MD5Utils;
21+
import com.alibaba.nacos.config.server.model.CacheItem;
22+
import com.alibaba.nacos.config.server.model.ConfigInfoGrayWrapper;
23+
import com.alibaba.nacos.config.server.service.ConfigCacheService;
24+
import com.alibaba.nacos.config.server.service.dump.ExternalDumpService;
25+
import com.alibaba.nacos.config.server.service.dump.disk.ConfigDiskServiceFactory;
26+
import com.alibaba.nacos.config.server.service.dump.task.DumpAllGrayTask;
27+
import com.alibaba.nacos.config.server.service.repository.ConfigInfoGrayPersistService;
28+
import com.alibaba.nacos.config.server.service.repository.ConfigInfoPersistService;
29+
import com.alibaba.nacos.config.server.utils.GroupKey2;
30+
import com.alibaba.nacos.config.server.utils.PropertyUtil;
31+
import com.alibaba.nacos.persistence.datasource.DataSourceService;
32+
import com.alibaba.nacos.persistence.datasource.DynamicDataSource;
33+
import com.alibaba.nacos.persistence.model.Page;
34+
import com.alibaba.nacos.plugin.datasource.constants.CommonConstant;
35+
import com.alibaba.nacos.sys.env.EnvUtil;
36+
import org.junit.jupiter.api.AfterEach;
37+
import org.junit.jupiter.api.BeforeEach;
38+
import org.junit.jupiter.api.Test;
39+
import org.junit.jupiter.api.extension.ExtendWith;
40+
import org.mockito.Mock;
41+
import org.mockito.MockedStatic;
42+
import org.mockito.Mockito;
43+
import org.mockito.junit.jupiter.MockitoExtension;
44+
45+
import java.util.ArrayList;
46+
import java.util.Arrays;
47+
import java.util.List;
48+
49+
import static org.junit.jupiter.api.Assertions.assertEquals;
50+
import static org.junit.jupiter.api.Assertions.assertFalse;
51+
import static org.junit.jupiter.api.Assertions.assertTrue;
52+
import static org.mockito.ArgumentMatchers.anyInt;
53+
import static org.mockito.ArgumentMatchers.eq;
54+
import static org.mockito.Mockito.atLeast;
55+
import static org.mockito.Mockito.mock;
56+
import static org.mockito.Mockito.never;
57+
import static org.mockito.Mockito.times;
58+
import static org.mockito.Mockito.verify;
59+
import static org.mockito.Mockito.when;
60+
61+
@ExtendWith(MockitoExtension.class)
62+
class DumpAllGrayProcessorTest {
63+
64+
private static final int PAGE_SIZE = 100;
65+
66+
@Mock
67+
DynamicDataSource dynamicDataSource;
68+
69+
@Mock
70+
DataSourceService dataSourceService;
71+
72+
@Mock
73+
ConfigInfoGrayPersistService configInfoGrayPersistService;
74+
75+
DumpAllProcessor dumpAllProcessor;
76+
77+
DumpAllGrayProcessor dumpAllGrayProcessor;
78+
79+
ExternalDumpService dumpService;
80+
81+
MockedStatic<DynamicDataSource> dynamicDataSourceMockedStatic;
82+
83+
MockedStatic<PropertyUtil> propertyUtilMockedStatic;
84+
85+
@Mock
86+
ConfigInfoPersistService configInfoPersistService;
87+
88+
MockedStatic<EnvUtil> envUtilMockedStatic;
89+
90+
private String mockMem = "tmpmocklimitfile.txt";
91+
92+
@BeforeEach
93+
void init() throws Exception {
94+
dynamicDataSourceMockedStatic = Mockito.mockStatic(DynamicDataSource.class);
95+
envUtilMockedStatic = Mockito.mockStatic(EnvUtil.class);
96+
propertyUtilMockedStatic = Mockito.mockStatic(PropertyUtil.class);
97+
propertyUtilMockedStatic.when(PropertyUtil::getAllDumpPageSize).thenReturn(100);
98+
dumpAllGrayProcessor = new DumpAllGrayProcessor(configInfoGrayPersistService);
99+
when(EnvUtil.getNacosHome()).thenReturn(System.getProperty("user.home"));
100+
when(EnvUtil.getProperty(eq(CommonConstant.NACOS_PLUGIN_DATASOURCE_LOG), eq(Boolean.class),
101+
eq(false))).thenReturn(false);
102+
dynamicDataSourceMockedStatic.when(DynamicDataSource::getInstance).thenReturn(dynamicDataSource);
103+
104+
when(dynamicDataSource.getDataSource()).thenReturn(dataSourceService);
105+
106+
dumpService = new ExternalDumpService(configInfoPersistService, null, null, configInfoGrayPersistService, null);
107+
108+
dumpAllProcessor = new DumpAllProcessor(configInfoPersistService);
109+
envUtilMockedStatic.when(() -> EnvUtil.getProperty(eq("memory_limit_file_path"),
110+
eq("/sys/fs/cgroup/memory/memory.limit_in_bytes"))).thenReturn(mockMem);
111+
112+
}
113+
114+
@AfterEach
115+
void after() {
116+
dynamicDataSourceMockedStatic.close();
117+
envUtilMockedStatic.close();
118+
propertyUtilMockedStatic.close();
119+
}
120+
121+
@Test
122+
void testProcessWithInvalidTaskType() {
123+
NacosTask invalidTask = mock(NacosTask.class);
124+
125+
boolean result = dumpAllGrayProcessor.process(invalidTask);
126+
127+
assertFalse(result);
128+
verify(configInfoGrayPersistService, never()).configInfoGrayCount();
129+
}
130+
131+
@Test
132+
void testProcessWithValidTaskType() throws Exception {
133+
final DumpAllGrayTask validTask = mock(DumpAllGrayTask.class);
134+
List<ConfigInfoGrayWrapper> configList = new ArrayList<>();
135+
configList.add(createGrayWrapper("dataId-1", "group-1"));
136+
137+
Page<ConfigInfoGrayWrapper> page = new Page<>();
138+
when(configInfoGrayPersistService.configInfoGrayCount()).thenReturn(1);
139+
when(configInfoGrayPersistService.findAllConfigInfoGrayForDumpAll(anyInt(), anyInt())).thenReturn(page);
140+
141+
boolean result = dumpAllGrayProcessor.process(validTask);
142+
143+
assertTrue(result);
144+
verify(configInfoGrayPersistService, times(1)).configInfoGrayCount();
145+
verify(configInfoGrayPersistService, times(1)).findAllConfigInfoGrayForDumpAll(anyInt(), anyInt());
146+
}
147+
148+
@Test
149+
void testPaginationLogic() {
150+
DumpAllGrayTask task = mock(DumpAllGrayTask.class);
151+
int totalConfigs = PAGE_SIZE * 2 + 50;
152+
when(configInfoGrayPersistService.configInfoGrayCount()).thenReturn(totalConfigs);
153+
154+
Page<ConfigInfoGrayWrapper> pageOne = new Page<>();
155+
when(configInfoGrayPersistService.findAllConfigInfoGrayForDumpAll(eq(1), anyInt())).thenReturn(pageOne);
156+
157+
boolean result = dumpAllGrayProcessor.process(task);
158+
159+
assertTrue(result);
160+
verify(configInfoGrayPersistService, atLeast(2)).findAllConfigInfoGrayForDumpAll(anyInt(), anyInt());
161+
}
162+
163+
@Test
164+
void testInteractionWithConfigCacheService() {
165+
DumpAllGrayTask task = mock(DumpAllGrayTask.class);
166+
Page<ConfigInfoGrayWrapper> page = new Page<>();
167+
when(configInfoGrayPersistService.configInfoGrayCount()).thenReturn(1);
168+
when(configInfoGrayPersistService.findAllConfigInfoGrayForDumpAll(anyInt(), anyInt())).thenReturn(page);
169+
boolean result = dumpAllGrayProcessor.process(task);
170+
assertTrue(result);
171+
}
172+
173+
/**
174+
* test dump all for all check task.
175+
*/
176+
@Test
177+
void testDumpAllGrayOnCheckAll() throws Exception {
178+
ConfigInfoGrayWrapper configInfoGrayWrapper1 = createGrayWrapper("data-1", "group-1");
179+
ConfigInfoGrayWrapper configInfoGrayWrapper2 = createGrayWrapper("data-2", "group-2");
180+
long timestamp = System.currentTimeMillis();
181+
configInfoGrayWrapper1.setLastModified(timestamp);
182+
configInfoGrayWrapper2.setLastModified(timestamp);
183+
184+
Page<ConfigInfoGrayWrapper> page = new Page<>();
185+
page.setTotalCount(2);
186+
page.setPagesAvailable(2);
187+
page.setPageNumber(1);
188+
List<ConfigInfoGrayWrapper> list = Arrays.asList(configInfoGrayWrapper1, configInfoGrayWrapper2);
189+
page.setPageItems(list);
190+
191+
Mockito.when(configInfoGrayPersistService.configInfoGrayCount()).thenReturn(2);
192+
Mockito.when(configInfoGrayPersistService.findAllConfigInfoGrayForDumpAll(anyInt(), anyInt())).thenReturn(page);
193+
194+
final String md51 = MD5Utils.md5Hex(configInfoGrayWrapper1.getContent(), "UTF-8");
195+
final String md52 = MD5Utils.md5Hex(configInfoGrayWrapper2.getContent(), "UTF-8");
196+
long latterTimestamp = timestamp + 999;
197+
long earlierTimestamp = timestamp - 999;
198+
String encryptedDataKey = "testEncryptedDataKey";
199+
200+
String dataId1 = configInfoGrayWrapper1.getDataId();
201+
String group1 = configInfoGrayWrapper1.getGroup();
202+
String grayName1 = configInfoGrayWrapper1.getGrayName();
203+
String grayRule1 = configInfoGrayWrapper1.getGrayRule();
204+
String tenant1 = configInfoGrayWrapper1.getTenant();
205+
String content1 = configInfoGrayWrapper1.getContent();
206+
207+
String dataId2 = configInfoGrayWrapper2.getDataId();
208+
String group2 = configInfoGrayWrapper2.getGroup();
209+
String grayName2 = configInfoGrayWrapper2.getGrayName();
210+
String grayRule2 = configInfoGrayWrapper2.getGrayRule();
211+
String tenant2 = configInfoGrayWrapper2.getTenant();
212+
String content2 = configInfoGrayWrapper2.getContent();
213+
214+
ConfigCacheService.dumpGray(dataId1, group1, tenant1, grayName1, grayRule1, content1, latterTimestamp, encryptedDataKey);
215+
ConfigCacheService.dumpGray(dataId2, group2, tenant2, grayName2, grayRule2, content2, earlierTimestamp, encryptedDataKey);
216+
217+
DumpAllGrayTask dumpAllTask = new DumpAllGrayTask();
218+
boolean process = dumpAllGrayProcessor.process(dumpAllTask);
219+
220+
assertTrue(process);
221+
222+
CacheItem contentCache1 = ConfigCacheService.getContentCache(
223+
GroupKey2.getKey(configInfoGrayWrapper1.getDataId(), configInfoGrayWrapper1.getGroup(),
224+
configInfoGrayWrapper1.getTenant()));
225+
assertEquals(md51, contentCache1.getConfigCacheGray().get(grayName1).getMd5());
226+
assertEquals(latterTimestamp, contentCache1.getConfigCacheGray().get(grayName1).getLastModifiedTs());
227+
228+
String contentFromDisk1 = ConfigDiskServiceFactory.getInstance()
229+
.getGrayContent(configInfoGrayWrapper1.getDataId(), configInfoGrayWrapper1.getGroup(),
230+
configInfoGrayWrapper1.getTenant(), configInfoGrayWrapper1.getGrayName());
231+
assertEquals(configInfoGrayWrapper1.getContent(), contentFromDisk1);
232+
233+
CacheItem contentCache2 = ConfigCacheService.getContentCache(
234+
GroupKey2.getKey(configInfoGrayWrapper2.getDataId(), configInfoGrayWrapper2.getGroup(),
235+
configInfoGrayWrapper2.getTenant()));
236+
assertEquals(md52, contentCache2.getConfigCacheGray().get(grayName2).getMd5());
237+
assertEquals(configInfoGrayWrapper2.getLastModified(), contentCache2.getConfigCacheGray().get(grayName2).getLastModifiedTs());
238+
239+
String contentFromDisk2 = ConfigDiskServiceFactory.getInstance()
240+
.getGrayContent(configInfoGrayWrapper2.getDataId(), configInfoGrayWrapper2.getGroup(),
241+
configInfoGrayWrapper2.getTenant(), configInfoGrayWrapper2.getGrayName());
242+
assertEquals(configInfoGrayWrapper2.getContent(), contentFromDisk2);
243+
}
244+
245+
private ConfigInfoGrayWrapper createGrayWrapper(String dataId, String group) {
246+
ConfigInfoGrayWrapper wrapper = new ConfigInfoGrayWrapper();
247+
wrapper.setDataId(dataId);
248+
wrapper.setGroup(group);
249+
wrapper.setTenant("tenant");
250+
wrapper.setGrayName("gray-" + dataId);
251+
String grayRule = "{\"type\":\"beta\",\"version\":\"1.0.0\",\"expr\":\"0 0/5 * * * ?\",\"priority\":1}";
252+
wrapper.setGrayRule(grayRule);
253+
wrapper.setContent("content");
254+
wrapper.setLastModified(System.currentTimeMillis());
255+
wrapper.setEncryptedDataKey("enc-key");
256+
wrapper.setMd5(MD5Utils.md5Hex(wrapper.getContent(), "UTF-8"));
257+
return wrapper;
258+
}
259+
}

0 commit comments

Comments
 (0)
Please sign in to comment.