Skip to content

Commit f632739

Browse files
ruanshudongruanshudong
committed
fix tarstat, tarsproperty, mac: mem, windows: mmap, linux: shm
1 parent 5f0e831 commit f632739

14 files changed

+70
-133
lines changed

‎.github/workflows/ccpp-win.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,5 +22,5 @@ jobs:
2222
- name: configure
2323
run: cd build; cmake --version; cmake .. -A X64
2424
- name: make
25-
run: cd build; cmake --build . --config release --target run-all
25+
run: cd build; cmake --build . --config release
2626

‎PropertyServer/PropertyDbManager.cpp

+5-5
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ PropertyDbManager::PropertyDbManager()
207207
{
208208
vector<size_t> &vDb = m_iter1->second;
209209

210-
FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|ip:" << m_iter1->first << "|DbNum:" << vDb.size() << endl;
210+
// FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|ip:" << m_iter1->first << "|DbNum:" << vDb.size() << endl;
211211

212212
++m_iter1;
213213
}
@@ -380,7 +380,7 @@ int PropertyDbManager::insert2Db(const PropertyMsg &mPropMsg, const string &sDat
380380

381381
TLOGDEBUG("insert " << strTbName << " affected:" << iCount << endl);
382382

383-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
383+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
384384

385385
iCount = 0;
386386
}
@@ -390,7 +390,7 @@ int PropertyDbManager::insert2Db(const PropertyMsg &mPropMsg, const string &sDat
390390
{
391391
pMysql->execute(osSql.str());
392392
TLOGDEBUG("insert " << strTbName << " affected:" << iCount << endl);
393-
FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
393+
// FDLOG("PropertyPool") << "statsec ip:" << ServerConfig::LocalIp << "|insert " << strTbName << "|insert affected:" << iCount << "|mysql affected:" << pMysql->getAffectedRows() << endl;
394394
}
395395
}
396396
catch (TC_Mysql_Exception& ex)
@@ -597,8 +597,8 @@ int PropertyDbManager::insert2MultiDbs(int iIndex, const PropertyMsg &propertyms
597597
int64_t iEnd = tars::TC_TimeProvider::getInstance()->getNowMs();
598598

599599
TLOGDEBUG("insert|" << iIndex << "|" << getIpAndPort(iIndex) << "|" << sDate << "|" << sFlag << "|" << propertymsg.size() << "|" << (iEnd - iBegin) << endl);
600-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag
601-
<< "|records:" << propertymsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl;
600+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|insert|dbIndex:" << iIndex << "|" << getIpAndPort(iIndex) << "|date:" << sDate << "|tflag:" << sFlag
601+
// << "|records:" << propertymsg.size() << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin(ms):" << iBegin << "|iEnd(ms):" << iEnd << endl;
602602
}
603603
}
604604
catch(TC_Mysql_Exception& ex)

‎PropertyServer/PropertyHashMap.h

+8-6
Original file line numberDiff line numberDiff line change
@@ -27,13 +27,15 @@ using namespace tars;
2727
typedef StatPropMsgBody PropBody;
2828
typedef StatPropMsgHead PropHead;
2929

30-
typedef TarsHashMap<PropHead,PropBody, ThreadLockPolicy, FileStorePolicy> PropHashMap;
30+
//typedef TarsHashMap<PropHead,PropBody, ThreadLockPolicy, FileStorePolicy> PropHashMap;
3131

32-
// #if TARGET_PLATFORM_IOS || TARGET_PLATFORM_WINDOWS
33-
// typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,MemStorePolicy> PropHashMap;//FileStorePolicy
34-
// #else
35-
// typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,ShmStorePolicy> PropHashMap;//FileStorePolicy
36-
// #endif
32+
#if TARGET_PLATFORM_IOS
33+
typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,MemStorePolicy> PropHashMap;//FileStorePolicy
34+
#elif TARGET_PLATFORM_WINDOWS
35+
typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,FileStorePolicy> PropHashMap;//FileStorePolicy
36+
#else
37+
typedef TarsHashMap<PropHead, PropBody, ThreadLockPolicy,ShmStorePolicy> PropHashMap;//FileStorePolicy
38+
#endif
3739

3840

3941
#if TARGET_PLATFORM_LINUX

‎PropertyServer/PropertyImp.cpp

+5-4
Original file line numberDiff line numberDiff line change
@@ -152,10 +152,12 @@ int PropertyImp::handlePropMsg(const map<StatPropMsgHead, StatPropMsgBody> &prop
152152
//////////////////////////////////////////////////////////////////////////////////////
153153
float rate = (pHashMap->getMapHead()._iUsedChunk) * 1.0/pHashMap->allBlockChunkCount();
154154

155-
if(rate >0.9)
155+
if(rate >0.95)
156156
{
157-
pHashMap->expand(pHashMap->getMapHead()._iMemSize * 2);
158-
TLOGERROR("PropertyImp::handlePropMsg hashmap expand to " << pHashMap->getMapHead()._iMemSize << endl);
157+
// pHashMap->expand(pHashMap->getMapHead()._iMemSize * 2);
158+
TLOGERROR("PropertyImp::handlePropMsg hashmap will full|_iMemSize:" << pHashMap->getMapHead()._iMemSize << endl);
159+
return -1;
160+
// TLOGERROR("PropertyImp::handlePropMsg hashmap expand to " << pHashMap->getMapHead()._iMemSize << endl);
159161
}
160162

161163
int iRet = pHashMap->add(tHead, body);
@@ -207,7 +209,6 @@ void PropertyImp::dump2file()
207209
g_app.setSelectBufferIndex(iSelectBuffer);
208210

209211
TLOGDEBUG("PropertyImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl);
210-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyImp::dump2file select buffer:" << iSelectBuffer << "|TimeInterv:" << tTimeInterv << "|now:" << tTimeNow << "|last:" << g_tLastDumpTime << endl;
211212
}
212213
}
213214
}

‎PropertyServer/PropertyReapThread.cpp

+11-11
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,12 @@ void ReapSSDProcThread::run()
9696
TLOGDEBUG("ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index)
9797
<< "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl);
9898

99-
FDLOG("PropertyPool") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index)
100-
<< "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl;
99+
// FDLOG("PropertyPool") << "ReapSSDProcThread::run stat ip:" << ServerConfig::LocalIp << "|dbIndex:" << item._index << "|" << PropertyDbManager::getInstance()->getIpAndPort(item._index)
100+
// << "|date:" << item._date << "|tflag:" << item._tflag << "|timecost(ms):" << (iEnd - iBegin) << "|iBegin:" << iBegin << "|iEnd:" << iEnd << endl;
101101

102102
if((iEnd - iBegin)/1000 > g_app.getInserInterv() * 40)
103103
{
104-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl;
104+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|ReapSSDProcThread::run timeout 8 minute." << endl;
105105
string sMsg("stat ip:");
106106
sMsg += ServerConfig::LocalIp;
107107
sMsg += " ReapSSDProcThread::run write db:";
@@ -125,7 +125,7 @@ void ReapSSDProcThread::run()
125125
catch(exception& e)
126126
{
127127
TLOGERROR("ReapSSDProcThread::run exception:" << e.what() << endl);
128-
FDLOG("PropertyPool") << "ReapSSDProcThread::run exception:" << e.what() << endl;
128+
// FDLOG("PropertyPool") << "ReapSSDProcThread::run exception:" << e.what() << endl;
129129
}
130130
}
131131
}
@@ -183,7 +183,7 @@ void PropertyReapThread::run()
183183
int iLastIndex = -1;
184184

185185
TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl);
186-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl;
186+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|PropertyReapThread::run iInsertDataThread:" << iInsertDataNum << "|dbNumber:" << dbNumber << endl;
187187

188188
while (!_terminate)
189189
{
@@ -211,10 +211,10 @@ void PropertyReapThread::run()
211211
int64_t tEnd = TNOWMS;
212212

213213
TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl);
214-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl;
214+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run getDataFromBuffer timecost(ms):" << (tEnd - tBegin) << endl;
215215

216216
TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl);
217-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl;
217+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert begin _vAllStatMsg.size:" << vAllPropertyMsg.size() << "|record num:" << iTotalNum << endl;
218218

219219
if(iTotalNum <= 0)
220220
{
@@ -284,7 +284,7 @@ void PropertyReapThread::run()
284284
}
285285

286286
TLOGDEBUG("propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl);
287-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl;
287+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iBufferIndex << "|PropertyReapThread::run insert record num:" << iTotalNum << "|tast patch finished." << endl;
288288
}
289289

290290
}
@@ -384,7 +384,7 @@ void PropertyReapThread::getDataFromBuffer(int iIndex, vector<PropertyMsg*> &vAl
384384
continue ;
385385
}
386386

387-
FDLOG("PropertyPool") << "property ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData load hashmap k:" << k << endl;
387+
// FDLOG("PropertyPool") << "property ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData load hashmap k:" << k << endl;
388388

389389
PropertyHashMap::lock_iterator it = pHashMap->beginSetTime();
390390
while ( it != pHashMap->end() )
@@ -428,12 +428,12 @@ void PropertyReapThread::getDataFromBuffer(int iIndex, vector<PropertyMsg*> &vAl
428428
iTotalNum = iCount;
429429

430430
TLOGDEBUG("PropertyReapThread::getDataFromBuffer Buffer Index:" << iIndex << "|get total size:" << iCount << endl);
431-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData get total size:" << iCount << "|end..." << endl;
431+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData get total size:" << iCount << "|end..." << endl;
432432
}
433433
catch (exception& ex)
434434
{
435435
TLOGERROR("PropertyReapThread::getDataFromBuffer exception:" << ex.what() << endl);
436-
FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData exception:" << ex.what() << endl;
436+
// FDLOG("PropertyPool") << "propertypool ip:" << ServerConfig::LocalIp << "|Buffer Index:" << iIndex << "|PropertyReapThread::getData exception:" << ex.what() << endl;
437437

438438
string sMsg("PropertyReapThread::getDataFromBuffer Buffer Index:");
439439
sMsg += TC_Common::tostr(iIndex);

‎PropertyServer/PropertyServer.cpp

+10-20
Original file line numberDiff line numberDiff line change
@@ -50,15 +50,13 @@ void PropertyServer::initialize()
5050
string s("");
5151
_selectBuffer = getSelectBufferFromFlag(s);
5252

53-
5453
for(size_t i =0; i < vec.size(); ++i)
5554
{
5655
vec[i].first = 0;
5756
vec[i].second = 0;
5857
}
5958
_buffer[_selectBuffer] = vec;
6059

61-
6260
for(size_t i =0; i < vec.size(); ++i)
6361
{
6462
vec[i].first = 0;
@@ -67,12 +65,10 @@ void PropertyServer::initialize()
6765
_buffer[!_selectBuffer] = vec;
6866

6967
TLOGDEBUG("PropertyServer::initialize iHandleNum:" << iHandleNum<< endl);
70-
FDLOG("PropertyPool") << "PropertyServer::initialize iHandleNum:" << iHandleNum << endl;
71-
68+
// FDLOG("PropertyPool") << "PropertyServer::initialize iHandleNum:" << iHandleNum << endl;
7269

7370
TLOGDEBUG("PropertyServer::initialize iSelectBuffer:" << _selectBuffer<< endl);
74-
FDLOG("PropertyPool") << "PropertyServer::initialize iSelectBuffer:" << _selectBuffer << endl;
75-
71+
// FDLOG("PropertyPool") << "PropertyServer::initialize iSelectBuffer:" << _selectBuffer << endl;
7672

7773
_randOrder = AppCache::getInstance()->get("RandOrder");
7874
TLOGDEBUG("PropertyServer::initialize randorder:" << _randOrder << endl);
@@ -203,15 +199,6 @@ void PropertyServer::initHashMap()
203199
float iFactor = TC_Common::strto<float>(g_pconf->get("/tars/hashmap<factor>","2"));
204200
int iSize = TC_Common::toSize(g_pconf->get("/tars/hashmap<size>"), 1024*1024*256);
205201

206-
207-
// _clonePath = ServerConfig::DataPath + "/" + g_pconf->get("/tars/hashmap<clonePatch>","clone");
208-
209-
// if(!TC_File::makeDirRecursive(_clonePath))
210-
// {
211-
// TLOGERROR("cannot create hashmap file " << _clonePath << endl);
212-
// exit(0);
213-
// }
214-
215202
TLOGDEBUG("PropertyServer::initHashMap init multi hashmap begin..." << endl);
216203

217204
for(int i = 0; i < 2; ++i)
@@ -245,11 +232,14 @@ void PropertyServer::initHashMap()
245232

246233
_hashmap[i][k].initDataBlockSize(iMinBlock,iMaxBlock,iFactor);
247234

248-
if(TC_File::isFileExist(sHashMapFile))
249-
{
250-
iSize = TC_File::getFileSize(sHashMapFile);
251-
}
252-
_hashmap[i][k].initStore( sHashMapFile.c_str(), iSize );
235+
#if TARGET_PLATFORM_IOS
236+
_hashmap[i][k].create(new char[iSize], iSize);
237+
#elif TARGET_PLATFORM_WINDOWS
238+
_hashmap[i][k].initStore(sHashMapFile.c_str(), iSize);
239+
#else
240+
key_t key = tars::hash<string>()(sHashMapFile);
241+
_hashmap[i][k].initStore(key, iSize);
242+
#endif
253243

254244
TLOGINFO("\n" << _hashmap[i][k].desc() << endl);
255245
}

0 commit comments

Comments
 (0)