Skip to content

Commit 9f2d16b

Browse files
stekyczstekycz
committed
Improved performance of journal cleaning
1 parent 28c26e0 commit 9f2d16b

File tree

4 files changed

+113
-30
lines changed

4 files changed

+113
-30
lines changed

‎src/Kdyby/Redis/RedisJournal.php‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,9 @@ class RedisJournal extends Nette\Object implements Nette\Caching\Storages\IJourn
3131
TAGS = 'tags',
3232
KEYS = 'keys';
3333

34+
/** @internal batch delete size */
35+
const BATCH_SIZE = 8000;
36+
3437
/**
3538
* @var RedisClient
3639
*/
@@ -108,34 +111,86 @@ private function cleanEntry($keys)
108111
* Cleans entries from journal.
109112
*
110113
* @param array $conds
114+
* @param \Nette\Caching\IStorage $storage
111115
*
112116
* @return array of removed items or NULL when performing a full cleanup
113117
*/
114-
public function clean(array $conds)
118+
public function clean(array $conds, Nette\Caching\IStorage $storage = NULL)
115119
{
116120
if (!empty($conds[Cache::ALL])) {
117121
$all = $this->client->keys(self::NS_NETTE . ':*');
122+
if ($storage instanceof RedisStorage) {
123+
$all = array_merge($all, $this->client->keys(RedisStorage::NS_NETTE . ':*'));
124+
}
118125

119-
$this->client->multi();
120126
call_user_func_array(array($this->client, 'del'), $all);
121-
$this->client->exec();
122127
return NULL;
123128
}
124129

125130
$entries = array();
126131
if (!empty($conds[Cache::TAGS])) {
132+
$removingTagKeys = array();
127133
foreach ((array)$conds[Cache::TAGS] as $tag) {
128-
$this->cleanEntry($found = $this->tagEntries($tag));
134+
$found = $this->tagEntries($tag);
135+
$removingTagKeys[] = $this->formatKey($tag, self::KEYS);
129136
$entries = array_merge($entries, $found);
130137
}
138+
if ($removingTagKeys) {
139+
call_user_func_array(array($this->client, 'del'), $removingTagKeys);
140+
}
131141
}
132142

133143
if (isset($conds[Cache::PRIORITY])) {
134-
$this->cleanEntry($found = $this->priorityEntries($conds[Cache::PRIORITY]));
144+
$found = $this->priorityEntries($conds[Cache::PRIORITY]);
145+
call_user_func_array(array($this->client, 'zRemRangeByScore'), array($this->formatKey(self::PRIORITY), 0, (int)$conds[Cache::PRIORITY]));
135146
$entries = array_merge($entries, $found);
136147
}
137148

138-
return array_unique($entries);
149+
$entries = array_unique($entries);
150+
151+
$removingKeys = array();
152+
$removingKeyTags = array();
153+
$removingKeyPriorities = array();
154+
foreach ($entries as $key) {
155+
if ($storage instanceof RedisStorage) {
156+
$removingKeys[] = $key;
157+
}
158+
$removingKeyTags[] = $this->formatKey($key, self::TAGS);
159+
$removingKeyPriorities[] = $this->formatKey($key, self::PRIORITY);
160+
if (count($removingKeyTags) >= self::BATCH_SIZE) {
161+
$this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries);
162+
$removingKeys = array();
163+
$removingKeyTags = array();
164+
$removingKeyPriorities = array();
165+
}
166+
}
167+
168+
$this->cleanBatchData($removingKeys, $removingKeyPriorities, $removingKeyTags, $entries);
169+
170+
return $storage instanceof RedisStorage ? array() : $entries;
171+
}
172+
173+
174+
175+
private function cleanBatchData(array $removingKeys, array $removingKeyPriorities, array $removingKeyTags, array $keys)
176+
{
177+
if ($removingKeyTags) {
178+
if ($keys) {
179+
$affectedTags = call_user_func_array(array($this->client, 'sunion'), array($removingKeyTags));
180+
foreach ($affectedTags as $tag) {
181+
if ($tag) {
182+
call_user_func_array(array($this->client, 'sRem'), array_merge(array($this->formatKey($tag, self::KEYS)), $keys));
183+
}
184+
}
185+
}
186+
call_user_func_array(array($this->client, 'del'), $removingKeyTags);
187+
}
188+
if ($removingKeyPriorities) {
189+
call_user_func_array(array($this->client, 'del'), $removingKeyPriorities);
190+
}
191+
if ($removingKeys) {
192+
call_user_func_array(array($this->client, 'del'), $removingKeys);
193+
}
139194
}
140195

141196

‎src/Kdyby/Redis/scripts/common.lua‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,23 @@ local tagEntries = function (tag)
2929
return redis.call('sMembers', formatKey(tag, "keys"))
3030
end
3131

32+
local range = function (from, to, step)
33+
step = step or 1
34+
local f =
35+
step > 0 and
36+
function(_, lastvalue)
37+
local nextvalue = lastvalue + step
38+
if nextvalue <= to then return nextvalue end
39+
end or
40+
step < 0 and
41+
function(_, lastvalue)
42+
local nextvalue = lastvalue + step
43+
if nextvalue >= to then return nextvalue end
44+
end or
45+
function(_, lastvalue) return lastvalue end
46+
return f, nil, from - step
47+
end
48+
3249
local cleanEntry = function (keys)
3350
for i, key in pairs(keys) do
3451
local tags = entryTags(key)
@@ -45,3 +62,34 @@ local cleanEntry = function (keys)
4562
-- redis.call('exec')
4663
end
4764
end
65+
66+
local mergeTables = function (first, second)
67+
for i, key in pairs(second) do
68+
first[#first + 1] = key
69+
end
70+
return first
71+
end
72+
73+
local batch = function (keys, callback)
74+
if #keys > 0 then
75+
-- redis.call('multi')
76+
-- the magic number 7998 becomes from Lua limitations, see http://stackoverflow.com/questions/19202367/how-to-avoid-redis-calls-in-lua-script-limitations
77+
local tmp = {}
78+
for i,key in pairs(keys) do
79+
tmp[#tmp + 1] = key
80+
if #tmp >= 7998 then
81+
callback(tmp)
82+
tmp = {}
83+
end
84+
end
85+
callback(tmp)
86+
-- redis.call('exec')
87+
end
88+
end
89+
90+
local batchDelete = function(keys)
91+
local delete = function (tmp)
92+
redis.call('del', unpack(tmp))
93+
end
94+
batch(keys, delete)
95+
end

‎src/Kdyby/Redis/scripts/journal.clean.lua‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,10 @@
22
local conds = cjson.decode(ARGV[1])
33

44
if conds["all"] ~= nil then
5-
-- redis.call('multi')
6-
for i, value in pairs(redis.call('keys', "Nette.Journal:*")) do
7-
redis.call('del', value)
5+
batchDelete(redis.call('keys', "Nette.Journal:*"))
6+
if conds["delete-entries"] ~= nil then
7+
batchDelete(redis.call('keys', "Nette.Storage:*"))
88
end
9-
-- redis.call('exec')
109

1110
return redis.status_reply("Ok")
1211
end

‎tests/KdybyTests/Redis/RedisJournal.phpt‎

Lines changed: 1 addition & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ class RedisJournalTest extends AbstractRedisTestCase
284284
Assert::null($result);
285285

286286
$result2 = $this->journal->clean(array(Cache::TAGS => 'test:all'));
287-
Assert::true(empty($result2));
287+
Assert::equal(array(), $result2);
288288
}
289289

290290

@@ -362,26 +362,7 @@ LUA;
362362
private function cacheGeneratorScripts()
363363
{
364364
$script = file_get_contents(__DIR__ . '/../../../src/Kdyby/Redis/scripts/common.lua');
365-
$script .= <<<LUA
366-
local range = function (from, to, step)
367-
step = step or 1
368-
local f =
369-
step > 0 and
370-
function(_, lastvalue)
371-
local nextvalue = lastvalue + step
372-
if nextvalue <= to then return nextvalue end
373-
end or
374-
step < 0 and
375-
function(_, lastvalue)
376-
local nextvalue = lastvalue + step
377-
if nextvalue >= to then return nextvalue end
378-
end or
379-
function(_, lastvalue) return lastvalue end
380-
return f, nil, from - step
381-
end
382365

383-
384-
LUA;
385366
return $script;
386367
}
387368

0 commit comments

Comments
 (0)