Skip to content

Commit 0cf061c

Browse files
sazzad16sazzad16
authored
Support Sentinel mode with UnifiedJedis (#3240)
* Support Sentinel mode with UnifiedJedis * modify
1 parent 130d4f5 commit 0cf061c

File tree

4 files changed

+488
-72
lines changed

4 files changed

+488
-72
lines changed

‎src/main/java/redis/clients/jedis/JedisSentineled.java

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package redis.clients.jedis;
2+
3+
import java.util.Set;
4+
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
5+
import redis.clients.jedis.providers.SentineledConnectionProvider;
6+
7+
public class JedisSentineled extends UnifiedJedis {
8+
9+
/**
10+
* This constructor is here for easier transition from {@link JedisSentinelPool#JedisSentinelPool(
11+
* java.lang.String, java.util.Set, redis.clients.jedis.JedisClientConfig, redis.clients.jedis.JedisClientConfig)}.
12+
*
13+
* @deprecated Use {@link #JedisSentineled(java.lang.String, redis.clients.jedis.JedisClientConfig,
14+
* java.util.Set, redis.clients.jedis.JedisClientConfig)}.
15+
*/
16+
@Deprecated
17+
// Legacy
18+
public JedisSentineled(String masterName, Set<HostAndPort> sentinels,
19+
final JedisClientConfig masterClientConfig, final JedisClientConfig sentinelClientConfig) {
20+
this(masterName, masterClientConfig, sentinels, sentinelClientConfig);
21+
}
22+
23+
public JedisSentineled(String masterName, final JedisClientConfig masterClientConfig,
24+
Set<HostAndPort> sentinels, final JedisClientConfig sentinelClientConfig) {
25+
this(new SentineledConnectionProvider(masterName, masterClientConfig, sentinels, sentinelClientConfig));
26+
}
27+
28+
/**
29+
* This constructor is here for easier transition from {@link JedisSentinelPool#JedisSentinelPool(
30+
* java.lang.String, java.util.Set, org.apache.commons.pool2.impl.GenericObjectPoolConfig,
31+
* redis.clients.jedis.JedisClientConfig, redis.clients.jedis.JedisClientConfig)}.
32+
*
33+
* @deprecated Use {@link #JedisSentineled(java.lang.String, redis.clients.jedis.JedisClientConfig,
34+
* org.apache.commons.pool2.impl.GenericObjectPoolConfig, java.util.Set, redis.clients.jedis.JedisClientConfig)}.
35+
*/
36+
@Deprecated
37+
// Legacy
38+
public JedisSentineled(String masterName, Set<HostAndPort> sentinels,
39+
final GenericObjectPoolConfig<Connection> poolConfig, final JedisClientConfig masterClientConfig,
40+
final JedisClientConfig sentinelClientConfig) {
41+
this(masterName, masterClientConfig, poolConfig, sentinels, sentinelClientConfig);
42+
}
43+
44+
public JedisSentineled(String masterName, final JedisClientConfig masterClientConfig,
45+
final GenericObjectPoolConfig<Connection> poolConfig,
46+
Set<HostAndPort> sentinels, final JedisClientConfig sentinelClientConfig) {
47+
this(new SentineledConnectionProvider(masterName, masterClientConfig, poolConfig, sentinels, sentinelClientConfig));
48+
}
49+
50+
public JedisSentineled(SentineledConnectionProvider sentineledConnectionProvider) {
51+
super(sentineledConnectionProvider);
52+
}
53+
54+
public HostAndPort getCurrentMaster() {
55+
return ((SentineledConnectionProvider) provider).getCurrentMaster();
56+
}
57+
}

‎src/main/java/redis/clients/jedis/providers/SentineledConnectionProvider.java

+280
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,280 @@
1+
package redis.clients.jedis.providers;
2+
3+
import java.util.ArrayList;
4+
import java.util.Collection;
5+
import java.util.List;
6+
import java.util.Set;
7+
import java.util.concurrent.atomic.AtomicBoolean;
8+
9+
import org.apache.commons.pool2.impl.GenericObjectPoolConfig;
10+
import org.slf4j.Logger;
11+
import org.slf4j.LoggerFactory;
12+
13+
import redis.clients.jedis.CommandArguments;
14+
import redis.clients.jedis.Connection;
15+
import redis.clients.jedis.ConnectionPool;
16+
import redis.clients.jedis.HostAndPort;
17+
import redis.clients.jedis.Jedis;
18+
import redis.clients.jedis.JedisClientConfig;
19+
import redis.clients.jedis.JedisPubSub;
20+
import redis.clients.jedis.exceptions.JedisConnectionException;
21+
import redis.clients.jedis.exceptions.JedisException;
22+
import redis.clients.jedis.util.IOUtils;
23+
24+
public class SentineledConnectionProvider implements ConnectionProvider {
25+
26+
private static final Logger LOG = LoggerFactory.getLogger(SentineledConnectionProvider.class);
27+
28+
protected static final long DEFAULT_SUBSCRIBE_RETRY_WAIT_TIME_MILLIS = 5000;
29+
30+
private volatile HostAndPort currentMaster;
31+
32+
private volatile ConnectionPool pool;
33+
34+
private final String masterName;
35+
36+
private final JedisClientConfig masterClientConfig;
37+
38+
private final GenericObjectPoolConfig<Connection> masterPoolConfig;
39+
40+
protected final Collection<SentinelListener> sentinelListeners = new ArrayList<>();
41+
42+
private final JedisClientConfig sentinelClientConfig;
43+
44+
private final long subscribeRetryWaitTimeMillis;
45+
46+
private final Object initPoolLock = new Object();
47+
48+
public SentineledConnectionProvider(String masterName, final JedisClientConfig masterClientConfig,
49+
Set<HostAndPort> sentinels, final JedisClientConfig sentinelClientConfig) {
50+
this(masterName, masterClientConfig, /*poolConfig*/ null, sentinels, sentinelClientConfig);
51+
}
52+
53+
public SentineledConnectionProvider(String masterName, final JedisClientConfig masterClientConfig,
54+
final GenericObjectPoolConfig<Connection> poolConfig,
55+
Set<HostAndPort> sentinels, final JedisClientConfig sentinelClientConfig) {
56+
this(masterName, masterClientConfig, poolConfig, sentinels, sentinelClientConfig,
57+
DEFAULT_SUBSCRIBE_RETRY_WAIT_TIME_MILLIS);
58+
}
59+
60+
public SentineledConnectionProvider(String masterName, final JedisClientConfig masterClientConfig,
61+
final GenericObjectPoolConfig<Connection> poolConfig,
62+
Set<HostAndPort> sentinels, final JedisClientConfig sentinelClientConfig,
63+
final long subscribeRetryWaitTimeMillis) {
64+
65+
this.masterName = masterName;
66+
this.masterClientConfig = masterClientConfig;
67+
this.masterPoolConfig = poolConfig;
68+
69+
this.sentinelClientConfig = sentinelClientConfig;
70+
this.subscribeRetryWaitTimeMillis = subscribeRetryWaitTimeMillis;
71+
72+
HostAndPort master = initSentinels(sentinels);
73+
initMaster(master);
74+
}
75+
76+
@Override
77+
public Connection getConnection() {
78+
return pool.getResource();
79+
}
80+
81+
@Override
82+
public Connection getConnection(CommandArguments args) {
83+
return pool.getResource();
84+
}
85+
86+
@Override
87+
public void close() {
88+
sentinelListeners.forEach(SentinelListener::shutdown);
89+
90+
pool.close();
91+
}
92+
93+
public HostAndPort getCurrentMaster() {
94+
return currentMaster;
95+
}
96+
97+
private void initMaster(HostAndPort master) {
98+
synchronized (initPoolLock) {
99+
if (!master.equals(currentMaster)) {
100+
currentMaster = master;
101+
102+
ConnectionPool newPool = masterPoolConfig != null
103+
? new ConnectionPool(currentMaster, masterClientConfig, masterPoolConfig)
104+
: new ConnectionPool(currentMaster, masterClientConfig);
105+
106+
ConnectionPool existingPool = pool;
107+
pool = newPool;
108+
LOG.info("Created connection pool to master at {}.", master);
109+
110+
if (existingPool != null) {
111+
// although we clear the pool, we still have to check the returned object in getResource,
112+
// this call only clears idle instances, not borrowed instances
113+
// existingPool.clear(); // necessary??
114+
existingPool.close();
115+
}
116+
}
117+
}
118+
}
119+
120+
private HostAndPort initSentinels(Set<HostAndPort> sentinels) {
121+
122+
HostAndPort master = null;
123+
boolean sentinelAvailable = false;
124+
125+
LOG.debug("Trying to find master from available sentinels...");
126+
127+
for (HostAndPort sentinel : sentinels) {
128+
129+
LOG.debug("Connecting to Sentinel {}...", sentinel);
130+
131+
try (Jedis jedis = new Jedis(sentinel, sentinelClientConfig)) {
132+
133+
List<String> masterAddr = jedis.sentinelGetMasterAddrByName(masterName);
134+
135+
// connected to sentinel...
136+
sentinelAvailable = true;
137+
138+
if (masterAddr == null || masterAddr.size() != 2) {
139+
LOG.warn("Sentinel {} is not monitoring master {}.", sentinel, masterName);
140+
continue;
141+
}
142+
143+
master = toHostAndPort(masterAddr);
144+
LOG.debug("Redis master reported at {}.", master);
145+
break;
146+
} catch (JedisException e) {
147+
// resolves #1036, it should handle JedisException there's another chance
148+
// of raising JedisDataException
149+
LOG.warn("Could not get master address from {}.", sentinel, e);
150+
}
151+
}
152+
153+
if (master == null) {
154+
if (sentinelAvailable) {
155+
// can connect to sentinel, but master name seems to not monitored
156+
throw new JedisException(
157+
"Can connect to sentinel, but " + masterName + " seems to be not monitored.");
158+
} else {
159+
throw new JedisConnectionException(
160+
"All sentinels down, cannot determine where " + masterName + " is running.");
161+
}
162+
}
163+
164+
LOG.info("Redis master running at {}. Starting sentinel listeners...", master);
165+
166+
for (HostAndPort sentinel : sentinels) {
167+
168+
SentinelListener listener = new SentinelListener(sentinel);
169+
// whether SentinelListener threads are alive or not, process can be stopped
170+
listener.setDaemon(true);
171+
sentinelListeners.add(listener);
172+
listener.start();
173+
}
174+
175+
return master;
176+
}
177+
178+
/**
179+
* Must be of size 2.
180+
*/
181+
private static HostAndPort toHostAndPort(List<String> masterAddr) {
182+
return toHostAndPort(masterAddr.get(0), masterAddr.get(1));
183+
}
184+
185+
private static HostAndPort toHostAndPort(String hostStr, String portStr) {
186+
return new HostAndPort(hostStr, Integer.parseInt(portStr));
187+
}
188+
189+
protected class SentinelListener extends Thread {
190+
191+
protected final HostAndPort node;
192+
protected volatile Jedis sentinelJedis;
193+
protected AtomicBoolean running = new AtomicBoolean(false);
194+
195+
public SentinelListener(HostAndPort node) {
196+
super(String.format("%s-SentinelListener-[%s]", masterName, node.toString()));
197+
this.node = node;
198+
}
199+
200+
@Override
201+
public void run() {
202+
203+
running.set(true);
204+
205+
while (running.get()) {
206+
207+
try {
208+
// double check that it is not being shutdown
209+
if (!running.get()) {
210+
break;
211+
}
212+
213+
sentinelJedis = new Jedis(node, sentinelClientConfig);
214+
215+
// code for active refresh
216+
List<String> masterAddr = sentinelJedis.sentinelGetMasterAddrByName(masterName);
217+
if (masterAddr == null || masterAddr.size() != 2) {
218+
LOG.warn("Can not get master {} address. Sentinel: {}.", masterName, node);
219+
} else {
220+
initMaster(toHostAndPort(masterAddr));
221+
}
222+
223+
sentinelJedis.subscribe(new JedisPubSub() {
224+
@Override
225+
public void onMessage(String channel, String message) {
226+
LOG.debug("Sentinel {} published: {}.", node, message);
227+
228+
String[] switchMasterMsg = message.split(" ");
229+
230+
if (switchMasterMsg.length > 3) {
231+
232+
if (masterName.equals(switchMasterMsg[0])) {
233+
initMaster(toHostAndPort(switchMasterMsg[3], switchMasterMsg[4]));
234+
} else {
235+
LOG.debug(
236+
"Ignoring message on +switch-master for master {}. Our master is {}.",
237+
switchMasterMsg[0], masterName);
238+
}
239+
240+
} else {
241+
LOG.error("Invalid message received on sentinel {} on channel +switch-master: {}.",
242+
node, message);
243+
}
244+
}
245+
}, "+switch-master");
246+
247+
} catch (JedisException e) {
248+
249+
if (running.get()) {
250+
LOG.error("Lost connection to sentinel {}. Sleeping {}ms and retrying.", node,
251+
subscribeRetryWaitTimeMillis, e);
252+
try {
253+
Thread.sleep(subscribeRetryWaitTimeMillis);
254+
} catch (InterruptedException se) {
255+
LOG.error("Sleep interrupted.", se);
256+
}
257+
} else {
258+
LOG.debug("Unsubscribing from sentinel {}.", node);
259+
}
260+
} finally {
261+
IOUtils.closeQuietly(sentinelJedis);
262+
}
263+
}
264+
}
265+
266+
// must not throw exception
267+
public void shutdown() {
268+
try {
269+
LOG.debug("Shutting down listener on {}.", node);
270+
running.set(false);
271+
// This isn't good, the Jedis object is not thread safe
272+
if (sentinelJedis != null) {
273+
sentinelJedis.close();
274+
}
275+
} catch (RuntimeException e) {
276+
LOG.error("Error while shutting down.", e);
277+
}
278+
}
279+
}
280+
}

0 commit comments

Comments
 (0)