Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
package cn.javayong.shardingjdbc4.spring.common.sharding;

import lombok.extern.slf4j.Slf4j;

import java.security.MessageDigest;
import java.util.Arrays;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;

/**
* 一致性哈希算法实现类
* 用于在分布式系统中实现数据或请求的均衡分配
*
* @author Durant
* @date 2024/9/24/周二
* @since 1.0.0
*/
@Slf4j
public class ConsistentHashingDatabase {
// 存储环上所有节点及其哈希值的映射,使用TreeMap自动排序
private final SortedMap<Long, String> circle = new TreeMap<>();
// 哈希函数接口实例,具体哈希算法通过构造函数指定
private final HashFunction hashFunction = new SHA256HashFunction();

/**
* 构造函数,初始化一致性哈希环
*
* @param nodes 分布式系统中的实际节点列表
* @param virtualNodeCount 每个实际节点对应的虚拟节点数量,用于提高哈希环的均衡度
*/
public void init(List<String> nodes, int virtualNodeCount) {
// 遍历所有节点,将其添加到哈希环中
for (String node : nodes) {
for (int i = 0; i < virtualNodeCount; i++) {
// 生成虚拟节点名称
String virtualNodeName = node + "VN" + i;
// 计算虚拟节点的哈希值,并添加到环上
long hash = hashFunction.hash(virtualNodeName);
// circle.put(hash, virtualNodeName); 这里本来取虚拟节点其实不对,应该取实际节点
circle.put(hash, node);
}
}
}

/**
* 根据给定的键(如数据键或请求键)找到对应的节点
*
* @param key 用于定位的键,可以是数据的标识符或请求的某个特征值
* @return 对应的节点名称,如果环为空则返回null
*/
public String getTargetNode(String key) {
String res = null;
if (circle.isEmpty()) {
return res;
}
// 计算键的哈希值
long hash = hashFunction.hash(key);
// 找到哈希环上大于等于hash值的节点
SortedMap<Long, String> tailMap = circle.tailMap(hash);
if (tailMap.isEmpty()) {
// 如果不存在,循环到环的起始位置
res = circle.get(circle.firstKey());
} else {
// 返回找到的节点
res = tailMap.get(tailMap.firstKey());
}
log.info("key, {}, hash:{}, res:{}", key, hash, res);
return res;
}

/**
* 哈希函数接口定义
*/
interface HashFunction {
long hash(String value);
}

/**
* 使用SHA-256算法实现的哈希函数
*/
static class SHA256HashFunction implements HashFunction {
@Override
public long hash(String value) {
try {
// 获取SHA-256消息摘要对象
MessageDigest digest = MessageDigest.getInstance("SHA-256");
// 计算哈希值
byte[] hash = digest.digest(value.getBytes());
// 将字节数组转换为长整型哈希值,注意这里的哈希有负数,可能有坑??注意注意
return ((long) (hash[3] & 0xFF) << 56)
| ((long) (hash[2] & 0xFF) << 48)
| ((long) (hash[1] & 0xFF) << 40)
| ((long) (hash[0] & 0xFF) << 32)
| ((long) (hash[7] & 0xFF) << 24)
| ((hash[6] & 0xFF) << 16)
| ((hash[5] & 0xFF) << 8)
| (hash[4] & 0xFF);
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}


public static void main(String[] args) {
ConsistentHashingDatabase consistentHashingDatabase = new ConsistentHashingDatabase();
consistentHashingDatabase.init(Arrays.asList("ds0", "ds1", "ds2", "ds3"), 8);

for (int i = 600; i < 650; i++) {
System.out.println(consistentHashingDatabase.getTargetNode(String.valueOf(i)));
}


}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package cn.javayong.shardingjdbc4.spring.common.sharding;

import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;

/**
* 一致性哈希算法实现类
* 用于在分布式系统中实现数据或请求的均衡分配
*
* @author Durant
* @date 2024/9/24/周二
* @since 1.0.0
*/
public class ConsistentHashingTables extends ConsistentHashingDatabase{
// 存储环上所有节点及其哈希值的映射,使用TreeMap自动排序
private final SortedMap<Long, String> circle = new TreeMap<>();

private final HashFunction hashFunction = new SHA256HashFunction();

/**
* 构造函数,初始化一致性哈希环, 分表策略, 分表策略中 没有虚拟节点的概念,这里virtualNodeCount=1
*
* @param nodes 分布式系统中的实际表列表
* @param virtualNodeCount 每个实际节点对应的虚拟节点数量,用于提高哈希环的均衡度
*/
public void init(List<String> nodes, int virtualNodeCount) {
// 遍历所有节点,将其添加到哈希环中
for (String node : nodes) {
for (int i = 0; i < virtualNodeCount; i++) {
// 生成虚拟节点名称
String virtualNodeName = node + "VN" + i;
// 计算虚拟节点的哈希值,并添加到环上
long hash = hashFunction.hash(virtualNodeName);
// circle.put(hash, virtualNodeName); 这里本来取虚拟节点其实不对,应该取实际节点
circle.put(hash, node);
}
}
}

@Override
public String getTargetNode(String key) {
if (this.circle.isEmpty()) {
return null;
}
// 计算键的哈希值
long hash = hashFunction.hash(key);
// 找到哈希环上大于等于hash值的节点
SortedMap<Long, String> tailMap = this.circle.tailMap(hash);
if (tailMap.isEmpty()) {
// 如果不存在,循环到环的起始位置
return this.circle.get(this.circle.firstKey());
} else {
// 返回找到的节点
return tailMap.get(tailMap.firstKey());
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
package cn.javayong.shardingjdbc4.spring.common.sharding;

import cn.javayong.shardingbase.ShardingConstants;
import cn.javayong.shardingbase.SnowFlakeIdGenerator;
import cn.javayong.shardingbase.StringHashUtil;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.shardingsphere.api.sharding.complex.ComplexKeysShardingAlgorithm;
import org.apache.shardingsphere.api.sharding.complex.ComplexKeysShardingValue;

import java.util.*;

/**
* 分库策略
* HashSlotAlgorithm 是4库8表
* 这里是 4节点 8虚拟节点,32表
*/
public class HashSlotAlgorithmDatabase implements ComplexKeysShardingAlgorithm {

ConsistentHashingDatabase consistentHashingDatabase = new ConsistentHashingDatabase();

// 虚拟节点
private static final int VN_NODE = 8;

/**
* 分片策略
*
* @param availableTargetNames available data sources or tables's names 发挥的是多少个库,ds0-3
* @param complexKeysShardingValue {@link ComplexKeysShardingValue} ex: ComplexKeysShardingValue(logicTableName=t_ent_order, columnNameAndShardingValuesMap={id=[674988670850883591]}, columnNameAndRangeValuesMap={})
* @return
*/
@Override
public Collection<String> doSharding(Collection availableTargetNames, ComplexKeysShardingValue complexKeysShardingValue) {
consistentHashingDatabase.init(new ArrayList<>(availableTargetNames), VN_NODE);
return getSharding(availableTargetNames, complexKeysShardingValue, consistentHashingDatabase);
}

//默认是否查询id
public String getQuerySnowFlakeIdColumn() {
return ShardingConstants.DEFAULT_PRIMARY_KEY;
}

//组合键数目 比如 sharding-columns:id,ent_id,region_code 组合键为:ent_id , region_code , 数目为 2
public int getCombineKeyLength() {
return ShardingConstants.DEFAULT_SINGLE_COMBINE_KEY_LENGTH;
}

List<String> doShardingValuesByQuerySnowFlakeId(ComplexKeysShardingValue complexKeysShardingValue) {
String querySnowFlakeIdColumn = getQuerySnowFlakeIdColumn();
return getShardingValueListByColumn(querySnowFlakeIdColumn, complexKeysShardingValue);
}

List<String> doShardingValuesByShardingColumns(ComplexKeysShardingValue complexKeysShardingValue) {
int combineKeyLength = getCombineKeyLength();
List<String> shardingColumnsArray = getShardingColumnsArray(complexKeysShardingValue);
if (shardingColumnsArray.size() != combineKeyLength) {
return Collections.EMPTY_LIST;
}
List<String> combinnationList = null;
//单分片键
if (combineKeyLength == ShardingConstants.DEFAULT_SINGLE_COMBINE_KEY_LENGTH) {
combinnationList = doSingleSharding(complexKeysShardingValue, shardingColumnsArray);
}
//多分片键
else {
combinnationList = doMultiSharding(complexKeysShardingValue, shardingColumnsArray);
}
return combinnationList;
}

private List<String> doSingleSharding(ComplexKeysShardingValue complexKeysShardingValue, List<String> shardingColumnsArray) {
String shardingColumn = shardingColumnsArray.get(0);
List<String> valueList = getShardingValueListByColumn(shardingColumn, complexKeysShardingValue);
return valueList;
}

private List<String> doMultiSharding(ComplexKeysShardingValue complexKeysShardingValue, List<String> shardingColumnsArray) {
List<List<String>> collection = new ArrayList<List<String>>();
for (int i = 0; i < shardingColumnsArray.size(); i++) {
String shardingColumn = shardingColumnsArray.get(i);
List<String> shardingValueList = getShardingValueListByColumn(shardingColumn, complexKeysShardingValue);
if (shardingValueList != null && shardingValueList.size() > 0) {
collection.add(shardingValueList);
}
}
return StringHashUtil.descartes(collection);
}

private List<String> getShardingValueListByColumn(String shardingColumn, ComplexKeysShardingValue complexKeysShardingValue) {
List<String> arrayList = new ArrayList<String>(1);
//先处理这个值
Map<String, Collection> columnNameAndShardingValuesMap = complexKeysShardingValue.getColumnNameAndShardingValuesMap();
columnNameAndShardingValuesMap.forEach((key, values) -> {
if (shardingColumn.equals(key)) {
for (Object value : values) {
if (value != null) {
arrayList.add(String.valueOf(value));
}
}
}
});
return arrayList;
}

private List<String> getShardingColumnsArray(ComplexKeysShardingValue complexKeysShardingValue) {
Map<String, Collection> columnNameAndShardingValuesMap = complexKeysShardingValue.getColumnNameAndShardingValuesMap();
int size = columnNameAndShardingValuesMap.size();
List<String> arr = new ArrayList<String>(size);
columnNameAndShardingValuesMap.forEach((key, values) -> {
arr.add(key);
});
//对分区键做一个排序,便于后续实现笛卡尔积
Collections.sort(arr);
return arr;
}

protected List<String> getSharding(Collection availableTargetNames, ComplexKeysShardingValue complexKeysShardingValue,
ConsistentHashingDatabase consistentHashing) {
List<Integer> slotList = new ArrayList<>();
// 真实分片数量
int count = availableTargetNames.size();
if ((count & (count - 1)) != 0) {
throw new RuntimeException("分区数必须是2的次幂,当前分区数是:" + count) {};
}
// 先判断是否走主键路由,若主键路由, 查询中到数据 按照主键路由 , 否则按照分表字段路由
boolean querySnowFlakeIdOK = false;
String querySnowFlakeIdColumn = getQuerySnowFlakeIdColumn();
if (StringUtils.isNotBlank(querySnowFlakeIdColumn)) {
List<String> valuesByQuerySnowFlakeId = doShardingValuesByQuerySnowFlakeId(complexKeysShardingValue);
if (CollectionUtils.isNotEmpty(valuesByQuerySnowFlakeId)) {
querySnowFlakeIdOK = true;
for (String idValue : valuesByQuerySnowFlakeId) {
slotList.add(SnowFlakeIdGenerator.getWorkerId(Long.valueOf(idValue)));
}
}
}
// 若主键路由失败 , 则通过分片组合字段路由
if (!querySnowFlakeIdOK) {
List<String> values = doShardingValuesByShardingColumns(complexKeysShardingValue);
if (CollectionUtils.isNotEmpty(values)) {
for (String value : values) {
int slot = StringHashUtil.hashSlot(value);
slotList.add(slot);
}
}
}
//返回结果
List<String> result = new ArrayList<>();
for (Integer slot : slotList) {
result.add(consistentHashing.getTargetNode(String.valueOf(slot)));
}
return result;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package cn.javayong.shardingjdbc4.spring.common.sharding;

import org.apache.shardingsphere.api.sharding.complex.ComplexKeysShardingValue;

import java.util.ArrayList;
import java.util.Collection;

/**
* 分表策略
* HashSlotAlgorithm 32 表
*/
public class HashSlotAlgorithmTables extends HashSlotAlgorithmDatabase{

ConsistentHashingTables consistentHashingTables = new ConsistentHashingTables();

/**
* 分片策略
*
* @param availableTargetNames available data sources or tables's names 发挥的是多少个库,t_ent_order_detail0-31
* @param complexKeysShardingValue {@link ComplexKeysShardingValue} ex: ComplexKeysShardingValue(logicTableName=t_ent_order, columnNameAndShardingValuesMap={id=[674988670850883591]}, columnNameAndRangeValuesMap={})
* @return
*/
@Override
public Collection<String> doSharding(Collection availableTargetNames, ComplexKeysShardingValue complexKeysShardingValue) {
consistentHashingTables.init(new ArrayList<>(availableTargetNames), 1);
return getSharding(availableTargetNames, complexKeysShardingValue, consistentHashingTables);
}

}
10 changes: 5 additions & 5 deletions shardingjdbc4-spring/src/main/resources/application-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ spring:
databaseStrategy:
complex:
sharding-columns: id,ent_id
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmDatabase
# 分表策略
tableStrategy:
none:
Expand All @@ -60,12 +60,12 @@ spring:
databaseStrategy:
complex:
sharding-columns: id,ent_id
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmDatabase
# 分表策略
tableStrategy:
complex:
sharding-columns: id,ent_id
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmTables
# 订单详情表
t_ent_order_detail:
# 真实表
Expand All @@ -74,12 +74,12 @@ spring:
databaseStrategy:
complex:
sharding-columns: id,ent_id
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmDatabase
# 分表策略
tableStrategy:
complex:
sharding-columns: id,ent_id
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm
algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmTables
bindingTables:
- t_ent_order,t_ent_order_detail

Expand Down