diff --git a/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/ConsistentHashingDatabase.java b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/ConsistentHashingDatabase.java new file mode 100644 index 0000000..e40c6a9 --- /dev/null +++ b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/ConsistentHashingDatabase.java @@ -0,0 +1,116 @@ +package cn.javayong.shardingjdbc4.spring.common.sharding; + +import lombok.extern.slf4j.Slf4j; + +import java.security.MessageDigest; +import java.util.Arrays; +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * 一致性哈希算法实现类 + * 用于在分布式系统中实现数据或请求的均衡分配 + * + * @author Durant + * @date 2024/9/24/周二 + * @since 1.0.0 + */ +@Slf4j +public class ConsistentHashingDatabase { + // 存储环上所有节点及其哈希值的映射,使用TreeMap自动排序 + private final SortedMap circle = new TreeMap<>(); + // 哈希函数接口实例,具体哈希算法通过构造函数指定 + private final HashFunction hashFunction = new SHA256HashFunction(); + + /** + * 构造函数,初始化一致性哈希环 + * + * @param nodes 分布式系统中的实际节点列表 + * @param virtualNodeCount 每个实际节点对应的虚拟节点数量,用于提高哈希环的均衡度 + */ + public void init(List nodes, int virtualNodeCount) { + // 遍历所有节点,将其添加到哈希环中 + for (String node : nodes) { + for (int i = 0; i < virtualNodeCount; i++) { + // 生成虚拟节点名称 + String virtualNodeName = node + "VN" + i; + // 计算虚拟节点的哈希值,并添加到环上 + long hash = hashFunction.hash(virtualNodeName); + // circle.put(hash, virtualNodeName); 这里本来取虚拟节点其实不对,应该取实际节点 + circle.put(hash, node); + } + } + } + + /** + * 根据给定的键(如数据键或请求键)找到对应的节点 + * + * @param key 用于定位的键,可以是数据的标识符或请求的某个特征值 + * @return 对应的节点名称,如果环为空则返回null + */ + public String getTargetNode(String key) { + String res = null; + if (circle.isEmpty()) { + return res; + } + // 计算键的哈希值 + long hash = hashFunction.hash(key); + // 找到哈希环上大于等于hash值的节点 + SortedMap tailMap = circle.tailMap(hash); + if (tailMap.isEmpty()) { + // 如果不存在,循环到环的起始位置 + res = circle.get(circle.firstKey()); + } else { + // 返回找到的节点 + res = tailMap.get(tailMap.firstKey()); + } + log.info("key, {}, hash:{}, res:{}", key, hash, res); + return res; + } + + /** + * 哈希函数接口定义 + */ + interface HashFunction { + long hash(String value); + } + + /** + * 使用SHA-256算法实现的哈希函数 + */ + static class SHA256HashFunction implements HashFunction { + @Override + public long hash(String value) { + try { + // 获取SHA-256消息摘要对象 + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + // 计算哈希值 + byte[] hash = digest.digest(value.getBytes()); + // 将字节数组转换为长整型哈希值,注意这里的哈希有负数,可能有坑??注意注意 + return ((long) (hash[3] & 0xFF) << 56) + | ((long) (hash[2] & 0xFF) << 48) + | ((long) (hash[1] & 0xFF) << 40) + | ((long) (hash[0] & 0xFF) << 32) + | ((long) (hash[7] & 0xFF) << 24) + | ((hash[6] & 0xFF) << 16) + | ((hash[5] & 0xFF) << 8) + | (hash[4] & 0xFF); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + + public static void main(String[] args) { + ConsistentHashingDatabase consistentHashingDatabase = new ConsistentHashingDatabase(); + consistentHashingDatabase.init(Arrays.asList("ds0", "ds1", "ds2", "ds3"), 8); + + for (int i = 600; i < 650; i++) { + System.out.println(consistentHashingDatabase.getTargetNode(String.valueOf(i))); + } + + + } +} diff --git a/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/ConsistentHashingTables.java b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/ConsistentHashingTables.java new file mode 100644 index 0000000..d505628 --- /dev/null +++ b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/ConsistentHashingTables.java @@ -0,0 +1,58 @@ +package cn.javayong.shardingjdbc4.spring.common.sharding; + +import java.util.List; +import java.util.SortedMap; +import java.util.TreeMap; + +/** + * 一致性哈希算法实现类 + * 用于在分布式系统中实现数据或请求的均衡分配 + * + * @author Durant + * @date 2024/9/24/周二 + * @since 1.0.0 + */ +public class ConsistentHashingTables extends ConsistentHashingDatabase{ + // 存储环上所有节点及其哈希值的映射,使用TreeMap自动排序 + private final SortedMap circle = new TreeMap<>(); + + private final HashFunction hashFunction = new SHA256HashFunction(); + + /** + * 构造函数,初始化一致性哈希环, 分表策略, 分表策略中 没有虚拟节点的概念,这里virtualNodeCount=1 + * + * @param nodes 分布式系统中的实际表列表 + * @param virtualNodeCount 每个实际节点对应的虚拟节点数量,用于提高哈希环的均衡度 + */ + public void init(List nodes, int virtualNodeCount) { + // 遍历所有节点,将其添加到哈希环中 + for (String node : nodes) { + for (int i = 0; i < virtualNodeCount; i++) { + // 生成虚拟节点名称 + String virtualNodeName = node + "VN" + i; + // 计算虚拟节点的哈希值,并添加到环上 + long hash = hashFunction.hash(virtualNodeName); + // circle.put(hash, virtualNodeName); 这里本来取虚拟节点其实不对,应该取实际节点 + circle.put(hash, node); + } + } + } + + @Override + public String getTargetNode(String key) { + if (this.circle.isEmpty()) { + return null; + } + // 计算键的哈希值 + long hash = hashFunction.hash(key); + // 找到哈希环上大于等于hash值的节点 + SortedMap tailMap = this.circle.tailMap(hash); + if (tailMap.isEmpty()) { + // 如果不存在,循环到环的起始位置 + return this.circle.get(this.circle.firstKey()); + } else { + // 返回找到的节点 + return tailMap.get(tailMap.firstKey()); + } + } +} diff --git a/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/HashSlotAlgorithmDatabase.java b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/HashSlotAlgorithmDatabase.java new file mode 100644 index 0000000..33a9b6d --- /dev/null +++ b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/HashSlotAlgorithmDatabase.java @@ -0,0 +1,154 @@ +package cn.javayong.shardingjdbc4.spring.common.sharding; + +import cn.javayong.shardingbase.ShardingConstants; +import cn.javayong.shardingbase.SnowFlakeIdGenerator; +import cn.javayong.shardingbase.StringHashUtil; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.shardingsphere.api.sharding.complex.ComplexKeysShardingAlgorithm; +import org.apache.shardingsphere.api.sharding.complex.ComplexKeysShardingValue; + +import java.util.*; + +/** + * 分库策略 + * HashSlotAlgorithm 是4库8表 + * 这里是 4节点 8虚拟节点,32表 + */ +public class HashSlotAlgorithmDatabase implements ComplexKeysShardingAlgorithm { + + ConsistentHashingDatabase consistentHashingDatabase = new ConsistentHashingDatabase(); + + // 虚拟节点 + private static final int VN_NODE = 8; + + /** + * 分片策略 + * + * @param availableTargetNames available data sources or tables's names 发挥的是多少个库,ds0-3 + * @param complexKeysShardingValue {@link ComplexKeysShardingValue} ex: ComplexKeysShardingValue(logicTableName=t_ent_order, columnNameAndShardingValuesMap={id=[674988670850883591]}, columnNameAndRangeValuesMap={}) + * @return + */ + @Override + public Collection doSharding(Collection availableTargetNames, ComplexKeysShardingValue complexKeysShardingValue) { + consistentHashingDatabase.init(new ArrayList<>(availableTargetNames), VN_NODE); + return getSharding(availableTargetNames, complexKeysShardingValue, consistentHashingDatabase); + } + + //默认是否查询id + public String getQuerySnowFlakeIdColumn() { + return ShardingConstants.DEFAULT_PRIMARY_KEY; + } + + //组合键数目 比如 sharding-columns:id,ent_id,region_code 组合键为:ent_id , region_code , 数目为 2 + public int getCombineKeyLength() { + return ShardingConstants.DEFAULT_SINGLE_COMBINE_KEY_LENGTH; + } + + List doShardingValuesByQuerySnowFlakeId(ComplexKeysShardingValue complexKeysShardingValue) { + String querySnowFlakeIdColumn = getQuerySnowFlakeIdColumn(); + return getShardingValueListByColumn(querySnowFlakeIdColumn, complexKeysShardingValue); + } + + List doShardingValuesByShardingColumns(ComplexKeysShardingValue complexKeysShardingValue) { + int combineKeyLength = getCombineKeyLength(); + List shardingColumnsArray = getShardingColumnsArray(complexKeysShardingValue); + if (shardingColumnsArray.size() != combineKeyLength) { + return Collections.EMPTY_LIST; + } + List combinnationList = null; + //单分片键 + if (combineKeyLength == ShardingConstants.DEFAULT_SINGLE_COMBINE_KEY_LENGTH) { + combinnationList = doSingleSharding(complexKeysShardingValue, shardingColumnsArray); + } + //多分片键 + else { + combinnationList = doMultiSharding(complexKeysShardingValue, shardingColumnsArray); + } + return combinnationList; + } + + private List doSingleSharding(ComplexKeysShardingValue complexKeysShardingValue, List shardingColumnsArray) { + String shardingColumn = shardingColumnsArray.get(0); + List valueList = getShardingValueListByColumn(shardingColumn, complexKeysShardingValue); + return valueList; + } + + private List doMultiSharding(ComplexKeysShardingValue complexKeysShardingValue, List shardingColumnsArray) { + List> collection = new ArrayList>(); + for (int i = 0; i < shardingColumnsArray.size(); i++) { + String shardingColumn = shardingColumnsArray.get(i); + List shardingValueList = getShardingValueListByColumn(shardingColumn, complexKeysShardingValue); + if (shardingValueList != null && shardingValueList.size() > 0) { + collection.add(shardingValueList); + } + } + return StringHashUtil.descartes(collection); + } + + private List getShardingValueListByColumn(String shardingColumn, ComplexKeysShardingValue complexKeysShardingValue) { + List arrayList = new ArrayList(1); + //先处理这个值 + Map columnNameAndShardingValuesMap = complexKeysShardingValue.getColumnNameAndShardingValuesMap(); + columnNameAndShardingValuesMap.forEach((key, values) -> { + if (shardingColumn.equals(key)) { + for (Object value : values) { + if (value != null) { + arrayList.add(String.valueOf(value)); + } + } + } + }); + return arrayList; + } + + private List getShardingColumnsArray(ComplexKeysShardingValue complexKeysShardingValue) { + Map columnNameAndShardingValuesMap = complexKeysShardingValue.getColumnNameAndShardingValuesMap(); + int size = columnNameAndShardingValuesMap.size(); + List arr = new ArrayList(size); + columnNameAndShardingValuesMap.forEach((key, values) -> { + arr.add(key); + }); + //对分区键做一个排序,便于后续实现笛卡尔积 + Collections.sort(arr); + return arr; + } + + protected List getSharding(Collection availableTargetNames, ComplexKeysShardingValue complexKeysShardingValue, + ConsistentHashingDatabase consistentHashing) { + List slotList = new ArrayList<>(); + // 真实分片数量 + int count = availableTargetNames.size(); + if ((count & (count - 1)) != 0) { + throw new RuntimeException("分区数必须是2的次幂,当前分区数是:" + count) {}; + } + // 先判断是否走主键路由,若主键路由, 查询中到数据 按照主键路由 , 否则按照分表字段路由 + boolean querySnowFlakeIdOK = false; + String querySnowFlakeIdColumn = getQuerySnowFlakeIdColumn(); + if (StringUtils.isNotBlank(querySnowFlakeIdColumn)) { + List valuesByQuerySnowFlakeId = doShardingValuesByQuerySnowFlakeId(complexKeysShardingValue); + if (CollectionUtils.isNotEmpty(valuesByQuerySnowFlakeId)) { + querySnowFlakeIdOK = true; + for (String idValue : valuesByQuerySnowFlakeId) { + slotList.add(SnowFlakeIdGenerator.getWorkerId(Long.valueOf(idValue))); + } + } + } + // 若主键路由失败 , 则通过分片组合字段路由 + if (!querySnowFlakeIdOK) { + List values = doShardingValuesByShardingColumns(complexKeysShardingValue); + if (CollectionUtils.isNotEmpty(values)) { + for (String value : values) { + int slot = StringHashUtil.hashSlot(value); + slotList.add(slot); + } + } + } + //返回结果 + List result = new ArrayList<>(); + for (Integer slot : slotList) { + result.add(consistentHashing.getTargetNode(String.valueOf(slot))); + } + return result; + } +} diff --git a/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/HashSlotAlgorithmTables.java b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/HashSlotAlgorithmTables.java new file mode 100644 index 0000000..5d1024c --- /dev/null +++ b/shardingjdbc4-spring/src/main/java/cn/javayong/shardingjdbc4/spring/common/sharding/HashSlotAlgorithmTables.java @@ -0,0 +1,29 @@ +package cn.javayong.shardingjdbc4.spring.common.sharding; + +import org.apache.shardingsphere.api.sharding.complex.ComplexKeysShardingValue; + +import java.util.ArrayList; +import java.util.Collection; + +/** + * 分表策略 + * HashSlotAlgorithm 32 表 + */ +public class HashSlotAlgorithmTables extends HashSlotAlgorithmDatabase{ + + ConsistentHashingTables consistentHashingTables = new ConsistentHashingTables(); + + /** + * 分片策略 + * + * @param availableTargetNames available data sources or tables's names 发挥的是多少个库,t_ent_order_detail0-31 + * @param complexKeysShardingValue {@link ComplexKeysShardingValue} ex: ComplexKeysShardingValue(logicTableName=t_ent_order, columnNameAndShardingValuesMap={id=[674988670850883591]}, columnNameAndRangeValuesMap={}) + * @return + */ + @Override + public Collection doSharding(Collection availableTargetNames, ComplexKeysShardingValue complexKeysShardingValue) { + consistentHashingTables.init(new ArrayList<>(availableTargetNames), 1); + return getSharding(availableTargetNames, complexKeysShardingValue, consistentHashingTables); + } + +} diff --git a/shardingjdbc4-spring/src/main/resources/application-test.yml b/shardingjdbc4-spring/src/main/resources/application-test.yml index 45e7420..f6a8c32 100644 --- a/shardingjdbc4-spring/src/main/resources/application-test.yml +++ b/shardingjdbc4-spring/src/main/resources/application-test.yml @@ -48,7 +48,7 @@ spring: databaseStrategy: complex: sharding-columns: id,ent_id - algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm + algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmDatabase # 分表策略 tableStrategy: none: @@ -60,12 +60,12 @@ spring: databaseStrategy: complex: sharding-columns: id,ent_id - algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm + algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmDatabase # 分表策略 tableStrategy: complex: sharding-columns: id,ent_id - algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm + algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmTables # 订单详情表 t_ent_order_detail: # 真实表 @@ -74,12 +74,12 @@ spring: databaseStrategy: complex: sharding-columns: id,ent_id - algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm + algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmDatabase # 分表策略 tableStrategy: complex: sharding-columns: id,ent_id - algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithm + algorithm-class-name: cn.javayong.shardingjdbc4.spring.common.sharding.HashSlotAlgorithmTables bindingTables: - t_ent_order,t_ent_order_detail