NCZY/WAI_Project_BACKEND
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

新增: 唯一id生成组件

Browse Source
This commit is contained in:
ruying408
2024-08-25 16:32:32 +08:00
parent 43347ba942
commit 82ef007ea0
17 changed files with 679 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/com/cool/CoolApplication.java
View File

@@ -21,7 +21,7 @@ import org.springframework.scheduling.annotation.EnableAsync;
@EnableAsync // 开启异步处理
@EnableCaching // 开启缓存
@SpringBootApplication
@MapperScan("com.cool.modules.*.mapper") // 扫描指定包中的MyBatis映射器
@MapperScan("com.cool.**.mapper") // 扫描指定包中的MyBatis映射器
public class CoolApplication {
private static volatile ConfigurableApplicationContext context;

24
src/main/java/com/cool/core/init/IDGenInit.java Normal file
View File

@@ -0,0 +1,24 @@
package com.cool.core.init;
import com.cool.core.leaf.IDGenService;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component;
/**
* 唯一ID 组件初始化
**/
@Slf4j
@Component
@RequiredArgsConstructor
public class IDGenInit implements ApplicationRunner {
final private IDGenService idGenService;
@Override
public void run(ApplicationArguments args) {
idGenService.init();
}
}

6
src/main/java/com/cool/core/leaf/IDGenService.java Normal file
View File

@@ -0,0 +1,6 @@
package com.cool.core.leaf;
public interface IDGenService {
long next(String key);
void init();
}

27
src/main/java/com/cool/core/leaf/common/CheckVO.java Normal file
View File

@@ -0,0 +1,27 @@
package com.cool.core.leaf.common;
public class CheckVO {
private long timestamp;
private int workID;
public CheckVO(long timestamp, int workID) {
this.timestamp = timestamp;
this.workID = workID;
}
public long getTimestamp() {
return timestamp;
}
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
public int getWorkID() {
return workID;
}
public void setWorkID(int workID) {
this.workID = workID;
}
}

39
src/main/java/com/cool/core/leaf/common/Result.java Normal file
View File

@@ -0,0 +1,39 @@
package com.cool.core.leaf.common;
public class Result {
private long id;
private Status status;
public Result() {
}
public Result(long id, Status status) {
this.id = id;
this.status = status;
}
public long getId() {
return id;
}
public void setId(long id) {
this.id = id;
}
public Status getStatus() {
return status;
}
public void setStatus(Status status) {
this.status = status;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("Result{");
sb.append("id=").append(id);
sb.append(", status=").append(status);
sb.append('}');
return sb.toString();
}
}

6
src/main/java/com/cool/core/leaf/common/Status.java Normal file
View File

@@ -0,0 +1,6 @@
package com.cool.core.leaf.common;
public enum Status {
SUCCESS,
EXCEPTION
}

5
src/main/java/com/cool/core/leaf/package-info.java Normal file
View File

@@ -0,0 +1,5 @@
/**
* 全局唯一id生成
* 来源美团：https://github.com/Meituan-Dianping/Leaf
*/
package com.cool.core.leaf;

309
src/main/java/com/cool/core/leaf/segment/SegmentIDGenImpl.java Normal file
View File

@@ -0,0 +1,309 @@
package com.cool.core.leaf.segment;
import static com.cool.core.leaf.segment.entity.table.LeafAllocEntityTableDef.LEAF_ALLOC_ENTITY;
import com.cool.core.exception.CoolPreconditions;
import com.cool.core.leaf.IDGenService;
import com.cool.core.leaf.common.Result;
import com.cool.core.leaf.common.Status;
import com.cool.core.leaf.segment.entity.LeafAllocEntity;
import com.cool.core.leaf.segment.mapper.LeafAllocMapper;
import com.cool.core.leaf.segment.model.Segment;
import com.cool.core.leaf.segment.model.SegmentBuffer;
import com.mybatisflex.core.query.QueryWrapper;
import com.mybatisflex.core.update.UpdateChain;
import java.util.*;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicLong;
import lombok.RequiredArgsConstructor;
import org.perf4j.StopWatch;
import org.perf4j.slf4j.Slf4JStopWatch;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Service
@RequiredArgsConstructor
public class SegmentIDGenImpl implements IDGenService {
private static final Logger logger = LoggerFactory.getLogger(SegmentIDGenImpl.class);
@Value("${leaf.segment.enable:false}")
private boolean enable;
/**
* IDCache未初始化成功时的异常码
*/
private static final long EXCEPTION_ID_IDCACHE_INIT_FALSE = -1;
/**
* key不存在时的异常码
*/
private static final long EXCEPTION_ID_KEY_NOT_EXISTS = -2;
/**
* SegmentBuffer中的两个Segment均未从DB中装载时的异常码
*/
private static final long EXCEPTION_ID_TWO_SEGMENTS_ARE_NULL = -3;
/**
* 最大步长不超过100,0000
*/
private static final int MAX_STEP = 1000000;
/**
* 一个Segment维持时间为15分钟
*/
private static final long SEGMENT_DURATION = 15 * 60 * 1000L;
private ExecutorService service = new ThreadPoolExecutor(5, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new UpdateThreadFactory());
private volatile boolean initOK = false;
private Map<String, SegmentBuffer> cache = new ConcurrentHashMap<String, SegmentBuffer>();
private final LeafAllocMapper leafAllocMapper;
public static class UpdateThreadFactory implements ThreadFactory {
private static int threadInitNumber = 0;
private static synchronized int nextThreadNum() {
return threadInitNumber++;
}
@Override
public Thread newThread(Runnable r) {
return new Thread(r, "Thread-Segment-Update-" + nextThreadNum());
}
}
@Override
public long next(String key) {
Result result = get(key);
CoolPreconditions.check(result.getId() < 0, "获取失败，code值: {}", result.getId());
return result.getId();
}
@Override
public void init() {
if (enable) {
// 确保加载到kv后才初始化成功
updateCacheFromDb();
initOK = true;
updateCacheFromDbAtEveryMinute();
logger.info("唯一ID组件初始化成功 ...");
}
}
private void updateCacheFromDbAtEveryMinute() {
ScheduledExecutorService service = Executors.newSingleThreadScheduledExecutor(r -> {
Thread t = new Thread(r);
t.setName("check-idCache-thread");
t.setDaemon(true);
return t;
});
service.scheduleWithFixedDelay(this::updateCacheFromDb, 60, 60, TimeUnit.SECONDS);
}
private void updateCacheFromDb() {
logger.info("update cache from db");
StopWatch sw = new Slf4JStopWatch();
try {
List<String> dbTags = leafAllocMapper.selectListByQuery(QueryWrapper.create().select(
LeafAllocEntity::getKey)).stream().map(LeafAllocEntity::getKey).toList();
if (dbTags == null || dbTags.isEmpty()) {
return;
}
List<String> cacheTags = new ArrayList<String>(cache.keySet());
Set<String> insertTagsSet = new HashSet<>(dbTags);
Set<String> removeTagsSet = new HashSet<>(cacheTags);
//db中新加的tags灌进cache
for(int i = 0; i < cacheTags.size(); i++){
String tmp = cacheTags.get(i);
if(insertTagsSet.contains(tmp)){
insertTagsSet.remove(tmp);
}
}
for (String tag : insertTagsSet) {
SegmentBuffer buffer = new SegmentBuffer();
buffer.setKey(tag);
Segment segment = buffer.getCurrent();
segment.setValue(new AtomicLong(0));
segment.setMax(0);
segment.setStep(0);
cache.put(tag, buffer);
logger.info("Add tag {} from db to IdCache, SegmentBuffer {}", tag, buffer);
}
//cache中已失效的tags从cache删除
for(int i = 0; i < dbTags.size(); i++){
String tmp = dbTags.get(i);
if(removeTagsSet.contains(tmp)){
removeTagsSet.remove(tmp);
}
}
for (String tag : removeTagsSet) {
cache.remove(tag);
logger.info("Remove tag {} from IdCache", tag);
}
} catch (Exception e) {
logger.warn("update cache from db exception", e);
} finally {
sw.stop("updateCacheFromDb");
}
}
private Result get(final String key) {
if (!initOK) {
return new Result(EXCEPTION_ID_IDCACHE_INIT_FALSE, Status.EXCEPTION);
}
CoolPreconditions.check(!initOK, "IDCache未初始化成功");
if (cache.containsKey(key)) {
SegmentBuffer buffer = cache.get(key);
if (!buffer.isInitOk()) {
synchronized (buffer) {
if (!buffer.isInitOk()) {
try {
updateSegmentFromDb(key, buffer.getCurrent());
logger.info("Init buffer. Update leafkey {} {} from db", key, buffer.getCurrent());
buffer.setInitOk(true);
} catch (Exception e) {
logger.warn("Init buffer {} exception", buffer.getCurrent(), e);
}
}
}
}
return getIdFromSegmentBuffer(cache.get(key));
}
return new Result(EXCEPTION_ID_KEY_NOT_EXISTS, Status.EXCEPTION);
}
public void updateSegmentFromDb(String key, Segment segment) {
StopWatch sw = new Slf4JStopWatch();
SegmentBuffer buffer = segment.getBuffer();
LeafAllocEntity leafAllocEntity;
if (!buffer.isInitOk()) {
leafAllocEntity = updateMaxIdAndGetLeafAlloc(key);
buffer.setStep(leafAllocEntity.getStep());
buffer.setMinStep(leafAllocEntity.getStep());//leafAlloc中的step为DB中的step
} else if (buffer.getUpdateTimestamp() == 0) {
leafAllocEntity = updateMaxIdAndGetLeafAlloc(key);
buffer.setUpdateTimestamp(System.currentTimeMillis());
buffer.setStep(leafAllocEntity.getStep());
buffer.setMinStep(leafAllocEntity.getStep());//leafAlloc中的step为DB中的step
} else {
long duration = System.currentTimeMillis() - buffer.getUpdateTimestamp();
int nextStep = buffer.getStep();
if (duration < SEGMENT_DURATION) {
if (nextStep * 2 > MAX_STEP) {
//do nothing
} else {
nextStep = nextStep * 2;
}
} else if (duration < SEGMENT_DURATION * 2) {
//do nothing with nextStep
} else {
nextStep = nextStep / 2 >= buffer.getMinStep() ? nextStep / 2 : nextStep;
}
logger.info("leafKey[{}], step[{}], duration[{}mins], nextStep[{}]", key, buffer.getStep(), String.format("%.2f",((double)duration / (1000 * 60))), nextStep);
LeafAllocEntity temp = new LeafAllocEntity();
temp.setKey(key);
temp.setStep(nextStep);
leafAllocEntity = updateMaxIdByCustomStepAndGetLeafAlloc(temp);
buffer.setUpdateTimestamp(System.currentTimeMillis());
buffer.setStep(nextStep);
buffer.setMinStep(leafAllocEntity.getStep());//leafAlloc的step为DB中的step
}
// must set value before set max
long value = leafAllocEntity.getMaxId() - buffer.getStep();
segment.getValue().set(value);
segment.setMax(leafAllocEntity.getMaxId());
segment.setStep(buffer.getStep());
sw.stop("updateSegmentFromDb", key + " " + segment);
}
private LeafAllocEntity updateMaxIdByCustomStepAndGetLeafAlloc(LeafAllocEntity temp) {
UpdateChain.of(LeafAllocEntity.class)
.setRaw(LeafAllocEntity::getMaxId, LEAF_ALLOC_ENTITY.MAX_ID.getName() + " + " + temp.getStep())
.where(LeafAllocEntity::getKey).eq(temp.getKey())
.update();
return leafAllocMapper.selectOneByQuery(QueryWrapper.create().select(
LEAF_ALLOC_ENTITY.KEY, LEAF_ALLOC_ENTITY.MAX_ID, LEAF_ALLOC_ENTITY.STEP).eq(LeafAllocEntity::getKey, temp.getKey()));
}
private LeafAllocEntity updateMaxIdAndGetLeafAlloc(String key) {
UpdateChain.of(LeafAllocEntity.class)
.setRaw(LeafAllocEntity::getMaxId, LEAF_ALLOC_ENTITY.MAX_ID.getName() + " + " + LEAF_ALLOC_ENTITY.STEP.getName())
.where(LeafAllocEntity::getKey).eq(key)
.update();
return leafAllocMapper.selectOneByQuery(QueryWrapper.create().select(
LEAF_ALLOC_ENTITY.KEY, LEAF_ALLOC_ENTITY.MAX_ID, LEAF_ALLOC_ENTITY.STEP).eq(LeafAllocEntity::getKey, key));
}
public Result getIdFromSegmentBuffer(final SegmentBuffer buffer) {
while (true) {
buffer.rLock().lock();
try {
final Segment segment = buffer.getCurrent();
if (!buffer.isNextReady() && (segment.getIdle() < 0.9 * segment.getStep()) && buffer.getThreadRunning().compareAndSet(false, true)) {
service.execute(new Runnable() {
@Override
public void run() {
Segment next = buffer.getSegments()[buffer.nextPos()];
boolean updateOk = false;
try {
updateSegmentFromDb(buffer.getKey(), next);
updateOk = true;
logger.info("update segment {} from db {}", buffer.getKey(), next);
} catch (Exception e) {
logger.warn(buffer.getKey() + " updateSegmentFromDb exception", e);
} finally {
if (updateOk) {
buffer.wLock().lock();
buffer.setNextReady(true);
buffer.getThreadRunning().set(false);
buffer.wLock().unlock();
} else {
buffer.getThreadRunning().set(false);
}
}
}
});
}
long value = segment.getValue().getAndIncrement();
if (value < segment.getMax()) {
return new Result(value, Status.SUCCESS);
}
} finally {
buffer.rLock().unlock();
}
waitAndSleep(buffer);
buffer.wLock().lock();
try {
final Segment segment = buffer.getCurrent();
long value = segment.getValue().getAndIncrement();
if (value < segment.getMax()) {
return new Result(value, Status.SUCCESS);
}
if (buffer.isNextReady()) {
buffer.switchPos();
buffer.setNextReady(false);
} else {
logger.error("Both two segments in {} are not ready!", buffer);
return new Result(EXCEPTION_ID_TWO_SEGMENTS_ARE_NULL, Status.EXCEPTION);
}
} finally {
buffer.wLock().unlock();
}
}
}
private void waitAndSleep(SegmentBuffer buffer) {
int roll = 0;
while (buffer.getThreadRunning().get()) {
roll += 1;
if(roll > 10000) {
try {
TimeUnit.MILLISECONDS.sleep(10);
break;
} catch (InterruptedException e) {
logger.warn("Thread {} Interrupted",Thread.currentThread().getName());
break;
}
}
}
}
}

27
src/main/java/com/cool/core/leaf/segment/entity/LeafAllocEntity.java Normal file
View File

@@ -0,0 +1,27 @@
package com.cool.core.leaf.segment.entity;
import com.cool.core.base.BaseEntity;
import com.mybatisflex.annotation.Table;
import com.tangzc.mybatisflex.autotable.annotation.ColumnDefine;
import com.tangzc.mybatisflex.autotable.annotation.UniIndex;
import lombok.Getter;
import lombok.Setter;
@Getter
@Setter
@Table(value = "leaf_alloc", comment = "唯一id分配")
public class LeafAllocEntity extends BaseEntity<LeafAllocEntity> {
@UniIndex
@ColumnDefine(comment = "业务key ，比如orderId", length = 20, notNull = true)
private String key;
@ColumnDefine(comment = "当前最大id", defaultValue = "1", notNull = true)
private Long maxId;
@ColumnDefine(comment = "步长", defaultValue = "500", notNull = true)
private Integer step;
@ColumnDefine(comment = "描述")
private String description;
}

7
src/main/java/com/cool/core/leaf/segment/mapper/LeafAllocMapper.java Normal file
View File

@@ -0,0 +1,7 @@
package com.cool.core.leaf.segment.mapper;
import com.cool.core.leaf.segment.entity.LeafAllocEntity;
import com.mybatisflex.core.BaseMapper;
public interface LeafAllocMapper extends BaseMapper<LeafAllocEntity> {
}

59
src/main/java/com/cool/core/leaf/segment/model/Segment.java Normal file
View File

@@ -0,0 +1,59 @@
package com.cool.core.leaf.segment.model;
import java.util.concurrent.atomic.AtomicLong;
public class Segment {
private AtomicLong value = new AtomicLong(0);
private volatile long max;
private volatile int step;
private SegmentBuffer buffer;
public Segment(SegmentBuffer buffer) {
this.buffer = buffer;
}
public AtomicLong getValue() {
return value;
}
public void setValue(AtomicLong value) {
this.value = value;
}
public long getMax() {
return max;
}
public void setMax(long max) {
this.max = max;
}
public int getStep() {
return step;
}
public void setStep(int step) {
this.step = step;
}
public SegmentBuffer getBuffer() {
return buffer;
}
public long getIdle() {
return this.getMax() - getValue().get();
}
@Override
public String toString() {
StringBuilder sb = new StringBuilder("Segment(");
sb.append("value:");
sb.append(value);
sb.append(",max:");
sb.append(max);
sb.append(",step:");
sb.append(step);
sb.append(")");
return sb.toString();
}
}

129
src/main/java/com/cool/core/leaf/segment/model/SegmentBuffer.java Normal file
View File

@@ -0,0 +1,129 @@
package com.cool.core.leaf.segment.model;
import java.util.Arrays;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* 双buffer
*/
public class SegmentBuffer {
private String key;
private Segment[] segments; //双buffer
private volatile int currentPos; //当前的使用的segment的index
private volatile boolean nextReady; //下一个segment是否处于可切换状态
private volatile boolean initOk; //是否初始化完成
private final AtomicBoolean threadRunning; //线程是否在运行中
private final ReadWriteLock lock;
private volatile int step;
private volatile int minStep;
private volatile long updateTimestamp;
public SegmentBuffer() {
segments = new Segment[]{new Segment(this), new Segment(this)};
currentPos = 0;
nextReady = false;
initOk = false;
threadRunning = new AtomicBoolean(false);
lock = new ReentrantReadWriteLock();
}
public String getKey() {
return key;
}
public void setKey(String key) {
this.key = key;
}
public Segment[] getSegments() {
return segments;
}
public Segment getCurrent() {
return segments[currentPos];
}
public int getCurrentPos() {
return currentPos;
}
public int nextPos() {
return (currentPos + 1) % 2;
}
public void switchPos() {
currentPos = nextPos();
}
public boolean isInitOk() {
return initOk;
}
public void setInitOk(boolean initOk) {
this.initOk = initOk;
}
public boolean isNextReady() {
return nextReady;
}
public void setNextReady(boolean nextReady) {
this.nextReady = nextReady;
}
public AtomicBoolean getThreadRunning() {
return threadRunning;
}
public Lock rLock() {
return lock.readLock();
}
public Lock wLock() {
return lock.writeLock();
}
public int getStep() {
return step;
}
public void setStep(int step) {
this.step = step;
}
public int getMinStep() {
return minStep;
}
public void setMinStep(int minStep) {
this.minStep = minStep;
}
public long getUpdateTimestamp() {
return updateTimestamp;
}
public void setUpdateTimestamp(long updateTimestamp) {
this.updateTimestamp = updateTimestamp;
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("SegmentBuffer{");
sb.append("key='").append(key).append('\'');
sb.append(", segments=").append(Arrays.toString(segments));
sb.append(", currentPos=").append(currentPos);
sb.append(", nextReady=").append(nextReady);
sb.append(", initOk=").append(initOk);
sb.append(", threadRunning=").append(threadRunning);
sb.append(", step=").append(step);
sb.append(", minStep=").append(minStep);
sb.append(", updateTimestamp=").append(updateTimestamp);
sb.append('}');
return sb.toString();
}
}

21
src/main/java/com/cool/core/util/MappingAlgorithm.java Normal file
View File

@@ -0,0 +1,21 @@
package com.cool.core.util;
/**
* 自定义映射算法
* 将 ID 转换为一个混淆形式的数字，然后能够逆向转换回原始 ID。
* 场景：混淆订单id
*/
public class MappingAlgorithm {
private static final long ENCRYPTION_KEY = 123456789L; // 任意密钥
// 将 ID 转换为混淆的数字
public static long encrypt(long id) {
return id ^ ENCRYPTION_KEY; // 使用异或操作进行混淆
}
// 将混淆的数字恢复为原始的 ID
public static long decrypt(long encryptedId) {
return encryptedId ^ ENCRYPTION_KEY; // 逆操作恢复原始 ID
}
}

6
src/main/java/com/cool/modules/base/controller/admin/AdminBaseCommController.java
View File

@@ -5,6 +5,7 @@ import com.cool.core.annotation.CoolRestController;
import com.cool.core.annotation.TokenIgnore;
import com.cool.core.eps.CoolEps;
import com.cool.core.file.FileUploadStrategyFactory;
import com.cool.core.leaf.IDGenService;
import com.cool.core.request.R;
import com.cool.modules.base.entity.sys.BaseSysUserEntity;
import com.cool.modules.base.service.sys.BaseSysLoginService;
@@ -41,10 +42,15 @@ public class AdminBaseCommController {
final private FileUploadStrategyFactory fileUploadStrategyFactory;
final private IDGenService idGenService;
@TokenIgnore
@Operation(summary = "实体信息与路径", description = "系统所有的实体信息与路径，供前端自动生成代码与服务")
@GetMapping("/eps")
public R eps() {
long orderId = idGenService.next("orderId");
System.out.println(orderId);
return R.ok(coolEps.getAdmin());
}

2
src/main/resources/application-local.yml
View File

@@ -18,7 +18,7 @@ auto-table:
# 建表的时候，父类的字段排序是在子类后面还是前面
superInsertPosition: before
# 模型包路径
model-package: com.cool.modules.*.entity.*
model-package: com.cool.**.entity.*
# Cool相关配置
cool:

7
src/main/resources/application.yml
View File

@@ -91,6 +91,7 @@ ignored:
# 忽略记录请求日志url
logUrls:
- /
- /**/eps
- /app/**
- /css/*
- /js/*
@@ -111,7 +112,7 @@ mybatis-flex:
# configuration:
#MyBatis Mapper 所对应的 XML 文件位置，如果在 Mapper 中有自定义的方法（XML 中有自定义的实现），需要进行该配置，指定 Mapper 所对应的 XML 文件位置
mapper-locations: [ "classpath*:/mapper/**/*.xml" ]
type-aliases-package: com.cool.modules.*.entity.*
type-aliases-package: com.cool.**.entity.*
global-config:
print-banner: false
@@ -150,3 +151,7 @@ cool:
# AutoTable配置，根据实体类自动生成表
auto-table:
show-banner: false
leaf:
segment:
enable: true