插件依赖类使用异步方式加载，解决主流程加载慢问题

2024-07-28 13:48:56 +08:00
parent a8c0954a39
commit c77be231fa
9 changed files with 182 additions and 102 deletions
--- a/src/main/java/com/cool/core/plugin/config/DynamicJarClassLoader.java
+++ b/src/main/java/com/cool/core/plugin/config/DynamicJarClassLoader.java
@@ -1,9 +1,16 @@
 package com.cool.core.plugin.config;

+import com.cool.core.exception.CoolPreconditions;
+import com.cool.core.util.AnnotationUtils;
+import com.cool.core.util.CompilerUtils;
 import java.net.URL;
 import java.net.URLClassLoader;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.jar.JarEntry;
 import lombok.extern.slf4j.Slf4j;

 /**
@@ -18,6 +25,8 @@ public class DynamicJarClassLoader extends URLClassLoader {
        super(urls, parent);
    }

+    private Boolean lock = false;
+
    @Override
    protected Class<?> findClass(String name) throws ClassNotFoundException {
        // 从已加载的类集合中获取指定名称的类
@@ -32,6 +41,8 @@ public class DynamicJarClassLoader extends URLClassLoader {
    }

    public void unload() {
+        CoolPreconditions.check(lock, "异步加载任务还未完成，请稍后重试......");
+        lock = true;
        new Thread(new Runnable() {
            @Override
            public void run() {
@@ -45,6 +56,77 @@ public class DynamicJarClassLoader extends URLClassLoader {
                    close();
                } catch (Exception e) {
                    log.error("unload error", e);
+                } finally{
+                    lock = false;
+                }
+            }
+        }).start();
+    }
+    public void loadClass(List<JarEntry> jarEntries, List<Class<?>> plugins) {
+        for (JarEntry jarEntry : jarEntries) {
+            loadClass(jarEntry, plugins);
+        }
+    }
+    /**
+     * 加载class
+     */
+    public void loadClass(JarEntry jarEntry, List<Class<?>> plugins) {
+        String entryName = jarEntry.getName();
+        String className = entryName.replace('/', '.').substring(0, entryName.length() - 6);
+        if (entryName.startsWith(CompilerUtils.META_INF_VERSIONS)) {
+            // 处理多版本类
+            String jdkVersion = CompilerUtils.getJdkVersion();
+            if (!entryName.startsWith(CompilerUtils.META_INF_VERSIONS + jdkVersion)) {
+                return;
+            }
+            // 替换版本目录
+            className = className.replace((CompilerUtils.META_INF_VERSIONS + jdkVersion).replace("/", ".") + ".", "");
+        }
+        try {
+            // 加载类
+            Class<?> clazz = super.loadClass(className);
+            if (plugins != null && AnnotationUtils.hasCoolPluginAnnotation(clazz)) {
+                // 添加插件
+                plugins.add(clazz);
+            }
+        } catch (ClassNotFoundException e) {
+            log.error("loadClassErr", e);
+        } catch ( NoClassDefFoundError | UnsupportedClassVersionError ignored) {
+        }
+    }
+
+    /**
+     * 异步加载
+     */
+    public void asyncLoadClass(List<JarEntry> list) {
+        CoolPreconditions.check(lock, "异步加载任务还未完成，请稍后重试......");
+        lock = true;
+        new Thread(new Runnable() {
+            @Override
+            public void run() {
+                log.info("开始异步加载....");
+                Instant start = Instant.now();
+                int size = list.size();
+                int currentProgress = 0;
+                int progressThreshold = 10; // 输出进度的阈值为10%
+                int count = 0;
+                try{
+                    for (JarEntry jarEntry : list) {
+                        count++;
+                        loadClass(jarEntry, null);
+                        // 计算进度百分比
+                        int progress = (int) ((count / (double) size) * 100);
+                        // 输出一次进度
+                        if (progress % progressThreshold == 0 && currentProgress != progress) {
+                            log.info("异步加载进度: {}%", progress);
+                            currentProgress = progress;
+                        }
+                    }
+                } finally{
+                    Instant end = Instant.now();
+                    Duration timeElapsed = Duration.between(start, end);
+                    log.info("异步加载完成本次共加载{}个文件 耗时: {}ms", count, timeElapsed.toMillis());
+                    lock = false;
                }
            }
        }).start();
--- a/src/main/java/com/cool/core/plugin/service/CoolPluginService.java
+++ b/src/main/java/com/cool/core/plugin/service/CoolPluginService.java
@@ -5,6 +5,7 @@ import cn.hutool.core.bean.copier.CopyOptions;
 import cn.hutool.core.codec.Base64Encoder;
 import cn.hutool.core.io.FileUtil;
 import cn.hutool.core.io.IoUtil;
+import cn.hutool.core.util.BooleanUtil;
 import cn.hutool.core.util.ObjUtil;
 import cn.hutool.core.util.StrUtil;
 import com.cool.core.config.PluginJson;
@@ -43,9 +44,12 @@ public class CoolPluginService {

    final private PluginInfoService pluginInfoService;

-    @Value("${cool.pluginPath}")
+    @Value("${cool.plugin.path}")
    private String pluginPath;

+    @Value("${cool.plugin.toDb:false}")
+    private Boolean toDb;
+
    public void init() {
        List<PluginInfoEntity> list = pluginInfoService
            .list(QueryWrapper
@@ -65,11 +69,15 @@ public class CoolPluginService {
    private void initInstall(PluginInfoEntity entity) {
        PluginJson pluginJson = entity.getPluginJson();
        File file = new File(pluginJson.getJarPath());
-        // 检查路径是否存在
+        // 检查文件是否存在
        if (!file.exists()) {
-            log.warn("插件不存在，请重新安装!");
+            PluginInfoEntity pluginInfoEntity = pluginInfoService.getById(entity.getId());
+            if (ObjUtil.isEmpty(pluginInfoEntity.getJarFile())) {
+                log.warn("插件文件不存在，请重新安装!");
                return;
            }
+            FileUtil.writeBytes(pluginInfoEntity.getJarFile(), file);
+        }
        file = new File(pluginJson.getJarPath());
        if (file.exists()) {
            ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();
@@ -100,7 +108,7 @@ public class CoolPluginService {
            PluginJson pluginJson = dynamicJarLoaderService.install(jarFilePath, force);
            key = pluginJson.getKey();
            // 保存插件信息入库
-            savePluginInfo(pluginJson, jarFilePath, force);
+            savePluginInfo(pluginJson, jarFilePath, jarFile, force);
            // 把 ApplicationContext 对象传递打插件类中，使其在插件中也能正常使用spring bean对象
            CoolPluginInvokers.setApplicationContext(pluginJson.getKey());
        } catch (PersistenceException persistenceException) {
@@ -159,20 +167,20 @@ public class CoolPluginService {
     * 卸载
     */
    public void uninstall(Long id) {
-        PluginInfoEntity pluginInfoEntity = pluginInfoService.getById(id);
+        PluginInfoEntity pluginInfoEntity = pluginInfoService.getPluginInfoEntityById(id);
        CoolPreconditions.checkEmpty(pluginInfoEntity, "插件不存在");
-        if (dynamicJarLoaderService.uninstall(pluginInfoEntity.getKey())) {
+        dynamicJarLoaderService.uninstall(pluginInfoEntity.getKey());
        boolean flag = pluginInfoEntity.removeById();
        if (flag) {
            FileUtil.del(pluginInfoEntity.getPluginJson().getJarPath());
        }
    }
-    }

    /**
     * 保存插件信息
     */
    private void savePluginInfo(PluginJson pluginJson, String jarFilePath ,
+        File jarFile,
        boolean force) {
        CoolPreconditions.checkEmpty(pluginJson, "插件安装失败");
        pluginJson.setJarPath(jarFilePath);
@@ -181,6 +189,10 @@ public class CoolPluginService {
        setLogoOrReadme(pluginJson, pluginInfo);
        pluginInfo.setKey(pluginJson.getKey());
        pluginInfo.setPluginJson(pluginJson);
+        if (BooleanUtil.isTrue(toDb)) {
+            // 转二进制
+            pluginInfo.setJarFile(FileUtil.readBytes(jarFile));
+        }
        if (force) {
            // 判断是否有同名插件， 有将其关闭
            closeSameNamePlugin(pluginJson);
@@ -213,6 +225,8 @@ public class CoolPluginService {
                // 覆盖时删除旧版本插件
                FileUtil.del(oldJarPath);
            }
+        } else {
+            pluginInfo.save();
        }
    }

@@ -268,7 +282,7 @@ public class CoolPluginService {
    }

    public void updatePlugin(PluginInfoEntity entity) {
-        PluginInfoEntity dbPluginInfoEntity = pluginInfoService.getById(
+        PluginInfoEntity dbPluginInfoEntity = pluginInfoService.getPluginInfoEntityById(
            entity.getId());
        // 调用插件更新配置标识
        boolean invokePluginConfig = false;
--- a/src/main/java/com/cool/core/plugin/service/DynamicJarLoaderService.java
+++ b/src/main/java/com/cool/core/plugin/service/DynamicJarLoaderService.java
@@ -8,16 +8,13 @@ import cn.hutool.json.JSONUtil;
 import com.cool.core.config.PluginJson;
 import com.cool.core.exception.CoolPreconditions;
 import com.cool.core.plugin.config.DynamicJarClassLoader;
-import com.cool.core.util.AnnotationUtils;
-import com.cool.core.util.CompilerUtils;
 import com.cool.modules.plugin.entity.PluginInfoEntity;
 import com.cool.modules.plugin.service.PluginInfoService;
 import java.io.File;
+import java.io.IOException;
 import java.io.InputStream;
 import java.net.JarURLConnection;
 import java.net.URL;
-import java.time.Duration;
-import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
@@ -48,41 +45,44 @@ public class DynamicJarLoaderService {
            Thread.currentThread().getContextClassLoader());
        Thread.currentThread().setContextClassLoader(dynamicJarClassLoader);
        PluginJson pluginJson = getPluginJsonAndCheck(force, dynamicJarClassLoader);
-        // 加载类
-        List<Class<?>> plugins = new ArrayList<>();
-        int count = 0;
+        // 历史如果有安装过，先卸载
        uninstall(pluginJson.getKey());
-        Instant start = Instant.now();
-        int progressThreshold = 10; // 输出进度的阈值为10%
-        try (JarFile jarFile = ((JarURLConnection) jarUrl.openConnection()).getJarFile()) {
-            List<JarEntry> list = jarFile.stream().toList();
-            int size = list.size();
-            int currentProgress = 0;
-            for (JarEntry jarEntry : list) {
-                count++;
-                loadClass(jarEntry, dynamicJarClassLoader, plugins);
-                // 计算进度百分比
-                int progress = (int) ((count / (double) size) * 100);
-                // 输出一次进度
-                if (progress % progressThreshold == 0 && currentProgress != progress) {
-                    log.info("安装进度: {}%", progress);
-                    currentProgress = progress;
-                }
-            }
-        } finally{
-            Instant end = Instant.now();
-            Duration timeElapsed = Duration.between(start, end);
-            log.info("本次共加载{}个文件 耗时: {}ms", count, timeElapsed.toMillis());
-        }
+        // 加载class
+        List<Class<?>> plugins = loadClass(jarUrl, dynamicJarClassLoader);
        // 校验插件
        checkPlugin(plugins);
+        // 注册插件,目前一个插件包，只允许有一个插件入口
        registerPlugin(pluginJson.getKey(), plugins.get(0), dynamicJarClassLoader, force);
-        dynamicJarClassLoaderMap.put(pluginJson.getKey(), dynamicJarClassLoader);
-
-        log.info("插件{}安装成功.", pluginJson.getKey());
        return pluginJson;
    }

+    /**
+     * 加载class
+     */
+    private static List<Class<?>> loadClass(URL jarUrl,
+        DynamicJarClassLoader dynamicJarClassLoader) throws IOException {
+        // 加载类
+        List<Class<?>> plugins = new ArrayList<>();
+        try (JarFile jarFile = ((JarURLConnection) jarUrl.openConnection()).getJarFile()) {
+            List<JarEntry> list = jarFile.stream().filter(o -> o.getName().endsWith(".class")).toList();
+            List<JarEntry> firstNElements = getFirstNElements(list, 100);
+            // 先加载前100个类，主够包含了插件主类，其余的异步加载
+            dynamicJarClassLoader.loadClass(firstNElements, plugins);
+            // 异步加载插件依赖类
+            dynamicJarClassLoader.asyncLoadClass(list);
+        }
+        return plugins;
+    }
+
+    public static <T> List<T> getFirstNElements(List<T> list, int n) {
+        // 确保 n 不超过 list 的大小
+        if (list.size() <= n) {
+            return new ArrayList<>(list); // 返回原列表的副本
+        } else {
+            return new ArrayList<>(list.subList(0, n)); // 返回前 N 个元素的副本
+        }
+    }
+
    /**
     * 校验，获取插件配置
     */
@@ -120,37 +120,6 @@ public class DynamicJarLoaderService {
        return pluginJson;
    }

-    /**
-     * 加载class
-     */
-    private static void loadClass(JarEntry jarEntry,
-        DynamicJarClassLoader dynamicJarClassLoader, List<Class<?>> plugins)
-        throws ClassNotFoundException {
-
-        String entryName = jarEntry.getName();
-        if (!entryName.endsWith(".class")) {
-            return;
-        }
-        String className = entryName.replace('/', '.').substring(0, entryName.length() - 6);
-        if (entryName.startsWith(CompilerUtils.META_INF_VERSIONS)) {
-            // 处理多版本类
-            String jdkVersion = CompilerUtils.getJdkVersion();
-            if (!entryName.startsWith(CompilerUtils.META_INF_VERSIONS + jdkVersion)) {
-                return;
-            }
-            // 替换版本目录
-            className = className.replace((CompilerUtils.META_INF_VERSIONS + jdkVersion).replace("/", ".") + ".", "");
-        }
-        try {
-            // 加载类
-            Class<?> clazz = dynamicJarClassLoader.loadClass(className);
-            if (AnnotationUtils.hasCoolPluginAnnotation(clazz)) {
-                plugins.add(clazz);
-            }
-        } catch (NoClassDefFoundError | UnsupportedClassVersionError ignored) {
-        }
-    }
-
    /**
     * 注册插件
     */
@@ -165,30 +134,23 @@ public class DynamicJarLoaderService {
        if (ObjUtil.isNotEmpty(key)) {
            pluginMap.remove(key);
            pluginMap.put(key, ReflectUtil.newInstance(pluginClazz));
+            dynamicJarClassLoaderMap.put(key, dynamicJarClassLoader);
        }
+        log.info("插件{}注册成功.", key);
    }

    /**
     * 卸载
     */
-    public boolean uninstall(String key) {
+    public void uninstall(String key) {
        DynamicJarClassLoader dynamicJarClassLoader = getDynamicJarClassLoader(key);
-//        ClassLoader originalClassLoader = Thread.currentThread().getContextClassLoader();
+        if (dynamicJarClassLoader == null) {
+            return;
+        }
        log.info("插件{}开始卸载", key);
-        try {
-//            Thread.currentThread().setContextClassLoader(dynamicJarClassLoader);
        pluginMap.remove(key);
-            if (dynamicJarClassLoader != null) {
        dynamicJarClassLoader.unload();
-            }
-        } catch (Exception e) {
-            log.error("uninstall {}失败", key, e);
-            CoolPreconditions.alwaysThrow("卸载失败");
-        } finally {
-//            Thread.currentThread().setContextClassLoader(originalClassLoader);
-        }
        log.info("插件{}卸载完成", key);
-        return true;
    }

    /**
--- a/src/main/java/com/cool/core/util/AnnotationUtils.java
+++ b/src/main/java/com/cool/core/util/AnnotationUtils.java
@@ -1,6 +1,8 @@
 package com.cool.core.util;

 import com.cool.core.annotation.CoolPlugin;
+import java.lang.annotation.Annotation;
+import java.lang.reflect.Modifier;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.stereotype.Component;
@@ -8,9 +10,6 @@ import org.springframework.stereotype.Controller;
 import org.springframework.stereotype.Repository;
 import org.springframework.stereotype.Service;

-import java.lang.annotation.Annotation;
-import java.lang.reflect.Modifier;
-
@Slf4j
 public class AnnotationUtils {

@@ -61,16 +60,13 @@ public class AnnotationUtils {
            return false;
        }
        try {
-//            if (clazz.getAnnotation(CoolPlugin.class) != null) {
-//                return true;
-//            }
            ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
            if (clazz.getAnnotation(
                    (Class<? extends Annotation>) contextClassLoader.loadClass(CoolPlugin.class.getName())) != null) {
                return true;
            }
        } catch (Exception e) {
-            log.error("出现异常：{}", e.getMessage());
+            log.error("出现异常：{}", e.getMessage(), e);
        }
        return false;
    }
--- a/src/main/java/com/cool/modules/plugin/controller/admin/AdminPluginInfoController.java
+++ b/src/main/java/com/cool/modules/plugin/controller/admin/AdminPluginInfoController.java
@@ -9,6 +9,7 @@ import com.cool.core.annotation.IgnoreRecycleData;
 import com.cool.core.base.BaseController;
 import com.cool.core.plugin.service.CoolPluginService;
 import com.cool.core.request.R;
+import com.cool.core.util.EntityUtils;
 import com.cool.modules.plugin.entity.PluginInfoEntity;
 import com.cool.modules.plugin.service.PluginInfoService;
 import com.mybatisflex.core.query.QueryWrapper;
@@ -34,8 +35,11 @@ public class AdminPluginInfoController extends BaseController<PluginInfoService,

    @Override
    protected void init(HttpServletRequest request, JSONObject requestParams) {
+
        setPageOption(createOp().queryWrapper(
-                QueryWrapper.create().orderBy(PLUGIN_INFO_ENTITY.UPDATE_TIME, false)));
+                QueryWrapper.create().orderBy(PLUGIN_INFO_ENTITY.UPDATE_TIME, false))
+            .select(EntityUtils.getFieldNamesWithSuperClass(PLUGIN_INFO_ENTITY.DEFAULT_COLUMNS,
+                PLUGIN_INFO_ENTITY.JAR_FILE.getName())));
    }

    @Override
--- a/src/main/java/com/cool/modules/plugin/entity/PluginInfoEntity.java
+++ b/src/main/java/com/cool/modules/plugin/entity/PluginInfoEntity.java
@@ -52,7 +52,7 @@ public class PluginInfoEntity extends BaseEntity<PluginInfoEntity> {
    @Column(typeHandler = Fastjson2TypeHandler.class)
    private PluginJson pluginJson;

-    @ColumnDefine(comment = "jar二进制文件-废弃", type = "longblob")
+    @ColumnDefine(comment = "jar二进制文件(可配置是否入库)", type = "longblob")
    private byte[] jarFile;

    @ColumnDefine(comment = "配置", type = "json")
--- a/src/main/java/com/cool/modules/plugin/service/PluginInfoService.java
+++ b/src/main/java/com/cool/modules/plugin/service/PluginInfoService.java
@@ -7,4 +7,6 @@ public interface PluginInfoService extends BaseService<PluginInfoEntity> {
    PluginInfoEntity getByKey(String key);

    PluginInfoEntity getPluginInfoEntityByHook(String hook);
+
+    PluginInfoEntity getPluginInfoEntityById(Long id);
 }
--- a/src/main/java/com/cool/modules/plugin/service/impl/PluginInfoServiceImpl.java
+++ b/src/main/java/com/cool/modules/plugin/service/impl/PluginInfoServiceImpl.java
@@ -3,6 +3,7 @@ package com.cool.modules.plugin.service.impl;
 import static com.cool.modules.plugin.entity.table.PluginInfoEntityTableDef.PLUGIN_INFO_ENTITY;

 import com.cool.core.base.BaseServiceImpl;
+import com.cool.core.util.EntityUtils;
 import com.cool.modules.plugin.entity.PluginInfoEntity;
 import com.cool.modules.plugin.mapper.PluginInfoMapper;
 import com.cool.modules.plugin.service.PluginInfoService;
@@ -31,8 +32,24 @@ public class PluginInfoServiceImpl extends BaseServiceImpl<PluginInfoMapper, Plu
     */
    @Override
    public PluginInfoEntity getPluginInfoEntityByHook(String hook) {
-        QueryWrapper queryWrapper = QueryWrapper.create().and(PLUGIN_INFO_ENTITY.HOOK.eq(hook))
+        QueryWrapper queryWrapper = getPluginInfoEntityQueryWrapper().and(PLUGIN_INFO_ENTITY.HOOK.eq(hook))
                .and(PLUGIN_INFO_ENTITY.STATUS.eq(1)).limit(1);
        return getOne(queryWrapper);
    }
+
+    /**
+     * 通过id获取插件信息,不带jar二进制
+     */
+    @Override
+    public PluginInfoEntity getPluginInfoEntityById(Long id) {
+        QueryWrapper queryWrapper = getPluginInfoEntityQueryWrapper().and(PLUGIN_INFO_ENTITY.ID.eq(id));
+        return getOne(queryWrapper);
+    }
+
+    /**
+     * 获取查询对象，排除掉 jar二进制
+     */
+    private QueryWrapper getPluginInfoEntityQueryWrapper() {
+        return QueryWrapper.create().select(EntityUtils.getFieldNamesWithSuperClass(PLUGIN_INFO_ENTITY.DEFAULT_COLUMNS, PLUGIN_INFO_ENTITY.JAR_FILE.getName()));
+    }
 }
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -111,8 +111,11 @@ mybatis-flex:
 cool:
  # 缓存名称
  cacheName: comm
+  plugin:
    # 插件安装位置
-  pluginPath: assets/plugin
+    path: assets/plugin
+    # 插件文件是否入库（默认否）,先预留该字段，如果是集群部署需要开启,
+    toDb: false
  # token 相关配置
  token:
    # 过期时间 单位：秒 半小时