Redkale 2.2.0 结束

bin/command.bat

@@ -0,0 +1,7 @@
+@ECHO OFF
+
+SET APP_HOME=%~dp0
+
+IF NOT EXIST "%APP_HOME%\conf\application.xml"  SET APP_HOME=%~dp0..
+
+java -DCMD=%1 -DAPP_HOME="%APP_HOME%" -classpath "%APP_HOME%"\lib\* org.redkale.boot.Application

bin/command.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+
+export LC_ALL="zh_CN.UTF-8"
+
+APP_HOME=`dirname "$0"`
+
+cd "$APP_HOME"/..
+
+APP_HOME=`pwd`
+
+if [ ! -f "$APP_HOME"/conf/application.xml ]; then 
+     APP_HOME="$APP_HOME"/..  
+fi
+
+lib="$APP_HOME"/lib
+for jar in `ls $APP_HOME/lib/*.jar`
+do
+    lib=$lib:$jar
+done
+export CLASSPATH=$CLASSPATH:$lib
+
+echo "$APP_HOME"
+java -DCMD=$1  -DAPP_HOME="$APP_HOME"  org.redkale.boot.Application

conf/application.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 
-<application port="2121">     
+<application nodeid="10000" port="2121">     
     
     <!--  详细配置说明见： http://redkale.org/redkale.html#redkale_confxml  -->
     

my/pom.xml

@@ -6,7 +6,7 @@
     <packaging>jar</packaging>
     <url>http://redkale.org</url>
     <description>redkale -- java framework</description>
-    <version>1.6.2</version>
+    <version>2.2.0</version>
     <licenses>     
         <license>      
             <name>Apache 2</name>       
@@ -37,8 +37,8 @@
     
     <properties>
         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-        <maven.compiler.source>1.8</maven.compiler.source>
+        <maven.compiler.source>11</maven.compiler.source>
-        <maven.compiler.target>1.8</maven.compiler.target>
+        <maven.compiler.target>11</maven.compiler.target>
     </properties>
     <name>Redkale</name>
     <distributionManagement>
@@ -61,7 +61,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
-                <version>3.5.1</version>                                
+                <version>3.8.0</version>                                
                 <configuration>
 					<compilerArgument>-parameters</compilerArgument>
                     <encoding>UTF-8</encoding>
@@ -74,7 +74,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-jar-plugin</artifactId>
-                <version>2.6</version>
+                <version>3.2.0</version>
                 <configuration>
                     <archive>
 						<addMavenDescriptor>false</addMavenDescriptor>
@@ -102,7 +102,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-source-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>3.2.0</version>
                 <executions>
                     <execution>
                         <goals>
@@ -114,7 +114,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-javadoc-plugin</artifactId>
-                <version>3.0.0</version>
+                <version>3.2.0</version>
                 <executions>
                     <execution>
                         <goals>
@@ -127,7 +127,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>   
-                <version>3.0.0</version>
+                <version>3.2.0</version>
                 <configuration>
                     <appendAssemblyId>false</appendAssemblyId>
                     <descriptors>   

src/META-INF/application-template.xml

@@ -19,11 +19,13 @@
   serviceid1_name1  serviceid1_name2   serviceid2_name1  serviceid2_name2    
 -->
 <!--     
+    nodeid: int     进程的节点ID，用于分布式环境，一个系统中节点ID必须全局唯一，使用cluster时框架会进行唯一性校验
+    name:           进程的名称，用于监控识别，命名规则: 字母、数字、下划线
     address:        本地局域网的IP地址， 默认值为默认网卡的ip，当不使用默认值需要指定值，如192.168.1.22
     port: required  程序的管理Server的端口，用于关闭或者与监管系统进行数据交互
     lib:            加上额外的lib路径,多个路径用分号;隔开； 默认为空。  例如: ${APP_HOME}/lib/a.jar;${APP_HOME}/lib2/b.jar;
 -->
-<application port="6560" lib="">  
+<application nodeid="1000" port="6560" lib="">  
     
     <!-- 
         【节点全局唯一】
@@ -43,12 +45,52 @@
         -->
         <transport bufferCapacity="32K" bufferPoolSize="32" threads="32" readTimeoutSeconds="6" writeTimeoutSeconds="6"/>
         
+        <!--
+           【节点全局唯一】
+            自动扫描时排除部分包路径
+            value：    排除lib.path与excludes中的正则表达式匹配的路径, 多个正则表达式用分号;隔开
+        -->
+        <excludelibs value="^.*mysql.*$;^.*kafka.*$"/>
+        
+        <!--
+           【节点全局唯一】
+            第三方服务发现管理接口
+            value：    类名，必须是org.redkale.cluster.ClusterAgent的子类
+            waits:     注销服务后是否需要等待检查周期时间后再进行Service销毁，默认值为：false
+                       当一个Service进行服务注销后，不能立刻销毁Service，因为健康检测是有间隔时间差的，
+                       需要等待一个健康检测周期时间，让其他进程都更新完服务列表。
+                       如果使用MQ，可以设置为false，如果对服务健壮性要求高，建议设置为true
+            protocols: 服务发现可以处理的协议， 默认值为: SNCP, 多个协议用分号;隔开
+            ports:     服务发现可以处理的端口， 多个端口用分号;隔开
+        -->
+        <!--
+        <cluster value="org.redkalex.cluster.consul.ConsulClusterAgent" waits="false" protocols="SNCP" ports="7070;7071">
+            <property name="xxxxxx" value="XXXXXXXX"/>
+        </cluster>
+        -->
+        <!--
+            MQ管理接口配置
+            不同MQ节点所配置的MQ集群不能重复。
+            name:   服务的名称，用于监控识别，多个mq节点时只能有一个name为空的节点，mq.name不能重复,命名规则: 字母、数字、下划线
+            value： 实现类名，必须是org.redkale.mq.MessageAgent的子类
+            MQ节点下的子节点配置没有固定格式, 根据MessageAgent实现方的定义来配置
+        -->
+        <!--
+        <mq name="" value="org.redkalex.mq.kafka.KafkaMessageAgent">
+            <servers value="127.0.0.1:9101"/>
+            <consumer>
+                <property name="xxxxxx" value="XXXXXXXX"/>
+            </consumer>
+            <producer>
+                <property name="xxxxxx" value="XXXXXXXX"/>
+            </producer>
+        </mq>
+        -->
         <!--
             一个组包含多个node， 同一Service服务可以由多个进程提供，这些进程称为一个GROUP，且同一GROUP内的进程必须在同一机房或局域网内
             一个group节点对应一个 Transport 对象。
             name:         服务组ID，长度不能超过11个字节. 默认为空字符串。 注意: name不能包含$符号。
-            protocol：值范围：UDP TCP， 默认TCP
+            protocol：    值范围：UDP TCP， 默认TCP
-            subprotocol: 子协议，预留字段。默认值为空
             注意: 一个node只能所属一个group。只要存在protocol=SNCP的Server节点信息， 就必须有group节点信息。
         -->
         <group name="" protocol="TCP">
@@ -64,7 +106,7 @@
         <!--
             全局的数据源设置, 可以是CacheSource、DataSource， JDBC的DataSource通常通过persistence.xml配置，此处多用于CacheSource的配置
             name:    资源名，用于依赖注入。
-            value：类名，必须是CacheSource或DataSource的子类，且必须实现Service接口。如果是DataSource.class，系统自动映射成DataJdbcSource.class
+            value：  类名，必须是CacheSource或DataSource的子类，且必须实现Service接口。如果是DataSource.class，系统自动映射成DataJdbcSource.class
             groups:  指定groups。
             xxx:     其他属性与子节点通过Service.init方法传入的AnyValue获取。
         -->
@@ -88,12 +130,9 @@
                System.setProperty("net.transport.poolmaxconns", "100");
                System.setProperty("net.transport.pinginterval", "30");
                System.setProperty("net.transport.checkinterval", "30");
-               System.setProperty("convert.json.tiny", "true");
+               System.setProperty("convert.tiny", "true");
-               System.setProperty("convert.bson.tiny", "true");
+               System.setProperty("convert.pool.size", "128");
-               System.setProperty("convert.json.pool.size", "128");
+               System.setProperty("convert.writer.buffer.defsize", "4096");
-               System.setProperty("convert.bson.pool.size", "128");
-               System.setProperty("convert.json.writer.buffer.defsize", "4096");
-               System.setProperty("convert.bson.writer.buffer.defsize", "4096");
                
             <properties>节点下也可包含非<property>节点.
             非<property>其节点可以通过@Resource(name="properties.xxxxxx")进行注入, 被注解的字段类型只能是AnyValue、AnyValue[]
@@ -117,7 +156,7 @@
         charset:             文本编码， 默认: UTF-8
         backlog:             默认10K
         threads：            线程数， 默认: CPU核数*2，最小8个
-        maxconns：最大连接数, 小于1表示无限制， 默认: 0
+        maxconns：           最大连接数, 小于1表示无限制， 默认: 0
         maxbody:             request.body最大值， 默认: 64K
         bufferCapacity:      ByteBuffer的初始化大小， TCP默认: 32K;  (HTTP 2.0、WebSocket，必须要16k以上); UDP默认: 1350B
         bufferPoolSize：     ByteBuffer池的大小，默认: 线程数*4
@@ -146,11 +185,13 @@
            在同一个进程中同一个name同一类型的Service将共用同一个实例
            autoload="true"  默认值. 自动加载classpath下所有的Service类  
            autoload="false" 需要显著的指定Service类
+           mq:        所属的MQ管理器，当 protocol == SNCP 时该值才有效, 存在该属性表示Service的SNCP协议采用消息总线代理模式
            includes： 当autoload="true"， 拉取类名与includes中的正则表达式匹配的类, 多个正则表达式用分号;隔开
            excludes： 当autoload="true"， 排除类名与excludes中的正则表达式匹配的类, 多个正则表达式用分号;隔开           
            groups:    所属组的节点,多个节点值用;隔开，如果配置文件中存在多个SNCP协议的Server节点，需要显式指定group属性.
                          当 protocol == SNCP 时 group表示当前Server与哪些节点组关联。
                          当 protocol != SNCP 时 group只能是空或者一个group的节点值，不能为多个节点值。
+                         特殊值"$cluster", 视为通过第三方服务注册发现管理工具来获取远程模式的ip端口信息
         -->
         <services autoload="true" includes="" excludes="">
 
@@ -198,8 +239,9 @@
         <!-- 
            REST的核心配置项
            当Server为HTTP协议时, rest节点才有效。存在[rest]节点则Server启动时会加载REST服务, 节点可以多个,(WATCH协议不需要设置，系统会自动生成)
-           path:     servlet的ContextPath前缀 默认为空
+           path:     servlet的ContextPath前缀 默认为空 【注: 开启MQ时,该字段失效】
            base:     REST服务的BaseServlet，必须是 org.redkale.net.http.HttpServlet 的子类，且子类必须标记@HttpUserType。
+           mq:        所属的MQ管理器, 存在该属性表示RestService的请求来自于消息总线 【注: 开启MQ时,path字段失效】
            autoload：默认值"true"  默认值. 加载当前server所能使用的Servce对象;    
            includes：当autoload="true"， 拉取类名与includes中的正则表达式匹配的类, 多个正则表达式用分号;隔开
            excludes：当autoload="true"， 排除类名与excludes中的正则表达式匹配的类, 多个正则表达式用分号;隔开

src/module-info.txt

@@ -22,6 +22,7 @@ module org.redkale {
     exports org.redkale.convert.bson;
     exports org.redkale.convert.ext;
     exports org.redkale.convert.json;
+    exports org.redkale.mq;
     exports org.redkale.net;
     exports org.redkale.net.http;
     exports org.redkale.net.sncp;
@@ -30,6 +31,9 @@ module org.redkale {
     exports org.redkale.util;
     exports org.redkale.watch;
     
+    uses org.redkale.cluster.ClusterAgent;
+    uses org.redkale.mq.MessageAgent;
+    uses org.redkale.source.CacheSource;
     uses org.redkale.source.SourceLoader;
     uses org.redkale.util.ResourceInjectLoader;
 }

src/org/redkale/asm/AnnotationVisitor.java

@@ -60,8 +60,8 @@ package org.redkale.asm;
 
 /**
  * A visitor to visit a Java annotation. The methods of this class must be
- * called in the following order: ( <tt>visit</tt> | <tt>visitEnum</tt> |
+ * called in the following order: ( <code>visit</code> | <code>visitEnum</code> |
- * <tt>visitAnnotation</tt> | <tt>visitArray</tt> )* <tt>visitEnd</tt>.
+ * <code>visitAnnotation</code> | <code>visitArray</code> )* <code>visitEnd</code>.
  *
  * @author Eric Bruneton
  * @author Eugene Kuleshov
@@ -151,7 +151,7 @@ public abstract class AnnotationVisitor {
      * @param desc
      *            the class descriptor of the nested annotation class.
      * @return a visitor to visit the actual nested annotation value, or
-     *         <tt>null</tt> if this visitor is not interested in visiting this
+     *         <code>null</code> if this visitor is not interested in visiting this
      *         nested annotation. <i>The nested annotation value must be fully
      *         visited before calling other methods on this annotation
      *         visitor</i>.
@@ -172,7 +172,7 @@ public abstract class AnnotationVisitor {
      * @param name
      *            the value name.
      * @return a visitor to visit the actual array value elements, or
-     *         <tt>null</tt> if this visitor is not interested in visiting these
+     *         <code>null</code> if this visitor is not interested in visiting these
      *         values. The 'name' parameters passed to the methods of this
      *         visitor are ignored. <i>All the array values must be visited
      *         before calling other methods on this annotation visitor</i>.

src/org/redkale/asm/AnnotationWriter.java

@@ -77,7 +77,7 @@ final class AnnotationWriter extends AnnotationVisitor {
     private int size;
 
     /**
-     * <tt>true<tt> if values are named, <tt>false</tt> otherwise. Annotation
+     * <code>true<code> if values are named, <code>false</code> otherwise. Annotation
      * writers used for annotation default and annotation arrays use unnamed
      * values.
      */
@@ -122,13 +122,13 @@ final class AnnotationWriter extends AnnotationVisitor {
      * @param cw
      *            the class writer to which this annotation must be added.
      * @param named
-     *            <tt>true<tt> if values are named, <tt>false</tt> otherwise.
+     *            <code>true<code> if values are named, <code>false</code> otherwise.
      * @param bv
      *            where the annotation values must be stored.
      * @param parent
      *            where the number of annotation values must be stored.
      * @param offset
-     *            where in <tt>parent</tt> the number of annotation values must
+     *            where in <code>parent</code> the number of annotation values must
      *            be stored.
      */
     AnnotationWriter(final ClassWriter cw, final boolean named,
@@ -354,7 +354,7 @@ final class AnnotationWriter extends AnnotationVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param out
      *            where the type reference and type path must be put.
      */

src/org/redkale/asm/Attribute.java

@@ -77,7 +77,7 @@ class Attribute {
     byte[] value;
 
     /**
-     * The next attribute in this attribute list. May be <tt>null</tt>.
+     * The next attribute in this attribute list. May be <code>null</code>.
      */
     Attribute next;
 
@@ -92,19 +92,19 @@ class Attribute {
     }
 
     /**
-     * Returns <tt>true</tt> if this type of attribute is unknown. The default
+     * Returns <code>true</code> if this type of attribute is unknown. The default
-     * implementation of this method always returns <tt>true</tt>.
+     * implementation of this method always returns <code>true</code>.
      *
-     * @return <tt>true</tt> if this type of attribute is unknown.
+     * @return <code>true</code> if this type of attribute is unknown.
      */
     public boolean isUnknown() {
         return true;
     }
 
     /**
-     * Returns <tt>true</tt> if this type of attribute is a code attribute.
+     * Returns <code>true</code> if this type of attribute is a code attribute.
      *
-     * @return <tt>true</tt> if this type of attribute is a code attribute.
+     * @return <code>true</code> if this type of attribute is a code attribute.
      */
     public boolean isCodeAttribute() {
         return false;
@@ -113,7 +113,7 @@ class Attribute {
     /**
      * Returns the labels corresponding to this attribute.
      *
-     * @return the labels corresponding to this attribute, or <tt>null</tt> if
+     * @return the labels corresponding to this attribute, or <code>null</code> if
      *         this attribute is not a code attribute that contains labels.
      */
     protected Label[] getLabels() {
@@ -123,7 +123,7 @@ class Attribute {
     /**
      * Reads a {@link #type type} attribute. This method must return a
      * <i>new</i> {@link Attribute} object, of type {@link #type type},
-     * corresponding to the <tt>len</tt> bytes starting at the given offset, in
+     * corresponding to the <code>len</code> bytes starting at the given offset, in
      * the given class reader.
      *
      * @param cr
@@ -146,7 +146,7 @@ class Attribute {
      *            containing the type and the length of the attribute, are not
      *            taken into account here.
      * @param labels
-     *            the labels of the method's code, or <tt>null</tt> if the
+     *            the labels of the method's code, or <code>null</code> if the
      *            attribute to be read is not a code attribute.
      * @return a <i>new</i> {@link Attribute} object corresponding to the given
      *         bytes.
@@ -169,11 +169,11 @@ class Attribute {
      *            class the items that corresponds to this attribute.
      * @param code
      *            the bytecode of the method corresponding to this code
-     *            attribute, or <tt>null</tt> if this attribute is not a code
+     *            attribute, or <code>null</code> if this attribute is not a code
      *            attributes.
      * @param len
      *            the length of the bytecode of the method corresponding to this
-     *            code attribute, or <tt>null</tt> if this attribute is not a
+     *            code attribute, or <code>null</code> if this attribute is not a
      *            code attribute.
      * @param maxStack
      *            the maximum stack size of the method corresponding to this
@@ -216,11 +216,11 @@ class Attribute {
      *            byte arrays, with the {@link #write write} method.
      * @param code
      *            the bytecode of the method corresponding to these code
-     *            attributes, or <tt>null</tt> if these attributes are not code
+     *            attributes, or <code>null</code> if these attributes are not code
      *            attributes.
      * @param len
      *            the length of the bytecode of the method corresponding to
-     *            these code attributes, or <tt>null</tt> if these attributes
+     *            these code attributes, or <code>null</code> if these attributes
      *            are not code attributes.
      * @param maxStack
      *            the maximum stack size of the method corresponding to these
@@ -254,11 +254,11 @@ class Attribute {
      *            byte arrays, with the {@link #write write} method.
      * @param code
      *            the bytecode of the method corresponding to these code
-     *            attributes, or <tt>null</tt> if these attributes are not code
+     *            attributes, or <code>null</code> if these attributes are not code
      *            attributes.
      * @param len
      *            the length of the bytecode of the method corresponding to
-     *            these code attributes, or <tt>null</tt> if these attributes
+     *            these code attributes, or <code>null</code> if these attributes
      *            are not code attributes.
      * @param maxStack
      *            the maximum stack size of the method corresponding to these

src/org/redkale/asm/ByteVector.java

@@ -332,7 +332,7 @@ public class ByteVector {
      * automatically enlarged if necessary.
      *
      * @param b
-     *            an array of bytes. May be <tt>null</tt> to put <tt>len</tt>
+     *            an array of bytes. May be <code>null</code> to put <code>len</code>
      *            null bytes into this byte vector.
      * @param off
      *            index of the fist byte of b that must be copied.

src/org/redkale/asm/ClassReader.java

@@ -267,7 +267,7 @@ public class ClassReader {
      * {@link Type#getInternalName() getInternalName}). For interfaces, the
      * super class is {@link Object}.
      *
-     * @return the internal name of super class, or <tt>null</tt> for
+     * @return the internal name of super class, or <code>null</code> for
      *         {@link Object} class.
      *
      * @see ClassVisitor#visit(int, int, String, String, String, String[])
@@ -281,7 +281,7 @@ public class ClassReader {
      * {@link Type#getInternalName() getInternalName}).
      *
      * @return the array of internal names for all implemented interfaces or
-     *         <tt>null</tt>.
+     *         <code>null</code>.
      *
      * @see ClassVisitor#visit(int, int, String, String, String, String[])
      */
@@ -1932,7 +1932,7 @@ public class ClassReader {
      * @param v
      *            start offset in {@link #b b} of the annotations to be read.
      * @param visible
-     *            <tt>true</tt> if the annotations to be read are visible at
+     *            <code>true</code> if the annotations to be read are visible at
      *            runtime.
      */
     private void readParameterAnnotations(final MethodVisitor mv,
@@ -2474,9 +2474,9 @@ public class ClassReader {
      *            and the length of the attribute, are not taken into account
      *            here.
      * @param labels
-     *            the labels of the method's code, or <tt>null</tt> if the
+     *            the labels of the method's code, or <code>null</code> if the
      *            attribute to be read is not a code attribute.
-     * @return the attribute that has been read, or <tt>null</tt> to skip this
+     * @return the attribute that has been read, or <code>null</code> to skip this
      *         attribute.
      */
     private Attribute readAttribute(final Attribute[] attrs, final String type,

src/org/redkale/asm/ClassVisitor.java

@@ -60,11 +60,11 @@ package org.redkale.asm;
 
 /**
  * A visitor to visit a Java class. The methods of this class must be called in
- * the following order: <tt>visit</tt> [ <tt>visitSource</tt> ] [
+ * the following order: <code>visit</code> [ <code>visitSource</code> ] [
- * <tt>visitModule</tt> ][ <tt>visitOuterClass</tt> ] ( <tt>visitAnnotation</tt> |
+ * <code>visitModule</code> ][ <code>visitOuterClass</code> ] ( <code>visitAnnotation</code> |
- * <tt>visitTypeAnnotation</tt> | <tt>visitAttribute</tt> )* (
+ * <code>visitTypeAnnotation</code> | <code>visitAttribute</code> )* (
- * <tt>visitInnerClass</tt> | <tt>visitField</tt> | <tt>visitMethod</tt> )*
+ * <code>visitInnerClass</code> | <code>visitField</code> | <code>visitMethod</code> )*
- * <tt>visitEnd</tt>.
+ * <code>visitEnd</code>.
  *
  * @author Eric Bruneton
  */
@@ -120,18 +120,18 @@ public abstract class ClassVisitor {
      *            the internal name of the class (see
      *            {@link Type#getInternalName() getInternalName}).
      * @param signature
-     *            the signature of this class. May be <tt>null</tt> if the class
+     *            the signature of this class. May be <code>null</code> if the class
      *            is not a generic one, and does not extend or implement generic
      *            classes or interfaces.
      * @param superName
      *            the internal of name of the super class (see
      *            {@link Type#getInternalName() getInternalName}). For
      *            interfaces, the super class is {@link Object}. May be
-     *            <tt>null</tt>, but only for the {@link Object} class.
+     *            <code>null</code>, but only for the {@link Object} class.
      * @param interfaces
      *            the internal names of the class's interfaces (see
      *            {@link Type#getInternalName() getInternalName}). May be
-     *            <tt>null</tt>.
+     *            <code>null</code>.
      */
     public void visit(int version, int access, String name, String signature,
             String superName, String[] interfaces) {
@@ -145,11 +145,11 @@ public abstract class ClassVisitor {
      *
      * @param source
      *            the name of the source file from which the class was compiled.
-     *            May be <tt>null</tt>.
+     *            May be <code>null</code>.
      * @param debug
      *            additional debug information to compute the correspondance
      *            between source and compiled elements of the class. May be
-     *            <tt>null</tt>.
+     *            <code>null</code>.
      */
     public void visitSource(String source, String debug) {
         if (cv != null) {
@@ -166,7 +166,7 @@ public abstract class ClassVisitor {
      *            and {@code ACC_MANDATED}.
      * @param version
      *            module version or null.
-     * @return a visitor to visit the module values, or <tt>null</tt> if
+     * @return a visitor to visit the module values, or <code>null</code> if
      *         this visitor is not interested in visiting this module.
      */
     public ModuleVisitor visitModule(String name, int access, String version) {
@@ -187,11 +187,11 @@ public abstract class ClassVisitor {
      *            internal name of the enclosing class of the class.
      * @param name
      *            the name of the method that contains the class, or
-     *            <tt>null</tt> if the class is not enclosed in a method of its
+     *            <code>null</code> if the class is not enclosed in a method of its
      *            enclosing class.
      * @param desc
      *            the descriptor of the method that contains the class, or
-     *            <tt>null</tt> if the class is not enclosed in a method of its
+     *            <code>null</code> if the class is not enclosed in a method of its
      *            enclosing class.
      */
     public void visitOuterClass(String owner, String name, String desc) {
@@ -206,8 +206,8 @@ public abstract class ClassVisitor {
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
@@ -231,12 +231,12 @@ public abstract class ClassVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitTypeAnnotation(int typeRef,
@@ -269,10 +269,10 @@ public abstract class ClassVisitor {
      * @param outerName
      *            the internal name of the class to which the inner class
      *            belongs (see {@link Type#getInternalName() getInternalName}).
-     *            May be <tt>null</tt> for not member classes.
+     *            May be <code>null</code> for not member classes.
      * @param innerName
      *            the (simple) name of the inner class inside its enclosing
-     *            class. May be <tt>null</tt> for anonymous inner classes.
+     *            class. May be <code>null</code> for anonymous inner classes.
      * @param access
      *            the access flags of the inner class as originally declared in
      *            the enclosing class.
@@ -295,20 +295,20 @@ public abstract class ClassVisitor {
      * @param desc
      *            the field's descriptor (see {@link Type Type}).
      * @param signature
-     *            the field's signature. May be <tt>null</tt> if the field's
+     *            the field's signature. May be <code>null</code> if the field's
      *            type does not use generic types.
      * @param value
      *            the field's initial value. This parameter, which may be
-     *            <tt>null</tt> if the field does not have an initial value,
+     *            <code>null</code> if the field does not have an initial value,
      *            must be an {@link Integer}, a {@link Float}, a {@link Long}, a
-     *            {@link Double} or a {@link String} (for <tt>int</tt>,
+     *            {@link Double} or a {@link String} (for <code>int</code>,
-     *            <tt>float</tt>, <tt>long</tt> or <tt>String</tt> fields
+     *            <code>float</code>, <code>long</code> or <code>String</code> fields
      *            respectively). <i>This parameter is only used for static
      *            fields</i>. Its value is ignored for non static fields, which
      *            must be initialized through bytecode instructions in
      *            constructors or methods.
      * @return a visitor to visit field annotations and attributes, or
-     *         <tt>null</tt> if this class visitor is not interested in visiting
+     *         <code>null</code> if this class visitor is not interested in visiting
      *         these annotations and attributes.
      */
     public FieldVisitor visitField(int access, String name, String desc,
@@ -321,7 +321,7 @@ public abstract class ClassVisitor {
 
     /**
      * Visits a method of the class. This method <i>must</i> return a new
-     * {@link MethodVisitor} instance (or <tt>null</tt>) each time it is called,
+     * {@link MethodVisitor} instance (or <code>null</code>) each time it is called,
      * i.e., it should not return a previously returned visitor.
      *
      * @param access
@@ -333,14 +333,14 @@ public abstract class ClassVisitor {
      * @param desc
      *            the method's descriptor (see {@link Type Type}).
      * @param signature
-     *            the method's signature. May be <tt>null</tt> if the method
+     *            the method's signature. May be <code>null</code> if the method
      *            parameters, return type and exceptions do not use generic
      *            types.
      * @param exceptions
      *            the internal names of the method's exception classes (see
      *            {@link Type#getInternalName() getInternalName}). May be
-     *            <tt>null</tt>.
+     *            <code>null</code>.
-     * @return an object to visit the byte code of the method, or <tt>null</tt>
+     * @return an object to visit the byte code of the method, or <code>null</code>
      *         if this class visitor is not interested in visiting the code of
      *         this method.
      */

src/org/redkale/asm/ClassWriter.java

@@ -391,8 +391,8 @@ public class ClassWriter extends ClassVisitor {
      * A type table used to temporarily store internal names that will not
      * necessarily be stored in the constant pool. This type table is used by
      * the control flow and data flow analysis algorithm used to compute stack
-     * map frames from scratch. This array associates to each index <tt>i</tt>
+     * map frames from scratch. This array associates to each index <code>i</code>
-     * the Item whose index is <tt>i</tt>. All Item objects stored in this array
+     * the Item whose index is <code>i</code>. All Item objects stored in this array
      * are also stored in the {@link #items} hash table. These two arrays allow
      * to retrieve an Item from its index or, conversely, to get the index of an
      * Item from its value. Each Item stores an internal name in its
@@ -556,7 +556,7 @@ public class ClassWriter extends ClassVisitor {
     private int compute;
 
     /**
-     * <tt>true</tt> if some methods have wide forward jumps using ASM pseudo
+     * <code>true</code> if some methods have wide forward jumps using ASM pseudo
      * instructions, which need to be expanded into sequences of standard
      * bytecode instructions. In this case the class is re-read and re-written
      * with a ClassReader -> ClassWriter chain to perform this transformation.
@@ -1486,7 +1486,7 @@ public class ClassWriter extends ClassVisitor {
      * @param desc
      *            the method's descriptor.
      * @param itf
-     *            <tt>true</tt> if <tt>owner</tt> is an interface.
+     *            <code>true</code> if <code>owner</code> is an interface.
      * @return a new or already existing method reference item.
      */
     Item newMethodItem(final String owner, final String name,
@@ -1515,7 +1515,7 @@ public class ClassWriter extends ClassVisitor {
      * @param desc
      *            the method's descriptor.
      * @param itf
-     *            <tt>true</tt> if <tt>owner</tt> is an interface.
+     *            <code>true</code> if <code>owner</code> is an interface.
      * @return the index of a new or already existing method reference item.
      */
     public int newMethod(final String owner, final String name,
@@ -1778,7 +1778,7 @@ public class ClassWriter extends ClassVisitor {
      * @param key
      *            a constant pool item.
      * @return the constant pool's hash table item which is equal to the given
-     *         item, or <tt>null</tt> if there is no such item.
+     *         item, or <code>null</code> if there is no such item.
      */
     private Item get(final Item key) {
         Item i = items[key.hashCode % items.length];

src/org/redkale/asm/FieldVisitor.java

@@ -60,8 +60,8 @@ package org.redkale.asm;
 
 /**
  * A visitor to visit a Java field. The methods of this class must be called in
- * the following order: ( <tt>visitAnnotation</tt> |
+ * the following order: ( <code>visitAnnotation</code> |
- * <tt>visitTypeAnnotation</tt> | <tt>visitAttribute</tt> )* <tt>visitEnd</tt>.
+ * <code>visitTypeAnnotation</code> | <code>visitAttribute</code> )* <code>visitEnd</code>.
  *
  * @author Eric Bruneton
  */
@@ -111,8 +111,8 @@ public abstract class FieldVisitor {
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
@@ -132,12 +132,12 @@ public abstract class FieldVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitTypeAnnotation(int typeRef,

src/org/redkale/asm/FieldWriter.java

@@ -100,28 +100,28 @@ final class FieldWriter extends FieldVisitor {
     private int value;
 
     /**
-     * The runtime visible annotations of this field. May be <tt>null</tt>.
+     * The runtime visible annotations of this field. May be <code>null</code>.
      */
     private AnnotationWriter anns;
 
     /**
-     * The runtime invisible annotations of this field. May be <tt>null</tt>.
+     * The runtime invisible annotations of this field. May be <code>null</code>.
      */
     private AnnotationWriter ianns;
 
     /**
-     * The runtime visible type annotations of this field. May be <tt>null</tt>.
+     * The runtime visible type annotations of this field. May be <code>null</code>.
      */
     private AnnotationWriter tanns;
 
     /**
      * The runtime invisible type annotations of this field. May be
-     * <tt>null</tt>.
+     * <code>null</code>.
      */
     private AnnotationWriter itanns;
 
     /**
-     * The non standard attributes of this field. May be <tt>null</tt>.
+     * The non standard attributes of this field. May be <code>null</code>.
      */
     private Attribute attrs;
 
@@ -141,9 +141,9 @@ final class FieldWriter extends FieldVisitor {
      * @param desc
      *            the field's descriptor (see {@link Type}).
      * @param signature
-     *            the field's signature. May be <tt>null</tt>.
+     *            the field's signature. May be <code>null</code>.
      * @param value
-     *            the field's constant value. May be <tt>null</tt>.
+     *            the field's constant value. May be <code>null</code>.
      */
     FieldWriter(final ClassWriter cw, final int access, final String name,
             final String desc, final String signature, final Object value) {

src/org/redkale/asm/Frame.java

@@ -1403,7 +1403,7 @@ class Frame {
 
     /**
      * Merges the input frame of the given basic block with the input and output
-     * frames of this basic block. Returns <tt>true</tt> if the input frame of
+     * frames of this basic block. Returns <code>true</code> if the input frame of
      * the given label has been changed by this operation.
      *
      * @param cw
@@ -1413,7 +1413,7 @@ class Frame {
      * @param edge
      *            the kind of the {@link Edge} between this label and 'label'.
      *            See {@link Edge#info}.
-     * @return <tt>true</tt> if the input frame of the given label has been
+     * @return <code>true</code> if the input frame of the given label has been
      *         changed by this operation.
      */
     final boolean merge(final ClassWriter cw, final Frame frame, final int edge) {
@@ -1511,7 +1511,7 @@ class Frame {
 
     /**
      * Merges the type at the given index in the given type array with the given
-     * type. Returns <tt>true</tt> if the type array has been modified by this
+     * type. Returns <code>true</code> if the type array has been modified by this
      * operation.
      *
      * @param cw
@@ -1522,7 +1522,7 @@ class Frame {
      *            an array of types.
      * @param index
      *            the index of the type that must be merged in 'types'.
-     * @return <tt>true</tt> if the type array has been modified by this
+     * @return <code>true</code> if the type array has been modified by this
      *         operation.
      */
     private static boolean merge(final ClassWriter cw, int t,

src/org/redkale/asm/Handler.java

@@ -82,7 +82,7 @@ class Handler {
 
     /**
      * Internal name of the type of exceptions handled by this handler, or
-     * <tt>null</tt> to catch any exceptions.
+     * <code>null</code> to catch any exceptions.
      */
     String desc;
 

src/org/redkale/asm/Item.java

@@ -306,8 +306,8 @@ final class Item {
      * @param i
      *            the item to be compared to this one. Both items must have the
      *            same {@link #type}.
-     * @return <tt>true</tt> if the given item if equal to this one,
+     * @return <code>true</code> if the given item if equal to this one,
-     *         <tt>false</tt> otherwise.
+     *         <code>false</code> otherwise.
      */
     boolean isEqualTo(final Item i) {
         switch (type) {

src/org/redkale/asm/Label.java

@@ -325,8 +325,8 @@ public class Label {
      *            the position of first byte of the bytecode instruction that
      *            contains this label.
      * @param wideOffset
-     *            <tt>true</tt> if the reference must be stored in 4 bytes, or
+     *            <code>true</code> if the reference must be stored in 4 bytes, or
-     *            <tt>false</tt> if it must be stored with 2 bytes.
+     *            <code>false</code> if it must be stored with 2 bytes.
      * @throws IllegalArgumentException
      *             if this label has not been created by the given code writer.
      */
@@ -389,7 +389,7 @@ public class Label {
      *            the position of this label in the bytecode.
      * @param data
      *            the bytecode of the method.
-     * @return <tt>true</tt> if a blank that was left for this label was too
+     * @return <code>true</code> if a blank that was left for this label was too
      *         small to store the offset. In such a case the corresponding jump
      *         instruction is replaced with a pseudo instruction (using unused
      *         opcodes) using an unsigned two bytes offset. These pseudo

src/org/redkale/asm/MethodVisitor.java

@@ -60,24 +60,24 @@ package org.redkale.asm;
 
 /**
  * A visitor to visit a Java method. The methods of this class must be called in
- * the following order: ( <tt>visitParameter</tt> )* [
+ * the following order: ( <code>visitParameter</code> )* [
- * <tt>visitAnnotationDefault</tt> ] ( <tt>visitAnnotation</tt> |
+ * <code>visitAnnotationDefault</code> ] ( <code>visitAnnotation</code> |
- * <tt>visitParameterAnnotation</tt> <tt>visitTypeAnnotation</tt> |
+ * <code>visitParameterAnnotation</code> <code>visitTypeAnnotation</code> |
- * <tt>visitAttribute</tt> )* [ <tt>visitCode</tt> ( <tt>visitFrame</tt> |
+ * <code>visitAttribute</code> )* [ <code>visitCode</code> ( <code>visitFrame</code> |
- * <tt>visit<i>X</i>Insn</tt> | <tt>visitLabel</tt> |
+ * <code>visit<i>X</i>Insn</code> | <code>visitLabel</code> |
- * <tt>visitInsnAnnotation</tt> | <tt>visitTryCatchBlock</tt> |
+ * <code>visitInsnAnnotation</code> | <code>visitTryCatchBlock</code> |
- * <tt>visitTryCatchAnnotation</tt> | <tt>visitLocalVariable</tt> |
+ * <code>visitTryCatchAnnotation</code> | <code>visitLocalVariable</code> |
- * <tt>visitLocalVariableAnnotation</tt> | <tt>visitLineNumber</tt> )*
+ * <code>visitLocalVariableAnnotation</code> | <code>visitLineNumber</code> )*
- * <tt>visitMaxs</tt> ] <tt>visitEnd</tt>. In addition, the
+ * <code>visitMaxs</code> ] <code>visitEnd</code>. In addition, the
- * <tt>visit<i>X</i>Insn</tt> and <tt>visitLabel</tt> methods must be called in
+ * <code>visit<i>X</i>Insn</code> and <code>visitLabel</code> methods must be called in
  * the sequential order of the bytecode instructions of the visited code,
- * <tt>visitInsnAnnotation</tt> must be called <i>after</i> the annotated
+ * <code>visitInsnAnnotation</code> must be called <i>after</i> the annotated
- * instruction, <tt>visitTryCatchBlock</tt> must be called <i>before</i> the
+ * instruction, <code>visitTryCatchBlock</code> must be called <i>before</i> the
  * labels passed as arguments have been visited,
- * <tt>visitTryCatchBlockAnnotation</tt> must be called <i>after</i> the
+ * <code>visitTryCatchBlockAnnotation</code> must be called <i>after</i> the
  * corresponding try catch block has been visited, and the
- * <tt>visitLocalVariable</tt>, <tt>visitLocalVariableAnnotation</tt> and
+ * <code>visitLocalVariable</code>, <code>visitLocalVariableAnnotation</code> and
- * <tt>visitLineNumber</tt> methods must be called <i>after</i> the labels
+ * <code>visitLineNumber</code> methods must be called <i>after</i> the labels
  * passed as arguments have been visited.
  *
  * @author Eric Bruneton
@@ -132,8 +132,8 @@ public abstract class MethodVisitor {
      * @param name
      *            parameter name or null if none is provided.
      * @param access
-     *            the parameter's access flags, only <tt>ACC_FINAL</tt>,
+     *            the parameter's access flags, only <code>ACC_FINAL</code>,
-     *            <tt>ACC_SYNTHETIC</tt> or/and <tt>ACC_MANDATED</tt> are
+     *            <code>ACC_SYNTHETIC</code> or/and <code>ACC_MANDATED</code> are
      *            allowed (see {@link Opcodes}).
      */
     public void visitParameter(String name, int access) {
@@ -146,7 +146,7 @@ public abstract class MethodVisitor {
      * Visits the default value of this annotation interface method.
      *
      * @return a visitor to the visit the actual default value of this
-     *         annotation interface method, or <tt>null</tt> if this visitor is
+     *         annotation interface method, or <code>null</code> if this visitor is
      *         not interested in visiting this default value. The 'name'
      *         parameters passed to the methods of this annotation visitor are
      *         ignored. Moreover, exacly one visit method must be called on this
@@ -165,8 +165,8 @@ public abstract class MethodVisitor {
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
@@ -193,12 +193,12 @@ public abstract class MethodVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitTypeAnnotation(int typeRef,
@@ -217,8 +217,8 @@ public abstract class MethodVisitor {
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitParameterAnnotation(int parameter,
@@ -569,7 +569,7 @@ public abstract class MethodVisitor {
      *            the constant to be loaded on the stack. This parameter must be
      *            a non null {@link Integer}, a {@link Float}, a {@link Long}, a
      *            {@link Double}, a {@link String}, a {@link Type} of OBJECT or
-     *            ARRAY sort for <tt>.class</tt> constants, for classes whose
+     *            ARRAY sort for <code>.class</code> constants, for classes whose
      *            version is 49.0, a {@link Type} of METHOD sort or a
      *            {@link Handle} for MethodType and MethodHandle constants, for
      *            classes whose version is 51.0.
@@ -604,8 +604,8 @@ public abstract class MethodVisitor {
      * @param dflt
      *            beginning of the default handler block.
      * @param labels
-     *            beginnings of the handler blocks. <tt>labels[i]</tt> is the
+     *            beginnings of the handler blocks. <code>labels[i]</code> is the
-     *            beginning of the handler block for the <tt>min + i</tt> key.
+     *            beginning of the handler block for the <code>min + i</code> key.
      */
     public void visitTableSwitchInsn(int min, int max, Label dflt,
             Label... labels) {
@@ -622,8 +622,8 @@ public abstract class MethodVisitor {
      * @param keys
      *            the values of the keys.
      * @param labels
-     *            beginnings of the handler blocks. <tt>labels[i]</tt> is the
+     *            beginnings of the handler blocks. <code>labels[i]</code> is the
-     *            beginning of the handler block for the <tt>keys[i]</tt> key.
+     *            beginning of the handler block for the <code>keys[i]</code> key.
      */
     public void visitLookupSwitchInsn(Label dflt, int[] keys, Label[] labels) {
         if (mv != null) {
@@ -668,12 +668,12 @@ public abstract class MethodVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitInsnAnnotation(int typeRef,
@@ -699,7 +699,7 @@ public abstract class MethodVisitor {
      *            beginning of the exception handler's code.
      * @param type
      *            internal name of the type of exceptions handled by the
-     *            handler, or <tt>null</tt> to catch any exceptions (for
+     *            handler, or <code>null</code> to catch any exceptions (for
      *            "finally" blocks).
      * @throws IllegalArgumentException
      *             if one of the labels has already been visited by this visitor
@@ -725,12 +725,12 @@ public abstract class MethodVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitTryCatchAnnotation(int typeRef,
@@ -750,7 +750,7 @@ public abstract class MethodVisitor {
      *            the type descriptor of this local variable.
      * @param signature
      *            the type signature of this local variable. May be
-     *            <tt>null</tt> if the local variable type does not use generic
+     *            <code>null</code> if the local variable type does not use generic
      *            types.
      * @param start
      *            the first instruction corresponding to the scope of this local
@@ -782,7 +782,7 @@ public abstract class MethodVisitor {
      * @param typePath
      *            the path to the annotated type argument, wildcard bound, array
      *            element type, or static inner type within 'typeRef'. May be
-     *            <tt>null</tt> if the annotation targets 'typeRef' as a whole.
+     *            <code>null</code> if the annotation targets 'typeRef' as a whole.
      * @param start
      *            the fist instructions corresponding to the continuous ranges
      *            that make the scope of this local variable (inclusive).
@@ -796,8 +796,8 @@ public abstract class MethodVisitor {
      * @param desc
      *            the class descriptor of the annotation class.
      * @param visible
-     *            <tt>true</tt> if the annotation is visible at runtime.
+     *            <code>true</code> if the annotation is visible at runtime.
-     * @return a visitor to visit the annotation values, or <tt>null</tt> if
+     * @return a visitor to visit the annotation values, or <code>null</code> if
      *         this visitor is not interested in visiting this annotation.
      */
     public AnnotationVisitor visitLocalVariableAnnotation(int typeRef,
@@ -819,7 +819,7 @@ public abstract class MethodVisitor {
      * @param start
      *            the first instruction corresponding to this line number.
      * @throws IllegalArgumentException
-     *             if <tt>start</tt> has not already been visited by this
+     *             if <code>start</code> has not already been visited by this
      *             visitor (by the {@link #visitLabel visitLabel} method).
      */
     public void visitLineNumber(int line, Label start) {

src/org/redkale/asm/MethodWriter.java

@@ -218,41 +218,41 @@ class MethodWriter extends MethodVisitor {
     int[] exceptions;
 
     /**
-     * The annotation default attribute of this method. May be <tt>null</tt>.
+     * The annotation default attribute of this method. May be <code>null</code>.
      */
     private ByteVector annd;
 
     /**
-     * The runtime visible annotations of this method. May be <tt>null</tt>.
+     * The runtime visible annotations of this method. May be <code>null</code>.
      */
     private AnnotationWriter anns;
 
     /**
-     * The runtime invisible annotations of this method. May be <tt>null</tt>.
+     * The runtime invisible annotations of this method. May be <code>null</code>.
      */
     private AnnotationWriter ianns;
 
     /**
-     * The runtime visible type annotations of this method. May be <tt>null</tt>
+     * The runtime visible type annotations of this method. May be <code>null</code>
      * .
      */
     private AnnotationWriter tanns;
 
     /**
      * The runtime invisible type annotations of this method. May be
-     * <tt>null</tt>.
+     * <code>null</code>.
      */
     private AnnotationWriter itanns;
 
     /**
      * The runtime visible parameter annotations of this method. May be
-     * <tt>null</tt>.
+     * <code>null</code>.
      */
     private AnnotationWriter[] panns;
 
     /**
      * The runtime invisible parameter annotations of this method. May be
-     * <tt>null</tt>.
+     * <code>null</code>.
      */
     private AnnotationWriter[] ipanns;
 
@@ -381,12 +381,12 @@ class MethodWriter extends MethodVisitor {
     private int lastCodeOffset;
 
     /**
-     * The runtime visible type annotations of the code. May be <tt>null</tt>.
+     * The runtime visible type annotations of the code. May be <code>null</code>.
      */
     private AnnotationWriter ctanns;
 
     /**
-     * The runtime invisible type annotations of the code. May be <tt>null</tt>.
+     * The runtime invisible type annotations of the code. May be <code>null</code>.
      */
     private AnnotationWriter ictanns;
 
@@ -446,7 +446,7 @@ class MethodWriter extends MethodVisitor {
      * is relative to the beginning of the current basic block, i.e., the true
      * stack size after the last visited instruction is equal to the
      * {@link Label#inputStackTop beginStackSize} of the current basic block
-     * plus <tt>stackSize</tt>.
+     * plus <code>stackSize</code>.
      */
     private int stackSize;
 
@@ -455,7 +455,7 @@ class MethodWriter extends MethodVisitor {
      * This size is relative to the beginning of the current basic block, i.e.,
      * the true maximum stack size after the last visited instruction is equal
      * to the {@link Label#inputStackTop beginStackSize} of the current basic
-     * block plus <tt>stackSize</tt>.
+     * block plus <code>stackSize</code>.
      */
     private int maxStackSize;
 
@@ -475,10 +475,10 @@ class MethodWriter extends MethodVisitor {
      * @param desc
      *            the method's descriptor (see {@link Type}).
      * @param signature
-     *            the method's signature. May be <tt>null</tt>.
+     *            the method's signature. May be <code>null</code>.
      * @param exceptions
      *            the internal names of the method's exceptions. May be
-     *            <tt>null</tt>.
+     *            <code>null</code>.
      * @param compute
      *            Indicates what must be automatically computed (see #compute).
      */

src/org/redkale/asm/ModuleVisitor.java

@@ -60,9 +60,9 @@ package org.redkale.asm;
 
 /**
  * A visitor to visit a Java module. The methods of this class must be called in
- * the following order: <tt>visitMainClass</tt> | ( <tt>visitPackage</tt> |
+ * the following order: <code>visitMainClass</code> | ( <code>visitPackage</code> |
- * <tt>visitRequire</tt> | <tt>visitExport</tt> | <tt>visitOpen</tt> |
+ * <code>visitRequire</code> | <code>visitExport</code> | <code>visitOpen</code> |
- * <tt>visitUse</tt> | <tt>visitProvide</tt> )* <tt>visitEnd</tt>.
+ * <code>visitUse</code> | <code>visitProvide</code> )* <code>visitEnd</code>.
  *
  * The methods {@link #visitRequire(String, int, String)}, {@link #visitExport(String, int, String...)},
  * {@link #visitOpen(String, int, String...)} and {@link #visitPackage(String)}
@@ -157,7 +157,7 @@ public abstract class ModuleVisitor {
      *        {@code ACC_MANDATED}.
      * @param modules the qualified names of the modules that can access to
      *        the public classes of the exported package or
-     *        <tt>null</tt>.
+     *        <code>null</code>.
      */
     public void visitExport(String packaze, int access, String... modules) {
         if (mv != null) {
@@ -174,7 +174,7 @@ public abstract class ModuleVisitor {
      *        {@code ACC_MANDATED}.
      * @param modules the qualified names of the modules that can use deep
      *        reflection to the classes of the open package or
-     *        <tt>null</tt>.
+     *        <code>null</code>.
      */
     public void visitOpen(String packaze, int access, String... modules) {
         if (mv != null) {

src/org/redkale/asm/Type.java

@@ -71,47 +71,47 @@ import java.lang.reflect.Method;
 public class Type {
 
     /**
-     * The sort of the <tt>void</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>void</code> type. See {@link #getSort getSort}.
      */
     public static final int VOID = 0;
 
     /**
-     * The sort of the <tt>boolean</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>boolean</code> type. See {@link #getSort getSort}.
      */
     public static final int BOOLEAN = 1;
 
     /**
-     * The sort of the <tt>char</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>char</code> type. See {@link #getSort getSort}.
      */
     public static final int CHAR = 2;
 
     /**
-     * The sort of the <tt>byte</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>byte</code> type. See {@link #getSort getSort}.
      */
     public static final int BYTE = 3;
 
     /**
-     * The sort of the <tt>short</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>short</code> type. See {@link #getSort getSort}.
      */
     public static final int SHORT = 4;
 
     /**
-     * The sort of the <tt>int</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>int</code> type. See {@link #getSort getSort}.
      */
     public static final int INT = 5;
 
     /**
-     * The sort of the <tt>float</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>float</code> type. See {@link #getSort getSort}.
      */
     public static final int FLOAT = 6;
 
     /**
-     * The sort of the <tt>long</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>long</code> type. See {@link #getSort getSort}.
      */
     public static final int LONG = 7;
 
     /**
-     * The sort of the <tt>double</tt> type. See {@link #getSort getSort}.
+     * The sort of the <code>double</code> type. See {@link #getSort getSort}.
      */
     public static final int DOUBLE = 8;
 
@@ -131,55 +131,55 @@ public class Type {
     public static final int METHOD = 11;
 
     /**
-     * The <tt>void</tt> type.
+     * The <code>void</code> type.
      */
     public static final Type VOID_TYPE = new Type(VOID, null, ('V' << 24)
             | (5 << 16) | (0 << 8) | 0, 1);
 
     /**
-     * The <tt>boolean</tt> type.
+     * The <code>boolean</code> type.
      */
     public static final Type BOOLEAN_TYPE = new Type(BOOLEAN, null, ('Z' << 24)
             | (0 << 16) | (5 << 8) | 1, 1);
 
     /**
-     * The <tt>char</tt> type.
+     * The <code>char</code> type.
      */
     public static final Type CHAR_TYPE = new Type(CHAR, null, ('C' << 24)
             | (0 << 16) | (6 << 8) | 1, 1);
 
     /**
-     * The <tt>byte</tt> type.
+     * The <code>byte</code> type.
      */
     public static final Type BYTE_TYPE = new Type(BYTE, null, ('B' << 24)
             | (0 << 16) | (5 << 8) | 1, 1);
 
     /**
-     * The <tt>short</tt> type.
+     * The <code>short</code> type.
      */
     public static final Type SHORT_TYPE = new Type(SHORT, null, ('S' << 24)
             | (0 << 16) | (7 << 8) | 1, 1);
 
     /**
-     * The <tt>int</tt> type.
+     * The <code>int</code> type.
      */
     public static final Type INT_TYPE = new Type(INT, null, ('I' << 24)
             | (0 << 16) | (0 << 8) | 1, 1);
 
     /**
-     * The <tt>float</tt> type.
+     * The <code>float</code> type.
      */
     public static final Type FLOAT_TYPE = new Type(FLOAT, null, ('F' << 24)
             | (2 << 16) | (2 << 8) | 1, 1);
 
     /**
-     * The <tt>long</tt> type.
+     * The <code>long</code> type.
      */
     public static final Type LONG_TYPE = new Type(LONG, null, ('J' << 24)
             | (1 << 16) | (1 << 8) | 2, 1);
 
     /**
-     * The <tt>double</tt> type.
+     * The <code>double</code> type.
      */
     public static final Type DOUBLE_TYPE = new Type(DOUBLE, null, ('D' << 24)
             | (3 << 16) | (3 << 8) | 2, 1);
@@ -439,8 +439,8 @@ public class Type {
      * @return the size of the arguments of the method (plus one for the
      *         implicit this argument), argSize, and the size of its return
      *         value, retSize, packed into a single int i =
-     *         <tt>(argSize &lt;&lt; 2) | retSize</tt> (argSize is therefore equal to
+     *         <code>(argSize &lt;&lt; 2) | retSize</code> (argSize is therefore equal to
-     *         <tt>i &gt;&gt; 2</tt>, and retSize to <tt>i &amp; 0x03</tt>).
+     *         <code>i &gt;&gt; 2</code>, and retSize to <code>i &amp; 0x03</code>).
      */
     public static int getArgumentsAndReturnSizes(final String desc) {
         int n = 1;
@@ -645,9 +645,9 @@ public class Type {
      * @return the size of the arguments (plus one for the implicit this
      *         argument), argSize, and the size of the return value, retSize,
      *         packed into a single
-     *         int i = <tt>(argSize &lt;&lt; 2) | retSize</tt>
+     *         int i = <code>(argSize &lt;&lt; 2) | retSize</code>
-     *         (argSize is therefore equal to <tt>i &gt;&gt; 2</tt>,
+     *         (argSize is therefore equal to <code>i &gt;&gt; 2</code>,
-     *         and retSize to <tt>i &amp; 0x03</tt>).
+     *         and retSize to <code>i &amp; 0x03</code>).
      */
     public int getArgumentsAndReturnSizes() {
         return getArgumentsAndReturnSizes(getDescriptor());
@@ -838,8 +838,8 @@ public class Type {
      * Returns the size of values of this type. This method must not be used for
      * method types.
      *
-     * @return the size of values of this type, i.e., 2 for <tt>long</tt> and
+     * @return the size of values of this type, i.e., 2 for <code>long</code> and
-     *         <tt>double</tt>, 0 for <tt>void</tt> and 1 otherwise.
+     *         <code>double</code>, 0 for <code>void</code> and 1 otherwise.
      */
     public int getSize() {
         // the size is in byte 0 of 'off' for primitive types (buf == null)
@@ -855,8 +855,8 @@ public class Type {
      *            ISTORE, IALOAD, IASTORE, IADD, ISUB, IMUL, IDIV, IREM, INEG,
      *            ISHL, ISHR, IUSHR, IAND, IOR, IXOR and IRETURN.
      * @return an opcode that is similar to the given opcode, but adapted to
-     *         this Java type. For example, if this type is <tt>float</tt> and
+     *         this Java type. For example, if this type is <code>float</code> and
-     *         <tt>opcode</tt> is IRETURN, this method returns FRETURN.
+     *         <code>opcode</code> is IRETURN, this method returns FRETURN.
      */
     public int getOpcode(final int opcode) {
         if (opcode == Opcodes.IALOAD || opcode == Opcodes.IASTORE) {
@@ -879,7 +879,7 @@ public class Type {
      *
      * @param o
      *            the object to be compared to this type.
-     * @return <tt>true</tt> if the given object is equal to this type.
+     * @return <code>true</code> if the given object is equal to this type.
      */
     @Override
     public boolean equals(final Object o) {

src/org/redkale/boot/Application.java

@@ -5,6 +5,7 @@
  */
 package org.redkale.boot;
 
+import org.redkale.cluster.ClusterAgent;
 import org.redkale.util.RedkaleClassLoader;
 import org.redkale.net.TransportGroupInfo;
 import java.io.*;
@@ -23,9 +24,10 @@ import javax.xml.parsers.*;
 import org.redkale.boot.ClassFilter.FilterEntry;
 import org.redkale.convert.Convert;
 import org.redkale.convert.bson.BsonFactory;
-import org.redkale.convert.json.JsonFactory;
+import org.redkale.convert.json.*;
+import org.redkale.mq.*;
 import org.redkale.net.*;
-import org.redkale.net.http.MimeType;
+import org.redkale.net.http.*;
 import org.redkale.net.sncp.*;
 import org.redkale.service.Service;
 import org.redkale.source.*;
@@ -56,6 +58,11 @@ public final class Application {
      */
     public static final String RESNAME_APP_TIME = "APP_TIME";
 
+    /**
+     * 当前进程的名称， 类型：String
+     */
+    public static final String RESNAME_APP_NAME = "APP_NAME";
+
     /**
      * 当前进程的根目录， 类型：String、File、Path、URI
      */
@@ -72,12 +79,12 @@ public final class Application {
     public static final String RESNAME_APP_GRES = "APP_GRES";
 
     /**
-     * 当前进程节点的name， 类型：String
+     * 当前进程节点的nodeid， 类型：int
      */
-    public static final String RESNAME_APP_NODE = "APP_NODE";
+    public static final String RESNAME_APP_NODEID = "APP_NODEID";
 
     /**
-     * 当前进程节点的IP地址， 类型：InetAddress、String
+     * 当前进程节点的IP地址， 类型：InetSocketAddress、InetAddress、String
      */
     public static final String RESNAME_APP_ADDR = "APP_ADDR";
 
@@ -106,8 +113,14 @@ public final class Application {
      */
     public static final String RESNAME_SERVER_RESFACTORY = Server.RESNAME_SERVER_RESFACTORY;
 
+    //本进程节点ID
+    final int nodeid;
+
+    //本进程节点ID
+    final String name;
+
     //本地IP地址
-    final InetAddress localAddress;
+    final InetSocketAddress localAddress;
 
     //CacheSource 资源
     final List<CacheSource> cacheSources = new CopyOnWriteArrayList<>();
@@ -115,18 +128,29 @@ public final class Application {
     //DataSource 资源
     final List<DataSource> dataSources = new CopyOnWriteArrayList<>();
 
-    //NodeServer 资源
+    //NodeServer 资源, 顺序必须是sncps, others, watchs
     final List<NodeServer> servers = new CopyOnWriteArrayList<>();
 
     //SNCP传输端的TransportFactory, 注意： 只给SNCP使用
     final TransportFactory sncpTransportFactory;
 
+    //第三方服务发现管理接口
+    //@since 2.1.0
+    final ClusterAgent clusterAgent;
+
+    //MQ管理接口
+    //@since 2.1.0
+    final MessageAgent[] messageAgents;
+
     //全局根ResourceFactory
     final ResourceFactory resourceFactory = ResourceFactory.root();
 
     //服务配置项
     final AnyValue config;
 
+    //排除的jar路径
+    final String excludelibs;
+
     //临时计数器
     CountDownLatch servicecdl;  //会出现两次赋值
 
@@ -192,24 +216,22 @@ public final class Application {
             throw new RuntimeException(e);
         }
         String localaddr = config.getValue("address", "").trim();
-        this.localAddress = localaddr.isEmpty() ? Utility.localInetAddress() : new InetSocketAddress(localaddr, config.getIntValue("port")).getAddress();
+        InetAddress addr = localaddr.isEmpty() ? Utility.localInetAddress() : new InetSocketAddress(localaddr, config.getIntValue("port")).getAddress();
-        this.resourceFactory.register(RESNAME_APP_ADDR, this.localAddress.getHostAddress());
+        this.localAddress = new InetSocketAddress(addr, config.getIntValue("port"));
-        this.resourceFactory.register(RESNAME_APP_ADDR, InetAddress.class, this.localAddress);
+        this.resourceFactory.register(RESNAME_APP_ADDR, addr.getHostAddress());
+        this.resourceFactory.register(RESNAME_APP_ADDR, InetAddress.class, addr);
+        this.resourceFactory.register(RESNAME_APP_ADDR, InetSocketAddress.class, this.localAddress);
+
         {
-            String node = config.getValue("node", "").trim();
+            int nid = config.getIntValue("nodeid", 0);
-            if (node.isEmpty()) {
+            this.nodeid = nid;
-                StringBuilder sb = new StringBuilder();
+            this.resourceFactory.register(RESNAME_APP_NODEID, nid);
-                byte[] bs = this.localAddress.getAddress();
+            System.setProperty(RESNAME_APP_NODEID, "" + nid);
-                int v1 = bs[bs.length - 2] & 0xff;
-                int v2 = bs[bs.length - 1] & 0xff;
-                if (v1 <= 0xf) sb.append('0');
-                sb.append(Integer.toHexString(v1));
-                if (v2 <= 0xf) sb.append('0');
-                sb.append(Integer.toHexString(v2));
-                node = sb.toString();
         }
-            this.resourceFactory.register(RESNAME_APP_NODE, node);
+        {
-            System.setProperty(RESNAME_APP_NODE, node);
+            this.name = checkName(config.getValue("name", ""));
+            this.resourceFactory.register(RESNAME_APP_NAME, name);
+            System.setProperty(RESNAME_APP_NAME, name);
         }
         //以下是初始化日志配置
         final URI logConfURI = "file".equals(confPath.getScheme()) ? new File(new File(confPath), "logging.properties").toURI()
@@ -271,6 +293,9 @@ public final class Application {
         AsynchronousChannelGroup transportGroup = null;
         final AnyValue resources = config.getAnyValue("resources");
         TransportStrategy strategy = null;
+        String excludelib0 = null;
+        ClusterAgent cluster = null;
+        MessageAgent[] mqs = null;
         int bufferCapacity = 32 * 1024;
         int bufferPoolSize = Runtime.getRuntime().availableProcessors() * 8;
         int readTimeoutSeconds = TransportFactory.DEFAULT_READTIMEOUTSECONDS;
@@ -278,6 +303,8 @@ public final class Application {
         AtomicLong createBufferCounter = new AtomicLong();
         AtomicLong cycleBufferCounter = new AtomicLong();
         if (resources != null) {
+            AnyValue excludelibConf = resources.getAnyValue("excludelibs");
+            if (excludelibConf != null) excludelib0 = excludelibConf.getValue("value");
             AnyValue transportConf = resources.getAnyValue("transport");
             int groupsize = resources.getAnyValues("group").length;
             if (groupsize > 0 && transportConf == null) transportConf = new DefaultAnyValue();
@@ -314,6 +341,80 @@ public final class Application {
                 }
                 logger.log(Level.INFO, Transport.class.getSimpleName() + " configure bufferCapacity = " + bufferCapacity / 1024 + "K; bufferPoolSize = " + bufferPoolSize + "; threads = " + threads + ";");
             }
+
+            AnyValue clusterConf = resources.getAnyValue("cluster");
+            if (clusterConf != null) {
+                try {
+                    String classval = clusterConf.getValue("value");
+                    if (classval == null || classval.isEmpty()) {
+                        Iterator<ClusterAgent> it = ServiceLoader.load(ClusterAgent.class, classLoader).iterator();
+                        while (it.hasNext()) {
+                            ClusterAgent agent = it.next();
+                            if (agent.match(clusterConf)) {
+                                cluster = agent;
+                                cluster.setConfig(clusterConf);
+                                break;
+                            }
+                        }
+                        if (cluster == null) logger.log(Level.SEVERE, "load application cluster resource, but not found name='value' value error: " + clusterConf);
+                    } else {
+                        Class type = classLoader.loadClass(clusterConf.getValue("value"));
+                        if (!ClusterAgent.class.isAssignableFrom(type)) {
+                            logger.log(Level.SEVERE, "load application cluster resource, but not found " + ClusterAgent.class.getSimpleName() + " implements class error: " + clusterConf);
+                        } else {
+                            cluster = (ClusterAgent) type.getDeclaredConstructor().newInstance();
+                            cluster.setConfig(clusterConf);
+                        }
+                    }
+                } catch (Exception e) {
+                    logger.log(Level.SEVERE, "load application cluster resource error: " + clusterConf, e);
+                }
+            }
+
+            AnyValue[] mqConfs = resources.getAnyValues("mq");
+            if (mqConfs != null && mqConfs.length > 0) {
+                mqs = new MessageAgent[mqConfs.length];
+                Set<String> mqnames = new HashSet<>();
+                for (int i = 0; i < mqConfs.length; i++) {
+                    AnyValue mqConf = mqConfs[0];
+                    String mqname = mqConf.getValue("name", "");
+                    if (mqnames.contains(mqname)) throw new RuntimeException("mq.name(" + mqname + ") is repeat");
+                    mqnames.add(mqname);
+                    String namex = mqConf.getValue("names");
+                    if (namex != null && !namex.isEmpty()) {
+                        for (String n : namex.split(";")) {
+                            if (n.trim().isEmpty()) continue;
+                            if (mqnames.contains(n.trim())) throw new RuntimeException("mq.name(" + n.trim() + ") is repeat");
+                            mqnames.add(n.trim());
+                        }
+                    }
+                    try {
+                        String classval = mqConf.getValue("value");
+                        if (classval == null || classval.isEmpty()) {
+                            Iterator<MessageAgent> it = ServiceLoader.load(MessageAgent.class, classLoader).iterator();
+                            while (it.hasNext()) {
+                                MessageAgent messageAgent = it.next();
+                                if (messageAgent.match(mqConf)) {
+                                    mqs[i] = messageAgent;
+                                    mqs[i].setConfig(mqConf);
+                                    break;
+                                }
+                            }
+                            if (mqs[i] == null) logger.log(Level.SEVERE, "load application mq resource, but not found name='value' value error: " + mqConf);
+                        } else {
+                            Class type = classLoader.loadClass(classval);
+                            if (!MessageAgent.class.isAssignableFrom(type)) {
+                                logger.log(Level.SEVERE, "load application mq resource, but not found " + MessageAgent.class.getSimpleName() + " implements class error: " + mqConf);
+                            } else {
+                                mqs[i] = (MessageAgent) type.getDeclaredConstructor().newInstance();
+                                mqs[i].setConfig(mqConf);
+                            }
+                        }
+                    } catch (Exception e) {
+                        logger.log(Level.SEVERE, "load application mq resource error: " + mqs[i], e);
+                    }
+                }
+            }
         }
         if (transportGroup == null) {
             final AtomicInteger counter = new AtomicInteger();
@@ -338,15 +439,29 @@ public final class Application {
                     return true;
                 });
         }
+        this.excludelibs = excludelib0;
         this.sncpTransportFactory = TransportFactory.create(transportExec, transportPool, transportGroup, (SSLContext) null, readTimeoutSeconds, writeTimeoutSeconds, strategy);
-        DefaultAnyValue tarnsportConf = DefaultAnyValue.create(TransportFactory.NAME_POOLMAXCONNS, System.getProperty("net.transport.poolmaxconns", "100"))
+        DefaultAnyValue tarnsportConf = DefaultAnyValue.create(TransportFactory.NAME_POOLMAXCONNS, System.getProperty("net.transport.pool.maxconns", "100"))
-            .addValue(TransportFactory.NAME_PINGINTERVAL, System.getProperty("net.transport.pinginterval", "30"))
+            .addValue(TransportFactory.NAME_PINGINTERVAL, System.getProperty("net.transport.ping.interval", "30"))
-            .addValue(TransportFactory.NAME_CHECKINTERVAL, System.getProperty("net.transport.checkinterval", "30"));
+            .addValue(TransportFactory.NAME_CHECKINTERVAL, System.getProperty("net.transport.check.interval", "30"));
         this.sncpTransportFactory.init(tarnsportConf, Sncp.PING_BUFFER, Sncp.PONG_BUFFER.remaining());
+        this.clusterAgent = cluster;
+        this.messageAgents = mqs;
         Thread.currentThread().setContextClassLoader(this.classLoader);
         this.serverClassLoader = new RedkaleClassLoader(this.classLoader);
     }
 
+    private String checkName(String name) {  //不能含特殊字符
+        if (name.isEmpty()) return name;
+        if (name.charAt(0) >= '0' && name.charAt(0) <= '9') throw new RuntimeException("name only 0-9 a-z A-Z _ cannot begin 0-9");
+        for (char ch : name.toCharArray()) {
+            if (!((ch >= '0' && ch <= '9') || ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))) { //不能含特殊字符
+                throw new RuntimeException("name only 0-9 a-z A-Z _ cannot begin 0-9");
+            }
+        }
+        return name;
+    }
+
     public ResourceFactory getResourceFactory() {
         return resourceFactory;
     }
@@ -355,6 +470,22 @@ public final class Application {
         return sncpTransportFactory;
     }
 
+    public ClusterAgent getClusterAgent() {
+        return clusterAgent;
+    }
+
+    public MessageAgent getMessageAgent(String name) {
+        if (messageAgents == null) return null;
+        for (MessageAgent agent : messageAgents) {
+            if (agent.getName().equals(name)) return agent;
+        }
+        return null;
+    }
+
+    public MessageAgent[] getMessageAgents() {
+        return messageAgents;
+    }
+
     public RedkaleClassLoader getClassLoader() {
         return classLoader;
     }
@@ -375,6 +506,14 @@ public final class Application {
         return new ArrayList<>(cacheSources);
     }
 
+    public int getNodeid() {
+        return nodeid;
+    }
+
+    public String getName() {
+        return name;
+    }
+
     public File getHome() {
         return home;
     }
@@ -395,12 +534,9 @@ public final class Application {
         System.setProperty("net.transport.poolmaxconns", "100");
         System.setProperty("net.transport.pinginterval", "30");
         System.setProperty("net.transport.checkinterval", "30");
-        System.setProperty("convert.bson.tiny", "true");
+        System.setProperty("convert.tiny", "true");
-        System.setProperty("convert.json.tiny", "true");
+        System.setProperty("convert.pool.size", "128");
-        System.setProperty("convert.bson.pool.size", "128");
+        System.setProperty("convert.writer.buffer.defsize", "4096");
-        System.setProperty("convert.json.pool.size", "128");
-        System.setProperty("convert.bson.writer.buffer.defsize", "4096");
-        System.setProperty("convert.json.writer.buffer.defsize", "4096");
 
         final String confpath = this.confPath.toString();
         final String homepath = this.home.getCanonicalPath();
@@ -418,7 +554,7 @@ public final class Application {
             pidstr = "APP_PID  = " + pid + "\r\n";
         } catch (Throwable t) {
         }
-        logger.log(Level.INFO, pidstr + "APP_JAVA = " + System.getProperty("java.version") + "\r\n" + RESNAME_APP_ADDR + " = " + this.localAddress.getHostAddress() + "\r\n" + RESNAME_APP_HOME + " = " + homepath + "\r\n" + RESNAME_APP_CONF + " = " + confpath);
+        logger.log(Level.INFO, pidstr + "APP_JAVA = " + System.getProperty("java.version") + "\r\n" + RESNAME_APP_NODEID + " = " + this.nodeid + "\r\n" + RESNAME_APP_ADDR + " = " + this.localAddress.getHostString() + ":" + this.localAddress.getPort() + "\r\n" + RESNAME_APP_HOME + " = " + homepath + "\r\n" + RESNAME_APP_CONF + " = " + confpath);
         String lib = config.getValue("lib", "${APP_HOME}/libs/*").trim().replace("${APP_HOME}", homepath);
         lib = lib.isEmpty() ? confpath : (lib + ";" + confpath);
         Server.loadLib(classLoader, logger, lib);
@@ -534,6 +670,41 @@ public final class Application {
 
         }, Application.class, ResourceFactory.class, TransportFactory.class, NodeSncpServer.class, NodeHttpServer.class, NodeWatchServer.class);
         //--------------------------------------------------------------------------
+        if (this.clusterAgent != null) {
+            if (logger.isLoggable(Level.FINER)) logger.log(Level.FINER, "ClusterAgent initing");
+            long s = System.currentTimeMillis();
+            clusterAgent.setTransportFactory(this.sncpTransportFactory);
+            this.resourceFactory.inject(clusterAgent);
+            clusterAgent.init(clusterAgent.getConfig());
+            this.resourceFactory.register(ClusterAgent.class, clusterAgent);
+            logger.info("ClusterAgent init in " + (System.currentTimeMillis() - s) + " ms");
+        }
+        if (this.messageAgents != null) {
+            if (logger.isLoggable(Level.FINER)) logger.log(Level.FINER, "MessageAgent initing");
+            long s = System.currentTimeMillis();
+            for (MessageAgent agent : this.messageAgents) {
+                this.resourceFactory.inject(agent);
+                agent.init(agent.getConfig());
+                this.resourceFactory.register(agent.getName(), MessageAgent.class, agent);
+                this.resourceFactory.register(agent.getName(), HttpMessageClient.class, agent.getHttpMessageClient());
+                //this.resourceFactory.register(agent.getName(), SncpMessageClient.class, agent.getSncpMessageClient()); //不需要给开发者使用
+            }
+            logger.info("MessageAgent init in " + (System.currentTimeMillis() - s) + " ms");
+
+        }
+        //------------------------------------- 注册 DataSource --------------------------------------------------------        
+        resourceFactory.register((ResourceFactory rf, final Object src, String resourceName, Field field, final Object attachment) -> {
+            try {
+                if (field.getAnnotation(Resource.class) == null) return;
+                if (clusterAgent == null) return;
+                HttpMessageClient messageClient = new HttpMessageClusterClient(clusterAgent);
+                field.set(src, messageClient);
+                rf.inject(messageClient, null); // 给其可能包含@Resource的字段赋值;
+                rf.register(resourceName, HttpMessageClient.class, messageClient);
+            } catch (Exception e) {
+                logger.log(Level.SEVERE, "[" + Thread.currentThread().getName() + "] DataSource inject error", e);
+            }
+        }, HttpMessageClient.class);
         initResources();
     }
 
@@ -548,7 +719,7 @@ public final class Application {
                 if (!"TCP".equalsIgnoreCase(protocol) && !"UDP".equalsIgnoreCase(protocol)) {
                     throw new RuntimeException("Not supported Transport Protocol " + conf.getValue("protocol"));
                 }
-                TransportGroupInfo ginfo = new TransportGroupInfo(group, protocol, conf.getValue("subprotocol", ""), new LinkedHashSet<>());
+                TransportGroupInfo ginfo = new TransportGroupInfo(group, protocol, new LinkedHashSet<>());
                 for (AnyValue node : conf.getAnyValues("node")) {
                     final InetSocketAddress addr = new InetSocketAddress(node.getValue("addr"), node.getIntValue("port"));
                     ginfo.putAddress(addr);
@@ -562,6 +733,7 @@ public final class Application {
                 if (!ApplicationListener.class.isAssignableFrom(clazz)) continue;
                 @SuppressWarnings("unchecked")
                 ApplicationListener listener = (ApplicationListener) clazz.getDeclaredConstructor().newInstance();
+                resourceFactory.inject(listener);
                 listener.init(config);
                 this.listeners.add(listener);
             }
@@ -602,7 +774,8 @@ public final class Application {
                         buffer.flip();
                         byte[] bytes = new byte[buffer.remaining()];
                         buffer.get(bytes);
-                        if ("SHUTDOWN".equalsIgnoreCase(new String(bytes))) {
+                        final String cmd = new String(bytes);
+                        if ("SHUTDOWN".equalsIgnoreCase(cmd)) {
                             try {
                                 long s = System.currentTimeMillis();
                                 logger.info(application.getClass().getSimpleName() + " shutdowning");
@@ -622,7 +795,7 @@ public final class Application {
                                 buffer.flip();
                                 channel.send(buffer, address);
                             }
-                        } else if ("APIDOC".equalsIgnoreCase(new String(bytes))) {
+                        } else if ("APIDOC".equalsIgnoreCase(cmd)) {
                             try {
                                 new ApiDocsService(application).run();
                                 buffer.clear();
@@ -635,6 +808,16 @@ public final class Application {
                                 buffer.flip();
                                 channel.send(buffer, address);
                             }
+                        } else {
+                            long s = System.currentTimeMillis();
+                            logger.info(application.getClass().getSimpleName() + " command " + cmd);
+                            application.command(cmd);
+                            buffer.clear();
+                            buffer.put("COMMAND OK".getBytes());
+                            buffer.flip();
+                            channel.send(buffer, address);
+                            long e = System.currentTimeMillis() - s;
+                            logger.info(application.getClass().getSimpleName() + " command in " + e + " ms");
                         }
                     }
                 } catch (Exception e) {
@@ -645,10 +828,10 @@ public final class Application {
         }.start();
     }
 
-    private void sendCommand(String command) throws Exception {
+    private static void sendCommand(Logger logger, int port, String command) throws Exception {
         final DatagramChannel channel = DatagramChannel.open();
         channel.configureBlocking(true);
-        channel.connect(new InetSocketAddress("127.0.0.1", config.getIntValue("port")));
+        channel.connect(new InetSocketAddress("127.0.0.1", port));
         ByteBuffer buffer = ByteBuffer.allocate(128);
         buffer.put(command.getBytes());
         buffer.flip();
@@ -661,7 +844,7 @@ public final class Application {
             byte[] bytes = new byte[buffer.remaining()];
             buffer.get(bytes);
             channel.close();
-            logger.info("Send: " + command + ", Reply: " + new String(bytes));
+            if (logger != null) logger.info("Send: " + command + ", Reply: " + new String(bytes));
             Thread.sleep(1000);
         } catch (Exception e) {
             if (e instanceof PortUnreachableException) {
@@ -670,7 +853,7 @@ public final class Application {
                     application.init();
                     application.start();
                     new ApiDocsService(application).run();
-                    logger.info("APIDOC OK");
+                    if (logger != null) logger.info("APIDOC OK");
                     return;
                 }
             }
@@ -679,6 +862,9 @@ public final class Application {
     }
 
     public void start() throws Exception {
+        if (!singletonrun && this.clusterAgent != null) {
+            this.clusterAgent.register(this);
+        }
         final AnyValue[] entrys = config.getAnyValues("server");
         CountDownLatch timecd = new CountDownLatch(entrys.length);
         final List<AnyValue> sncps = new ArrayList<>();
@@ -700,12 +886,43 @@ public final class Application {
         runServers(timecd, others);
         runServers(timecd, watchs); //必须在所有服务都启动后再启动WATCH服务
         timecd.await();
+        if (this.clusterAgent != null) this.clusterAgent.start();
+        if (this.messageAgents != null) {
+            if (logger.isLoggable(Level.FINER)) logger.log(Level.FINER, "MessageAgent starting");
+            long s = System.currentTimeMillis();
+            final StringBuffer sb = new StringBuffer();
+            Set<String> names = new HashSet<>();
+            for (MessageAgent agent : this.messageAgents) {
+                names.add(agent.getName());
+                Map<String, Long> map = agent.start().join();
+                AtomicInteger maxlen = new AtomicInteger();
+                map.keySet().forEach(str -> {
+                    if (str.length() > maxlen.get()) maxlen.set(str.length());
+                });
+                new TreeMap<>(map).forEach((topic, ms) -> sb.append("MessageConsumer(topic=").append(alignString(topic, maxlen.get())).append(") init and start in ").append(ms).append(" ms\r\n")
+                );
+            }
+            if (sb.length() > 0) logger.info(sb.toString().trim());
+            logger.info("MessageAgent(names=" + JsonConvert.root().convertTo(names) + ") start in " + (System.currentTimeMillis() - s) + " ms");
+        }
         //if (!singletonrun) signalHandle();
         //if (!singletonrun) clearPersistData();
         logger.info(this.getClass().getSimpleName() + " started in " + (System.currentTimeMillis() - startTime) + " ms\r\n");
+        for (ApplicationListener listener : this.listeners) {
+            listener.postStart(this);
+        }
         if (!singletonrun) this.serversLatch.await();
     }
 
+    private static String alignString(String value, int maxlen) {
+        StringBuilder sb = new StringBuilder(maxlen);
+        sb.append(value);
+        for (int i = 0; i < maxlen - value.length(); i++) {
+            sb.append(' ');
+        }
+        return sb.toString();
+    }
+
 //    private void clearPersistData() {
 //        File cachedir = new File(home, "cache");
 //        if (!cachedir.isDirectory()) return;
@@ -784,13 +1001,12 @@ public final class Application {
                                 synchronized (nodeClasses) {
                                     if (!inited.getAndSet(true)) { //加载自定义的协议，如：SOCKS
                                         ClassFilter profilter = new ClassFilter(classLoader, NodeProtocol.class, NodeServer.class, (Class[]) null);
-                                        ClassFilter.Loader.load(home, serconf.getValue("excludelibs", "").split(";"), profilter);
+                                        ClassFilter.Loader.load(home, ((excludelibs != null ? (excludelibs + ";") : "") + serconf.getValue("excludelibs", "")).split(";"), profilter);
                                         final Set<FilterEntry<NodeServer>> entrys = profilter.getFilterEntrys();
                                         for (FilterEntry<NodeServer> entry : entrys) {
                                             final Class<? extends NodeServer> type = entry.getType();
                                             NodeProtocol pros = type.getAnnotation(NodeProtocol.class);
-                                            for (String p : pros.value()) {
+                                            String p = pros.value().toUpperCase();
-                                                p = p.toUpperCase();
                                             if ("SNCP".equals(p) || "HTTP".equals(p)) continue;
                                             final Class<? extends NodeServer> old = nodeClasses.get(p);
                                             if (old != null && old != type) {
@@ -801,7 +1017,6 @@ public final class Application {
                                     }
                                 }
                             }
-                            }
                             Class<? extends NodeServer> nodeClass = nodeClasses.get(protocol);
                             if (nodeClass != null) server = NodeServer.create(nodeClass, Application.this, serconf);
                         }
@@ -832,14 +1047,14 @@ public final class Application {
     public static <T extends Service> T singleton(String name, Class<T> serviceClass, Class<? extends Service>... extServiceClasses) throws Exception {
         if (serviceClass == null) throw new IllegalArgumentException("serviceClass is null");
         final Application application = Application.create(true);
-        System.setProperty("red" + "kale-singleton-serviceclass", serviceClass.getName());
+        System.setProperty("red" + "kale.singleton.serviceclass", serviceClass.getName());
         if (extServiceClasses != null && extServiceClasses.length > 0) {
             StringBuilder sb = new StringBuilder();
             for (Class clazz : extServiceClasses) {
                 if (sb.length() > 0) sb.append(',');
                 sb.append(clazz.getName());
             }
-            System.setProperty("red" + "kale-singleton-extserviceclasses", sb.toString());
+            System.setProperty("red" + "kale.singleton.extserviceclasses", sb.toString());
         }
         application.init();
         application.start();
@@ -853,6 +1068,10 @@ public final class Application {
     }
 
     public static Application create(final boolean singleton) throws IOException {
+        return new Application(singleton, loadAppXml());
+    }
+
+    private static AnyValue loadAppXml() throws IOException {
         final String home = new File(System.getProperty(RESNAME_APP_HOME, "")).getCanonicalPath().replace('\\', '/');
         System.setProperty(RESNAME_APP_HOME, home);
         String confsubpath = System.getProperty(RESNAME_APP_CONF, "conf");
@@ -864,21 +1083,19 @@ public final class Application {
         } else {
             appconf = new File(new File(home, confsubpath), "application.xml").toURI();
         }
-        return new Application(singleton, load(appconf.toURL().openStream()));
+        return load(appconf.toURL().openStream());
     }
 
     public static void main(String[] args) throws Exception {
         Utility.midnight(); //先初始化一下Utility
         Thread.currentThread().setName("Redkale-Application-Main-Thread");
         //运行主程序
-        final Application application = Application.create(false);
         if (System.getProperty("CMD") != null) {
-            application.sendCommand(System.getProperty("CMD"));
+            AnyValue config = loadAppXml();
-            return;
+            Application.sendCommand(null, config.getIntValue("port"), System.getProperty("CMD"));
-        } else if (System.getProperty("SHUTDOWN") != null) { //兼容旧接口
-            application.sendCommand("SHUTDOWN");
             return;
         }
+        final Application application = Application.create(false);
         application.init();
         application.startSelfServer();
         try {
@@ -902,6 +1119,17 @@ public final class Application {
         return null;
     }
 
+    public void command(String cmd) {
+        List<NodeServer> localServers = new ArrayList<>(servers); //顺序sncps, others, watchs        
+        localServers.stream().forEach((server) -> {
+            try {
+                server.command(cmd);
+            } catch (Exception t) {
+                logger.log(Level.WARNING, " command server(" + server.getSocketAddress() + ") error", t);
+            }
+        });
+    }
+
     public void shutdown() throws Exception {
         for (ApplicationListener listener : this.listeners) {
             try {
@@ -910,8 +1138,26 @@ public final class Application {
                 logger.log(Level.WARNING, listener.getClass() + " preShutdown erroneous", e);
             }
         }
-
+        List<NodeServer> localServers = new ArrayList<>(servers); //顺序sncps, others, watchs
-        servers.stream().forEach((server) -> {
+        Collections.reverse(localServers); //倒序， 必须让watchs先关闭，watch包含服务发现和注销逻辑
+        if (this.messageAgents != null) {
+            Set<String> names = new HashSet<>();
+            if (logger.isLoggable(Level.FINER)) logger.log(Level.FINER, "MessageAgent stopping");
+            long s = System.currentTimeMillis();
+            for (MessageAgent agent : this.messageAgents) {
+                names.add(agent.getName());
+                agent.stop().join();
+            }
+            logger.info("MessageAgent(names=" + JsonConvert.root().convertTo(names) + ") stop in " + (System.currentTimeMillis() - s) + " ms");
+        }
+        if (clusterAgent != null) {
+            if (logger.isLoggable(Level.FINER)) logger.log(Level.FINER, "ClusterAgent destroying");
+            long s = System.currentTimeMillis();
+            clusterAgent.deregister(this);
+            clusterAgent.destroy(clusterAgent.getConfig());
+            logger.info("ClusterAgent destroy in " + (System.currentTimeMillis() - s) + " ms");
+        }
+        localServers.stream().forEach((server) -> {
             try {
                 server.shutdown();
             } catch (Exception t) {
@@ -920,7 +1166,16 @@ public final class Application {
                 serversLatch.countDown();
             }
         });
-
+        if (this.messageAgents != null) {
+            Set<String> names = new HashSet<>();
+            if (logger.isLoggable(Level.FINER)) logger.log(Level.FINER, "MessageAgent destroying");
+            long s = System.currentTimeMillis();
+            for (MessageAgent agent : this.messageAgents) {
+                names.add(agent.getName());
+                agent.destroy(agent.getConfig());
+            }
+            logger.info("MessageAgent(names=" + JsonConvert.root().convertTo(names) + ") destroy in " + (System.currentTimeMillis() - s) + " ms");
+        }
         for (DataSource source : dataSources) {
             if (source == null) continue;
             try {

src/org/redkale/boot/ApplicationListener.java

@@ -35,6 +35,14 @@ public interface ApplicationListener {
     default void preStart(Application application) {
     }
 
+    /**
+     * Application 在运行start后调用
+     *
+     * @param application Application
+     */
+    default void postStart(Application application) {
+    }
+
     /**
      * Application 在运行shutdown前调用
      *

src/org/redkale/boot/ClassFilter.java

@@ -175,8 +175,9 @@ public final class ClassFilter<T> {
         }
         if (!r && ors != null) {
             for (ClassFilter filter : ors) {
-                if (filter.accept(property, clazzname)) {
+                if (filter.accept(filter.conf, clazzname)) {
                     cf = filter;
+                    property = cf.conf;
                     break;
                 }
             }
@@ -189,11 +190,11 @@ public final class ClassFilter<T> {
                 if (property == null) {
                     property = cf.conf;
                 } else if (property instanceof DefaultAnyValue) {
-                    ((DefaultAnyValue) property).addAll(cf.conf);
+                    ((DefaultAnyValue) property).addAllStringSet(cf.conf);
                 } else {
                     DefaultAnyValue dav = new DefaultAnyValue();
-                    dav.addAll(property);
+                    dav.addAllStringSet(property);
-                    dav.addAll(cf.conf);
+                    dav.addAllStringSet(cf.conf);
                     property = dav;
                 }
             }
@@ -243,7 +244,7 @@ public final class ClassFilter<T> {
         }
         if (!r && ors != null) {
             for (ClassFilter filter : ors) {
-                if (filter.accept(property, classname)) return true;
+                if (filter.accept(filter.conf, classname)) return true;
             }
         }
         return r;
@@ -407,7 +408,7 @@ public final class ClassFilter<T> {
                 str = str.trim();
                 if (str.endsWith(";")) str = str.substring(0, str.length() - 1);
             }
-            if (str != null) groups.addAll(Arrays.asList(str.split(";")));
+            if (str != null) this.groups.addAll(Arrays.asList(str.split(";")));
             this.property = property;
             this.autoload = autoload;
             this.expect = expect;
@@ -417,7 +418,7 @@ public final class ClassFilter<T> {
         @Override
         public String toString() {
             return this.getClass().getSimpleName() + "[thread=" + Thread.currentThread().getName()
-                + ", type=" + this.type.getSimpleName() + ", name=" + name + ", groups=" + groups + "]";
+                + ", type=" + this.type.getSimpleName() + ", name=" + name + ", groups=" + this.groups + "]";
         }
 
         @Override
@@ -444,6 +445,10 @@ public final class ClassFilter<T> {
             return property;
         }
 
+        public boolean containsGroup(String group) {
+            return groups != null && groups.contains(group);
+        }
+
         public boolean isEmptyGroups() {
             return groups == null || groups.isEmpty();
         }
@@ -495,7 +500,7 @@ public final class ClassFilter<T> {
                     boolean skip = false;
                     for (Pattern p : excludePatterns) {
                         if (p.matcher(url.toString()).matches()) {
-                            skip = false;
+                            skip = true;
                             break;
                         }
                     }

src/org/redkale/boot/NodeHttpServer.java

@@ -14,6 +14,8 @@ import java.util.logging.Level;
 import javax.annotation.*;
 import static org.redkale.boot.Application.RESNAME_SNCP_ADDR;
 import org.redkale.boot.ClassFilter.FilterEntry;
+import org.redkale.cluster.ClusterAgent;
+import org.redkale.mq.MessageAgent;
 import org.redkale.net.*;
 import org.redkale.net.http.*;
 import org.redkale.net.sncp.Sncp;
@@ -30,7 +32,7 @@ import org.redkale.watch.*;
  *
  * @author zhangjx
  */
-@NodeProtocol({"HTTP"})
+@NodeProtocol("HTTP")
 public class NodeHttpServer extends NodeServer {
 
     protected final boolean rest; //是否加载REST服务， 为true加载rest节点信息并将所有可REST化的Service生成RestServlet
@@ -120,7 +122,15 @@ public class NodeHttpServer extends NodeServer {
                         resourceFactory.register(RESNAME_SNCP_ADDR, String.class, sncpResFactory.find(RESNAME_SNCP_ADDR, String.class));
                     }
                     if (nodeService == null) {
-                        nodeService = Sncp.createLocalService(serverClassLoader, resourceName, WebSocketNodeService.class, application.getResourceFactory(), application.getSncpTransportFactory(), (InetSocketAddress) null, (Set<String>) null, (AnyValue) null);
+                        MessageAgent messageAgent = null;
+                        try {
+                            Field c = WebSocketServlet.class.getDeclaredField("messageAgent");
+                            c.setAccessible(true);
+                            messageAgent = (MessageAgent) c.get(src);
+                        } catch (Exception ex) {
+                            logger.log(Level.WARNING, "WebSocketServlet getMessageAgent error", ex);
+                        }
+                        nodeService = Sncp.createLocalService(serverClassLoader, resourceName, WebSocketNodeService.class, messageAgent, application.getResourceFactory(), application.getSncpTransportFactory(), (InetSocketAddress) null, (Set<String>) null, (AnyValue) null);
                         regFactory.register(resourceName, WebSocketNode.class, nodeService);
                     }
                     resourceFactory.inject(nodeService, self);
@@ -136,7 +146,7 @@ public class NodeHttpServer extends NodeServer {
     @SuppressWarnings("unchecked")
     protected void loadHttpFilter(final AnyValue filtersConf, final ClassFilter<? extends Filter> classFilter) throws Exception {
         final StringBuilder sb = logger.isLoggable(Level.INFO) ? new StringBuilder() : null;
-        final String threadName = "[" + Thread.currentThread().getName() + "] ";
+        final String localThreadName = "[" + Thread.currentThread().getName() + "] ";
         List<FilterEntry<? extends Filter>> list = new ArrayList(classFilter.getFilterEntrys());
         for (FilterEntry<? extends Filter> en : list) {
             Class<HttpFilter> clazz = (Class<HttpFilter>) en.getType();
@@ -145,7 +155,7 @@ public class NodeHttpServer extends NodeServer {
             resourceFactory.inject(filter, this);
             DefaultAnyValue filterConf = (DefaultAnyValue) en.getProperty();
             this.httpServer.addHttpFilter(filter, filterConf);
-            if (sb != null) sb.append(threadName).append(" Load ").append(clazz.getName()).append(LINE_SEPARATOR);
+            if (sb != null) sb.append(localThreadName).append(" Load ").append(clazz.getName()).append(LINE_SEPARATOR);
         }
         if (sb != null && sb.length() > 0) logger.log(Level.INFO, sb.toString());
     }
@@ -158,7 +168,7 @@ public class NodeHttpServer extends NodeServer {
         if (!prefix0.isEmpty() && prefix0.charAt(prefix0.length() - 1) == '/') prefix0 = prefix0.substring(0, prefix0.length() - 1);
         if (!prefix0.isEmpty() && prefix0.charAt(0) != '/') prefix0 = '/' + prefix0;
         final String prefix = prefix0;
-        final String threadName = "[" + Thread.currentThread().getName() + "] ";
+        final String localThreadName = "[" + Thread.currentThread().getName() + "] ";
         List<FilterEntry<? extends Servlet>> list = new ArrayList(servletFilter.getFilterEntrys());
         list.sort((FilterEntry<? extends Servlet> o1, FilterEntry<? extends Servlet> o2) -> {  //必须保证WebSocketServlet优先加载， 因为要确保其他的HttpServlet可以注入本地模式的WebSocketNode
             boolean ws1 = WebSocketServlet.class.isAssignableFrom(o1.getType());
@@ -176,7 +186,11 @@ public class NodeHttpServer extends NodeServer {
             Class<HttpServlet> clazz = (Class<HttpServlet>) en.getType();
             if (Modifier.isAbstract(clazz.getModifiers())) continue;
             WebServlet ws = clazz.getAnnotation(WebServlet.class);
-            if (ws == null || ws.value().length == 0) continue;
+            if (ws == null) continue;
+            if (ws.value().length == 0) {
+                logger.log(Level.INFO, "not found @WebServlet.value in " + clazz.getName());
+                continue;
+            }
             final HttpServlet servlet = clazz.getDeclaredConstructor().newInstance();
             resourceFactory.inject(servlet, this);
             final String[] mappings = ws.value();
@@ -197,7 +211,7 @@ public class NodeHttpServer extends NodeServer {
                 if (as.getKey().length() > max) max = as.getKey().length();
             }
             for (AbstractMap.SimpleEntry<String, String[]> as : ss) {
-                sb.append(threadName).append(" Load ").append(as.getKey());
+                sb.append(localThreadName).append(" Load ").append(as.getKey());
                 for (int i = 0; i < max - as.getKey().length(); i++) {
                     sb.append(' ');
                 }
@@ -222,11 +236,18 @@ public class NodeHttpServer extends NodeServer {
         String prefix0 = restConf.getValue("path", "");
         if (!prefix0.isEmpty() && prefix0.charAt(prefix0.length() - 1) == '/') prefix0 = prefix0.substring(0, prefix0.length() - 1);
         if (!prefix0.isEmpty() && prefix0.charAt(0) != '/') prefix0 = '/' + prefix0;
-        final String prefix = prefix0;
 
-        final String threadName = "[" + Thread.currentThread().getName() + "] ";
+        final String localThreadName = "[" + Thread.currentThread().getName() + "] ";
         final List<AbstractMap.SimpleEntry<String, String[]>> ss = sb == null ? null : new ArrayList<>();
-
+        String mqname = restConf.getValue("mq");
+        MessageAgent agent0 = null;
+        if (mqname != null) {
+            agent0 = application.getMessageAgent(mqname);
+            if (agent0 == null) throw new RuntimeException("not found " + MessageAgent.class.getSimpleName() + " config for (name=" + mqname + ")");
+        }
+        final MessageAgent messageAgent = agent0;
+        if (messageAgent != null) prefix0 = ""; //开启MQ时,prefix字段失效
+        final String prefix = prefix0;
         final boolean autoload = restConf.getBoolValue("autoload", true);
         {  //加载RestService
             String userTypeStr = restConf.getValue("usertype");
@@ -269,7 +290,9 @@ public class NodeHttpServer extends NodeServer {
                     WebServlet ws = servlet.getClass().getAnnotation(WebServlet.class);
                     if (ws != null && !ws.repair()) prefix2 = "";
                     resourceFactory.inject(servlet, NodeHttpServer.this);
-                    //if (finest) logger.finest(threadName + " Create RestServlet(resource.name='" + name + "') = " + servlet);
+                    dynServletMap.put(service, servlet);
+                    if (messageAgent != null) messageAgent.putService(this, service, servlet);
+                    //if (finest) logger.finest(localThreadName + " Create RestServlet(resource.name='" + name + "') = " + servlet);
                     if (ss != null) {
                         String[] mappings = servlet.getClass().getAnnotation(WebServlet.class).value();
                         for (int i = 0; i < mappings.length; i++) {
@@ -322,13 +345,13 @@ public class NodeHttpServer extends NodeServer {
                     return;
                 }
                 restedObjects.add(stype); //避免重复创建Rest对象
-                HttpServlet servlet = httpServer.addRestWebSocketServlet(serverClassLoader, stype, prefix, en.getProperty());
+                WebSocketServlet servlet = httpServer.addRestWebSocketServlet(serverClassLoader, stype, messageAgent, prefix, en.getProperty());
                 if (servlet == null) return; //没有RestOnMessage方法的HttpServlet调用Rest.createRestWebSocketServlet就会返回null 
                 String prefix2 = prefix;
                 WebServlet ws = servlet.getClass().getAnnotation(WebServlet.class);
                 if (ws != null && !ws.repair()) prefix2 = "";
                 resourceFactory.inject(servlet, NodeHttpServer.this);
-                if (finest) logger.finest(threadName + " " + stype.getName() + " create a RestWebSocketServlet");
+                if (finest) logger.finest(localThreadName + " " + stype.getName() + " create a RestWebSocketServlet");
                 if (ss != null) {
                     String[] mappings = servlet.getClass().getAnnotation(WebServlet.class).value();
                     for (int i = 0; i < mappings.length; i++) {
@@ -338,6 +361,7 @@ public class NodeHttpServer extends NodeServer {
                 }
             }
         }
+        if (messageAgent != null) this.messageAgents.put(messageAgent.getName(), messageAgent);
         //输出信息
         if (ss != null && !ss.isEmpty() && sb != null) {
             ss.sort((AbstractMap.SimpleEntry<String, String[]> o1, AbstractMap.SimpleEntry<String, String[]> o2) -> o1.getKey().compareTo(o2.getKey()));
@@ -346,13 +370,30 @@ public class NodeHttpServer extends NodeServer {
                 if (as.getKey().length() > max) max = as.getKey().length();
             }
             for (AbstractMap.SimpleEntry<String, String[]> as : ss) {
-                sb.append(threadName).append(" Load ").append(as.getKey());
+                sb.append(localThreadName).append(" Load ").append(as.getKey());
                 for (int i = 0; i < max - as.getKey().length(); i++) {
                     sb.append(' ');
                 }
                 sb.append("  mapping to  ").append(Arrays.toString(as.getValue())).append(LINE_SEPARATOR);
             }
-            sb.append(threadName).append(" All HttpServlets load cost " + (System.currentTimeMillis() - starts) + " ms" + LINE_SEPARATOR);
+            sb.append(localThreadName).append(" All HttpServlets load cost ").append(System.currentTimeMillis() - starts).append(" ms").append(LINE_SEPARATOR);
         }
     }
+
+    @Override //loadServlet执行之后调用
+    protected void postLoadServlets() {
+        final ClusterAgent cluster = application.clusterAgent;
+        if (cluster != null) {
+            NodeProtocol pros = getClass().getAnnotation(NodeProtocol.class);
+            String protocol = pros.value().toUpperCase();
+            if (!cluster.containsProtocol(protocol)) return;
+            if (!cluster.containsPort(server.getSocketAddress().getPort())) return;
+            cluster.register(this, protocol, dynServletMap.keySet(), new HashSet<>());
+        }
+    }
+
+    @Override
+    protected void afterClusterDeregisterOnPreDestroyServices(ClusterAgent cluster, String protocol) {
+        cluster.deregister(this, protocol, dynServletMap.keySet(), new HashSet<>());
+    }
 }

src/org/redkale/boot/NodeProtocol.java

@@ -20,5 +20,5 @@ import java.lang.annotation.*;
 @Documented
 public @interface NodeProtocol {
 
-    String[] value();
+    String value();
 }

src/org/redkale/boot/NodeServer.java

@@ -5,6 +5,8 @@
  */
 package org.redkale.boot;
 
+import org.redkale.cluster.ClusterAgent;
+import org.redkale.mq.MessageAgent;
 import org.redkale.util.RedkaleClassLoader;
 import java.io.*;
 import java.lang.annotation.Annotation;
@@ -27,7 +29,6 @@ import org.redkale.net.sncp.*;
 import org.redkale.service.*;
 import org.redkale.source.*;
 import org.redkale.util.*;
-import org.redkale.util.AnyValue.DefaultAnyValue;
 
 /**
  * Server节点的初始化配置类
@@ -67,11 +68,13 @@ public abstract class NodeServer {
     private InetSocketAddress sncpAddress;
 
     //加载Service时的处理函数
-    protected Consumer<Service> consumer;
+    protected BiConsumer<MessageAgent, Service> consumer;
 
     //server节点的配置
     protected AnyValue serverConf;
 
+    protected final String threadName;
+
     //加载server节点后的拦截器
     protected NodeInterceptor interceptor;
 
@@ -84,11 +87,20 @@ public abstract class NodeServer {
     //远程模式的Service对象集合
     protected final Set<Service> remoteServices = new LinkedHashSet<>();
 
+    protected final Map<Service, Servlet> dynServletMap = new LinkedHashMap<>();
+
+    //MessageAgent对象集合
+    protected final Map<String, MessageAgent> messageAgents = new HashMap<>();
+
+    //需要远程模式Service的MessageAgent对象集合
+    protected final Map<String, MessageAgent> sncpRemoteAgents = new HashMap<>();
+
     private volatile int maxClassNameLength = 0;
 
     private volatile int maxNameLength = 0;
 
     public NodeServer(Application application, Server server) {
+        this.threadName = Thread.currentThread().getName();
         this.application = application;
         this.server = server;
         this.resourceFactory = server.getResourceFactory();
@@ -111,7 +123,7 @@ public abstract class NodeServer {
         this.serverConf = config == null ? AnyValue.create() : config;
         if (isSNCP()) { // SNCP协议
             String host = this.serverConf.getValue("host", isWATCH() ? "127.0.0.1" : "0.0.0.0").replace("0.0.0.0", "");
-            this.sncpAddress = new InetSocketAddress(host.isEmpty() ? application.localAddress.getHostAddress() : host, this.serverConf.getIntValue("port"));
+            this.sncpAddress = new InetSocketAddress(host.isEmpty() ? application.localAddress.getAddress().getHostAddress() : host, this.serverConf.getIntValue("port"));
             this.sncpGroup = application.sncpTransportFactory.findGroupName(this.sncpAddress);
             //单向SNCP服务不需要对等group
             //if (this.sncpGroup == null) throw new RuntimeException("Server (" + String.valueOf(config).replaceAll("\\s+", " ") + ") not found <group> info");
@@ -142,9 +154,9 @@ public abstract class NodeServer {
         //必须要进行初始化， 构建Service时需要使用Context中的ExecutorService
         server.init(this.serverConf);
         //init之后才有Executor
-        resourceFactory.register(Server.RESNAME_SERVER_EXECUTOR, Executor.class, server.getExecutor());
+        resourceFactory.register(Server.RESNAME_SERVER_EXECUTOR, Executor.class, server.getWorkExecutor());
-        resourceFactory.register(Server.RESNAME_SERVER_EXECUTOR, ExecutorService.class, server.getExecutor());
+        resourceFactory.register(Server.RESNAME_SERVER_EXECUTOR, ExecutorService.class, server.getWorkExecutor());
-        resourceFactory.register(Server.RESNAME_SERVER_EXECUTOR, ThreadPoolExecutor.class, server.getExecutor());
+        resourceFactory.register(Server.RESNAME_SERVER_EXECUTOR, ThreadPoolExecutor.class, server.getWorkExecutor());
 
         initResource(); //给 DataSource、CacheSource 注册依赖注入时的监听回调事件。
         String interceptorClass = this.serverConf.getValue("interceptor", "");
@@ -155,8 +167,8 @@ public abstract class NodeServer {
 
         ClassFilter<Service> serviceFilter = createServiceClassFilter();
         if (application.singletonrun) { //singleton模式下只加载指定的Service
-            final String ssc = System.getProperty("red" + "kale-singleton-serviceclass");
+            final String ssc = System.getProperty("red" + "kale.singleton.serviceclass");
-            final String extssc = System.getProperty("red" + "kale-singleton-extserviceclasses");
+            final String extssc = System.getProperty("red" + "kale.singleton.extserviceclasses");
             if (ssc != null) {
                 final List<String> sscList = new ArrayList<>();
                 sscList.add(ssc);
@@ -172,13 +184,14 @@ public abstract class NodeServer {
         ClassFilter<Servlet> servletFilter = createServletClassFilter();
         ClassFilter otherFilter = createOtherClassFilter();
         long s = System.currentTimeMillis();
-        ClassFilter.Loader.load(application.getHome(), serverConf.getValue("excludelibs", "").split(";"), serviceFilter, filterFilter, servletFilter, otherFilter);
+        ClassFilter.Loader.load(application.getHome(), ((application.excludelibs != null ? (application.excludelibs + ";") : "") + serverConf.getValue("excludelibs", "")).split(";"), serviceFilter, filterFilter, servletFilter, otherFilter);
         long e = System.currentTimeMillis() - s;
         logger.info(this.getClass().getSimpleName() + " load filter class in " + e + " ms");
         loadService(serviceFilter, otherFilter); //必须在servlet之前
         if (!application.singletonrun) { //非singleton模式下才加载Filter、Servlet
             loadFilter(filterFilter, otherFilter);
             loadServlet(servletFilter, otherFilter);
+            postLoadServlets();
         }
         if (this.interceptor != null) this.resourceFactory.inject(this.interceptor);
     }
@@ -198,7 +211,20 @@ public abstract class NodeServer {
         if (resources != null) {
             for (AnyValue sourceConf : resources.getAnyValues("source")) {
                 try {
-                    Class type = serverClassLoader.loadClass(sourceConf.getValue("value"));
+                    String classval = sourceConf.getValue("value");
+                    Class type = null;
+                    if (classval == null || classval.isEmpty()) {
+                        Iterator<CacheSource> it = ServiceLoader.load(CacheSource.class, serverClassLoader).iterator();
+                        while (it.hasNext()) {
+                            CacheSource s = it.next();
+                            if (s.match(sourceConf)) {
+                                type = s.getClass();
+                                break;
+                            }
+                        }
+                    } else {
+                        type = serverClassLoader.loadClass(classval);
+                    }
                     if (type == DataSource.class) {
                         type = DataMemorySource.class;
                         for (AnyValue itemConf : sourceConf.getAnyValues("property")) {
@@ -271,9 +297,7 @@ public abstract class NodeServer {
                             SncpClient client = Sncp.getSncpClient(srcService);
                             final InetSocketAddress sncpAddr = client == null ? null : client.getClientAddress();
                             final Set<String> groups = new HashSet<>();
-                            if (client != null && client.getSameGroup() != null) groups.add(client.getSameGroup());
+                            source = (DataSource) Sncp.createLocalService(serverClassLoader, resourceName, sourceType, client == null ? null : client.getMessageAgent(), appResFactory, appSncpTranFactory, sncpAddr, groups, Sncp.getConf(srcService));
-                            if (client != null && client.getDiffGroups() != null) groups.addAll(client.getDiffGroups());
-                            source = (DataSource) Sncp.createLocalService(serverClassLoader, resourceName, sourceType, appResFactory, appSncpTranFactory, sncpAddr, groups, Sncp.getConf(srcService));
                         }
                     }
                 }
@@ -284,26 +308,12 @@ public abstract class NodeServer {
                 application.dataSources.add(source);
                 appResFactory.register(resourceName, DataSource.class, source);
 
-                SncpClient client = Sncp.getSncpClient((Service) src);
-                final InetSocketAddress sncpAddr = client == null ? null : client.getClientAddress();
-                if ((src instanceof DataSource) && sncpAddr != null && resourceFactory.find(resourceName, DataCacheListener.class) == null) { //只有DataSourceService 才能赋值 DataCacheListener   
-                    final NodeSncpServer sncpServer = application.findNodeSncpServer(sncpAddr);
-                    final Set<String> groups = new HashSet<>();
-                    if (client != null && client.getSameGroup() != null) groups.add(client.getSameGroup());
-                    if (client != null && client.getDiffGroups() != null) groups.addAll(client.getDiffGroups());
-                    Service cacheListenerService = Sncp.createLocalService(serverClassLoader, resourceName, DataCacheListenerService.class, appResFactory, appSncpTranFactory, sncpAddr, groups, Sncp.getConf((Service) src));
-                    appResFactory.register(resourceName, DataCacheListener.class, cacheListenerService);
-                    localServices.add(cacheListenerService);
-                    sncpServer.consumerAccept(cacheListenerService);
-                    rf.inject(cacheListenerService, self);
-                    logger.info("[" + Thread.currentThread().getName() + "] Load Service " + cacheListenerService);
-                }
                 field.set(src, source);
                 rf.inject(source, self); // 给其可能包含@Resource的字段赋值;
                 //NodeServer.this.watchFactory.inject(src);
                 if (source instanceof Service && needinit) ((Service) source).init(sourceConf);
             } catch (Exception e) {
-                logger.log(Level.SEVERE, "[" + Thread.currentThread().getName() + "] DataSource inject error", e);
+                logger.log(Level.SEVERE, "[" + Thread.currentThread().getName() + "] DataSource inject to " + src + " error", e);
             }
         }, DataSource.class);
 
@@ -318,20 +328,32 @@ public abstract class NodeServer {
                     final Service srcService = (Service) src;
                     SncpClient client = Sncp.getSncpClient(srcService);
                     final InetSocketAddress sncpAddr = client == null ? null : client.getClientAddress();
-                    final Set<String> groups = new HashSet<>();
-                    if (client != null && client.getSameGroup() != null) groups.add(client.getSameGroup());
-                    if (client != null && client.getDiffGroups() != null) groups.addAll(client.getDiffGroups());
-
                     SimpleEntry<Class, AnyValue> resEntry = cacheResource.get(resourceName);
                     AnyValue sourceConf = resEntry == null ? null : resEntry.getValue();
                     if (sourceConf == null) {
                         SimpleEntry<Class, AnyValue> resEntry2 = dataResources.get(resourceName);
                         sourceConf = resEntry2 == null ? null : resEntry2.getValue();
                     }
-                    final Class sourceType = sourceConf == null ? CacheMemorySource.class : serverClassLoader.loadClass(sourceConf.getValue("value"));
+                    Class sourceType0 = CacheMemorySource.class;
+                    if (sourceConf != null) {
+                        String classval = sourceConf.getValue("value");
+                        if (classval == null || classval.isEmpty()) {
+                            Iterator<CacheSource> it = ServiceLoader.load(CacheSource.class, serverClassLoader).iterator();
+                            while (it.hasNext()) {
+                                CacheSource s = it.next();
+                                if (s.match(sourceConf)) {
+                                    sourceType0 = s.getClass();
+                                    break;
+                                }
+                            }
+                        } else {
+                            sourceType0 = serverClassLoader.loadClass(classval);
+                        }
+                    }
+                    final Class sourceType = sourceType0;
                     Object source = null;
                     if (CacheSource.class.isAssignableFrom(sourceType)) { // CacheSource
-                        source = (CacheSource) Sncp.createLocalService(serverClassLoader, resourceName, sourceType, appResFactory, appSncpTranFactory, sncpAddr, groups, Sncp.getConf(srcService));
+                        source = Modifier.isFinal(sourceType.getModifiers()) ? sourceType.getConstructor().newInstance() : (CacheSource) Sncp.createLocalService(serverClassLoader, resourceName, sourceType, client == null ? null : client.getMessageAgent(), appResFactory, appSncpTranFactory, sncpAddr, null, Sncp.getConf(srcService));
                         Type genericType = field.getGenericType();
                         ParameterizedType pt = (genericType instanceof ParameterizedType) ? (ParameterizedType) genericType : null;
                         Type valType = pt == null ? null : pt.getActualTypeArguments()[0];
@@ -376,18 +398,21 @@ public abstract class NodeServer {
                     if (nodeService == null) {
                         final HashSet<String> groups = new HashSet<>();
                         if (groups.isEmpty() && isSNCP() && NodeServer.this.sncpGroup != null) groups.add(NodeServer.this.sncpGroup);
-                        nodeService = Sncp.createLocalService(serverClassLoader, resourceName, WebSocketNodeService.class, application.getResourceFactory(), application.getSncpTransportFactory(), NodeServer.this.sncpAddress, groups, (AnyValue) null);
+                        nodeService = Sncp.createLocalService(serverClassLoader, resourceName, WebSocketNodeService.class, Sncp.getMessageAgent((Service) src), application.getResourceFactory(), application.getSncpTransportFactory(), NodeServer.this.sncpAddress, groups, (AnyValue) null);
                         (isSNCP() ? appResFactory : resourceFactory).register(resourceName, WebSocketNode.class, nodeService);
+                        ((WebSocketNodeService) nodeService).setName(resourceName);
                     }
                     resourceFactory.inject(nodeService, self);
+                    MessageAgent messageAgent = Sncp.getMessageAgent((Service) src);
+                    if (messageAgent != null && Sncp.getMessageAgent(nodeService) == null) Sncp.setMessageAgent(nodeService, messageAgent);
                     field.set(src, nodeService);
                     if (Sncp.isRemote(nodeService)) {
                         remoteServices.add(nodeService);
                     } else {
-                        if (field != null) rf.inject(nodeService); //动态加载的Service也存在按需加载的注入资源
+                        rf.inject(nodeService); //动态加载的Service也存在按需加载的注入资源
                         localServices.add(nodeService);
                         interceptorServices.add(nodeService);
-                        if (consumer != null) consumer.accept(nodeService);
+                        if (consumer != null) consumer.accept(null, nodeService);
                     }
                 } catch (Exception e) {
                     logger.log(Level.SEVERE, "WebSocketNode inject error", e);
@@ -405,7 +430,7 @@ public abstract class NodeServer {
     protected void loadService(ClassFilter<? extends Service> serviceFilter, ClassFilter otherFilter) throws Exception {
         if (serviceFilter == null) return;
         final long starts = System.currentTimeMillis();
-        final String threadName = "[" + Thread.currentThread().getName() + "] ";
+        final String localThreadName = "[" + Thread.currentThread().getName() + "] ";
         final Set<FilterEntry<? extends Service>> entrys = (Set) serviceFilter.getAllFilterEntrys();
         ResourceFactory regFactory = isSNCP() ? application.getResourceFactory() : resourceFactory;
         final ResourceFactory appResourceFactory = application.getResourceFactory();
@@ -418,12 +443,11 @@ public abstract class NodeServer {
                 if (Modifier.isAbstract(serviceImplClass.getModifiers())) continue; //修饰abstract的类跳过
                 if (DataSource.class.isAssignableFrom(serviceImplClass)) continue;
                 if (CacheSource.class.isAssignableFrom(serviceImplClass)) continue;
-                if (DataCacheListener.class.isAssignableFrom(serviceImplClass)) continue;
             }
             if (entry.getName().contains("$")) throw new RuntimeException("<name> value cannot contains '$' in " + entry.getProperty());
             Service oldother = resourceFactory.find(entry.getName(), serviceImplClass);
             if (oldother != null) { //Server加载Service时需要判断是否已经加载过了。
-                interceptorServices.add(oldother);
+                if (!Sncp.isRemote(oldother)) interceptorServices.add(oldother);
                 continue;
             }
             final HashSet<String> groups = entry.getGroups(); //groups.isEmpty()表示<services>没有配置groups属性。
@@ -437,16 +461,28 @@ public abstract class NodeServer {
             final ResourceFactory.ResourceLoader resourceLoader = (ResourceFactory rf, final Object src, final String resourceName, Field field, final Object attachment) -> {
                 try {
                     if (SncpClient.parseMethod(serviceImplClass).isEmpty() && serviceImplClass.getAnnotation(Priority.class) == null) {  //class没有可用的方法且没有标记启动优先级的， 通常为BaseService
-                        if (!serviceImplClass.getName().startsWith("org.redkale.")) logger.log(Level.FINE, serviceImplClass + " cannot load because not found less one public non-final method");
+                        if (!serviceImplClass.getName().startsWith("org.redkale.") && !serviceImplClass.getSimpleName().contains("Base")) {
+                            logger.log(Level.FINE, serviceImplClass + " cannot load because not found less one public non-final method");
+                        }
                         return;
                     }
 
+                    MessageAgent agent = null;
+                    if (entry.getProperty() != null && entry.getProperty().getValue("mq") != null) {
+                        agent = application.getMessageAgent(entry.getProperty().getValue("mq"));
+                        if (agent != null) messageAgents.put(agent.getName(), agent);
+                    }
+
                     Service service;
-                    boolean ws = src instanceof WebSocketServlet;
+                    final boolean ws = src instanceof WebSocketServlet;
                     if (ws || localed) { //本地模式
-                        service = Sncp.createLocalService(serverClassLoader, resourceName, serviceImplClass, appResourceFactory, appSncpTransFactory, NodeServer.this.sncpAddress, groups, entry.getProperty());
+                        service = Sncp.createLocalService(serverClassLoader, resourceName, serviceImplClass, agent, appResourceFactory, appSncpTransFactory, NodeServer.this.sncpAddress, groups, entry.getProperty());
                     } else {
-                        service = Sncp.createRemoteService(serverClassLoader, resourceName, serviceImplClass, appSncpTransFactory, NodeServer.this.sncpAddress, groups, entry.getProperty());
+                        service = Sncp.createRemoteService(serverClassLoader, resourceName, serviceImplClass, agent, appSncpTransFactory, NodeServer.this.sncpAddress, groups, entry.getProperty());
+                    }
+                    if (service instanceof WebSocketNodeService) {
+                        ((WebSocketNodeService) service).setName(resourceName);
+                        if (agent != null) Sncp.setMessageAgent(service, agent);
                     }
                     final Class restype = Sncp.getResourceType(service);
                     if (rf.find(resourceName, restype) == null) {
@@ -456,11 +492,12 @@ public abstract class NodeServer {
                     }
                     if (Sncp.isRemote(service)) {
                         remoteServices.add(service);
+                        if (agent != null) sncpRemoteAgents.put(agent.getName(), agent);
                     } else {
                         if (field != null) rf.inject(service); //动态加载的Service也存在按需加载的注入资源
                         localServices.add(service);
                         interceptorServices.add(service);
-                        if (consumer != null) consumer.accept(service);
+                        if (consumer != null) consumer.accept(agent, service);
                     }
                 } catch (RuntimeException ex) {
                     throw ex;
@@ -493,7 +530,13 @@ public abstract class NodeServer {
 
         if (sb != null) {
             remoteServices.forEach(y -> {
-                sb.append(threadName).append(Sncp.toSimpleString(y, maxNameLength, maxClassNameLength)).append(" load and inject").append(LINE_SEPARATOR);
+                sb.append(localThreadName).append(Sncp.toSimpleString(y, maxNameLength, maxClassNameLength)).append(" load and inject").append(LINE_SEPARATOR);
+            });
+        }
+        if (isSNCP() && !sncpRemoteAgents.isEmpty()) {
+            sncpRemoteAgents.values().forEach(agent -> {
+                // agent.putSncpResp((NodeSncpServer) this);
+                //  agent.startSncpRespConsumer();
             });
         }
         //----------------- init -----------------
@@ -510,21 +553,25 @@ public abstract class NodeServer {
         localServices.clear();
         localServices.addAll(swlist);
         //this.loadPersistData();
+        long preinits = System.currentTimeMillis();
+        preInitServices(localServices, remoteServices);
+        long preinite = System.currentTimeMillis() - preinits;
         final List<String> slist = sb == null ? null : new CopyOnWriteArrayList<>();
         localServices.stream().forEach(y -> {
             long s = System.currentTimeMillis();
             y.init(Sncp.getConf(y));
             long e = System.currentTimeMillis() - s;
             String serstr = Sncp.toSimpleString(y, maxNameLength, maxClassNameLength);
-            if (slist != null) slist.add(new StringBuilder().append(threadName).append(serstr).append(" load and init in ").append(e).append(" ms").append(LINE_SEPARATOR).toString());
+            if (slist != null) slist.add(new StringBuilder().append(localThreadName).append(serstr).append(" load and init in ").append(e).append(" ms").append(LINE_SEPARATOR).toString());
         });
         if (slist != null && sb != null) {
             List<String> wlist = new ArrayList<>(slist); //直接使用CopyOnWriteArrayList偶尔会出现莫名的异常(CopyOnWriteArrayList源码1185行)
             for (String s : wlist) {
                 sb.append(s);
             }
-            sb.append(threadName).append("All Services load cost " + (System.currentTimeMillis() - starts) + " ms" + LINE_SEPARATOR);
+            sb.append(localThreadName).append("All Services load cost ").append(System.currentTimeMillis() - starts).append(" ms" + LINE_SEPARATOR);
         }
+        if (sb != null && preinite > 10) sb.append(localThreadName).append(ClusterAgent.class.getSimpleName()).append(" register ").append(preinite).append(" ms" + LINE_SEPARATOR);
         if (sb != null && sb.length() > 0) logger.log(Level.INFO, sb.toString());
     }
 
@@ -533,6 +580,44 @@ public abstract class NodeServer {
         maxClassNameLength = Math.max(maxClassNameLength, Sncp.getResourceType(y).getName().length() + 1);
     }
 
+    //Service.init执行之前调用
+    protected void preInitServices(Set<Service> localServices, Set<Service> remoteServices) {
+        final ClusterAgent cluster = application.clusterAgent;
+        if (cluster == null) return;
+        NodeProtocol pros = getClass().getAnnotation(NodeProtocol.class);
+        String protocol = pros.value().toUpperCase();
+        if (!cluster.containsProtocol(protocol)) return;
+        if (!cluster.containsPort(server.getSocketAddress().getPort())) return;
+        cluster.register(this, protocol, localServices, remoteServices);
+    }
+
+    //loadServlet执行之后调用
+    protected void postLoadServlets() {
+
+    }
+
+    //Service.destroy执行之前调用
+    protected void preDestroyServices(Set<Service> localServices, Set<Service> remoteServices) {
+        if (application.clusterAgent != null) { //服务注销
+            final ClusterAgent cluster = application.clusterAgent;
+            NodeProtocol pros = getClass().getAnnotation(NodeProtocol.class);
+            String protocol = pros.value().toUpperCase();
+            if (cluster.containsProtocol(protocol) && cluster.containsPort(server.getSocketAddress().getPort())) {
+                cluster.deregister(this, protocol, localServices, remoteServices);
+                afterClusterDeregisterOnPreDestroyServices(cluster, protocol);
+            }
+        }
+        if (!this.messageAgents.isEmpty()) { //MQ
+        }
+    }
+
+    protected void afterClusterDeregisterOnPreDestroyServices(ClusterAgent cluster, String protocol) {
+    }
+
+    //Server.start执行之后调用
+    protected void postStartServer(Set<Service> localServices, Set<Service> remoteServices) {
+    }
+
     protected abstract ClassFilter<Filter> createFilterClassFilter();
 
     protected abstract ClassFilter<Servlet> createServletClassFilter();
@@ -563,22 +648,24 @@ public abstract class NodeServer {
         }
         cf = null;
         for (AnyValue list : proplist) {
-            DefaultAnyValue prop = null;
+            AnyValue.DefaultAnyValue prop = null;
             String sc = list.getValue("groups");
+            String mq = list.getValue("mq");
             if (sc != null) {
                 sc = sc.trim();
                 if (sc.endsWith(";")) sc = sc.substring(0, sc.length() - 1);
             }
             if (sc == null) sc = localGroup;
-            if (sc != null) {
+            if (sc != null || mq != null) {
                 prop = new AnyValue.DefaultAnyValue();
-                prop.addValue("groups", sc);
+                if (sc != null) prop.addValue("groups", sc);
+                if (mq != null) prop.addValue("mq", mq);
             }
             ClassFilter filter = new ClassFilter(this.serverClassLoader, ref, inter, excludeSuperClasses, prop);
             for (AnyValue av : list.getAnyValues(property)) { // <service>、<filter>、<servlet> 节点
                 final AnyValue[] items = av.getAnyValues("property");
-                if (av instanceof DefaultAnyValue && items.length > 0) { //存在 <property>节点
+                if (av instanceof AnyValue.DefaultAnyValue && items.length > 0) { //存在 <property>节点
-                    DefaultAnyValue dav = DefaultAnyValue.create();
+                    AnyValue.DefaultAnyValue dav = AnyValue.DefaultAnyValue.create();
                     final AnyValue.Entry<String>[] strings = av.getStringEntrys();
                     if (strings != null) {  //将<service>、<filter>、<servlet>节点的属性值传给dav
                         for (AnyValue.Entry<String> en : strings) {
@@ -591,7 +678,7 @@ public abstract class NodeServer {
                             if (!"property".equals(en.name)) dav.addValue(en.name, en.getValue());
                         }
                     }
-                    DefaultAnyValue ps = DefaultAnyValue.create();
+                    AnyValue.DefaultAnyValue ps = AnyValue.DefaultAnyValue.create();
                     for (AnyValue item : items) {
                         ps.addValue(item.getValue("name"), item.getValue("value"));
                     }
@@ -627,6 +714,10 @@ public abstract class NodeServer {
         return false;
     }
 
+    public Application getApplication() {
+        return application;
+    }
+
     public ResourceFactory getResourceFactory() {
         return resourceFactory;
     }
@@ -660,12 +751,14 @@ public abstract class NodeServer {
     public void start() throws IOException {
         if (interceptor != null) interceptor.preStart(this);
         server.start();
+        postStartServer(localServices, remoteServices);
     }
 
     public void shutdown() throws IOException {
         if (interceptor != null) interceptor.preShutdown(this);
         final StringBuilder sb = logger.isLoggable(Level.INFO) ? new StringBuilder() : null;
         final boolean finest = logger.isLoggable(Level.FINEST);
+        preDestroyServices(localServices, remoteServices);
         localServices.forEach(y -> {
             long s = System.currentTimeMillis();
             if (finest) logger.finest(y + " is destroying");
@@ -680,6 +773,42 @@ public abstract class NodeServer {
         server.shutdown();
     }
 
+    public void command(String cmd) throws IOException {
+        final StringBuilder sb = logger.isLoggable(Level.INFO) ? new StringBuilder() : null;
+        final boolean finest = logger.isLoggable(Level.FINEST);
+        localServices.forEach(y -> {
+            Set<Method> methods = new HashSet<>();
+            Class loop = y.getClass();
+            //do {   public方法不用递归
+            for (Method m : loop.getMethods()) {
+                Command c = m.getAnnotation(Command.class);
+                if (c == null) continue;
+                if (Modifier.isStatic(m.getModifiers())) continue;
+                if (m.getReturnType() != void.class) continue;
+                if (m.getParameterCount() != 1) continue;
+                if (m.getParameterTypes()[0] != String.class) continue;
+                methods.add(m);
+            }
+            //} while ((loop = loop.getSuperclass()) != Object.class);
+            if (methods.isEmpty()) return;
+            long s = System.currentTimeMillis();
+            Method one = null;
+            try {
+                for (Method method : methods) {
+                    one = method;
+                    method.invoke(y, cmd);
+                }
+            } catch (Exception ex) {
+                logger.log(Level.SEVERE, one + " run error, cmd = " + cmd, ex);
+            }
+            long e = System.currentTimeMillis() - s;
+            if (e > 10 && sb != null) {
+                sb.append(Sncp.toSimpleString(y, maxNameLength, maxClassNameLength)).append(" command (").append(cmd).append(") ").append(e).append("ms").append(LINE_SEPARATOR);
+            }
+        });
+        if (sb != null && sb.length() > 0) logger.log(Level.INFO, sb.toString());
+    }
+
     public <T extends Server> T getServer() {
         return (T) server;
     }
@@ -696,4 +825,7 @@ public abstract class NodeServer {
         return new LinkedHashSet<>(remoteServices);
     }
 
+    public String getThreadName() {
+        return this.threadName;
+    }
 }

src/org/redkale/boot/NodeSncpServer.java

@@ -10,9 +10,10 @@ import java.net.*;
 import java.util.*;
 import java.util.logging.Level;
 import org.redkale.boot.ClassFilter.FilterEntry;
+import org.redkale.mq.MessageAgent;
 import org.redkale.net.*;
 import org.redkale.net.sncp.*;
-import org.redkale.service.Service;
+import org.redkale.service.*;
 import org.redkale.util.*;
 import org.redkale.util.AnyValue.DefaultAnyValue;
 
@@ -24,7 +25,7 @@ import org.redkale.util.AnyValue.DefaultAnyValue;
  *
  * @author zhangjx
  */
-@NodeProtocol({"SNCP"})
+@NodeProtocol("SNCP")
 public class NodeSncpServer extends NodeServer {
 
     protected final SncpServer sncpServer;
@@ -32,14 +33,15 @@ public class NodeSncpServer extends NodeServer {
     private NodeSncpServer(Application application, AnyValue serconf) {
         super(application, createServer(application, serconf));
         this.sncpServer = (SncpServer) this.server;
-        this.consumer = sncpServer == null || application.singletonrun ? null : x -> sncpServer.addSncpServlet(x); //singleton模式下不生成SncpServlet
+        this.consumer = sncpServer == null || application.singletonrun ? null : (agent, x) -> {//singleton模式下不生成SncpServlet
+            if (x.getClass().getAnnotation(Local.class) != null) return;
+            SncpDynServlet servlet = sncpServer.addSncpServlet(x);
+            dynServletMap.put(x, servlet);
+            if (agent != null) agent.putService(this, x, servlet);
+        };
     }
 
     public static NodeServer createNodeServer(Application application, AnyValue serconf) {
-        if (serconf != null && serconf.getAnyValue("rest") != null) {
-            ((AnyValue.DefaultAnyValue) serconf).addValue("_$sncp", "true");
-            return new NodeHttpServer(application, serconf);
-        }
         return new NodeSncpServer(application, serconf);
     }
 
@@ -52,8 +54,8 @@ public class NodeSncpServer extends NodeServer {
         return sncpServer == null ? null : sncpServer.getSocketAddress();
     }
 
-    public void consumerAccept(Service service) {
+    public void consumerAccept(MessageAgent messageAgent, Service service) {
-        if (this.consumer != null) this.consumer.accept(service);
+        if (this.consumer != null) this.consumer.accept(messageAgent, service);
     }
 
     @Override
@@ -62,11 +64,11 @@ public class NodeSncpServer extends NodeServer {
         //-------------------------------------------------------------------
         if (sncpServer == null) return; //调试时server才可能为null
         final StringBuilder sb = logger.isLoggable(Level.FINE) ? new StringBuilder() : null;
-        final String threadName = "[" + Thread.currentThread().getName() + "] ";
+        final String localThreadName = "[" + Thread.currentThread().getName() + "] ";
         List<SncpServlet> servlets = sncpServer.getSncpServlets();
         Collections.sort(servlets);
         for (SncpServlet en : servlets) {
-            if (sb != null) sb.append(threadName).append(" Load ").append(en).append(LINE_SEPARATOR);
+            if (sb != null) sb.append(localThreadName).append(" Load ").append(en).append(LINE_SEPARATOR);
         }
         if (sb != null && sb.length() > 0) logger.log(Level.FINE, sb.toString());
     }
@@ -88,7 +90,7 @@ public class NodeSncpServer extends NodeServer {
     @SuppressWarnings("unchecked")
     protected void loadSncpFilter(final AnyValue servletsConf, final ClassFilter<? extends Filter> classFilter) throws Exception {
         final StringBuilder sb = logger.isLoggable(Level.INFO) ? new StringBuilder() : null;
-        final String threadName = "[" + Thread.currentThread().getName() + "] ";
+        final String localThreadName = "[" + Thread.currentThread().getName() + "] ";
         List<FilterEntry<? extends Filter>> list = new ArrayList(classFilter.getFilterEntrys());
         for (FilterEntry<? extends Filter> en : list) {
             Class<SncpFilter> clazz = (Class<SncpFilter>) en.getType();
@@ -97,7 +99,7 @@ public class NodeSncpServer extends NodeServer {
             resourceFactory.inject(filter, this);
             DefaultAnyValue filterConf = (DefaultAnyValue) en.getProperty();
             this.sncpServer.addSncpFilter(filter, filterConf);
-            if (sb != null) sb.append(threadName).append(" Load ").append(clazz.getName()).append(LINE_SEPARATOR);
+            if (sb != null) sb.append(localThreadName).append(" Load ").append(clazz.getName()).append(LINE_SEPARATOR);
         }
         if (sb != null && sb.length() > 0) logger.log(Level.INFO, sb.toString());
     }

src/org/redkale/boot/NodeWatchServer.java

@@ -16,7 +16,7 @@ import org.redkale.watch.*;
  *
  * @author zhangjx
  */
-@NodeProtocol({"WATCH"})
+@NodeProtocol("WATCH")
 public class NodeWatchServer extends NodeHttpServer {
 
     public NodeWatchServer(Application application, AnyValue serconf) {

src/org/redkale/boot/watch/ServerWatchService.java

@@ -81,7 +81,7 @@ public class ServerWatchService extends AbstractWatchService {
             protocol += "/HTTP";
         } else {
             NodeProtocol np = node.getClass().getAnnotation(NodeProtocol.class);
-            if (np != null && np.value().length > 0) protocol += "/" + np.value()[0];
+            protocol += "/" + np.value();
         }
         rs.put("name", server.getName());
         rs.put("protocol", protocol);

src/org/redkale/boot/watch/TransportWatchService.java

@@ -72,15 +72,6 @@ public class TransportWatchService extends AbstractWatchService {
                     if (client.getRemoteGroups() != null && client.getRemoteGroups().contains(group)) {
                         client.getRemoteGroupTransport().addRemoteAddresses(address);
                     }
-                } else {
-                    if (group.equals(client.getSameGroup())) {
-                        client.getSameGroupTransport().addRemoteAddresses(address);
-                    }
-                    if (client.getDiffGroups() != null && client.getDiffGroups().contains(group)) {
-                        for (Transport transport : client.getDiffGroupTransports()) {
-                            transport.addRemoteAddresses(address);
-                        }
-                    }
                 }
             }
             DefaultAnyValue node = DefaultAnyValue.create("addr", addr).addValue("port", port);
@@ -114,15 +105,6 @@ public class TransportWatchService extends AbstractWatchService {
                     if (client.getRemoteGroups() != null && client.getRemoteGroups().contains(group)) {
                         client.getRemoteGroupTransport().removeRemoteAddresses(address);
                     }
-                } else {
-                    if (group.equals(client.getSameGroup())) {
-                        client.getSameGroupTransport().removeRemoteAddresses(address);
-                    }
-                    if (client.getDiffGroups() != null && client.getDiffGroups().contains(group)) {
-                        for (Transport transport : client.getDiffGroupTransports()) {
-                            transport.removeRemoteAddresses(address);
-                        }
-                    }
                 }
             }
             for (AnyValue groupconf : application.getAppConfig().getAnyValue("resources").getAnyValues("group")) {

src/org/redkale/cluster/ClusterAgent.java

@@ -0,0 +1,333 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.cluster;
+
+import java.lang.ref.WeakReference;
+import java.net.InetSocketAddress;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.logging.Logger;
+import javax.annotation.Resource;
+import org.redkale.boot.*;
+import static org.redkale.boot.Application.*;
+import org.redkale.convert.json.JsonConvert;
+import org.redkale.mq.MessageMultiConsumer;
+import org.redkale.net.*;
+import org.redkale.net.http.*;
+import org.redkale.net.sncp.*;
+import org.redkale.service.*;
+import org.redkale.util.*;
+
+/**
+ * 第三方服务发现管理接口cluster
+ *
+ *
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public abstract class ClusterAgent {
+
+    protected final Logger logger = Logger.getLogger(this.getClass().getSimpleName());
+
+    @Resource(name = RESNAME_APP_NODEID)
+    protected int nodeid;
+
+    @Resource(name = RESNAME_APP_NAME)
+    protected String appName = "";
+
+    @Resource(name = RESNAME_APP_ADDR)
+    protected InetSocketAddress appAddress;
+
+    protected String name;
+
+    protected boolean waits;
+
+    protected String[] protocols; //必须全大写
+
+    protected int[] ports;
+
+    protected AnyValue config;
+
+    protected TransportFactory transportFactory;
+
+    protected final ConcurrentHashMap<String, ClusterEntry> localEntrys = new ConcurrentHashMap<>();
+
+    protected final ConcurrentHashMap<String, ClusterEntry> remoteEntrys = new ConcurrentHashMap<>();
+
+    public void init(AnyValue config) {
+        this.config = config;
+        this.name = config.getValue("name", "");
+        this.waits = config.getBoolValue("waits", false);
+        {
+            String ps = config.getValue("protocols", "").toUpperCase();
+            if (ps == null || ps.isEmpty()) ps = "SNCP;HTTP";
+            this.protocols = ps.split(";");
+        }
+        String ts = config.getValue("ports", "");
+        if (ts != null && !ts.isEmpty()) {
+            String[] its = ts.split(";");
+            List<Integer> list = new ArrayList<>();
+            for (String str : its) {
+                if (str.trim().isEmpty()) continue;
+                list.add(Integer.parseInt(str.trim()));
+            }
+            if (!list.isEmpty()) this.ports = list.stream().mapToInt(x -> x).toArray();
+        }
+    }
+
+    public void destroy(AnyValue config) {
+    }
+
+    //ServiceLoader时判断配置是否符合当前实现类
+    public abstract boolean match(AnyValue config);
+
+    public boolean containsProtocol(String protocol) {
+        if (protocol == null || protocol.isEmpty()) return false;
+        return protocols == null || Utility.contains(protocols, protocol.toUpperCase());
+    }
+
+    public boolean containsPort(int port) {
+        if (ports == null || ports.length == 0) return true;
+        return Utility.contains(ports, port);
+    }
+
+    public abstract void register(Application application);
+
+    public abstract void deregister(Application application);
+
+    //注册服务, 在NodeService调用Service.init方法之前调用
+    public void register(NodeServer ns, String protocol, Set<Service> localServices, Set<Service> remoteServices) {
+        if (localServices.isEmpty()) return;
+        //注册本地模式
+        for (Service service : localServices) {
+            if (!canRegister(protocol, service)) continue;
+            register(ns, protocol, service);
+            ClusterEntry htentry = new ClusterEntry(ns, protocol, service);
+            localEntrys.put(htentry.serviceid, htentry);
+            if (protocol.toLowerCase().startsWith("http")) {
+                MessageMultiConsumer mmc = service.getClass().getAnnotation(MessageMultiConsumer.class);
+                if (mmc != null) {
+                    register(ns, "mqtp", service);
+                    ClusterEntry mqentry = new ClusterEntry(ns, "mqtp", service);
+                    localEntrys.put(mqentry.serviceid, mqentry);
+                }
+            }
+        }
+        //远程模式加载IP列表, 只支持SNCP协议    
+        if (ns.isSNCP()) {
+            for (Service service : remoteServices) {
+                ClusterEntry entry = new ClusterEntry(ns, protocol, service);
+                updateSncpTransport(entry);
+                remoteEntrys.put(entry.serviceid, entry);
+            }
+        }
+    }
+
+    //注销服务, 在NodeService调用Service.destroy 方法之前调用
+    public void deregister(NodeServer ns, String protocol, Set<Service> localServices, Set<Service> remoteServices) {
+        //注销本地模式  远程模式不注册
+        for (Service service : localServices) {
+            if (!canRegister(protocol, service)) continue;
+            deregister(ns, protocol, service);
+        }
+        afterDeregister(ns, protocol);
+    }
+
+    protected boolean canRegister(String protocol, Service service) {
+        if ("SNCP".equalsIgnoreCase(protocol) && service.getClass().getAnnotation(Local.class) != null) return false;
+        if (service instanceof WebSocketNode) {
+            if (((WebSocketNode) service).getLocalWebSocketEngine() == null) return false;
+        }
+        return true;
+    }
+
+    public void start() {
+    }
+
+    protected void afterDeregister(NodeServer ns, String protocol) {
+        if (!this.waits) return;
+        int s = intervalCheckSeconds();
+        if (s > 0) {  //暂停，弥补其他依赖本进程服务的周期偏差
+            try {
+                Thread.sleep(s * 1000);
+            } catch (InterruptedException ex) {
+            }
+            logger.info(this.getClass().getSimpleName() + " wait for " + s * 1000 + "ms after deregister");
+        }
+    }
+
+    public int intervalCheckSeconds() {
+        return 10;
+    }
+
+    //获取MQTP的HTTP远程服务的可用ip列表, key = servicename的后半段
+    public abstract CompletableFuture<Map<String, Collection<InetSocketAddress>>> queryMqtpAddress(String protocol, String module, String resname);
+
+    //获取HTTP远程服务的可用ip列表
+    public abstract CompletableFuture<Collection<InetSocketAddress>> queryHttpAddress(String protocol, String module, String resname);
+
+    //获取远程服务的可用ip列表
+    protected abstract CompletableFuture<Collection<InetSocketAddress>> queryAddress(ClusterEntry entry);
+
+    //注册服务
+    protected abstract void register(NodeServer ns, String protocol, Service service);
+
+    //注销服务
+    protected abstract void deregister(NodeServer ns, String protocol, Service service);
+
+    //格式: protocol:classtype-resourcename
+    protected void updateSncpTransport(ClusterEntry entry) {
+        Service service = entry.serviceref.get();
+        if (service == null) return;
+        Collection<InetSocketAddress> addrs = ClusterAgent.this.queryAddress(entry).join();
+        Sncp.updateTransport(service, transportFactory, Sncp.getResourceType(service).getName() + "-" + Sncp.getResourceName(service), entry.netprotocol, entry.address, null, addrs);
+    }
+
+    protected String generateApplicationServiceName() {
+        return "application" + (appName == null || appName.isEmpty() ? "" : ("." + appName)) + ".node" + this.nodeid;
+    }
+
+    protected String generateApplicationServiceId() { //与servicename相同
+        return generateApplicationServiceName();
+    }
+
+    protected String generateApplicationCheckName() {
+        return "check-" + generateApplicationServiceName();
+    }
+
+    protected String generateApplicationCheckId() {
+        return "check-" + generateApplicationServiceId();
+    }
+
+    //也会提供给HttpMessageClusterAgent适用
+    public String generateHttpServiceName(String protocol, String module, String resname) {
+        return protocol.toLowerCase() + ":" + module + (resname == null || resname.isEmpty() ? "" : ("-" + resname));
+    }
+
+    //格式: protocol:classtype-resourcename
+    protected String generateServiceName(NodeServer ns, String protocol, Service service) {
+        if (protocol.toLowerCase().startsWith("http")) {  //HTTP使用RestService.name方式是为了与MessageClient中的module保持一致, 因为HTTP依靠的url中的module，无法知道Service类名
+            String resname = Sncp.getResourceName(service);
+            String module = Rest.getRestModule(service).toLowerCase();
+            return protocol.toLowerCase() + ":" + module + (resname.isEmpty() ? "" : ("-" + resname));
+        }
+        if ("mqtp".equalsIgnoreCase(protocol)) {
+            MessageMultiConsumer mmc = service.getClass().getAnnotation(MessageMultiConsumer.class);
+            String selfmodule = Rest.getRestModule(service).toLowerCase();
+            return protocol.toLowerCase() + ":" + mmc.module() + ":" + selfmodule;
+        }
+        if (!Sncp.isSncpDyn(service)) return protocol.toLowerCase() + ":" + service.getClass().getName();
+        String resname = Sncp.getResourceName(service);
+        return protocol.toLowerCase() + ":" + Sncp.getResourceType(service).getName() + (resname.isEmpty() ? "" : ("-" + resname));
+    }
+
+    //格式: protocol:classtype-resourcename:nodeid
+    protected String generateServiceId(NodeServer ns, String protocol, Service service) {
+        return generateServiceName(ns, protocol, service) + ":" + this.nodeid;
+    }
+
+    protected String generateCheckName(NodeServer ns, String protocol, Service service) {
+        return "check-" + generateServiceName(ns, protocol, service);
+    }
+
+    protected String generateCheckId(NodeServer ns, String protocol, Service service) {
+        return "check-" + generateServiceId(ns, protocol, service);
+    }
+
+    protected ConcurrentHashMap<String, ClusterEntry> getLocalEntrys() {
+        return localEntrys;
+    }
+
+    protected ConcurrentHashMap<String, ClusterEntry> getRemoteEntrys() {
+        return remoteEntrys;
+    }
+
+    @Override
+    public String toString() {
+        return JsonConvert.root().convertTo(this);
+    }
+
+    public TransportFactory getTransportFactory() {
+        return transportFactory;
+    }
+
+    public void setTransportFactory(TransportFactory transportFactory) {
+        this.transportFactory = transportFactory;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(String name) {
+        this.name = name;
+    }
+
+    public String[] getProtocols() {
+        return protocols;
+    }
+
+    public void setProtocols(String[] protocols) {
+        this.protocols = protocols;
+    }
+
+    public int[] getPorts() {
+        return ports;
+    }
+
+    public void setPorts(int[] ports) {
+        this.ports = ports;
+    }
+
+    public AnyValue getConfig() {
+        return config;
+    }
+
+    public void setConfig(AnyValue config) {
+        this.config = config;
+    }
+
+    public class ClusterEntry {
+
+        public String serviceid;
+
+        public String servicename;
+
+        public String checkid;
+
+        public String checkname;
+
+        public String protocol;
+
+        public String netprotocol;
+
+        public WeakReference<Service> serviceref;
+
+        public InetSocketAddress address;
+
+        public boolean canceled;
+
+        public ClusterEntry(NodeServer ns, String protocol, Service service) {
+            this.serviceid = generateServiceId(ns, protocol, service);
+            this.servicename = generateServiceName(ns, protocol, service);
+            this.checkid = generateCheckId(ns, protocol, service);
+            this.checkname = generateCheckName(ns, protocol, service);
+            this.protocol = protocol;
+            this.address = ns.getSocketAddress();
+            this.serviceref = new WeakReference(service);
+            Server server = ns.getServer();
+            this.netprotocol = server instanceof SncpServer ? ((SncpServer) server).getNetprotocol() : Transport.DEFAULT_PROTOCOL;
+        }
+
+        @Override
+        public String toString() {
+            return JsonConvert.root().convertTo(this);
+        }
+    }
+}

src/org/redkale/convert/ArrayDecoder.java

@@ -119,6 +119,7 @@ public class ArrayDecoder<T> implements Decodeable<Reader, T[]> {
     }
 
     protected T readMemberValue(Reader in, DeMember member, Decodeable<Reader, T> decoder, boolean first) {
+        if (in == null) return null;
         return decoder.convertFrom(in);
     }
 

src/org/redkale/convert/ArrayEncoder.java

@@ -66,7 +66,7 @@ public class ArrayEncoder<T> implements Encodeable<Writer, T[]> {
             return;
         }
         if (value.length == 0) {
-            out.writeArrayB(0, componentEncoder, value);
+            out.writeArrayB(0, this, componentEncoder, value); 
             out.writeArrayE();
             return;
         }
@@ -81,7 +81,7 @@ public class ArrayEncoder<T> implements Encodeable<Writer, T[]> {
                 }
             }
         }
-        if (out.writeArrayB(value.length, componentEncoder, value) < 0) {
+        if (out.writeArrayB(value.length, this, componentEncoder, value) < 0) { 
             final Type comp = this.componentType;
             boolean first = true;
             for (Object v : value) {

src/org/redkale/convert/CollectionDecoder.java

@@ -129,6 +129,7 @@ public class CollectionDecoder<T> implements Decodeable<Reader, Collection<T>> {
     }
 
     protected T readMemberValue(Reader in, DeMember member, Decodeable<Reader, T> decoder, boolean first) {
+        if (in == null) return null;
         return decoder.convertFrom(in);
     }
 

src/org/redkale/convert/CollectionEncoder.java

@@ -61,7 +61,7 @@ public class CollectionEncoder<T> implements Encodeable<Writer, Collection<T>> {
             return;
         }
         if (value.isEmpty()) {
-            out.writeArrayB(0, componentEncoder, value);
+            out.writeArrayB(0, this, componentEncoder, value);
             out.writeArrayE();
             return;
         }
@@ -76,18 +76,18 @@ public class CollectionEncoder<T> implements Encodeable<Writer, Collection<T>> {
                 }
             }
         }
-        if (out.writeArrayB(value.size(), componentEncoder, value) < 0) {
+        if (out.writeArrayB(value.size(), this, componentEncoder, value) < 0) {
             boolean first = true;
             for (Object v : value) {
                 if (!first) out.writeArrayMark();
-                writeValue(out, member, v);
+                writeMemberValue(out, member, v, first);
                 if (first) first = false;
             }
         }
         out.writeArrayE();
     }
 
-    protected void writeValue(Writer out, EnMember member, Object value) {
+    protected void writeMemberValue(Writer out, EnMember member, Object value, boolean first) {
         componentEncoder.convertTo(out, value);
     }
 
@@ -96,6 +96,11 @@ public class CollectionEncoder<T> implements Encodeable<Writer, Collection<T>> {
         return type;
     }
 
+    @Override
+    public String toString() {
+        return this.getClass().getSimpleName() + "{componentType:" + this.type + ", encoder:" + this.componentEncoder + "}";
+    }
+
     public Encodeable<Writer, Object> getComponentEncoder() {
         return componentEncoder;
     }

src/org/redkale/convert/Convert.java

@@ -50,10 +50,17 @@ public abstract class Convert<R extends Reader, W extends Writer> {
 
     public abstract <T> T convertFrom(final Type type, final byte[] bytes);
 
+    //@since 2.2.0
+    public abstract <T> T convertFrom(final Type type, final byte[] bytes, final int offset, final int length);
+
     public abstract <T> T convertFrom(final Type type, final ByteBuffer... buffers);
 
     public abstract <T> T convertFrom(final Type type, final ConvertMask mask, final ByteBuffer... buffers);
 
+    public abstract byte[] convertToBytes(final Object value);
+
+    public abstract byte[] convertToBytes(final Type type, final Object value);
+
     public abstract ByteBuffer[] convertTo(final Supplier<ByteBuffer> supplier, final Object value);
 
     public abstract ByteBuffer[] convertTo(final Supplier<ByteBuffer> supplier, final Type type, final Object value);

src/org/redkale/convert/ConvertFactory.java

@@ -16,7 +16,9 @@ import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.atomic.*;
 import java.util.regex.Pattern;
 import java.util.stream.*;
+import org.redkale.convert.bson.BsonConvert;
 import org.redkale.convert.ext.*;
+import org.redkale.convert.json.JsonConvert;
 import org.redkale.util.*;
 
 /**
@@ -32,6 +34,10 @@ import org.redkale.util.*;
 @SuppressWarnings("unchecked")
 public abstract class ConvertFactory<R extends Reader, W extends Writer> {
 
+    private static final AtomicBoolean loaderInited = new AtomicBoolean();
+
+    private static Convert defProtobufConvert;
+
     private final ConvertFactory parent;
 
     protected Convert<R, W> convert;
@@ -188,6 +194,30 @@ public abstract class ConvertFactory<R extends Reader, W extends Writer> {
         return this.parent;
     }
 
+    public static Convert findConvert(ConvertType type) {
+        if (type == null) return null;
+        if (type == ConvertType.JSON) return JsonConvert.root();
+        if (type == ConvertType.BSON) return BsonConvert.root();
+        if (loaderInited.get()) {
+            if (type == ConvertType.PROTOBUF) return defProtobufConvert;
+        }
+        synchronized (loaderInited) {
+            if (!loaderInited.get()) {
+                Iterator<ConvertLoader> it = ServiceLoader.load(ConvertLoader.class).iterator();
+                while (it.hasNext()) {
+                    ConvertLoader cl = it.next();
+                    if (cl.type() == ConvertType.PROTOBUF) defProtobufConvert = cl.convert();
+                }
+                loaderInited.set(true);
+            }
+        }
+        return type == ConvertType.PROTOBUF ? defProtobufConvert : null;
+    }
+
+    protected static boolean getSystemPropertyBoolean(String key, String parentkey, boolean defvalue) {
+        return Boolean.parseBoolean(System.getProperty(key, System.getProperty(parentkey, String.valueOf(defvalue))));
+    }
+
     public abstract ConvertType getConvertType();
 
     public abstract boolean isReversible(); //是否可逆的
@@ -282,10 +312,14 @@ public abstract class ConvertFactory<R extends Reader, W extends Writer> {
             final Method method = (Method) element;
             fieldName = readGetSetFieldName(method);
             if (fieldName != null) {
+                Class mclz = method.getDeclaringClass();
+                do {
                     try {
-                    ccs = method.getDeclaringClass().getDeclaredField(fieldName).getAnnotationsByType(ConvertColumn.class);
+                        ccs = mclz.getDeclaredField(fieldName).getAnnotationsByType(ConvertColumn.class);
+                        break;
                     } catch (Exception e) { //说明没有该字段，忽略异常
                     }
+                } while (mclz != Object.class && (mclz = mclz.getSuperclass()) != Object.class);
             }
         }
         if (onlyColumns != null && fieldName == null) {

src/org/redkale/convert/ConvertLoader.java

@@ -0,0 +1,23 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.convert;
+
+/**
+ * Convert的扩展实现类加载器
+ *
+ *
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.2.0
+ */
+public interface ConvertLoader {
+
+    public ConvertType type();
+
+    public Convert convert();
+}

src/org/redkale/convert/ConvertType.java

@@ -17,8 +17,10 @@ public enum ConvertType {
 
     JSON(1),
     BSON(2),
-    DIY(64),
+    PROTOBUF(64),
-    ALL(127);
+    PROTOBUF_JSON(64 + 1),
+    DIY(256),
+    ALL(1023);
 
     private final int value;
 
@@ -26,8 +28,19 @@ public enum ConvertType {
         this.value = v;
     }
 
+    public int getValue() {
+        return value;
+    }
+
     public boolean contains(ConvertType type) {
         if (type == null) return false;
         return this.value >= type.value && (this.value & type.value) > 0;
     }
+
+    public static ConvertType find(int value) {
+        for (ConvertType t : ConvertType.values()) {
+            if (value == t.value) return t;
+        }
+        return null;
+    }
 }

src/org/redkale/convert/DeMember.java

@@ -26,6 +26,8 @@ public final class DeMember<R extends Reader, T, F> {
 
     protected int position; //从1开始
 
+    protected int tag; //主要给protobuf使用
+
     protected final Attribute<T, F> attribute;
 
     protected Decodeable<R, F> decoder;
@@ -84,8 +86,13 @@ public final class DeMember<R extends Reader, T, F> {
         return this.position;
     }
 
+    public int getTag() {
+        return this.tag;
+    }
+
     public int compareTo(boolean fieldSort, DeMember<R, T, F> o) {
         if (o == null) return -1;
+        if (this.position != o.position) return (this.position == 0 ? Integer.MAX_VALUE : this.position) - (o.position == 0 ? Integer.MAX_VALUE : o.position);
         if (this.index != o.index) return (this.index == 0 ? Integer.MAX_VALUE : this.index) - (o.index == 0 ? Integer.MAX_VALUE : o.index);
         if (this.index != 0) throw new RuntimeException("fields (" + attribute.field() + ", " + o.attribute.field() + ") have same ConvertColumn.index(" + this.index + ") in " + attribute.declaringClass());
         return fieldSort ? this.attribute.field().compareTo(o.attribute.field()) : 0;
@@ -106,6 +113,6 @@ public final class DeMember<R extends Reader, T, F> {
 
     @Override
     public String toString() {
-        return "DeMember{" + "attribute=" + attribute.field() + ", decoder=" + (decoder == null ? null : decoder.getClass().getName()) + '}';
+        return "DeMember{" + "attribute=" + attribute.field() + ", position=" + position + ", tag=" + tag + ", decoder=" + (decoder == null ? null : decoder.getClass().getName()) + '}';
     }
 }

src/org/redkale/convert/EnMember.java

@@ -26,21 +26,23 @@ public final class EnMember<W extends Writer, T, F> {
 
     final Encodeable<W, F> encoder;
 
-    final boolean istring;
+    final boolean string;
 
     //final boolean isnumber;
-    final boolean isbool;
+    final boolean bool;
 
     protected int index;
 
     protected int position; //从1开始
 
+    protected int tag; //主要给protobuf使用
+
     public EnMember(Attribute<T, F> attribute, Encodeable<W, F> encoder) {
         this.attribute = attribute;
         this.encoder = encoder;
         Class t = attribute.type();
-        this.istring = CharSequence.class.isAssignableFrom(t);
+        this.string = CharSequence.class.isAssignableFrom(t);
-        this.isbool = t == Boolean.class || t == boolean.class;
+        this.bool = t == Boolean.class || t == boolean.class;
         //this.isnumber = Number.class.isAssignableFrom(t) || (!this.isbool && t.isPrimitive());
     }
 
@@ -74,11 +76,11 @@ public final class EnMember<W extends Writer, T, F> {
     }
 
     public boolean isStringType() {
-        return istring;
+        return string;
     }
 
     public boolean isBoolType() {
-        return isbool;
+        return bool;
     }
 
     public int getIndex() {
@@ -89,8 +91,13 @@ public final class EnMember<W extends Writer, T, F> {
         return this.position;
     }
 
+    public int getTag() {
+        return this.tag;
+    }
+
     public int compareTo(boolean fieldSort, EnMember<W, T, F> o) {
         if (o == null) return -1;
+        if (this.position != o.position) return (this.position == 0 ? Integer.MAX_VALUE : this.position) - (o.position == 0 ? Integer.MAX_VALUE : o.position);
         if (this.index != o.index) return (this.index == 0 ? Integer.MAX_VALUE : this.index) - (o.index == 0 ? Integer.MAX_VALUE : o.index);
         if (this.index != 0) throw new RuntimeException("fields (" + attribute.field() + ", " + o.attribute.field() + ") have same ConvertColumn.index(" + this.index + ") in " + attribute.declaringClass());
         return fieldSort ? this.attribute.field().compareTo(o.attribute.field()) : 0;
@@ -111,6 +118,6 @@ public final class EnMember<W extends Writer, T, F> {
 
     @Override
     public String toString() {
-        return "EnMember{" + "attribute=" + attribute.field() + ", encoder=" + (encoder == null ? null : encoder.getClass().getName()) + '}';
+        return "EnMember{" + "attribute=" + attribute.field() + ", position=" + position + ", encoder=" + (encoder == null ? null : encoder.getClass().getName()) + '}';
     }
 }

src/org/redkale/convert/ObjectDecoder.java

@@ -29,7 +29,7 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
 
     protected Creator<T> creator;
 
-    protected DeMember<R, T, ?>[] creatorConstructorMembers = new DeMember[0];
+    protected DeMember<R, T, ?>[] creatorConstructorMembers = null;
 
     protected DeMember[] members;
 
@@ -101,7 +101,7 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
                         Type t = TypeToken.createClassType(TypeToken.getGenericType(field.getGenericType(), this.type), this.type);
                         fieldCoder = factory.loadDecoder(t);
                     }
-                    DeMember member = new DeMember(ObjectEncoder.createAttribute(factory, clazz, field, null, null), fieldCoder);
+                    DeMember member = new DeMember(ObjectEncoder.createAttribute(factory, type, clazz, field, null, null), fieldCoder);
                     if (ref != null) member.index = ref.getIndex();
                     list.add(member);
                 }
@@ -131,7 +131,7 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
                         Type t = TypeToken.createClassType(TypeToken.getGenericType(method.getGenericParameterTypes()[0], this.type), this.type);
                         fieldCoder = factory.loadDecoder(t);
                     }
-                    DeMember member = new DeMember(ObjectEncoder.createAttribute(factory, clazz, null, null, method), fieldCoder);
+                    DeMember member = new DeMember(ObjectEncoder.createAttribute(factory, type, clazz, null, null, method), fieldCoder);
                     if (ref != null) member.index = ref.getIndex();
                     list.add(member);
                 }
@@ -149,7 +149,7 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
                         try {
                             Field f = clazz.getDeclaredField(constructorField);
                             Type t = TypeToken.createClassType(f.getGenericType(), this.type);
-                            list.add(new DeMember(ObjectEncoder.createAttribute(factory, clazz, f, null, null), factory.loadDecoder(t)));
+                            list.add(new DeMember(ObjectEncoder.createAttribute(factory, type, clazz, f, null, null), factory.loadDecoder(t)));
                         } catch (NoSuchFieldException nsfe) { //不存在field， 可能存在getter方法
                             char[] fs = constructorField.toCharArray();
                             fs[0] = Character.toUpperCase(fs[0]);
@@ -161,11 +161,12 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
                                 getter = clazz.getMethod("is" + mn);
                             }
                             Type t = TypeToken.createClassType(TypeToken.getGenericType(getter.getGenericParameterTypes()[0], this.type), this.type);
-                            list.add(new DeMember(ObjectEncoder.createAttribute(factory, clazz, null, getter, null), factory.loadDecoder(t)));
+                            list.add(new DeMember(ObjectEncoder.createAttribute(factory, type, clazz, null, getter, null), factory.loadDecoder(t)));
                         }
                     }
                 }
                 this.members = list.toArray(new DeMember[list.size()]);
+                //先排序一次
                 Arrays.sort(this.members, (a, b) -> a.compareTo(factory.isFieldSort(), b));
                 Set<Integer> pos = new HashSet<>();
                 for (int i = 0; i < this.members.length; i++) {
@@ -179,7 +180,9 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
                         while (pos.contains(++pidx));
                         member.position = pidx;
                     }
+                    initForEachDeMember(factory, member);
                 }
+                Arrays.sort(this.members, (a, b) -> a.compareTo(factory.isFieldSort(), b));
 
                 if (cps != null) {
                     final String[] fields = cps;
@@ -205,6 +208,13 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
         }
     }
 
+    protected void initForEachDeMember(ConvertFactory factory, DeMember member) {
+    }
+
+    protected void setTag(DeMember member, int tag) {
+        member.tag = tag;
+    }
+
     /**
      * 对象格式: [0x1][short字段个数][字段名][字段值]...[0x2]
      *
@@ -214,9 +224,10 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
      */
     @Override
     public T convertFrom(final R in) {
-        final String clazz = in.readObjectB(typeClass);
+        R objin = objectReader(in);
+        final String clazz = objin.readObjectB(typeClass);
         if (clazz == null) return null;
-        if (!clazz.isEmpty()) return (T) factory.loadDecoder(factory.getEntityAlias(clazz)).convertFrom(in);
+        if (!clazz.isEmpty()) return (T) factory.loadDecoder(factory.getEntityAlias(clazz)).convertFrom(objin);
         if (!this.inited) {
             synchronized (lock) {
                 try {
@@ -234,17 +245,17 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
         if (this.creatorConstructorMembers == null) {  //空构造函数
             final T result = this.creator == null ? null : this.creator.create();
             boolean first = true;
-            while (hasNext(in, first)) {
+            while (hasNext(objin, first)) {
-                DeMember member = in.readFieldName(members);
+                DeMember member = objin.readFieldName(members);
-                in.readBlank();
+                objin.readBlank();
                 if (member == null) {
-                    in.skipValue(); //跳过不存在的属性的值
+                    objin.skipValue(); //跳过不存在的属性的值
                 } else {
-                    readMemberValue(in, member, result, first);
+                    readMemberValue(objin, member, result, first);
                 }
                 first = false;
             }
-            in.readObjectE(typeClass);
+            objin.readObjectE(typeClass);
             return result;
         } else {  //带参数的构造函数
             final DeMember<R, T, ?>[] fields = this.creatorConstructorMembers;
@@ -252,13 +263,13 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
             final Object[][] otherParams = new Object[this.members.length][2];
             int oc = 0;
             boolean first = true;
-            while (hasNext(in, first)) {
+            while (hasNext(objin, first)) {
-                DeMember member = in.readFieldName(members);
+                DeMember member = objin.readFieldName(members);
-                in.readBlank();
+                objin.readBlank();
                 if (member == null) {
-                    in.skipValue(); //跳过不存在的属性的值
+                    objin.skipValue(); //跳过不存在的属性的值
                 } else {
-                    Object val = readMemberValue(in, member, first);
+                    Object val = readMemberValue(objin, member, first);
                     boolean flag = true;
                     for (int i = 0; i < fields.length; i++) {
                         if (member == fields[i]) {
@@ -272,7 +283,7 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
                 }
                 first = false;
             }
-            in.readObjectE(typeClass);
+            objin.readObjectE(typeClass);
             if (this.creator == null) return null;
             final T result = this.creator.create(constructorParams);
             for (int i = 0; i < oc; i++) {
@@ -282,6 +293,10 @@ public class ObjectDecoder<R extends Reader, T> implements Decodeable<R, T> {
         }
     }
 
+    protected R objectReader(R in) {
+        return in;
+    }
+
     protected boolean hasNext(R in, boolean first) {
         return in.hasNext();
     }

src/org/redkale/convert/ObjectEncoder.java

@@ -82,7 +82,7 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
                         Type t = TypeToken.createClassType(TypeToken.getGenericType(field.getGenericType(), this.type), this.type);
                         fieldCoder = factory.loadEncoder(t);
                     }
-                    EnMember member = new EnMember(createAttribute(factory, clazz, field, null, null), fieldCoder);
+                    EnMember member = new EnMember(createAttribute(factory, type, clazz, field, null, null), fieldCoder);
                     if (ref != null) member.index = ref.getIndex();
                     list.add(member);
                 }
@@ -111,11 +111,12 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
                         Type t = TypeToken.createClassType(TypeToken.getGenericType(method.getGenericReturnType(), this.type), this.type);
                         fieldCoder = factory.loadEncoder(t);
                     }
-                    EnMember member = new EnMember(createAttribute(factory, clazz, null, method, null), fieldCoder);
+                    EnMember member = new EnMember(createAttribute(factory, type, clazz, null, method, null), fieldCoder);
                     if (ref != null) member.index = ref.getIndex();
                     list.add(member);
                 }
                 this.members = list.toArray(new EnMember[list.size()]);
+                //先排序一次
                 Arrays.sort(this.members, (a, b) -> a.compareTo(factory.isFieldSort(), b));
                 Set<Integer> pos = new HashSet<>();
                 for (int i = 0; i < this.members.length; i++) {
@@ -129,7 +130,9 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
                         while (pos.contains(++pidx));
                         member.position = pidx;
                     }
+                    initForEachEnMember(factory, member);
                 }
+                Arrays.sort(this.members, (a, b) -> a.compareTo(factory.isFieldSort(), b));
 
             } catch (Exception ex) {
                 throw new ConvertException(ex);
@@ -142,6 +145,13 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
         }
     }
 
+    protected void initForEachEnMember(ConvertFactory factory, EnMember member) {
+    }
+
+    protected void setTag(EnMember member, int tag) {
+        member.tag = tag;
+    }
+
     @Override
     public void convertTo(W out, T value) {
         if (value == null) {
@@ -163,25 +173,30 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
             factory.loadEncoder(clz).convertTo(out, value);
             return;
         }
-        if (out.writeObjectB(value) < 0) {
+        W objout = objectWriter(out, value);
+        if (objout.writeObjectB(value) < 0) {
             int maxPosition = 0;
             for (EnMember member : members) {
                 maxPosition = member.getPosition();
-                out.writeObjectField(member, value);
+                objout.writeObjectField(member, value);
             }
-            if (out.objExtFunc != null) {
+            if (objout.objExtFunc != null) {
-                ConvertField[] extFields = out.objExtFunc.apply(value);
+                ConvertField[] extFields = objout.objExtFunc.apply(value);
                 if (extFields != null) {
                     Encodeable<W, ?> anyEncoder = factory.getAnyEncoder();
                     for (ConvertField en : extFields) {
                         if (en == null) continue;
                         maxPosition++;
-                        out.writeObjectField(en.getName(), en.getType(), Math.max(en.getPosition(), maxPosition), anyEncoder, en.getValue());
+                        objout.writeObjectField(en.getName(), en.getType(), Math.max(en.getPosition(), maxPosition), anyEncoder, en.getValue());
                     }
                 }
             }
         }
-        out.writeObjectE(value);
+        objout.writeObjectE(value);
+    }
+
+    protected W objectWriter(W out, T value) {
+        return out;
     }
 
     @Override
@@ -263,7 +278,7 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
         }
     }
 
-    static Attribute createAttribute(final ConvertFactory factory, Class clazz, final Field field, final Method getter, final Method setter) {
+    static Attribute createAttribute(final ConvertFactory factory, Type realType, Class clazz, final Field field, final Method getter, final Method setter) {
         String fieldalias;
         if (field != null) { // public field
             ConvertColumnEntry ref = factory.findRef(clazz, field);
@@ -289,7 +304,7 @@ public class ObjectEncoder<W extends Writer, T> implements Encodeable<W, T> {
             }
             fieldalias = ref == null || ref.name().isEmpty() ? mfieldname : ref.name();
         }
-        return Attribute.create(clazz, fieldalias, field, getter, setter);
+        return Attribute.create(realType, clazz, fieldalias, null, field, getter, setter, null);
     }
 
 }

src/org/redkale/convert/OptionalCoder.java

@@ -20,9 +20,7 @@ import java.util.*;
  */
 public class OptionalCoder<R extends Reader, W extends Writer, T> extends SimpledCoder<R, W, Optional<T>> {
 
-    private final Type type;
+    protected final Type componentType;
-
-    private final Type componentType;
 
     protected final Class componentClass;
 

src/org/redkale/convert/StreamEncoder.java

@@ -62,7 +62,7 @@ public class StreamEncoder<T> implements Encodeable<Writer, Stream<T>> {
         }
         Object[] array = value.toArray();
         if (array.length == 0) {
-            out.writeArrayB(0, componentEncoder, array);
+            out.writeArrayB(0, this, componentEncoder, array);
             out.writeArrayE();
             return;
         }
@@ -77,7 +77,7 @@ public class StreamEncoder<T> implements Encodeable<Writer, Stream<T>> {
                 }
             }
         }
-        if (out.writeArrayB(array.length, componentEncoder, array) < 0) {
+        if (out.writeArrayB(array.length, this, componentEncoder, array) < 0) {
             boolean first = true;
             for (Object v : array) {
                 if (!first) out.writeArrayMark();

src/org/redkale/convert/Writer.java

@@ -126,9 +126,9 @@ public abstract class Writer {
         }
         if (value == null) return;
         if (tiny()) {
-            if (member.istring) {
+            if (member.string) {
                 if (((CharSequence) value).length() == 0) return;
-            } else if (member.isbool) {
+            } else if (member.bool) {
                 if (!((Boolean) value)) return;
             }
         }
@@ -186,12 +186,13 @@ public abstract class Writer {
      * 输出一个数组前的操作
      *
      * @param size             数组长度
+     * @param arrayEncoder     Encodeable 可能是ArrayEncoder、CollectionEncoder或StreamEncoder
      * @param componentEncoder Encodeable
      * @param obj              对象, 不一定是数组、Collection对象，也可能是伪Collection对象
      *
      * @return 返回-1表示还没有写入对象内容，大于-1表示已写入对象内容，返回对象内容大小
      */
-    public abstract int writeArrayB(int size, Encodeable<Writer, Object> componentEncoder, Object obj);
+    public abstract int writeArrayB(int size, Encodeable arrayEncoder, Encodeable<Writer, Object> componentEncoder, Object obj);
 
     /**
      * 输出数组元素间的间隔符

src/org/redkale/convert/bson/BsonConvert.java

@@ -39,9 +39,9 @@ import org.redkale.util.*;
  */
 public class BsonConvert extends BinaryConvert<BsonReader, BsonWriter> {
 
-    private static final ObjectPool<BsonReader> readerPool = BsonReader.createPool(Integer.getInteger("convert.bson.pool.size", 16));
+    private static final ObjectPool<BsonReader> readerPool = BsonReader.createPool(Integer.getInteger("convert.bson.pool.size", Integer.getInteger("convert.pool.size", 16)));
 
-    private static final ObjectPool<BsonWriter> writerPool = BsonWriter.createPool(Integer.getInteger("convert.bson.pool.size", 16));
+    private static final ObjectPool<BsonWriter> writerPool = BsonWriter.createPool(Integer.getInteger("convert.bson.pool.size", Integer.getInteger("convert.pool.size", 16)));
 
     private final boolean tiny;
 
@@ -115,11 +115,12 @@ public class BsonConvert extends BinaryConvert<BsonReader, BsonWriter> {
         return convertFrom(type, bytes, 0, bytes.length);
     }
 
+    @Override
     @SuppressWarnings("unchecked")
-    public <T> T convertFrom(final Type type, final byte[] bytes, final int start, final int len) {
+    public <T> T convertFrom(final Type type, final byte[] bytes, final int offset, final int len) {
         if (type == null) return null;
         final BsonReader in = readerPool.get();
-        in.setBytes(bytes, start, len);
+        in.setBytes(bytes, offset, len);
         @SuppressWarnings("unchecked")
         T rs = (T) factory.loadDecoder(type).convertFrom(in);
         readerPool.accept(in);
@@ -177,6 +178,16 @@ public class BsonConvert extends BinaryConvert<BsonReader, BsonWriter> {
         return result;
     }
 
+    @Override
+    public byte[] convertToBytes(final Object value) {
+        return convertTo(value);
+    }
+
+    @Override
+    public byte[] convertToBytes(final Type type, final Object value) {
+        return convertTo(type, value);
+    }
+
     @Override
     public byte[] convertMapTo(final Object... values) {
         if (values == null) return null;

src/org/redkale/convert/bson/BsonFactory.java

@@ -24,7 +24,7 @@ import org.redkale.util.*;
 @SuppressWarnings("unchecked")
 public final class BsonFactory extends ConvertFactory<BsonReader, BsonWriter> {
 
-    private static final BsonFactory instance = new BsonFactory(null, Boolean.getBoolean("convert.bson.tiny"));
+    private static final BsonFactory instance = new BsonFactory(null, getSystemPropertyBoolean("convert.bson.tiny", "convert.tiny", true));
 
     static final Decodeable objectDecoder = instance.loadDecoder(Object.class);
 
@@ -69,7 +69,7 @@ public final class BsonFactory extends ConvertFactory<BsonReader, BsonWriter> {
     }
 
     public static BsonFactory create() {
-        return new BsonFactory(null, Boolean.getBoolean("convert.bson.tiny"));
+        return new BsonFactory(null, getSystemPropertyBoolean("convert.bson.tiny", "convert.tiny", true));
     }
 
     @Override

src/org/redkale/convert/bson/BsonWriter.java

@@ -20,7 +20,7 @@ import org.redkale.util.*;
  */
 public class BsonWriter extends Writer {
 
-    private static final int defaultSize = Integer.getInteger("convert.bson.writer.buffer.defsize", 1024);
+    private static final int defaultSize = Integer.getInteger("convert.bson.writer.buffer.defsize", Integer.getInteger("convert.writer.buffer.defsize", 1024));
 
     private byte[] content;
 
@@ -135,7 +135,7 @@ public class BsonWriter extends Writer {
             writeNull();
             return;
         }
-        writeArrayB(values.length, null, values);
+        writeArrayB(values.length, null, null, values);
         boolean flag = false;
         for (byte v : values) {
             if (flag) writeArrayMark();
@@ -254,7 +254,7 @@ public class BsonWriter extends Writer {
     }
 
     @Override
-    public final int writeArrayB(int size, Encodeable<Writer, Object> componentEncoder, Object obj) {
+    public final int writeArrayB(int size, Encodeable arrayEncoder, Encodeable<Writer, Object> componentEncoder, Object obj) {
         writeInt(size);
         if (componentEncoder != null && componentEncoder != ByteSimpledCoder.instance) {
             writeByte(BsonFactory.typeEnum(componentEncoder.getType()));

src/org/redkale/convert/ext/BoolArraySimpledCoder.java

@@ -29,7 +29,7 @@ public final class BoolArraySimpledCoder<R extends Reader, W extends Writer> ext
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, BoolSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, BoolSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (boolean v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/ByteBufferSimpledCoder.java

@@ -30,7 +30,7 @@ public final class ByteBufferSimpledCoder<R extends Reader, W extends Writer> ex
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(value.remaining(), ByteSimpledCoder.instance, value) < 0) {
+        if (out.writeArrayB(value.remaining(), this, ByteSimpledCoder.instance, value) < 0) {
             boolean flag = false;
             for (byte v : value.array()) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/CharArraySimpledCoder.java

@@ -29,7 +29,7 @@ public final class CharArraySimpledCoder<R extends Reader, W extends Writer> ext
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, CharSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, CharSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (char v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/DoubleArraySimpledCoder.java

@@ -30,7 +30,7 @@ public final class DoubleArraySimpledCoder<R extends Reader, W extends Writer> e
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, DoubleSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, DoubleSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (double v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/EnumSimpledCoder.java

@@ -22,8 +22,6 @@ import org.redkale.convert.Writer;
  */
 public final class EnumSimpledCoder<R extends Reader, W extends Writer, E extends Enum> extends SimpledCoder<R, W, E> {
 
-    private final Class<E> type;
-
     public EnumSimpledCoder(Class<E> type) {
         this.type = type;
     }
@@ -42,11 +40,11 @@ public final class EnumSimpledCoder<R extends Reader, W extends Writer, E extend
     public E convertFrom(final R in) {
         String value = in.readSmallString();
         if (value == null) return null;
-        return (E) Enum.valueOf(type, value);
+        return (E) Enum.valueOf((Class<E>) type, value);
     }
 
     @Override
     public Class<E> getType() {
-        return type;
+        return (Class<E>) type;
     }
 }

src/org/redkale/convert/ext/FloatArraySimpledCoder.java

@@ -29,7 +29,7 @@ public final class FloatArraySimpledCoder<R extends Reader, W extends Writer> ex
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, FloatSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, FloatSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (float v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/IntArraySimpledCoder.java

@@ -30,7 +30,7 @@ public final class IntArraySimpledCoder<R extends Reader, W extends Writer> exte
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, IntSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, IntSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (int v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/LongArraySimpledCoder.java

@@ -30,7 +30,7 @@ public final class LongArraySimpledCoder<R extends Reader, W extends Writer> ext
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, LongSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, LongSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (long v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/ShortArraySimpledCoder.java

@@ -29,7 +29,7 @@ public final class ShortArraySimpledCoder<R extends Reader, W extends Writer> ex
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, ShortSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, ShortSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (short v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/ext/StringArraySimpledCoder.java

@@ -27,7 +27,7 @@ public final class StringArraySimpledCoder<R extends Reader, W extends Writer> e
             out.writeNull();
             return;
         }
-        if (out.writeArrayB(values.length, StringSimpledCoder.instance, values) < 0) {
+        if (out.writeArrayB(values.length, this, StringSimpledCoder.instance, values) < 0) {
             boolean flag = false;
             for (String v : values) {
                 if (flag) out.writeArrayMark();

src/org/redkale/convert/json/JsonByteBufferReader.java

@@ -69,6 +69,10 @@ public class JsonByteBufferReader extends JsonReader {
      */
     @Override
     protected final char nextChar() {
+        return nextChar(null);
+    }
+
+    protected final char nextChar(StringBuilder sb) {
         if (currentChar != 0) {
             char ch = currentChar;
             this.currentChar = 0;
@@ -78,14 +82,18 @@ public class JsonByteBufferReader extends JsonReader {
             int remain = this.currentBuffer.remaining();
             if (remain == 0 && this.currentIndex + 1 >= this.buffers.length) return 0;
         }
-        byte b1 = nextByte();
+        byte b = nextByte();
-        if (b1 >= 0) {// 1 byte, 7 bits: 0xxxxxxx
+        if (b >= 0) {// 1 byte, 7 bits: 0xxxxxxx
-            return (char) b1;
+            return (char) b;
-        } else if ((b1 >> 5) == -2 && (b1 & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx
+        } else if ((b >> 5) == -2 && (b & 0x1e) != 0) { // 2 bytes, 11 bits: 110xxxxx 10xxxxxx
-            return (char) (((b1 << 6) ^ nextByte()) ^ (((byte) 0xC0 << 6) ^ ((byte) 0x80)));
+            return (char) (((b << 6) ^ nextByte()) ^ (((byte) 0xC0 << 6) ^ ((byte) 0x80)));
-        } else if ((b1 >> 4) == -2) { // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
+        } else if ((b >> 4) == -2) { // 3 bytes, 16 bits: 1110xxxx 10xxxxxx 10xxxxxx
-            return (char) ((b1 << 12) ^ (nextByte() << 6) ^ (nextByte() ^ (((byte) 0xE0 << 12) ^ ((byte) 0x80 << 6) ^ ((byte) 0x80))));
+            return (char) ((b << 12) ^ (nextByte() << 6) ^ (nextByte() ^ (((byte) 0xE0 << 12) ^ ((byte) 0x80 << 6) ^ ((byte) 0x80))));
-        } else { // 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+        } else if ((b >> 3) == -2) {// 4 bytes, 21 bits: 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
+            int uc = ((b << 18) ^ (nextByte() << 12) ^ (nextByte() << 6) ^ (nextByte() ^ (((byte) 0xF0 << 18) ^ ((byte) 0x80 << 12) ^ ((byte) 0x80 << 6) ^ ((byte) 0x80))));
+            if (sb != null) sb.append(Character.highSurrogate(uc));
+            return Character.lowSurrogate(uc);
+        } else {
             throw new RuntimeException(new UnmappableCharacterException(4));
         }
     }
@@ -208,9 +216,9 @@ public class JsonByteBufferReader extends JsonReader {
         if (ch == '"' || ch == '\'') {
             final char quote = ch;
             for (;;) {
-                ch = nextChar();
+                ch = nextChar(sb);
                 if (ch == '\\') {
-                    char c = nextChar();
+                    char c = nextChar(sb);
                     switch (c) {
                         case '"':
                         case '\'':
@@ -249,9 +257,9 @@ public class JsonByteBufferReader extends JsonReader {
         } else {
             sb.append(ch);
             for (;;) {
-                ch = nextChar();
+                ch = nextChar(sb);
                 if (ch == '\\') {
-                    char c = nextChar();
+                    char c = nextChar(sb);
                     switch (c) {
                         case '"':
                         case '\'':

src/org/redkale/convert/json/JsonConvert.java

@@ -11,6 +11,7 @@ import java.nio.*;
 import java.nio.charset.*;
 import java.util.function.*;
 import org.redkale.convert.*;
+import org.redkale.service.RetResult;
 import org.redkale.util.*;
 
 /**
@@ -26,9 +27,12 @@ public class JsonConvert extends TextConvert<JsonReader, JsonWriter> {
     public static final Type TYPE_MAP_STRING_STRING = new TypeToken<java.util.HashMap<String, String>>() {
     }.getType();
 
-    private static final ObjectPool<JsonReader> readerPool = JsonReader.createPool(Integer.getInteger("convert.json.pool.size", 16));
+    public static final Type TYPE_RETRESULT_STRING = new TypeToken<RetResult<String>>() {
+    }.getType();
 
-    private static final ObjectPool<JsonWriter> writerPool = JsonWriter.createPool(Integer.getInteger("convert.json.pool.size", 16));
+    private static final ObjectPool<JsonReader> readerPool = JsonReader.createPool(Integer.getInteger("convert.json.pool.size", Integer.getInteger("convert.pool.size", 16)));
+
+    private static final ObjectPool<JsonWriter> writerPool = JsonWriter.createPool(Integer.getInteger("convert.json.pool.size", Integer.getInteger("convert.pool.size", 16)));
 
     private final boolean tiny;
 
@@ -106,6 +110,12 @@ public class JsonConvert extends TextConvert<JsonReader, JsonWriter> {
         return convertFrom(type, new String(bytes, StandardCharsets.UTF_8));
     }
 
+    @Override
+    public <T> T convertFrom(final Type type, final byte[] bytes, final int offset, final int length) {
+        if (bytes == null) return null;
+        return convertFrom(type, new String(bytes, offset, length, StandardCharsets.UTF_8));
+    }
+
     public <T> T convertFrom(final Type type, final String text) {
         if (text == null) return null;
         return convertFrom(type, Utility.charArray(text));
@@ -116,10 +126,10 @@ public class JsonConvert extends TextConvert<JsonReader, JsonWriter> {
         return convertFrom(type, text, 0, text.length);
     }
 
-    public <T> T convertFrom(final Type type, final char[] text, final int start, final int len) {
+    public <T> T convertFrom(final Type type, final char[] text, final int offset, final int length) {
         if (text == null || type == null) return null;
         final JsonReader in = readerPool.get();
-        in.setText(text, start, len);
+        in.setText(text, offset, length);
         T rs = (T) factory.loadDecoder(type).convertFrom(in);
         readerPool.accept(in);
         return rs;
@@ -162,10 +172,10 @@ public class JsonConvert extends TextConvert<JsonReader, JsonWriter> {
     }
 
     //返回非null的值是由String、ArrayList、HashMap任意组合的对象
-    public <V> V convertFrom(final char[] text, final int start, final int len) {
+    public <V> V convertFrom(final char[] text, final int offset, final int length) {
         if (text == null) return null;
         final JsonReader in = readerPool.get();
-        in.setText(text, start, len);
+        in.setText(text, offset, length);
         Object rs = new AnyDecoder(factory).convertFrom(in);
         readerPool.accept(in);
         return (V) rs;
@@ -214,6 +224,25 @@ public class JsonConvert extends TextConvert<JsonReader, JsonWriter> {
         return result;
     }
 
+    @Override
+    public byte[] convertToBytes(final Object value) {
+        if (value == null) return null;
+        String result = convertTo(value.getClass(), value);
+        return result == null ? null : result.getBytes(StandardCharsets.UTF_8);
+    }
+
+    @Override
+    public byte[] convertToBytes(final Type type, final Object value) {
+        if (type == null) return null;
+        if (value == null) return null;
+        final JsonWriter writer = pollJsonWriter();
+        writer.specify(type);
+        factory.loadEncoder(type).convertTo(writer, value);
+        String result = writer.toString();
+        writerPool.accept(writer);
+        return result == null ? null : result.getBytes(StandardCharsets.UTF_8);
+    }
+
     @Override
     public String convertMapTo(final Object... values) {
         if (values == null) return "null";

src/org/redkale/convert/json/JsonFactory.java

@@ -23,7 +23,7 @@ import org.redkale.util.*;
 @SuppressWarnings("unchecked")
 public final class JsonFactory extends ConvertFactory<JsonReader, JsonWriter> {
 
-    private static final JsonFactory instance = new JsonFactory(null, Boolean.getBoolean("convert.json.tiny"));
+    private static final JsonFactory instance = new JsonFactory(null, getSystemPropertyBoolean("convert.json.tiny", "convert.tiny", true));
 
     static {
 
@@ -60,7 +60,7 @@ public final class JsonFactory extends ConvertFactory<JsonReader, JsonWriter> {
     }
 
     public static JsonFactory create() {
-        return new JsonFactory(null, Boolean.getBoolean("convert.json.tiny"));
+        return new JsonFactory(null, getSystemPropertyBoolean("convert.json.tiny", "convert.tiny", true));
     }
 
     @Override

src/org/redkale/convert/json/JsonWriter.java

@@ -24,7 +24,7 @@ public class JsonWriter extends Writer {
 
     private static final char[] CHARS_FALSEVALUE = "false".toCharArray();
 
-    private static final int defaultSize = Integer.getInteger("convert.json.writer.buffer.defsize", 1024);
+    private static final int defaultSize = Integer.getInteger("convert.json.writer.buffer.defsize", Integer.getInteger("convert.writer.buffer.defsize", 1024));
 
     private int count;
 
@@ -197,7 +197,7 @@ public class JsonWriter extends Writer {
             writeNull();
             return;
         }
-        writeArrayB(values.length, null, values);
+        writeArrayB(values.length, null, null, values);
         boolean flag = false;
         for (byte v : values) {
             if (flag) writeArrayMark();
@@ -354,7 +354,7 @@ public class JsonWriter extends Writer {
     }
 
     @Override
-    public final int writeArrayB(int size, Encodeable<Writer, Object> componentEncoder, Object obj) {
+    public final int writeArrayB(int size, Encodeable arrayEncoder, Encodeable<Writer, Object> componentEncoder, Object obj) {
         writeTo('[');
         return -1;
     }

src/org/redkale/mq/HttpMessageClient.java

@@ -0,0 +1,189 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.lang.reflect.Type;
+import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Logger;
+import org.redkale.convert.json.JsonConvert;
+import org.redkale.net.http.*;
+
+/**
+ * 不依赖MessageRecord则可兼容RPC方式
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpMessageClient extends MessageClient {
+
+    protected final Logger logger = Logger.getLogger(this.getClass().getSimpleName());
+
+    protected HttpMessageClient(MessageAgent messageAgent) {
+        super(messageAgent);
+        if (messageAgent != null) { // //RPC方式下无messageAgent
+            this.respTopic = messageAgent.generateHttpRespTopic();
+        }
+    }
+
+    //格式: http.req.user
+    public String generateHttpReqTopic(String module) {
+        return MessageAgent.generateHttpReqTopic(module);
+    }
+
+    //格式: http.req.user-n10
+    public String generateHttpReqTopic(String module, String resname) {
+        return MessageAgent.generateHttpReqTopic(module, resname);
+    }
+
+    public String generateHttpReqTopic(HttpSimpleRequest request, String path) {
+        String module = request.getRequestURI();
+        if (path != null && !path.isEmpty() && module.startsWith(path)) module = module.substring(path.length());
+        module = module.substring(1); //去掉/
+        module = module.substring(0, module.indexOf('/'));
+        Map<String, String> headers = request.getHeaders();
+        String resname = headers == null ? "" : headers.getOrDefault(Rest.REST_HEADER_RESOURCE_NAME, "");
+        return MessageAgent.generateHttpReqTopic(module, resname);
+    }
+
+    public final void produceMessage(HttpSimpleRequest request) {
+        produceMessage(generateHttpReqTopic(request, null), 0, null, request, null);
+    }
+
+    public final void produceMessage(HttpSimpleRequest request, AtomicLong counter) {
+        produceMessage(generateHttpReqTopic(request, null), 0, null, request, counter);
+    }
+
+    public final void produceMessage(int userid, HttpSimpleRequest request) {
+        produceMessage(generateHttpReqTopic(request, null), userid, null, request, null);
+    }
+
+    public final void produceMessage(int userid, String groupid, HttpSimpleRequest request) {
+        produceMessage(generateHttpReqTopic(request, null), userid, groupid, request, null);
+    }
+
+    public final void produceMessage(int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        produceMessage(generateHttpReqTopic(request, null), userid, groupid, request, counter);
+    }
+
+    public final void produceMessage(String topic, HttpSimpleRequest request) {
+        produceMessage(topic, 0, null, request, null);
+    }
+
+    public final void produceMessage(String topic, HttpSimpleRequest request, AtomicLong counter) {
+        produceMessage(topic, 0, null, request, counter);
+    }
+
+    public final void produceMessage(String topic, int userid, String groupid, HttpSimpleRequest request) {
+        produceMessage(topic, userid, groupid, request, null);
+    }
+
+    public final void broadcastMessage(HttpSimpleRequest request) {
+        broadcastMessage(generateHttpReqTopic(request, null), 0, null, request, null);
+    }
+
+    public final void broadcastMessage(HttpSimpleRequest request, AtomicLong counter) {
+        broadcastMessage(generateHttpReqTopic(request, null), 0, null, request, counter);
+    }
+
+    public final void broadcastMessage(int userid, HttpSimpleRequest request) {
+        broadcastMessage(generateHttpReqTopic(request, null), userid, null, request, null);
+    }
+
+    public final void broadcastMessage(int userid, String groupid, HttpSimpleRequest request) {
+        broadcastMessage(generateHttpReqTopic(request, null), userid, groupid, request, null);
+    }
+
+    public final void broadcastMessage(int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        broadcastMessage(generateHttpReqTopic(request, null), userid, groupid, request, counter);
+    }
+
+    public final void broadcastMessage(String topic, HttpSimpleRequest request) {
+        broadcastMessage(topic, 0, null, request, null);
+    }
+
+    public final void broadcastMessage(String topic, HttpSimpleRequest request, AtomicLong counter) {
+        broadcastMessage(topic, 0, null, request, counter);
+    }
+
+    public final void broadcastMessage(String topic, int userid, String groupid, HttpSimpleRequest request) {
+        broadcastMessage(topic, userid, groupid, request, null);
+    }
+
+    public final <T> CompletableFuture<T> sendMessage(HttpSimpleRequest request, Type type) {
+        return sendMessage(generateHttpReqTopic(request, null), 0, null, request, null).thenApply((HttpResult<byte[]> httbs) -> {
+            if (httbs == null || httbs.getResult() == null) return null;
+            return JsonConvert.root().convertFrom(type, httbs.getResult());
+        });
+    }
+
+    public final <T> CompletableFuture<T> sendMessage(int userid, HttpSimpleRequest request, Type type) {
+        return sendMessage(generateHttpReqTopic(request, null), userid, null, request, null).thenApply((HttpResult<byte[]> httbs) -> {
+            if (httbs == null || httbs.getResult() == null) return null;
+            return JsonConvert.root().convertFrom(type, httbs.getResult());
+        });
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(HttpSimpleRequest request) {
+        return sendMessage(generateHttpReqTopic(request, null), 0, null, request, null);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(HttpSimpleRequest request, AtomicLong counter) {
+        return sendMessage(generateHttpReqTopic(request, null), 0, null, request, counter);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(int userid, HttpSimpleRequest request) {
+        return sendMessage(generateHttpReqTopic(request, null), userid, null, request, null);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(int userid, String groupid, HttpSimpleRequest request) {
+        return sendMessage(generateHttpReqTopic(request, null), userid, groupid, request, null);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        return sendMessage(generateHttpReqTopic(request, null), userid, groupid, request, counter);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(String topic, HttpSimpleRequest request) {
+        return sendMessage(topic, 0, null, request, null);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(String topic, HttpSimpleRequest request, AtomicLong counter) {
+        return sendMessage(topic, 0, null, request, counter);
+    }
+
+    public final CompletableFuture<HttpResult<byte[]>> sendMessage(String topic, int userid, String groupid, HttpSimpleRequest request) {
+        return sendMessage(topic, userid, null, request, (AtomicLong) null);
+    }
+
+    public CompletableFuture<HttpResult<byte[]>> sendMessage(String topic, int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        MessageRecord message = new MessageRecord(topic, null, HttpSimpleRequestCoder.getInstance().encode(request));
+        message.userid(userid).groupid(groupid);
+        return sendMessage(message, true, counter).thenApply(r -> r.decodeContent(HttpResultCoder.getInstance()));
+    }
+
+    public void broadcastMessage(String topic, int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        MessageRecord message = new MessageRecord(topic, null, HttpSimpleRequestCoder.getInstance().encode(request));
+        message.userid(userid).groupid(groupid);
+        sendMessage(message, false, counter);
+    }
+
+    public void produceMessage(String topic, int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        MessageRecord message = new MessageRecord(topic, null, HttpSimpleRequestCoder.getInstance().encode(request));
+        message.userid(userid).groupid(groupid);
+        sendMessage(message, false, counter);
+    }
+
+    @Override
+    protected MessageProducers getProducer() {
+        return messageAgent.getHttpProducer();
+    }
+}

src/org/redkale/mq/HttpMessageClusterClient.java

@@ -0,0 +1,179 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.net.*;
+import java.nio.charset.StandardCharsets;
+import java.time.Duration;
+import java.util.*;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.*;
+import java.util.logging.Level;
+import org.redkale.cluster.ClusterAgent;
+import org.redkale.net.http.*;
+
+/**
+ * 没有配置MQ的情况下依赖ClusterAgent实现的默认HttpMessageClient实例
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpMessageClusterClient extends HttpMessageClient {
+
+    //jdk.internal.net.http.common.Utils.DISALLOWED_HEADERS_SET
+    private static final Set<String> DISALLOWED_HEADERS_SET = Set.of("connection", "content-length",
+        "date", "expect", "from", "host", "origin",
+        "referer", "upgrade", "via", "warning");
+
+    protected ClusterAgent clusterAgent;
+
+    protected java.net.http.HttpClient httpClient;
+
+    public HttpMessageClusterClient(ClusterAgent clusterAgent) {
+        super(null);
+        Objects.requireNonNull(clusterAgent);
+        this.clusterAgent = clusterAgent;
+        this.httpClient = java.net.http.HttpClient.newHttpClient();
+    }
+
+    @Override
+    public CompletableFuture<HttpResult<byte[]>> sendMessage(String topic, int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        return httpAsync(userid, request);
+    }
+
+    @Override
+    public void produceMessage(String topic, int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        httpAsync(userid, request);
+    }
+
+    @Override
+    public void broadcastMessage(String topic, int userid, String groupid, HttpSimpleRequest request, AtomicLong counter) {
+        mqtpAsync(userid, request);
+    }
+
+    private CompletableFuture<HttpResult<byte[]>> mqtpAsync(int userid, HttpSimpleRequest req) {
+        final boolean finest = logger.isLoggable(Level.FINEST);
+        String module = req.getRequestURI();
+        module = module.substring(1); //去掉/
+        module = module.substring(0, module.indexOf('/'));
+        Map<String, String> headers = req.getHeaders();
+        String resname = headers == null ? "" : headers.getOrDefault(Rest.REST_HEADER_RESOURCE_NAME, "");
+        return clusterAgent.queryMqtpAddress("mqtp", module, resname).thenCompose(addrmap -> {
+            if (addrmap == null || addrmap.isEmpty()) return new HttpResult().status(404).toAnyFuture();
+            java.net.http.HttpRequest.Builder builder = java.net.http.HttpRequest.newBuilder().timeout(Duration.ofMillis(30000));
+            if (req.isRpc()) builder.header(Rest.REST_HEADER_RPC_NAME, "true");
+            if (req.isFrombody()) builder.header(Rest.REST_HEADER_PARAM_FROM_BODY, "true");
+            if (req.getReqConvertType() != null) builder.header(Rest.REST_HEADER_REQ_CONVERT_TYPE, req.getReqConvertType().toString());
+            if (req.getRespConvertType() != null) builder.header(Rest.REST_HEADER_RESP_CONVERT_TYPE, req.getRespConvertType().toString());
+            if (userid != 0) builder.header(Rest.REST_HEADER_CURRUSERID_NAME, "" + userid);
+            if (headers != null) headers.forEach((n, v) -> {
+                    if (!DISALLOWED_HEADERS_SET.contains(n.toLowerCase())) builder.header(n, v);
+                });
+            builder.header("Content-Type", "x-www-form-urlencoded");
+            if (req.getBody() != null && req.getBody().length > 0) {
+                String paramstr = req.getParametersToString();
+                if (paramstr != null) {
+                    if (req.getRequestURI().indexOf('?') > 0) {
+                        req.setRequestURI(req.getRequestURI() + "&" + paramstr);
+                    } else {
+                        req.setRequestURI(req.getRequestURI() + "?" + paramstr);
+                    }
+                }
+                builder.POST(java.net.http.HttpRequest.BodyPublishers.ofByteArray(req.getBody()));
+            } else {
+                String paramstr = req.getParametersToString();
+                if (paramstr != null) builder.POST(java.net.http.HttpRequest.BodyPublishers.ofString(paramstr));
+            }
+            List<CompletableFuture> futures = new ArrayList<>();
+            for (Map.Entry<String, Collection<InetSocketAddress>> en : addrmap.entrySet()) {
+                String realmodule = en.getKey();
+                Collection<InetSocketAddress> addrs = en.getValue();
+                if (addrs == null || addrs.isEmpty()) continue;
+                String suburi = req.getRequestURI();
+                suburi = suburi.substring(1); //跳过 /
+                suburi = "/" + realmodule + suburi.substring(suburi.indexOf('/'));
+                futures.add(forEachCollectionFuture(finest, userid, req, (req.getPath() != null && !req.getPath().isEmpty() ? req.getPath() : "") + suburi, builder, addrs.iterator()));
+            }
+            if (futures.isEmpty()) return CompletableFuture.completedFuture(null);
+            return CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).thenApply(v -> null);
+        });
+    }
+
+    private CompletableFuture<HttpResult<byte[]>> httpAsync(int userid, HttpSimpleRequest req) {
+        final boolean finest = logger.isLoggable(Level.FINEST);
+        String module = req.getRequestURI();
+        module = module.substring(1); //去掉/
+        module = module.substring(0, module.indexOf('/'));
+        Map<String, String> headers = req.getHeaders();
+        String resname = headers == null ? "" : headers.getOrDefault(Rest.REST_HEADER_RESOURCE_NAME, "");
+        return clusterAgent.queryHttpAddress("http", module, resname).thenCompose(addrs -> {
+            if (addrs == null || addrs.isEmpty()) return new HttpResult().status(404).toAnyFuture();
+            java.net.http.HttpRequest.Builder builder = java.net.http.HttpRequest.newBuilder().timeout(Duration.ofMillis(30000));
+            if (req.isRpc()) builder.header(Rest.REST_HEADER_RPC_NAME, "true");
+            if (req.isFrombody()) builder.header(Rest.REST_HEADER_PARAM_FROM_BODY, "true");
+            if (req.getReqConvertType() != null) builder.header(Rest.REST_HEADER_REQ_CONVERT_TYPE, req.getReqConvertType().toString());
+            if (req.getRespConvertType() != null) builder.header(Rest.REST_HEADER_RESP_CONVERT_TYPE, req.getRespConvertType().toString());
+            if (userid != 0) builder.header(Rest.REST_HEADER_CURRUSERID_NAME, "" + userid);
+            if (headers != null) headers.forEach((n, v) -> {
+                    if (!DISALLOWED_HEADERS_SET.contains(n.toLowerCase())) builder.header(n, v);
+                });
+            builder.header("Content-Type", "x-www-form-urlencoded");
+            if (req.getBody() != null && req.getBody().length > 0) {
+                String paramstr = req.getParametersToString();
+                if (paramstr != null) {
+                    if (req.getRequestURI().indexOf('?') > 0) {
+                        req.setRequestURI(req.getRequestURI() + "&" + paramstr);
+                    } else {
+                        req.setRequestURI(req.getRequestURI() + "?" + paramstr);
+                    }
+                }
+                builder.POST(java.net.http.HttpRequest.BodyPublishers.ofByteArray(req.getBody()));
+            } else {
+                String paramstr = req.getParametersToString();
+                if (paramstr != null) builder.POST(java.net.http.HttpRequest.BodyPublishers.ofString(paramstr));
+            }
+            return forEachCollectionFuture(finest, userid, req, (req.getPath() != null && !req.getPath().isEmpty() ? req.getPath() : "") + req.getRequestURI(), builder, addrs.iterator());
+        });
+    }
+
+    private CompletableFuture<HttpResult<byte[]>> forEachCollectionFuture(boolean finest, int userid, HttpSimpleRequest req, String requesturi, java.net.http.HttpRequest.Builder builder, Iterator<InetSocketAddress> it) {
+        if (!it.hasNext()) return CompletableFuture.completedFuture(null);
+        InetSocketAddress addr = it.next();
+        String url = "http://" + addr.getHostString() + ":" + addr.getPort() + requesturi;
+        return httpClient.sendAsync(builder.copy().uri(URI.create(url)).build(), java.net.http.HttpResponse.BodyHandlers.ofByteArray()).thenCompose(resp -> {
+            if (resp.statusCode() != 200) return forEachCollectionFuture(finest, userid, req, requesturi, builder, it);
+            HttpResult rs = new HttpResult();
+            java.net.http.HttpHeaders hs = resp.headers();
+            if (hs != null) {
+                Map<String, List<String>> hm = hs.map();
+                if (hm != null) {
+                    for (Map.Entry<String, List<String>> en : hm.entrySet()) {
+                        if ("date".equals(en.getKey()) || "content-type".equals(en.getKey())
+                            || "server".equals(en.getKey()) || "connection".equals(en.getKey())) continue;
+                        List<String> val = en.getValue();
+                        if (val != null && val.size() == 1) {
+                            rs.header(en.getKey(), val.get(0));
+                        }
+                    }
+                }
+            }
+            rs.setResult(resp.body());
+            if (finest) {
+                StringBuilder sb = new StringBuilder();
+                Map<String, String> params = req.getParams();
+                if (params != null && !params.isEmpty()) {
+                    params.forEach((n, v) -> sb.append('&').append(n).append('=').append(v));
+                }
+                logger.log(Level.FINEST, url + "?userid=" + userid + sb + ", result = " + new String(resp.body(), StandardCharsets.UTF_8));
+            }
+            return CompletableFuture.completedFuture(rs);
+        });
+    }
+}

src/org/redkale/mq/HttpMessageProcessor.java

@@ -0,0 +1,136 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.*;
+import java.util.logging.*;
+import org.redkale.boot.NodeHttpServer;
+import org.redkale.net.http.*;
+import org.redkale.service.Service;
+import org.redkale.util.ThreadHashExecutor;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpMessageProcessor implements MessageProcessor {
+
+    protected final boolean finest;
+
+    protected final boolean finer;
+
+    protected final Logger logger;
+
+    protected final MessageProducers producer;
+
+    protected final NodeHttpServer server;
+
+    protected final ThreadHashExecutor workExecutor;
+
+    protected final Service service;
+
+    protected final HttpServlet servlet;
+
+    protected final boolean multiconsumer;
+
+    protected final String restmodule; //  前后有/, 例如: /user/
+
+    protected final String multimodule; //  前后有/, 例如: /userstat/
+
+    protected CountDownLatch cdl;
+
+    protected final Runnable innerCallback = () -> {
+        if (cdl != null) cdl.countDown();
+    };
+
+    public HttpMessageProcessor(Logger logger, ThreadHashExecutor workExecutor, MessageProducers producer, NodeHttpServer server, Service service, HttpServlet servlet) {
+        this.logger = logger;
+        this.finest = logger.isLoggable(Level.FINEST);
+        this.finer = logger.isLoggable(Level.FINER);
+        this.producer = producer;
+        this.server = server;
+        this.service = service;
+        this.servlet = servlet;
+        MessageMultiConsumer mmc = service.getClass().getAnnotation(MessageMultiConsumer.class);
+        this.multiconsumer = mmc != null;
+        this.restmodule = "/" + Rest.getRestModule(service) + "/";
+        this.multimodule = mmc != null ? ("/" + mmc.module() + "/") : null;
+        this.workExecutor = workExecutor;
+    }
+
+    @Override
+    public void begin(final int size) {
+        if (this.workExecutor != null) this.cdl = new CountDownLatch(size);
+    }
+
+    @Override
+    public void process(final MessageRecord message, final Runnable callback) {
+        if (this.workExecutor == null) {
+            execute(message, innerCallback);
+        } else {
+            this.workExecutor.execute(message.hash(), () -> execute(message, innerCallback));
+        }
+    }
+
+    private void execute(final MessageRecord message, final Runnable callback) {
+        HttpMessageRequest request = null;
+        try {
+            long cha = System.currentTimeMillis() - message.createtime;
+            if (cha > 50 || finer) {
+                logger.log(Level.FINER, "HttpMessageProcessor.process (mq.delay = " + cha + " ms) message: " + message);
+            } else if (finest) {
+                logger.log(Level.FINEST, "HttpMessageProcessor.process (mq.delay = " + cha + " ms) message: " + message);
+            }
+            if (multiconsumer) message.setResptopic(null); //不容许有响应
+            HttpContext context = server.getHttpServer().getContext();
+            request = new HttpMessageRequest(context, message);
+            if (multiconsumer) {
+                request.setRequestURI(request.getRequestURI().replaceFirst(this.multimodule, this.restmodule));
+            }
+            HttpMessageResponse response = new HttpMessageResponse(context, request, callback, null, null, producer.getProducer(message));
+            servlet.execute(request, response);
+        } catch (Throwable ex) {
+            if (message.getResptopic() != null && !message.getResptopic().isEmpty()) {
+                HttpMessageResponse.finishHttpResult(finest, request == null ? null : request.getRespConvert(),
+                    message, callback, producer.getProducer(message), message.getResptopic(), new HttpResult().status(500));
+            }
+            logger.log(Level.SEVERE, HttpMessageProcessor.class.getSimpleName() + " process error, message=" + message, ex instanceof CompletionException ? ((CompletionException) ex).getCause() : ex);
+        }
+    }
+
+    @Override
+    public void commit() {
+        if (this.cdl != null) {
+            try {
+                this.cdl.await(30, TimeUnit.SECONDS);
+            } catch (Exception ex) {
+            }
+            this.cdl = null;
+        }
+    }
+
+    public MessageProducers getProducer() {
+        return producer;
+    }
+
+    public NodeHttpServer getServer() {
+        return server;
+    }
+
+    public Service getService() {
+        return service;
+    }
+
+    public HttpServlet getServlet() {
+        return servlet;
+    }
+
+}

src/org/redkale/mq/HttpMessageRequest.java

@@ -0,0 +1,38 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import org.redkale.convert.Convert;
+import org.redkale.net.http.*;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpMessageRequest extends HttpRequest {
+
+    protected MessageRecord message;
+
+    public HttpMessageRequest(HttpContext context, MessageRecord message) {
+        super(context, message.decodeContent(HttpSimpleRequestCoder.getInstance()));
+        this.message = message;
+        this.currentUserid = message.getUserid();
+    }
+
+    public void setRequestURI(String uri) {
+        this.requestURI = uri;
+    }
+
+    @Override
+    public Convert getRespConvert() {
+        return this.respConvert;
+    }
+}

src/org/redkale/mq/HttpMessageResponse.java

@@ -0,0 +1,167 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.logging.Level;
+import org.redkale.convert.*;
+import org.redkale.net.Response;
+import org.redkale.net.http.*;
+import org.redkale.service.RetResult;
+import org.redkale.util.ObjectPool;
+
+/**
+ *
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpMessageResponse extends HttpResponse {
+
+    protected MessageRecord message;
+
+    protected MessageProducer producer;
+
+    protected boolean finest;
+
+    protected Runnable callback;
+
+    public HttpMessageResponse(HttpContext context, HttpMessageRequest request, Runnable callback,
+        ObjectPool<Response> responsePool, HttpResponseConfig config, MessageProducer producer) {
+        super(context, request, responsePool, config);
+        this.message = request.message;
+        this.callback = callback;
+        this.producer = producer;
+        this.finest = producer.logger.isLoggable(Level.FINEST);
+    }
+
+    public HttpMessageResponse(HttpContext context, MessageRecord message, Runnable callback, HttpResponseConfig config, MessageProducer producer) {
+        super(context, new HttpMessageRequest(context, message), null, config);
+        this.message = message;
+        this.callback = callback;
+        this.producer = producer;
+    }
+
+    public void finishHttpResult(HttpResult result) {
+        finishHttpResult(this.finest, ((HttpMessageRequest) this.request).getRespConvert(), this.message, this.callback, this.producer, message.getResptopic(), result);
+    }
+
+    public static void finishHttpResult(boolean finest, Convert respConvert, MessageRecord msg, Runnable callback, MessageProducer producer, String resptopic, HttpResult result) {
+        if (callback != null) callback.run();
+        if (resptopic == null || resptopic.isEmpty()) return;
+        if (result.getResult() instanceof RetResult) {
+            RetResult ret = (RetResult) result.getResult();
+            //必须要塞入retcode， 开发者可以无需反序列化ret便可确定操作是否返回成功
+            if (!ret.isSuccess()) result.header("retcode", String.valueOf(ret.getRetcode()));
+        }
+        if (result.convert() == null && respConvert != null) result.convert(respConvert);
+        if (finest) {
+            Object innerrs = result.getResult();
+            if (innerrs instanceof byte[]) innerrs = new String((byte[]) innerrs, StandardCharsets.UTF_8);
+            producer.logger.log(Level.FINEST, "HttpMessageProcessor.process seqid=" + msg.getSeqid() + ", content: " + innerrs + ", status: " + result.getStatus() + ", headers: " + result.getHeaders());
+        }
+        byte[] content = HttpResultCoder.getInstance().encode(result);
+        producer.apply(new MessageRecord(msg.getSeqid(), resptopic, null, content));
+    }
+
+    @Override
+    public void finishJson(org.redkale.service.RetResult ret) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        finishHttpResult(new HttpResult(ret.clearConvert(), ret));
+    }
+
+    @Override
+    public void finish(String obj) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        finishHttpResult(new HttpResult(obj == null ? "" : obj));
+    }
+
+    @Override
+    public void finish404() {
+        finish(404, null);
+    }
+
+    @Override
+    public void finish(int status, String message) {
+        if (finest) producer.logger.log(Level.FINEST, "HttpMessageResponse.finish status: " + status);
+        if (this.message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        finishHttpResult(new HttpResult(message == null ? "" : message).status(status));
+    }
+
+    @Override
+    public void finish(final Convert convert, HttpResult result) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        if (convert != null) result.convert(convert);
+        finishHttpResult(result);
+    }
+
+    @Override
+    public void finish(final byte[] bs) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        finishHttpResult(new HttpResult(bs));
+    }
+
+    @Override
+    public void finish(final String contentType, final byte[] bs) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        finishHttpResult(new HttpResult(bs).contentType(contentType));
+    }
+
+    @Override
+    public void finish(boolean kill, ByteBuffer buffer) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        byte[] bs = new byte[buffer.remaining()];
+        buffer.get(bs);
+        finishHttpResult(new HttpResult(bs));
+    }
+
+    @Override
+    public void finish(boolean kill, ByteBuffer... buffers) {
+        if (message.isEmptyResptopic()) {
+            if (callback != null) callback.run();
+            return;
+        }
+        int size = 0;
+        for (ByteBuffer buf : buffers) {
+            size += buf.remaining();
+        }
+        byte[] bs = new byte[size];
+        int index = 0;
+        for (ByteBuffer buf : buffers) {
+            int r = buf.remaining();
+            buf.get(bs, index, r);
+            index += r;
+        }
+        finishHttpResult(new HttpResult(bs));
+    }
+
+}

src/org/redkale/mq/HttpResultCoder.java

@@ -0,0 +1,132 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.net.HttpCookie;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.redkale.convert.*;
+import org.redkale.convert.json.JsonConvert;
+import static org.redkale.mq.MessageCoder.*;
+import org.redkale.net.http.HttpResult;
+import org.redkale.util.Utility;
+
+/**
+ * HttpResult的MessageCoder实现
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpResultCoder implements MessageCoder<HttpResult> {
+
+    private static final HttpResultCoder instance = new HttpResultCoder();
+
+    public static HttpResultCoder getInstance() {
+        return instance;
+    }
+
+    @Override
+    public byte[] encode(HttpResult data) {
+        if (data == null) return null;
+        byte[] contentType = MessageCoder.getBytes(data.getContentType());
+        byte[] headers = MessageCoder.getBytes(data.getHeaders());
+        byte[] cookies = getBytes(data.getCookies());
+        byte[] content;
+        if (data.getResult() == null) {
+            content = new byte[0]; //""
+        } else if (data.getResult() instanceof byte[]) {
+            content = (byte[]) data.getResult();
+        } else if (data.getResult() instanceof CharSequence) {
+            content = MessageCoder.getBytes(data.getResult().toString());
+        } else {
+            Convert cc = data.convert();
+            if (cc == null) cc = JsonConvert.root();
+            content = cc.convertToBytes(data.getResult());
+        }
+        int count = 4 + 2 + contentType.length + headers.length + cookies.length + 4 + (content == null ? 0 : content.length);
+        final byte[] bs = new byte[count];
+        ByteBuffer buffer = ByteBuffer.wrap(bs);
+        buffer.putInt(data.getStatus());
+        buffer.putChar((char) contentType.length);
+        if (contentType.length > 0) buffer.put(contentType);
+        buffer.put(headers);
+        buffer.put(cookies);
+        if (content == null || content.length == 0) {
+            buffer.putInt(0);
+        } else {
+            buffer.putInt(content.length);
+            buffer.put(content);
+        }
+        return bs;
+    }
+
+    @Override
+    public HttpResult<byte[]> decode(byte[] data) {
+        if (data == null) return null;
+        ByteBuffer buffer = ByteBuffer.wrap(data);
+        HttpResult result = new HttpResult();
+        result.setStatus(buffer.getInt());
+        result.setContentType(MessageCoder.getShortString(buffer));
+        result.setHeaders(MessageCoder.getMap(buffer));
+        result.setCookies(getCookieList(buffer));
+        int len = buffer.getInt();
+        if (len > 0) {
+            byte[] bs = new byte[len];
+            buffer.get(bs);
+            result.setResult(bs);
+        }
+        return result;
+    }
+
+    public static byte[] getBytes(final List<HttpCookie> list) {
+        if (list == null || list.isEmpty()) return new byte[2];
+        final AtomicInteger len = new AtomicInteger(2);
+        list.forEach(cookie -> {
+            len.addAndGet(2 + (cookie.getName() == null ? 0 : Utility.encodeUTF8Length(cookie.getName())));
+            len.addAndGet(2 + (cookie.getValue() == null ? 0 : Utility.encodeUTF8Length(cookie.getValue())));
+            len.addAndGet(2 + (cookie.getDomain() == null ? 0 : Utility.encodeUTF8Length(cookie.getDomain())));
+            len.addAndGet(2 + (cookie.getPath() == null ? 0 : Utility.encodeUTF8Length(cookie.getPath())));
+            len.addAndGet(2 + (cookie.getPortlist() == null ? 0 : Utility.encodeUTF8Length(cookie.getPortlist())));
+            len.addAndGet(8 + 1 + 1); //maxage Secure HttpOnly
+        });
+        final byte[] bs = new byte[len.get()];
+        final ByteBuffer buffer = ByteBuffer.wrap(bs);
+        buffer.putChar((char) list.size());
+        list.forEach(cookie -> {
+            putShortString(buffer, cookie.getName());
+            putShortString(buffer, cookie.getValue());
+            putShortString(buffer, cookie.getDomain());
+            putShortString(buffer, cookie.getPath());
+            putShortString(buffer, cookie.getPortlist());
+            buffer.putLong(cookie.getMaxAge());
+            buffer.put(cookie.getSecure() ? (byte) 1 : (byte) 0);
+            buffer.put(cookie.isHttpOnly() ? (byte) 1 : (byte) 0);
+        });
+        return bs;
+    }
+
+    public static List<HttpCookie> getCookieList(ByteBuffer buffer) {
+        int len = buffer.getChar();
+        if (len == 0) return null;
+        final List<HttpCookie> list = new ArrayList<>(len);
+        for (int i = 0; i < len; i++) {
+            HttpCookie cookie = new HttpCookie(getShortString(buffer), getShortString(buffer));
+            cookie.setDomain(getShortString(buffer));
+            cookie.setPath(getShortString(buffer));
+            cookie.setPortlist(getShortString(buffer));
+            cookie.setMaxAge(buffer.getLong());
+            cookie.setSecure(buffer.get() == 1);
+            cookie.setHttpOnly(buffer.get() == 1);
+            list.add(cookie);
+        }
+        return list;
+    }
+}

src/org/redkale/mq/HttpSimpleRequestCoder.java

@@ -0,0 +1,106 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import org.redkale.convert.ConvertType;
+import org.redkale.net.http.HttpSimpleRequest;
+
+/**
+ * HttpSimpleRequest的MessageCoder实现
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class HttpSimpleRequestCoder implements MessageCoder<HttpSimpleRequest> {
+
+    private static final HttpSimpleRequestCoder instance = new HttpSimpleRequestCoder();
+
+    public static HttpSimpleRequestCoder getInstance() {
+        return instance;
+    }
+
+    @Override
+    public byte[] encode(HttpSimpleRequest data) {
+        byte[] requestURI = MessageCoder.getBytes(data.getRequestURI()); //long-string
+        byte[] path = MessageCoder.getBytes(data.getRequestURI()); //short-string
+        byte[] remoteAddr = MessageCoder.getBytes(data.getRemoteAddr());//short-string
+        byte[] sessionid = MessageCoder.getBytes(data.getSessionid());//short-string
+        byte[] contentType = MessageCoder.getBytes(data.getContentType());//short-string
+        byte[] headers = MessageCoder.getBytes(data.getHeaders());
+        byte[] params = MessageCoder.getBytes(data.getParams());
+        byte[] body = MessageCoder.getBytes(data.getBody());
+        int count = 1 //rpc
+            + 1 //frombody
+            + 4 //reqConvertType
+            + 4 //respConvertType
+            + 4 + requestURI.length + 2 + path.length + 2 + remoteAddr.length + 2 + sessionid.length
+            + 2 + contentType.length + 4 + headers.length + params.length + 4 + body.length;
+        byte[] bs = new byte[count];
+        ByteBuffer buffer = ByteBuffer.wrap(bs);
+        buffer.put((byte) (data.isRpc() ? 'T' : 'F'));
+        buffer.put((byte) (data.isFrombody() ? 'T' : 'F'));
+        buffer.putInt(data.getReqConvertType() == null ? 0 : data.getReqConvertType().getValue());
+        buffer.putInt(data.getRespConvertType() == null ? 0 : data.getRespConvertType().getValue());
+        buffer.putInt(requestURI.length);
+        if (requestURI.length > 0) buffer.put(requestURI);
+        buffer.putChar((char) path.length);
+        if (path.length > 0) buffer.put(path);
+        buffer.putChar((char) remoteAddr.length);
+        if (remoteAddr.length > 0) buffer.put(remoteAddr);
+        buffer.putChar((char) sessionid.length);
+        if (sessionid.length > 0) buffer.put(sessionid);
+        buffer.putChar((char) contentType.length);
+        if (contentType.length > 0) buffer.put(contentType);
+        buffer.putInt(data.getCurrentUserid());
+        buffer.put(headers);
+        buffer.put(params);
+        buffer.putInt(body.length);
+        if (body.length > 0) buffer.put(body);
+        return bs;
+    }
+
+    @Override
+    public HttpSimpleRequest decode(byte[] data) {
+        if (data == null) return null;
+        ByteBuffer buffer = ByteBuffer.wrap(data);
+        HttpSimpleRequest req = new HttpSimpleRequest();
+        req.setRpc(buffer.get() == 'T');
+        req.setFrombody(buffer.get() == 'T');
+        int reqformat = buffer.getInt();
+        int respformat = buffer.getInt();
+        if (reqformat != 0) req.setReqConvertType(ConvertType.find(reqformat));
+        if (respformat != 0) req.setRespConvertType(ConvertType.find(respformat));
+        req.setRequestURI(MessageCoder.getLongString(buffer));
+        req.setPath(MessageCoder.getShortString(buffer));
+        req.setRemoteAddr(MessageCoder.getShortString(buffer));
+        req.setSessionid(MessageCoder.getShortString(buffer));
+        req.setContentType(MessageCoder.getShortString(buffer));
+        req.setCurrentUserid(buffer.getInt());
+        req.setHeaders(MessageCoder.getMap(buffer));
+        req.setParams(MessageCoder.getMap(buffer));
+        int len = buffer.getInt();
+        if (len > 0) {
+            byte[] bs = new byte[len];
+            buffer.get(bs);
+            req.setBody(bs);
+        }
+        return req;
+    }
+
+    protected static String getString(ByteBuffer buffer) {
+        int len = buffer.getInt();
+        if (len == 0) return null;
+        byte[] bs = new byte[len];
+        buffer.get(bs);
+        return new String(bs, StandardCharsets.UTF_8);
+    }
+}

src/org/redkale/mq/MessageAgent.java

@@ -0,0 +1,317 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import javax.annotation.Resource;
+import org.redkale.boot.*;
+import static org.redkale.boot.Application.RESNAME_APP_NODEID;
+import org.redkale.net.Servlet;
+import org.redkale.net.http.*;
+import org.redkale.net.sncp.*;
+import org.redkale.service.Service;
+import org.redkale.util.*;
+
+/**
+ * MQ管理器
+ *
+ *
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public abstract class MessageAgent {
+
+    protected final Logger logger = Logger.getLogger(this.getClass().getSimpleName());
+
+    @Resource(name = RESNAME_APP_NODEID)
+    protected int nodeid;
+
+    protected String name;
+
+    protected AnyValue config;
+
+    protected MessageProducers httpProducer;
+
+    protected MessageProducers sncpProducer;
+
+    protected final Object httpProducerLock = new Object();
+
+    protected final Object sncpProducerLock = new Object();
+
+    protected HttpMessageClient httpMessageClient;
+
+    protected SncpMessageClient sncpMessageClient;
+
+    protected ScheduledThreadPoolExecutor timeoutExecutor;
+
+    protected ThreadHashExecutor workExecutor;
+
+    protected int producerCount = 1;
+
+    //本地Service消息接收处理器， key:consumer
+    protected HashMap<String, MessageConsumerNode> messageNodes = new LinkedHashMap<>();
+
+    public void init(AnyValue config) {
+        this.name = checkName(config.getValue("name", ""));
+        this.httpMessageClient = new HttpMessageClient(this);
+        this.sncpMessageClient = new SncpMessageClient(this);
+        this.producerCount = config.getIntValue("producers", Runtime.getRuntime().availableProcessors());
+        if ("hash".equalsIgnoreCase(config.getValue("pool", "hash"))) {
+            this.workExecutor = new ThreadHashExecutor(Math.max(4, config.getIntValue("threads", Runtime.getRuntime().availableProcessors())));
+        }
+        // application (it doesn't execute completion handlers).
+        this.timeoutExecutor = (ScheduledThreadPoolExecutor) Executors.newScheduledThreadPool(1, (Runnable r) -> {
+            Thread t = new Thread(r);
+            t.setName("MessageAgent-Timeout-Thread");
+            t.setDaemon(true);
+            return t;
+        });
+        this.timeoutExecutor.setRemoveOnCancelPolicy(true);
+    }
+
+    public boolean isHashPool() {
+        return this.workExecutor != null;
+    }
+
+    public CompletableFuture<Map<String, Long>> start() {
+        final LinkedHashMap<String, Long> map = new LinkedHashMap<>();
+        final List<CompletableFuture> futures = new ArrayList<>();
+        this.messageNodes.values().forEach(node -> {
+            long s = System.currentTimeMillis();
+            futures.add(node.consumer.startup().whenComplete((r, t) -> map.put(node.consumer.consumerid, System.currentTimeMillis() - s)));
+        });
+        return CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()])).thenApply(r -> map);
+    }
+
+    //Application.shutdown  在执行server.shutdown之前执行
+    public CompletableFuture<Void> stop() {
+        List<CompletableFuture> futures = new ArrayList<>();
+        this.messageNodes.values().forEach(node -> {
+            futures.add(node.consumer.shutdown());
+        });
+        return CompletableFuture.allOf(futures.toArray(new CompletableFuture[futures.size()]));
+    }
+
+    //Application.shutdown 在所有server.shutdown执行后执行
+    public void destroy(AnyValue config) {
+        this.httpMessageClient.close().join();
+        this.sncpMessageClient.close().join();
+        this.workExecutor.shutdown();
+        if (this.timeoutExecutor != null) this.timeoutExecutor.shutdown();
+        if (this.sncpProducer != null) this.sncpProducer.shutdown().join();
+        if (this.httpProducer != null) this.httpProducer.shutdown().join();
+    }
+
+    protected List<MessageConsumer> getAllMessageConsumer() {
+        List<MessageConsumer> consumers = new ArrayList<>();
+        MessageConsumer one = this.httpMessageClient == null ? null : this.httpMessageClient.respConsumer;
+        if (one != null) consumers.add(one);
+        one = this.sncpMessageClient == null ? null : this.sncpMessageClient.respConsumer;
+        if (one != null) consumers.add(one);
+        consumers.addAll(messageNodes.values().stream().map(mcn -> mcn.consumer).collect(Collectors.toList()));
+        return consumers;
+    }
+
+    protected List<MessageProducer> getAllMessageProducer() {
+        List<MessageProducer> producers = new ArrayList<>();
+        if (this.httpProducer != null) producers.addAll(List.of(this.httpProducer.producers));
+        if (this.sncpProducer != null) producers.addAll(List.of(this.sncpProducer.producers));
+        MessageProducers one = this.httpMessageClient == null ? null : this.httpMessageClient.getProducer();
+        if (one != null) producers.addAll(List.of(one.producers));
+        one = this.sncpMessageClient == null ? null : this.sncpMessageClient.getProducer();
+        if (one != null) producers.addAll(List.of(one.producers));
+        return producers;
+    }
+
+    public Logger getLogger() {
+        return logger;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public AnyValue getConfig() {
+        return config;
+    }
+
+    public void setConfig(AnyValue config) {
+        this.config = config;
+    }
+
+    public HttpMessageClient getHttpMessageClient() {
+        return httpMessageClient;
+    }
+
+    public SncpMessageClient getSncpMessageClient() {
+        return sncpMessageClient;
+    }
+
+    protected String checkName(String name) {  //不能含特殊字符
+        if (name.isEmpty()) return name;
+        if (name.charAt(0) >= '0' && name.charAt(0) <= '9') throw new RuntimeException("name only 0-9 a-z A-Z _ cannot begin 0-9");
+        for (char ch : name.toCharArray()) {
+            if (!((ch >= '0' && ch <= '9') || ch == '_' || (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z'))) { //不能含特殊字符
+                throw new RuntimeException("name only 0-9 a-z A-Z _ cannot begin 0-9");
+            }
+        }
+        return name;
+    }
+
+    //获取指定topic的生产处理器
+    public MessageProducers getSncpProducer() {
+        if (this.sncpProducer == null) {
+            synchronized (sncpProducerLock) {
+                if (this.sncpProducer == null) {
+                    MessageProducer[] producers = new MessageProducer[producerCount];
+                    for (int i = 0; i < producers.length; i++) {
+                        MessageProducer producer = createProducer("SncpProducer");
+                        producer.startup().join();
+                        producers[i] = producer;
+                    }
+                    this.sncpProducer = new MessageProducers(producers);
+                }
+            }
+        }
+        return this.sncpProducer;
+    }
+
+    public MessageProducers getHttpProducer() {
+        if (this.httpProducer == null) {
+            synchronized (httpProducerLock) {
+                if (this.httpProducer == null) {
+                    MessageProducer[] producers = new MessageProducer[producerCount];
+                    for (int i = 0; i < producers.length; i++) {
+                        MessageProducer producer = createProducer("HttpProducer");
+                        producer.startup().join();
+                        producers[i] = producer;
+                    }
+                    this.httpProducer = new MessageProducers(producers);
+                }
+            }
+        }
+        return this.httpProducer;
+    }
+
+    //创建指定topic的生产处理器
+    protected abstract MessageProducer createProducer(String name);
+
+    //创建topic，如果已存在则跳过
+    public abstract boolean createTopic(String... topics);
+
+    //删除topic，如果不存在则跳过
+    public abstract boolean deleteTopic(String... topics);
+
+    //查询所有topic
+    public abstract List<String> queryTopic();
+
+    //ServiceLoader时判断配置是否符合当前实现类
+    public abstract boolean match(AnyValue config);
+
+    //创建指定topic的消费处理器
+    public abstract MessageConsumer createConsumer(String[] topics, String group, MessageProcessor processor);
+
+    public final synchronized void putService(NodeHttpServer ns, Service service, HttpServlet servlet) {
+        String[] topics = generateHttpReqTopics(service);
+        String consumerid = generateHttpConsumerid(topics, service);
+        if (messageNodes.containsKey(consumerid)) throw new RuntimeException("consumerid(" + consumerid + ") is repeat");
+        HttpMessageProcessor processor = new HttpMessageProcessor(this.logger, this.workExecutor, getHttpProducer(), ns, service, servlet);
+        this.messageNodes.put(consumerid, new MessageConsumerNode(ns, service, servlet, processor, createConsumer(topics, consumerid, processor)));
+    }
+
+    public final synchronized void putService(NodeSncpServer ns, Service service, SncpServlet servlet) {
+        String topic = generateSncpReqTopic(service);
+        String consumerid = generateSncpConsumerid(topic, service);
+        if (messageNodes.containsKey(consumerid)) throw new RuntimeException("consumerid(" + consumerid + ") is repeat");
+        SncpMessageProcessor processor = new SncpMessageProcessor(this.logger, this.workExecutor, getSncpProducer(), ns, service, servlet);
+        this.messageNodes.put(consumerid, new MessageConsumerNode(ns, service, servlet, processor, createConsumer(new String[]{topic}, consumerid, processor)));
+    }
+
+    //格式: sncp.req.user
+    public final String generateSncpReqTopic(Service service) {
+        if (service instanceof WebSocketNode) {
+            String resname = Sncp.getResourceName(service);
+            return "sncp.req.ws" + (resname.isEmpty() ? "" : ("-" + resname)) + ".node" + nodeid;
+        }
+        String resname = Sncp.getResourceName(service);
+        return "sncp.req." + Sncp.getResourceType(service).getSimpleName().replaceAll("Service.*$", "").toLowerCase() + (resname.isEmpty() ? "" : ("-" + resname));
+    }
+
+    //格式: consumer-sncp.req.user  不提供外部使用
+    protected final String generateSncpConsumerid(String topic, Service service) {
+        return "consumer-" + topic;
+    }
+
+    //格式: http.req.user
+    public static String generateHttpReqTopic(String module) {
+        return "http.req." + module.toLowerCase();
+    }
+
+    //格式: http.req.user
+    public static String generateHttpReqTopic(String module, String resname) {
+        return "http.req." + module.toLowerCase() + (resname == null || resname.isEmpty() ? "" : ("-" + resname));
+    }
+
+    //格式: sncp.resp.node10
+    protected String generateSncpRespTopic() {
+        return "sncp.resp.node" + nodeid;
+    }
+
+    //格式: http.resp.node10
+    protected String generateHttpRespTopic() {
+        return "http.resp.node" + nodeid;
+    }
+
+    //格式: http.req.user
+    protected String[] generateHttpReqTopics(Service service) {
+        String resname = Sncp.getResourceName(service);
+        String module = Rest.getRestModule(service).toLowerCase();
+        MessageMultiConsumer mmc = service.getClass().getAnnotation(MessageMultiConsumer.class);
+        if (mmc != null) return new String[]{generateHttpReqTopic(mmc.module()) + (resname.isEmpty() ? "" : ("-" + resname))};
+        return new String[]{"http.req." + module + (resname.isEmpty() ? "" : ("-" + resname))};
+    }
+
+    //格式: consumer-http.req.user
+    protected String generateHttpConsumerid(String[] topics, Service service) {
+        String resname = Sncp.getResourceName(service);
+        String key = Rest.getRestModule(service).toLowerCase();
+        return "consumer-http.req." + key + (resname.isEmpty() ? "" : ("-" + resname));
+
+    }
+
+    //格式: xxxx.resp.node10
+    protected String generateRespTopic(String protocol) {
+        return protocol + ".resp.node" + nodeid;
+    }
+
+    protected static class MessageConsumerNode {
+
+        public final NodeServer server;
+
+        public final Service service;
+
+        public final Servlet servlet;
+
+        public final MessageProcessor processor;
+
+        public final MessageConsumer consumer;
+
+        public MessageConsumerNode(NodeServer server, Service service, Servlet servlet, MessageProcessor processor, MessageConsumer consumer) {
+            this.server = server;
+            this.service = service;
+            this.servlet = servlet;
+            this.processor = processor;
+            this.consumer = consumer;
+        }
+
+    }
+}

src/org/redkale/mq/MessageClient.java

@@ -0,0 +1,100 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public abstract class MessageClient {
+
+    protected final ConcurrentHashMap<Long, MessageRespFutureNode> respNodes = new ConcurrentHashMap<>();
+
+    protected final MessageAgent messageAgent;
+
+    protected MessageConsumer respConsumer;
+
+    protected String respTopic;
+
+    protected String respConsumerid;
+
+    protected boolean finest;
+
+    protected boolean finer;
+
+    protected MessageClient(MessageAgent messageAgent) {
+        this.messageAgent = messageAgent;
+        this.finest = messageAgent == null ? false : messageAgent.logger.isLoggable(Level.FINEST);
+        this.finer = messageAgent == null ? false : messageAgent.logger.isLoggable(Level.FINER);
+    }
+
+    protected CompletableFuture<Void> close() {
+        if (this.respConsumer == null) return CompletableFuture.completedFuture(null);
+        return this.respConsumer.shutdown();
+    }
+
+    protected CompletableFuture<MessageRecord> sendMessage(MessageRecord message, boolean needresp, AtomicLong counter) {
+        CompletableFuture<MessageRecord> future = new CompletableFuture<>();
+        try {
+            if (this.respConsumer == null) {
+                synchronized (this) {
+                    if (this.respConsumerid == null) this.respConsumerid = "consumer-" + this.respTopic;
+                    if (this.respConsumer == null) {
+                        MessageProcessor processor = (msg, callback) -> {
+                            long now = System.currentTimeMillis();
+                            MessageRespFutureNode node = respNodes.remove(msg.getSeqid());
+                            if (node == null) {
+                                messageAgent.logger.log(Level.WARNING, MessageClient.this.getClass().getSimpleName() + " process " + msg + " error， not found msgnode");
+                                return;
+                            }
+                            AtomicLong ncer = node.getCounter();
+                            if (ncer != null) ncer.decrementAndGet();
+                            node.future.complete(msg);
+                            long cha = now - msg.createtime;
+                            if (cha > 50 || finer) {
+                                messageAgent.logger.log(Level.FINER, "MessageRespFutureNode.process (mq.delay = " + cha + "ms) message: " + msg);
+                            } else if (finest) {
+                                messageAgent.logger.log(Level.FINEST, "MessageRespFutureNode.process (mq.delay = " + cha + "ms) message: " + msg);
+                            }
+
+                        };
+                        MessageConsumer one = messageAgent.createConsumer(new String[]{respTopic}, respConsumerid, processor);
+                        one.startup().join();
+                        this.respConsumer = one;
+                    }
+                }
+            }
+            if (needresp && (message.getResptopic() == null || message.getResptopic().isEmpty())) {
+                message.setResptopic(respTopic);
+            }
+            if (counter != null) counter.incrementAndGet();
+            getProducer().apply(message);
+            if (needresp) {
+                MessageRespFutureNode node = new MessageRespFutureNode(message, respNodes, counter, future);
+                respNodes.put(message.getSeqid(), node);
+                ScheduledThreadPoolExecutor executor = messageAgent.timeoutExecutor;
+                if (executor != null) executor.schedule(node, 30, TimeUnit.SECONDS);
+            } else {
+                future.complete(null);
+            }
+        } catch (Exception ex) {
+            future.completeExceptionally(ex);
+        } finally {
+            return future;
+        }
+    }
+
+    protected abstract MessageProducers getProducer();
+}

src/org/redkale/mq/MessageCoder.java

@@ -0,0 +1,107 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.redkale.util.Utility;
+
+/**
+ * 将MessageRecord.content内容加解密
+ *
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ * 
+ * @param <T> 泛型
+ */
+public interface MessageCoder<T> {
+
+    //编码
+    public byte[] encode(T data);
+
+    //解码
+    public T decode(byte[] data);
+
+    public static byte[] getBytes(byte[] value) {
+        if (value == null) return MessageRecord.EMPTY_BYTES;
+        return value;
+    }
+
+    public static byte[] getBytes(String value) {
+        if (value == null || value.isEmpty()) return MessageRecord.EMPTY_BYTES;
+        return value.getBytes(StandardCharsets.UTF_8);
+    }
+
+    public static byte[] getBytes(final Map<String, String> map) {
+        if (map == null || map.isEmpty()) return new byte[2];
+        final AtomicInteger len = new AtomicInteger(2);
+        map.forEach((key, value) -> {
+            len.addAndGet(2 + (key == null ? 0 : Utility.encodeUTF8Length(key)));
+            len.addAndGet(4 + (value == null ? 0 : Utility.encodeUTF8Length(value)));
+        });
+        final byte[] bs = new byte[len.get()];
+        final ByteBuffer buffer = ByteBuffer.wrap(bs);
+        buffer.putChar((char) map.size());
+        map.forEach((key, value) -> {
+            putShortString(buffer, key);
+            putLongString(buffer, value);
+        });
+        return bs;
+    }
+
+    public static void putLongString(ByteBuffer buffer, String value) {
+        if (value == null || value.isEmpty()) {
+            buffer.putInt(0);
+        } else {
+            byte[] bs = value.getBytes(StandardCharsets.UTF_8);
+            buffer.putInt(bs.length);
+            buffer.put(bs);
+        }
+    }
+
+    public static String getLongString(ByteBuffer buffer) {
+        int len = buffer.getInt();
+        if (len == 0) return null;
+        byte[] bs = new byte[len];
+        buffer.get(bs);
+        return new String(bs, StandardCharsets.UTF_8);
+    }
+
+    public static void putShortString(ByteBuffer buffer, String value) {
+        if (value == null || value.isEmpty()) {
+            buffer.putChar((char) 0);
+        } else {
+            byte[] bs = value.getBytes(StandardCharsets.UTF_8);
+            buffer.putChar((char) bs.length);
+            buffer.put(bs);
+        }
+    }
+
+    public static String getShortString(ByteBuffer buffer) {
+        int len = buffer.getChar();
+        if (len == 0) return null;
+        byte[] bs = new byte[len];
+        buffer.get(bs);
+        return new String(bs, StandardCharsets.UTF_8);
+    }
+
+    public static Map<String, String> getMap(ByteBuffer buffer) {
+        int len = buffer.getChar();
+        if (len == 0) return null;
+        Map<String, String> map = new HashMap<>(len);
+        for (int i = 0; i < len; i++) {
+            map.put(getShortString(buffer), getLongString(buffer));
+        }
+        return map;
+    }
+}

src/org/redkale/mq/MessageConsumer.java

@@ -0,0 +1,63 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
+import java.util.logging.Logger;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public abstract class MessageConsumer {
+
+    protected final String[] topics;
+
+    protected final String consumerid;
+
+    protected MessageAgent messageAgent;
+
+    protected final MessageProcessor processor;
+
+    protected final Logger logger;
+
+    protected volatile boolean closed;
+
+    protected MessageConsumer(MessageAgent messageAgent, String[] topics, final String consumerid, MessageProcessor processor) {
+        Objects.requireNonNull(messageAgent);
+        Objects.requireNonNull(topics);
+        Objects.requireNonNull(consumerid);
+        Objects.requireNonNull(processor);
+        this.messageAgent = messageAgent;
+        this.logger = messageAgent.logger;
+        this.topics = topics;
+        this.consumerid = consumerid;
+        this.processor = processor;
+    }
+
+    public MessageProcessor getProcessor() {
+        return processor;
+    }
+
+    public String[] getTopics() {
+        return topics;
+    }
+
+    public abstract CompletableFuture<Void> startup();
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+    public abstract CompletableFuture<Void> shutdown();
+}

src/org/redkale/mq/MessageMultiConsumer.java

@@ -0,0 +1,61 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.lang.annotation.*;
+import static java.lang.annotation.ElementType.*;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+/**
+ * 多消费组，需要同 @RestService 一起使用
+ * <p>
+ * 通常一个topic只会被一个RestService消费， 当一个topic需要被其他RestService消费时，就需要使用&#64;MessageMultiConsumer
+ *
+ * <blockquote><pre>
+ * &#64;RestService(name = "user", comment = "用户服务")
+ * public class UserService implements Service{
+ *
+ *      &#64;RestMapping(comment = "用户登录")
+ *      public RetResult login(LoginBean bean){
+ *          //do something
+ *      }
+ * }
+ * </pre></blockquote>
+ *
+ * 需求：统计用户登录次数， 可以创建一个MessageMultiConsumer 的 RestService：
+ * <blockquote><pre>
+ * <b>&#64;MessageMultiConsumer(module = "user") </b>
+ * &#64;RestService(name = "loginstat", comment = "用户统计服务")
+ * public class LoginStatService implements Service{
+ *
+ *      private AtomicLong counter = new AtomicLong();
+ *
+ *      &#64;RestMapping(name = "login", comment = "用户登录统计")
+ *      public void stat(LoginBean bean){     //参数必须和UserService.login方法一致
+ *          counter.incrementAndGet();
+ *      }
+ * }
+ * </pre></blockquote>
+ *
+ * <p>
+ * 注： 标记 &#64;MessageMultiConsumer 的Service的&#64;RestMapping方法都只能是void返回类型
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+@Inherited
+@Documented
+@Target({TYPE})
+@Retention(RUNTIME)
+public @interface MessageMultiConsumer {
+
+    String module();
+}

src/org/redkale/mq/MessageProcessor.java

@@ -0,0 +1,26 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public interface MessageProcessor {
+
+    default void begin(int size) {
+    }
+
+    public void process(MessageRecord message, Runnable callback);
+
+    default void commit() {
+    }
+}

src/org/redkale/mq/MessageProducer.java

@@ -0,0 +1,42 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.logging.Logger;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public abstract class MessageProducer {
+
+    protected final Logger logger;
+
+    protected final String name;
+
+    protected volatile boolean closed;
+
+    protected MessageProducer(String name, Logger logger) {
+        this.name = name;
+        this.logger = logger;
+    }
+
+    public abstract CompletableFuture<Void> apply(MessageRecord message);
+
+    public abstract CompletableFuture<Void> startup();
+
+    public boolean isClosed() {
+        return closed;
+    }
+
+    public abstract CompletableFuture<Void> shutdown();
+}

src/org/redkale/mq/MessageProducers.java

@@ -0,0 +1,62 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class MessageProducers {
+
+    protected final MessageProducer[] producers;
+
+    protected final AtomicInteger index = new AtomicInteger();
+
+    public MessageProducers(MessageProducer[] producers) {
+        this.producers = producers;
+    }
+
+    public MessageProducer getProducer(MessageRecord message) {
+        if (this.producers.length == 1) return this.producers[0];
+        int hash = index.incrementAndGet();
+        if (index.get() > 1000 * producers.length) {
+            synchronized (index) {
+                if (index.get() > 1000 * producers.length) {
+                    index.addAndGet(-1000 * producers.length);
+                }
+            }
+        }
+        return producers[hash % producers.length];
+    }
+
+    public CompletableFuture<Void> apply(MessageRecord message) {
+        return getProducer(message).apply(message);
+    }
+
+    public CompletableFuture<Void> startup() {
+        CompletableFuture[] futures = new CompletableFuture[producers.length];
+        for (int i = 0; i < producers.length; i++) {
+            futures[i] = producers[i].startup();
+        }
+        return CompletableFuture.allOf(futures);
+    }
+
+    public CompletableFuture<Void> shutdown() {
+        CompletableFuture[] futures = new CompletableFuture[producers.length];
+        for (int i = 0; i < producers.length; i++) {
+            futures[i] = producers[i].shutdown();
+        }
+        return CompletableFuture.allOf(futures);
+    }
+}

src/org/redkale/mq/MessageRecord.java

@@ -0,0 +1,296 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.io.Serializable;
+import java.nio.charset.StandardCharsets;
+import org.redkale.convert.*;
+import org.redkale.util.Comment;
+
+/**
+ * 存在MQ里面的数据结构<p>
+ * groupid + userid 来确定partition， 优先使用 groupid
+ *
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class MessageRecord implements Serializable {
+
+    static final byte[] EMPTY_BYTES = new byte[0];
+
+    @ConvertColumn(index = 1)
+    @Comment("消息序列号")
+    protected long seqid;
+
+    @ConvertColumn(index = 2)
+    @Comment("版本")
+    protected int version;
+
+    @ConvertColumn(index = 3)
+    @Comment("标记位, 自定义时使用")
+    protected int flag;
+
+    @ConvertColumn(index = 4)
+    @Comment("创建时间")
+    protected long createtime;
+
+    @ConvertColumn(index = 5)
+    @Comment("用户ID，无用户信息视为0")
+    protected int userid;
+
+    @ConvertColumn(index = 6)
+    @Comment("组ID")
+    protected String groupid;
+
+    @ConvertColumn(index = 7)
+    @Comment("当前topic")
+    protected String topic;
+
+    @ConvertColumn(index = 8)
+    @Comment("目标topic, 为空表示无目标topic")
+    protected String resptopic;
+
+    @ConvertColumn(index = 9)
+    @Comment("消息内容")
+    protected byte[] content;
+
+    public MessageRecord() {
+    }
+
+    public MessageRecord(String resptopic, String content) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), 0, null, null, resptopic, content == null ? null : content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    public MessageRecord(String topic, String resptopic, String content) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), 0, null, topic, resptopic, content == null ? null : content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    public MessageRecord(int userid, String topic, String resptopic, String content) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), userid, null, topic, resptopic, content == null ? null : content.getBytes(StandardCharsets.UTF_8));
+    }
+
+    public MessageRecord(String topic, String resptopic, Convert convert, Object bean) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), 0, null, topic, resptopic, convert.convertToBytes(bean));
+    }
+
+    public MessageRecord(int userid, String topic, String resptopic, Convert convert, Object bean) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), userid, null, topic, resptopic, convert.convertToBytes(bean));
+    }
+
+    public MessageRecord(int userid, String groupid, String topic, String resptopic, Convert convert, Object bean) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), userid, groupid, topic, resptopic, convert.convertToBytes(bean));
+    }
+
+    public MessageRecord(int flag, int userid, String groupid, String topic, String resptopic, Convert convert, Object bean) {
+        this(System.nanoTime(), 1, flag, System.currentTimeMillis(), userid, groupid, topic, resptopic, convert.convertToBytes(bean));
+    }
+
+    public MessageRecord(String topic, String resptopic, byte[] content) {
+        this(System.nanoTime(), 1, 0, System.currentTimeMillis(), 0, null, topic, resptopic, content);
+    }
+
+    public MessageRecord(long seqid, String topic, String resptopic, byte[] content) {
+        this(seqid, 1, 0, System.currentTimeMillis(), 0, null, topic, resptopic, content);
+    }
+
+    public MessageRecord(long seqid, int flag, int userid, String groupid, String topic, String resptopic, byte[] content) {
+        this(seqid, 1, flag, System.currentTimeMillis(), userid, groupid, topic, resptopic, content);
+    }
+
+    public MessageRecord(long seqid, int version, int flag, long createtime, int userid, String groupid, String topic, String resptopic, byte[] content) {
+        this.seqid = seqid;
+        this.version = version;
+        this.flag = flag;
+        this.createtime = createtime;
+        this.userid = userid;
+        this.groupid = groupid;
+        this.topic = topic;
+        this.resptopic = resptopic;
+        this.content = content;
+    }
+
+    public String contentString() {
+        return content == null ? null : new String(content, StandardCharsets.UTF_8);
+    }
+
+    @ConvertDisabled
+    public boolean isEmptyTopic() {
+        return this.topic == null || this.topic.isEmpty();
+    }
+
+    @ConvertDisabled
+    public boolean isEmptyResptopic() {
+        return this.resptopic == null || this.resptopic.isEmpty();
+    }
+
+    public <T> T convertFromContent(Convert convert, java.lang.reflect.Type type) {
+        if (this.content == null || this.content.length == 0) return null;
+        return (T) convert.convertFrom(type, this.content);
+    }
+
+    public <T> T decodeContent(MessageCoder<T> coder) {
+        if (this.content == null || this.content.length == 0) return null;
+        return (T) coder.decode(this.content);
+    }
+
+    public <T> MessageRecord encodeContent(MessageCoder<T> coder, T data) {
+        this.content = coder.encode(data);
+        return this;
+    }
+
+    public int hash() {
+        if (groupid != null && !groupid.isEmpty()) {
+            return groupid.hashCode();
+        } else if (userid > 0) {
+            return userid;
+        } else {
+            return 0;
+        }
+    }
+
+    public MessageRecord version(int version) {
+        this.version = version;
+        return this;
+    }
+
+    public MessageRecord flag(int flag) {
+        this.flag = flag;
+        return this;
+    }
+
+    public MessageRecord createtime(long createtime) {
+        this.createtime = createtime;
+        return this;
+    }
+
+    public MessageRecord userid(int userid) {
+        this.userid = userid;
+        return this;
+    }
+
+    public MessageRecord groupid(String groupid) {
+        this.groupid = groupid;
+        return this;
+    }
+
+    public MessageRecord topic(String topic) {
+        this.topic = topic;
+        return this;
+    }
+
+    public MessageRecord resptopic(String resptopic) {
+        this.resptopic = resptopic;
+        return this;
+    }
+
+    public MessageRecord content(byte[] content) {
+        this.content = content;
+        return this;
+    }
+
+    public MessageRecord contentString(String content) {
+        this.content = content == null ? null : content.getBytes(StandardCharsets.UTF_8);
+        return this;
+    }
+
+    public long getSeqid() {
+        return seqid;
+    }
+
+    public void setSeqid(long seqid) {
+        this.seqid = seqid;
+    }
+
+    public int getVersion() {
+        return version;
+    }
+
+    public void setVersion(int version) {
+        this.version = version;
+    }
+
+    public int getFlag() {
+        return flag;
+    }
+
+    public void setFlag(int flag) {
+        this.flag = flag;
+    }
+
+    public long getCreatetime() {
+        return createtime;
+    }
+
+    public void setCreatetime(long createtime) {
+        this.createtime = createtime;
+    }
+
+    public int getUserid() {
+        return userid;
+    }
+
+    public void setUserid(int userid) {
+        this.userid = userid;
+    }
+
+    public String getGroupid() {
+        return groupid;
+    }
+
+    public void setGroupid(String groupid) {
+        this.groupid = groupid;
+    }
+
+    public String getTopic() {
+        return topic;
+    }
+
+    public void setTopic(String topic) {
+        this.topic = topic;
+    }
+
+    public String getResptopic() {
+        return resptopic;
+    }
+
+    public void setResptopic(String resptopic) {
+        this.resptopic = resptopic;
+    }
+
+    public byte[] getContent() {
+        return content;
+    }
+
+    public void setContent(byte[] content) {
+        this.content = content;
+    }
+
+    @Override
+    public String toString() {
+        //return JsonConvert.root().convertTo(this);
+        StringBuilder sb = new StringBuilder(128);
+        sb.append("{\"seqid\":").append(this.seqid);
+        sb.append(",\"version\":").append(this.version);
+        if (this.flag != 0) sb.append(",\"flag\":").append(this.flag);
+        if (this.createtime != 0) sb.append(",\"createtime\":").append(this.createtime);
+        if (this.userid != 0) sb.append(",\"userid\":").append(this.userid);
+        if (this.groupid != null) sb.append(",\"groupid\":\"").append(this.groupid).append("\"");
+        if (this.topic != null) sb.append(",\"topic\":\"").append(this.topic).append("\"");
+        if (this.resptopic != null) sb.append(",\"resptopic\":\"").append(this.resptopic).append("\"");
+        if (this.content != null) sb.append(",\"content\":").append(new String(this.content, StandardCharsets.UTF_8)).append("\"");
+        sb.append("}");
+        return sb.toString();
+    }
+
+//    public static void main(String[] args) throws Throwable {
+//        System.out.println(new MessageRecord(333, 2, 3, null, "tt", null, "xxx".getBytes()));
+//    }
+}

src/org/redkale/mq/MessageRecordCoder.java

@@ -0,0 +1,80 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.nio.ByteBuffer;
+
+/**
+ * MessageRecord的MessageCoder实现
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class MessageRecordCoder implements MessageCoder<MessageRecord> {
+
+    private static final MessageRecordCoder instance = new MessageRecordCoder();
+
+    public static MessageRecordCoder getInstance() {
+        return instance;
+    }
+
+    @Override
+    public byte[] encode(MessageRecord data) {
+        if (data == null) return null;
+        byte[] stopics = MessageCoder.getBytes(data.getTopic());
+        byte[] dtopics = MessageCoder.getBytes(data.getResptopic());
+        byte[] groupid = MessageCoder.getBytes(data.getGroupid());
+        int count = 8 + 4 + 4 + 8 + 4 + 2 + stopics.length + 2 + dtopics.length + 2 + groupid.length + 4 + (data.getContent() == null ? 0 : data.getContent().length);
+        final byte[] bs = new byte[count];
+        ByteBuffer buffer = ByteBuffer.wrap(bs);
+        buffer.putLong(data.getSeqid());
+        buffer.putInt(data.getVersion());
+        buffer.putInt(data.getFlag());
+        buffer.putLong(data.getCreatetime());
+        buffer.putInt(data.getUserid());
+        buffer.putChar((char) groupid.length);
+        if (groupid.length > 0) buffer.put(groupid);
+        buffer.putChar((char) stopics.length);
+        if (stopics.length > 0) buffer.put(stopics);
+        buffer.putChar((char) dtopics.length);
+        if (dtopics.length > 0) buffer.put(dtopics);
+        if (data.getContent() == null) {
+            buffer.putInt(0);
+        } else {
+            buffer.putInt(data.getContent().length);
+            buffer.put(data.getContent());
+        }
+        return bs;
+    }
+
+    @Override
+    public MessageRecord decode(byte[] data) {
+        if (data == null) return null;
+        ByteBuffer buffer = ByteBuffer.wrap(data);
+        long seqid = buffer.getLong();
+        int version = buffer.getInt();
+        int flag = buffer.getInt();
+        long createtime = buffer.getLong();
+        int userid = buffer.getInt();
+
+        String groupid = MessageCoder.getShortString(buffer);
+        String topic = MessageCoder.getShortString(buffer);
+        String resptopic = MessageCoder.getShortString(buffer);
+
+        byte[] content = null;
+        int contentlen = buffer.getInt();
+        if (contentlen > 0) {
+            content = new byte[contentlen];
+            buffer.get(content);
+        }
+        return new MessageRecord(seqid, version, flag, createtime, userid, groupid, topic, resptopic, content);
+    }
+
+}

src/org/redkale/mq/MessageRespFutureNode.java

@@ -0,0 +1,62 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * MQ管理器
+ *
+ *
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class MessageRespFutureNode implements Runnable {
+
+    protected final long seqid;
+
+    protected final long createtime;
+
+    protected final AtomicLong counter;
+
+    protected final CompletableFuture<MessageRecord> future;
+
+    protected final ConcurrentHashMap<Long, MessageRespFutureNode> respNodes;
+
+    public MessageRespFutureNode(MessageRecord message, ConcurrentHashMap<Long, MessageRespFutureNode> respNodes, AtomicLong counter, CompletableFuture<MessageRecord> future) {
+        this.seqid = message.getSeqid();
+        this.respNodes = respNodes;
+        this.counter = counter;
+        this.future = future;
+        this.createtime = System.currentTimeMillis();
+    }
+
+    @Override  //超时后被timeoutExecutor调用
+    public void run() { //timeout
+        respNodes.remove(this.seqid);
+        future.completeExceptionally(new TimeoutException());
+    }
+
+    public long getSeqid() {
+        return seqid;
+    }
+
+    public long getCreatetime() {
+        return createtime;
+    }
+
+    public AtomicLong getCounter() {
+        return counter;
+    }
+
+    public CompletableFuture<MessageRecord> getFuture() {
+        return future;
+    }
+}

src/org/redkale/mq/MessageResponse.java

@@ -0,0 +1,20 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public interface MessageResponse {
+
+    public void finish(MessageRecord message);
+}

src/org/redkale/mq/SncpMessageClient.java

@@ -0,0 +1,56 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class SncpMessageClient extends MessageClient {
+
+    protected SncpMessageClient(MessageAgent messageAgent) {
+        super(messageAgent);
+        this.respTopic = messageAgent.generateSncpRespTopic();
+    }
+
+    @Override
+    protected MessageProducers getProducer() {
+        return messageAgent.getSncpProducer();
+    }
+
+    public String getRespTopic() {
+        return this.respTopic;
+    }
+
+    //只发送消息，不需要响应
+    public final void produceMessage(MessageRecord message) {
+        produceMessage(message, null);
+    }
+
+    //只发送消息，不需要响应
+    public final void produceMessage(MessageRecord message, AtomicLong counter) {
+        sendMessage(message, false, counter);
+    }
+
+    //发送消息，需要响应
+    public final CompletableFuture<MessageRecord> sendMessage(MessageRecord message) {
+        return sendMessage(message, null);
+    }
+
+    //发送消息，需要响应
+    public final CompletableFuture<MessageRecord> sendMessage(MessageRecord message, AtomicLong counter) {
+        return sendMessage(message, true, counter);
+    }
+
+}

src/org/redkale/mq/SncpMessageProcessor.java

@@ -0,0 +1,107 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.util.concurrent.*;
+import java.util.logging.*;
+import org.redkale.boot.NodeSncpServer;
+import org.redkale.net.sncp.*;
+import org.redkale.service.Service;
+import org.redkale.util.ThreadHashExecutor;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class SncpMessageProcessor implements MessageProcessor {
+
+    protected final Logger logger;
+
+    protected final MessageProducers producer;
+
+    protected final NodeSncpServer server;
+
+    protected final ThreadHashExecutor workExecutor;
+
+    protected final Service service;
+
+    protected final SncpServlet servlet;
+
+    protected CountDownLatch cdl;
+
+    protected final Runnable innerCallback = () -> {
+        if (cdl != null) cdl.countDown();
+    };
+
+    public SncpMessageProcessor(Logger logger, ThreadHashExecutor workExecutor, MessageProducers producer, NodeSncpServer server, Service service, SncpServlet servlet) {
+        this.logger = logger;
+        this.producer = producer;
+        this.server = server;
+        this.service = service;
+        this.servlet = servlet;
+        this.workExecutor = workExecutor;
+    }
+
+    @Override
+    public void begin(final int size) {
+        if (this.workExecutor != null) this.cdl = new CountDownLatch(size);
+    }
+
+    @Override
+    public void process(final MessageRecord message, final Runnable callback) {
+        if (this.workExecutor == null) {
+            execute(message, innerCallback);
+        } else {
+            this.workExecutor.execute(message.hash(), () -> execute(message, innerCallback));
+        }
+    }
+
+    private void execute(final MessageRecord message, final Runnable callback) {
+        SncpMessageResponse response = null;
+        try {
+            SncpContext context = server.getSncpServer().getContext();
+            SncpMessageRequest request = new SncpMessageRequest(context, message);
+            response = new SncpMessageResponse(context, request, callback, null, producer.getProducer(message));
+            servlet.execute(request, response);
+        } catch (Throwable ex) {
+            if (response != null) response.finish(SncpResponse.RETCODE_ILLSERVICEID, null);
+            logger.log(Level.SEVERE, SncpMessageProcessor.class.getSimpleName() + " process error, message=" + message, ex instanceof CompletionException ? ((CompletionException) ex).getCause() : ex);
+        }
+    }
+
+    @Override
+    public void commit() {
+        if (this.cdl != null) {
+            try {
+                this.cdl.await(30, TimeUnit.SECONDS);
+            } catch (Exception ex) {
+            }
+            this.cdl = null;
+        }
+    }
+
+    public MessageProducers getProducer() {
+        return producer;
+    }
+
+    public NodeSncpServer getServer() {
+        return server;
+    }
+
+    public Service getService() {
+        return service;
+    }
+
+    public SncpServlet getServlet() {
+        return servlet;
+    }
+
+}

src/org/redkale/mq/SncpMessageRequest.java

@@ -0,0 +1,31 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.nio.ByteBuffer;
+import org.redkale.net.sncp.*;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class SncpMessageRequest extends SncpRequest {
+
+    protected MessageRecord message;
+
+    @SuppressWarnings("OverridableMethodCallInConstructor")
+    public SncpMessageRequest(SncpContext context, MessageRecord message) {
+        super(context, null);
+        this.message = message;
+        readHeader(ByteBuffer.wrap(message.getContent()));
+    }
+
+}

src/org/redkale/mq/SncpMessageResponse.java

@@ -0,0 +1,60 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.mq;
+
+import java.nio.ByteBuffer;
+import org.redkale.convert.bson.BsonWriter;
+import org.redkale.net.Response;
+import org.redkale.net.sncp.*;
+import static org.redkale.net.sncp.SncpRequest.HEADER_SIZE;
+import org.redkale.util.ObjectPool;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class SncpMessageResponse extends SncpResponse {
+
+    protected MessageRecord message;
+
+    protected MessageProducer producer;
+
+    protected Runnable callback;
+
+    public SncpMessageResponse(SncpContext context, SncpMessageRequest request, Runnable callback, ObjectPool<Response> responsePool, MessageProducer producer) {
+        super(context, request, responsePool);
+        this.message = request.message;
+        this.callback = callback;
+        this.producer = producer;
+    }
+
+    public SncpMessageResponse(SncpContext context, MessageRecord message, Runnable callback, ObjectPool<Response> responsePool, MessageProducer producer) {
+        super(context, new SncpMessageRequest(context, message), responsePool);
+        this.message = message;
+        this.callback = callback;
+        this.producer = producer;
+    }
+
+    @Override
+    public void finish(final int retcode, final BsonWriter out) {
+        if (callback != null) callback.run();
+        if (out == null) {
+            final byte[] result = new byte[SncpRequest.HEADER_SIZE];
+            fillHeader(ByteBuffer.wrap(result), 0, retcode);
+            producer.apply(new MessageRecord(message.getSeqid(), message.getResptopic(), null, (byte[]) null));
+            return;
+        }
+        final int respBodyLength = out.count(); //body总长度
+        final byte[] result = out.toArray();
+        fillHeader(ByteBuffer.wrap(result), respBodyLength - HEADER_SIZE, retcode);
+        producer.apply(new MessageRecord(message.getSeqid(), message.getResptopic(), null, result));
+    }
+}

src/org/redkale/net/AsyncConnection.java

@@ -5,7 +5,7 @@
  */
 package org.redkale.net;
 
-import java.io.IOException;
+import java.io.*;
 import java.net.*;
 import java.nio.*;
 import java.nio.channels.*;
@@ -23,45 +23,49 @@ import org.redkale.util.*;
  *
  * @author zhangjx
  */
-public abstract class AsyncConnection implements ReadableByteChannel, WritableByteChannel, AutoCloseable {
+public abstract class AsyncConnection implements AutoCloseable {
 
-    protected SSLContext sslContext;
+    private SSLContext sslContext;
 
-    protected Map<String, Object> attributes; //用于存储绑定在Connection上的对象集合
+    private Map<String, Object> attributes; //用于存储绑定在Connection上的对象集合
 
-    protected Object subobject; //用于存储绑定在Connection上的对象， 同attributes， 只绑定单个对象时尽量使用subobject而非attributes
+    private Object subobject; //用于存储绑定在Connection上的对象， 同attributes， 只绑定单个对象时尽量使用subobject而非attributes
 
     protected volatile long readtime;
 
     protected volatile long writetime;
 
-    protected final Supplier<ByteBuffer> bufferSupplier;
+    private final Supplier<ByteBuffer> bufferSupplier;
 
-    protected final Consumer<ByteBuffer> bufferConsumer;
+    private final Consumer<ByteBuffer> bufferConsumer;
 
-    protected ByteBuffer readBuffer;
+    private ByteBuffer readBuffer;
 
     //在线数
-    protected AtomicLong livingCounter;
+    private AtomicLong livingCounter;
 
     //关闭数
-    protected AtomicLong closedCounter;
+    private AtomicLong closedCounter;
 
-    protected Consumer<AsyncConnection> beforeCloseListener;
+    private Consumer<AsyncConnection> beforeCloseListener;
 
     //关联的事件数， 小于1表示没有事件
-    protected final AtomicInteger eventing = new AtomicInteger();
+    private final AtomicInteger eventing = new AtomicInteger();
 
-    protected AsyncConnection(ObjectPool<ByteBuffer> bufferPool, SSLContext sslContext) {
+    protected AsyncConnection(ObjectPool<ByteBuffer> bufferPool, SSLContext sslContext,
-        this(bufferPool, bufferPool, sslContext);
+        final AtomicLong livingCounter, final AtomicLong closedCounter) {
+        this(bufferPool, bufferPool, sslContext, livingCounter, closedCounter);
     }
 
-    protected AsyncConnection(Supplier<ByteBuffer> bufferSupplier, Consumer<ByteBuffer> bufferConsumer, SSLContext sslContext) {
+    protected AsyncConnection(Supplier<ByteBuffer> bufferSupplier, Consumer<ByteBuffer> bufferConsumer, SSLContext sslContext,
+        final AtomicLong livingCounter, final AtomicLong closedCounter) {
         Objects.requireNonNull(bufferSupplier);
         Objects.requireNonNull(bufferConsumer);
         this.bufferSupplier = bufferSupplier;
         this.bufferConsumer = bufferConsumer;
         this.sslContext = sslContext;
+        this.livingCounter = livingCounter;
+        this.closedCounter = closedCounter;
     }
 
     public Supplier<ByteBuffer> getBufferSupplier() {
@@ -88,7 +92,6 @@ public abstract class AsyncConnection implements ReadableByteChannel, WritableBy
         return eventing.decrementAndGet();
     }
 
-    @Override
     public abstract boolean isOpen();
 
     public abstract boolean isTCP();
@@ -113,13 +116,11 @@ public abstract class AsyncConnection implements ReadableByteChannel, WritableBy
 
     public abstract void setWriteTimeoutSeconds(int writeTimeoutSeconds);
 
-    @Override
+    public abstract ReadableByteChannel readableByteChannel();
-    public abstract int read(ByteBuffer dst) throws IOException;
 
     public abstract void read(CompletionHandler<Integer, ByteBuffer> handler);
 
-    @Override
+    public abstract WritableByteChannel writableByteChannel();
-    public abstract int write(ByteBuffer src) throws IOException;
 
     public abstract <A> void write(ByteBuffer src, A attachment, CompletionHandler<Integer, ? super A> handler);
 
@@ -140,40 +141,23 @@ public abstract class AsyncConnection implements ReadableByteChannel, WritableBy
             this.readBuffer = null;
             return rs;
         }
-//        Thread thread = Thread.currentThread();
-//        if (thread instanceof IOThread) {
-//            return ((IOThread) thread).getBufferPool().get();
-//        }
         return bufferSupplier.get();
     }
 
     public void offerBuffer(ByteBuffer buffer) {
         if (buffer == null) return;
-//        Thread thread = Thread.currentThread();
-//        if (thread instanceof IOThread) {
-//            ((IOThread) thread).getBufferPool().accept((ByteBuffer) buffer);
-//            return;
-//        }
         bufferConsumer.accept(buffer);
     }
 
     public void offerBuffer(ByteBuffer... buffers) {
         if (buffers == null) return;
         Consumer<ByteBuffer> consumer = this.bufferConsumer;
-//        Thread thread = Thread.currentThread();
-//        if (thread instanceof IOThread) {
-//            consumer = ((IOThread) thread).getBufferPool();
-//        }
         for (ByteBuffer buffer : buffers) {
             consumer.accept(buffer);
         }
     }
 
     public ByteBuffer pollWriteBuffer() {
-//        Thread thread = Thread.currentThread();
-//        if (thread instanceof IOThread) {
-//            return ((IOThread) thread).getBufferPool().get();
-//        }
         return bufferSupplier.get();
     }
 

src/org/redkale/net/IOThread.java

@@ -1,61 +0,0 @@
-/*
- * To change this license header, choose License Headers in Project Properties.
- * To change this template file, choose Tools | Templates
- * and open the template in the editor.
- */
-package org.redkale.net;
-
-import java.nio.ByteBuffer;
-import java.util.concurrent.ExecutorService;
-import org.redkale.util.*;
-
-/**
- * 协议处理的IO线程类
- *
- * <p>
- * 详情见: https://redkale.org
- *
- * @author zhangjx
- */
-public class IOThread extends Thread {
-
-    protected Thread localThread;
-
-    protected final ExecutorService executor;
-
-    protected ObjectPool<ByteBuffer> bufferPool;
-
-    public IOThread(ExecutorService executor, ObjectPool<ByteBuffer> bufferPool, Runnable runner) {
-        super(runner);
-        this.executor = executor;
-        this.bufferPool = bufferPool;
-        this.setDaemon(true);
-    }
-
-    public void runAsync(Runnable runner) {
-        executor.execute(runner);
-    }
-
-    public ExecutorService getExecutor() {
-        return executor;
-    }
-
-    public ObjectPool<ByteBuffer> getBufferPool() {
-        return bufferPool;
-    }
-
-    @Override
-    public void run() {
-        this.localThread = Thread.currentThread();
-        super.run();
-    }
-
-    public boolean inSameThread() {
-        return this.localThread == Thread.currentThread();
-    }
-
-    public boolean inSameThread(Thread thread) {
-        return this.localThread == thread;
-    }
-
-}

src/org/redkale/net/ProtocolServer.java

@@ -73,7 +73,7 @@ public abstract class ProtocolServer {
             } else if ("aio".equalsIgnoreCase(netimpl)) {
                 return new TcpAioProtocolServer(context);
             } else if ("nio".equalsIgnoreCase(netimpl)) {
-                return null;//  return new TcpNioProtocolServer(context);
+                return new TcpNioProtocolServer(context);
             }
         } else if ("UDP".equalsIgnoreCase(protocol)) {
             if (netimpl == null || netimpl.isEmpty()) {

src/org/redkale/net/Response.java

@@ -27,7 +27,7 @@ public abstract class Response<C extends Context, R extends Request<C>> {
 
     protected final C context;
 
-    protected final ObjectPool<Response> responsePool;
+    protected final ObjectPool<Response> responsePool; //虚拟构建的Response可能不存在responsePool
 
     protected final R request;
 
@@ -223,7 +223,7 @@ public abstract class Response<C extends Context, R extends Request<C>> {
     public void finish(final byte[] bs) {
         if (!this.inited) return; //避免重复关闭
         if (this.context.bufferCapacity == bs.length) {
-            ByteBuffer buffer = channel.bufferSupplier.get();
+            ByteBuffer buffer = channel.getBufferSupplier().get();
             buffer.put(bs);
             buffer.flip();
             this.finish(buffer);

src/org/redkale/net/Server.java

@@ -86,7 +86,7 @@ public abstract class Server<K extends Serializable, C extends Context, R extend
     protected int threads;
 
     //线程池
-    protected ThreadPoolExecutor executor;
+    protected ThreadPoolExecutor workExecutor;
 
     //ByteBuffer池大小
     protected int bufferPoolSize;
@@ -151,8 +151,8 @@ public abstract class Server<K extends Serializable, C extends Context, R extend
         final AtomicInteger counter = new AtomicInteger();
         final Format f = createFormat();
         final String n = name;
-        this.executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads, (Runnable r) -> {
+        this.workExecutor = (ThreadPoolExecutor) Executors.newFixedThreadPool(threads, (Runnable r) -> {
-            Thread t = new WorkThread(executor, r);
+            Thread t = new WorkThread(workExecutor, r);
             t.setName("Redkale-" + n + "-ServletThread-" + f.format(counter.incrementAndGet()));
             return t;
         });
@@ -187,8 +187,8 @@ public abstract class Server<K extends Serializable, C extends Context, R extend
         return resourceFactory;
     }
 
-    public ThreadPoolExecutor getExecutor() {
+    public ThreadPoolExecutor getWorkExecutor() {
-        return executor;
+        return workExecutor;
     }
 
     public InetSocketAddress getSocketAddress() {
@@ -285,7 +285,7 @@ public abstract class Server<K extends Serializable, C extends Context, R extend
         final String threadName = "[" + Thread.currentThread().getName() + "] ";
         postStart();
         logger.info(threadName + this.getClass().getSimpleName() + ("TCP".equalsIgnoreCase(protocol) ? "" : ("." + protocol)) + " listen: " + address
-            + ", threads: " + threads + ", maxbody: " + formatLenth(context.maxbody) + ", bufferCapacity: " + formatLenth(bufferCapacity) + ", bufferPoolSize: " + bufferPoolSize + ", responsePoolSize: " + responsePoolSize
+            + ", cpu: " + Runtime.getRuntime().availableProcessors() + ", threads: " + threads + ", maxbody: " + formatLenth(context.maxbody) + ", bufferCapacity: " + formatLenth(bufferCapacity) + ", bufferPoolSize: " + bufferPoolSize + ", responsePoolSize: " + responsePoolSize
             + ", started in " + (System.currentTimeMillis() - context.getServerStartTime()) + " ms");
     }
 

src/org/redkale/net/TcpAioAsyncConnection.java

@@ -5,7 +5,7 @@
  */
 package org.redkale.net;
 
-import java.io.IOException;
+import java.io.*;
 import java.net.*;
 import java.nio.ByteBuffer;
 import java.nio.channels.*;
@@ -22,10 +22,9 @@ import javax.net.ssl.SSLContext;
  *
  * @author zhangjx
  */
-public class TcpAioAsyncConnection extends AsyncConnection {
+class TcpAioAsyncConnection extends AsyncConnection {
 
     //private final Semaphore semaphore = new Semaphore(1);
-
     private int readTimeoutSeconds;
 
     private int writeTimeoutSeconds;
@@ -40,7 +39,7 @@ public class TcpAioAsyncConnection extends AsyncConnection {
         final AsynchronousSocketChannel ch, final SSLContext sslContext, final SocketAddress addr0,
         final int readTimeoutSeconds, final int writeTimeoutSeconds,
         final AtomicLong livingCounter, final AtomicLong closedCounter) {
-        super(bufferSupplier, bufferConsumer, sslContext);
+        super(bufferSupplier, bufferConsumer, sslContext, livingCounter, closedCounter);
         this.channel = ch;
         this.readTimeoutSeconds = readTimeoutSeconds;
         this.writeTimeoutSeconds = writeTimeoutSeconds;
@@ -53,8 +52,6 @@ public class TcpAioAsyncConnection extends AsyncConnection {
             }
         }
         this.remoteAddress = addr;
-        this.livingCounter = livingCounter;
-        this.closedCounter = closedCounter;
     }
 
     @Override
@@ -132,7 +129,6 @@ public class TcpAioAsyncConnection extends AsyncConnection {
 //            semaphore.release();
 //        }
 //    }
-
     @Override
     public <A> void write(ByteBuffer src, A attachment, CompletionHandler<Integer, ? super A> handler) {
         write(true, src, attachment, handler);
@@ -232,23 +228,57 @@ public class TcpAioAsyncConnection extends AsyncConnection {
     }
 
     @Override
-    public final int read(ByteBuffer dst) throws IOException {
+    public final ReadableByteChannel readableByteChannel() {
+        return new ReadableByteChannel() {
+            @Override
+            public int read(ByteBuffer dst) throws IOException {
                 try {
-            return channel.read(dst).get();
+                    return channel.read(dst).get(readTimeoutSeconds > 0 ? readTimeoutSeconds : 6, TimeUnit.SECONDS);
-        } catch (InterruptedException | ExecutionException e) {
+                } catch (Exception e) {
                     throw new IOException(e);
                 }
             }
 
             @Override
-    public final int write(ByteBuffer src) throws IOException {
+            @SuppressWarnings("InfiniteRecursion")
+            public boolean isOpen() {
+                return isOpen();
+            }
+
+            @Override
+            @SuppressWarnings("InfiniteRecursion")
+            public void close() throws IOException {
+                close();
+            }
+        };
+    }
+
+    @Override
+    public final WritableByteChannel writableByteChannel() {
+        return new WritableByteChannel() {
+            @Override
+            public int write(ByteBuffer src) throws IOException {
                 try {
-            return channel.write(src).get();
+                    return channel.write(src).get(readTimeoutSeconds > 0 ? readTimeoutSeconds : 6, TimeUnit.SECONDS);
-        } catch (InterruptedException | ExecutionException e) {
+                } catch (Exception e) {
                     throw new IOException(e);
                 }
             }
 
+            @Override
+            @SuppressWarnings("InfiniteRecursion")
+            public boolean isOpen() {
+                return isOpen();
+            }
+
+            @Override
+            @SuppressWarnings("InfiniteRecursion")
+            public void close() throws IOException {
+                close();
+            }
+        };
+    }
+
     @Override
     public final void close() throws IOException {
         super.close();

src/org/redkale/net/TcpNioAsyncConnection.java

@@ -0,0 +1,545 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.net;
+
+import java.io.IOException;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.*;
+import java.util.*;
+import java.util.concurrent.*;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.*;
+import javax.net.ssl.SSLContext;
+import org.redkale.net.AsyncConnection;
+import org.redkale.net.nio.NioCompletionHandler;
+import org.redkale.net.nio.NioThread;
+import org.redkale.net.nio.NioThreadGroup;
+import org.redkale.util.ObjectPool;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class TcpNioAsyncConnection extends AsyncConnection {
+
+    private int readTimeoutSeconds;
+
+    private int writeTimeoutSeconds;
+
+    private final SocketAddress remoteAddress;
+
+    final SocketChannel channel;
+
+    final NioThread ioThread;
+
+    final NioThreadGroup ioGroup;
+
+    final ExecutorService workExecutor;
+
+    //连
+    private Object connectAttachment;
+
+    private CompletionHandler<Void, Object> connectCompletionHandler;
+
+    private boolean connectPending;
+
+    private SelectionKey connectKey;
+
+    //读操作
+    private ByteBuffer readByteBuffer;
+
+    private CompletionHandler<Integer, ByteBuffer> readCompletionHandler;
+
+    private boolean readPending;
+
+    private SelectionKey readKey;
+
+    //写操作, 二选一，要么writeByteBuffer有值，要么writeByteBuffers、writeOffset、writeLength有值
+    private ByteBuffer writeByteBuffer;
+
+    private ByteBuffer[] writeByteBuffers;
+
+    private int writeOffset;
+
+    private int writeLength;
+
+    private Object writeAttachment;
+
+    private CompletionHandler<Integer, Object> writeCompletionHandler;
+
+    private boolean writePending;
+
+    private SelectionKey writeKey;
+
+    public TcpNioAsyncConnection(NioThreadGroup ioGroup, NioThread ioThread, ExecutorService workExecutor,
+        ObjectPool<ByteBuffer> bufferPool, SocketChannel ch,
+        SSLContext sslContext, final SocketAddress addr0, AtomicLong livingCounter, AtomicLong closedCounter) {
+        super(bufferPool, sslContext, livingCounter, closedCounter);
+        this.ioGroup = ioGroup;
+        this.ioThread = ioThread;
+        this.workExecutor = workExecutor;
+        this.channel = ch;
+        SocketAddress addr = addr0;
+        if (addr == null) {
+            try {
+                addr = ch.getRemoteAddress();
+            } catch (Exception e) {
+                //do nothing
+            }
+        }
+        this.remoteAddress = addr;
+    }
+
+    public TcpNioAsyncConnection(NioThreadGroup ioGroup, NioThread ioThread, ExecutorService workExecutor,
+        Supplier<ByteBuffer> bufferSupplier, Consumer<ByteBuffer> bufferConsumer,
+        SocketChannel ch, SSLContext sslContext, final SocketAddress addr0,
+        AtomicLong livingCounter, AtomicLong closedCounter) {
+        super(bufferSupplier, bufferConsumer, sslContext, livingCounter, closedCounter);
+        this.ioGroup = ioGroup;
+        this.ioThread = ioThread;
+        this.workExecutor = workExecutor;
+        this.channel = ch;
+        SocketAddress addr = addr0;
+        if (addr == null) {
+            try {
+                addr = ch.getRemoteAddress();
+            } catch (Exception e) {
+                //do nothing
+            }
+        }
+        this.remoteAddress = addr;
+    }
+
+    @Override
+    public boolean isOpen() {
+        return this.channel.isOpen();
+    }
+
+    @Override
+    public boolean isTCP() {
+        return true;
+    }
+
+    @Override
+    public boolean shutdownInput() {
+        try {
+            this.channel.shutdownInput();
+            return true;
+        } catch (IOException e) {
+            return false;
+        }
+    }
+
+    @Override
+    public boolean shutdownOutput() {
+        try {
+            this.channel.shutdownOutput();
+            return true;
+        } catch (IOException e) {
+            return false;
+        }
+    }
+
+    @Override
+    public <T> boolean setOption(SocketOption<T> name, T value) {
+        try {
+            this.channel.setOption(name, value);
+            return true;
+        } catch (IOException e) {
+            return false;
+        }
+    }
+
+    @Override
+    public Set<SocketOption<?>> supportedOptions() {
+        return this.channel.supportedOptions();
+    }
+
+    @Override
+    public SocketAddress getRemoteAddress() {
+        return remoteAddress;
+    }
+
+    @Override
+    public SocketAddress getLocalAddress() {
+        try {
+            return channel.getLocalAddress();
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
+    @Override
+    public void setReadTimeoutSeconds(int readTimeoutSeconds) {
+        this.readTimeoutSeconds = readTimeoutSeconds;
+    }
+
+    @Override
+    public void setWriteTimeoutSeconds(int writeTimeoutSeconds) {
+        this.writeTimeoutSeconds = writeTimeoutSeconds;
+    }
+
+    @Override
+    public int getReadTimeoutSeconds() {
+        return this.readTimeoutSeconds;
+    }
+
+    @Override
+    public int getWriteTimeoutSeconds() {
+        return this.writeTimeoutSeconds;
+    }
+
+    @Override
+    public ReadableByteChannel readableByteChannel() {
+        return this.channel;
+    }
+
+    @Override
+    public void read(CompletionHandler<Integer, ByteBuffer> handler) {
+        Objects.requireNonNull(handler);
+        if (!this.channel.isConnected()) {
+            if (this.workExecutor == null) {
+                handler.failed(new NotYetConnectedException(), pollReadBuffer());
+            } else {
+                this.workExecutor.execute(() -> handler.failed(new NotYetConnectedException(), pollReadBuffer()));
+            }
+            return;
+        }
+        if (this.readPending) {
+            if (this.workExecutor == null) {
+                handler.failed(new ReadPendingException(), pollReadBuffer());
+            } else {
+                this.workExecutor.execute(() -> handler.failed(new ReadPendingException(), pollReadBuffer()));
+            }
+            return;
+        }
+        this.readPending = true;
+        if (this.readTimeoutSeconds > 0) {
+            NioCompletionHandler newhandler = new NioCompletionHandler(handler, this.readByteBuffer);
+            this.readCompletionHandler = newhandler;
+            newhandler.timeoutFuture = ioGroup.scheduleTimeout(newhandler, this.readTimeoutSeconds, TimeUnit.SECONDS);
+        } else {
+            this.readCompletionHandler = handler;
+        }
+        doRead();
+    }
+
+    @Override
+    public WritableByteChannel writableByteChannel() {
+        return this.channel;
+    }
+
+    @Override
+    public <A> void write(ByteBuffer src, A attachment, CompletionHandler<Integer, ? super A> handler) {
+        Objects.requireNonNull(src);
+        Objects.requireNonNull(handler);
+        if (!this.channel.isConnected()) {
+            if (this.workExecutor == null) {
+                handler.failed(new NotYetConnectedException(), attachment);
+            } else {
+                this.workExecutor.execute(() -> handler.failed(new NotYetConnectedException(), attachment));
+            }
+            return;
+        }
+        if (this.writePending) {
+            if (this.workExecutor == null) {
+                handler.failed(new WritePendingException(), attachment);
+            } else {
+                this.workExecutor.execute(() -> handler.failed(new WritePendingException(), attachment));
+            }
+            return;
+        }
+        this.writePending = true;
+        this.writeByteBuffer = src;
+        this.writeAttachment = attachment;
+        if (this.writeTimeoutSeconds > 0) {
+            NioCompletionHandler newhandler = new NioCompletionHandler(handler, attachment);
+            this.writeCompletionHandler = newhandler;
+            newhandler.timeoutFuture = ioGroup.scheduleTimeout(newhandler, this.writeTimeoutSeconds, TimeUnit.SECONDS);
+        } else {
+            this.writeCompletionHandler = (CompletionHandler) handler;
+        }
+        doWrite();
+    }
+
+    @Override
+    public <A> void write(ByteBuffer[] srcs, int offset, int length, A attachment, CompletionHandler<Integer, ? super A> handler) {
+        Objects.requireNonNull(srcs);
+        Objects.requireNonNull(handler);
+        if (!this.channel.isConnected()) {
+            if (this.workExecutor == null) {
+                handler.failed(new NotYetConnectedException(), attachment);
+            } else {
+                this.workExecutor.execute(() -> handler.failed(new NotYetConnectedException(), attachment));
+            }
+            return;
+        }
+        if (this.writePending) {
+            if (this.workExecutor == null) {
+                handler.failed(new WritePendingException(), attachment);
+            } else {
+                this.workExecutor.execute(() -> handler.failed(new WritePendingException(), attachment));
+            }
+            return;
+        }
+        this.writePending = true;
+        this.writeByteBuffers = srcs;
+        this.writeOffset = offset;
+        this.writeLength = length;
+        this.writeAttachment = attachment;
+        if (this.writeTimeoutSeconds > 0) {
+            NioCompletionHandler newhandler = new NioCompletionHandler(handler, attachment);
+            this.writeCompletionHandler = newhandler;
+            newhandler.timeoutFuture = ioGroup.scheduleTimeout(newhandler, this.writeTimeoutSeconds, TimeUnit.SECONDS);
+        } else {
+            this.writeCompletionHandler = (CompletionHandler) handler;
+        }
+        doWrite();
+    }
+
+    public void doConnect() {
+        try {
+            boolean connected = channel.isConnectionPending();
+            if (connected || channel.connect(remoteAddress)) {
+                connected = channel.finishConnect();
+            }
+            if (connected) {
+                CompletionHandler handler = this.connectCompletionHandler;
+                Object attach = this.connectAttachment;
+                clearConnect();
+                if (handler != null) {
+                    if (this.workExecutor == null) {
+                        handler.completed(null, attach);
+                    } else {
+                        this.workExecutor.execute(() -> handler.completed(null, attach));
+                    }
+                }
+            } else if (connectKey == null) {
+                ioThread.register(selector -> {
+                    try {
+                        connectKey = channel.register(selector, SelectionKey.OP_CONNECT);
+                        connectKey.attach(this);
+                    } catch (ClosedChannelException e) {
+                        CompletionHandler handler = this.connectCompletionHandler;
+                        Object attach = this.connectAttachment;
+                        clearConnect();
+                        if (handler != null) {
+                            if (this.workExecutor == null) {
+                                handler.failed(e, attach);
+                            } else {
+                                this.workExecutor.execute(() -> handler.failed(e, attach));
+                            }
+                        }
+                    }
+                });
+            } else {
+                CompletionHandler handler = this.connectCompletionHandler;
+                Object attach = this.connectAttachment;
+                clearConnect();
+                if (handler != null) {
+                    IOException e = new IOException();
+                    if (this.workExecutor == null) {
+                        handler.failed(e, attach);
+                    } else {
+                        this.workExecutor.execute(() -> handler.failed(e, attach));
+                    }
+                }
+            }
+        } catch (IOException e) {
+            CompletionHandler handler = this.connectCompletionHandler;
+            Object attach = this.connectAttachment;
+            clearConnect();
+            if (handler != null) {
+                if (this.workExecutor == null) {
+                    handler.failed(e, attach);
+                } else {
+                    this.workExecutor.execute(() -> handler.failed(e, attach));
+                }
+            }
+        }
+    }
+
+    private void clearConnect() {
+        this.connectCompletionHandler = null;
+        this.connectAttachment = null;
+        this.connectPending = false;//必须放最后
+    }
+
+    public void doRead() {
+        try {
+            final boolean invokeDirect = this.ioThread.inSameThread();
+            int totalCount = 0;
+            boolean hasRemain = true;
+            if (invokeDirect && this.readByteBuffer == null) {
+                this.readByteBuffer = pollReadBuffer();
+                if (this.readTimeoutSeconds > 0) {
+                    ((NioCompletionHandler) this.readCompletionHandler).setAttachment(this.readByteBuffer);
+                }
+            }
+            while (invokeDirect && hasRemain) {
+                int readCount = this.channel.read(readByteBuffer);
+                hasRemain = readByteBuffer.hasRemaining();
+                if (readCount <= 0) {
+                    if (totalCount == 0) totalCount = readCount;
+                    break;
+                }
+                totalCount += readCount;
+            }
+            if (totalCount != 0 || !hasRemain) {
+                if (readKey != null) readKey.interestOps(readKey.interestOps() & ~SelectionKey.OP_READ);
+                CompletionHandler<Integer, ByteBuffer> handler = this.readCompletionHandler;
+                ByteBuffer attach = this.readByteBuffer;
+                clearRead();
+                if (handler != null) {
+                    if (this.workExecutor == null) {
+                        handler.completed(totalCount, attach);
+                    } else {
+                        final int totalCount0 = totalCount;
+                        this.workExecutor.execute(() -> handler.completed(totalCount0, attach));
+                    }
+                }
+            } else if (readKey == null) {
+                ioThread.register(selector -> {
+                    try {
+                        readKey = channel.register(selector, SelectionKey.OP_READ);
+                        readKey.attach(this);
+                    } catch (ClosedChannelException e) {
+                        CompletionHandler<Integer, ByteBuffer> handler = this.readCompletionHandler;
+                        ByteBuffer attach = this.readByteBuffer;
+                        clearRead();
+                        if (handler != null) {
+                            if (this.workExecutor == null) {
+                                handler.failed(e, attach);
+                            } else {
+                                this.workExecutor.execute(() -> handler.failed(e, attach));
+                            }
+                        }
+                    }
+                });
+            } else {
+                ioGroup.interestOpsOr(ioThread, readKey, SelectionKey.OP_READ);
+            }
+        } catch (Exception e) {
+            CompletionHandler<Integer, ByteBuffer> handler = this.readCompletionHandler;
+            ByteBuffer attach = this.readByteBuffer;
+            clearRead();
+            if (handler != null) {
+                if (this.workExecutor == null) {
+                    handler.failed(e, attach);
+                } else {
+                    this.workExecutor.execute(() -> handler.failed(e, attach));
+                }
+            }
+        }
+    }
+
+    private void clearRead() {
+        this.readCompletionHandler = null;
+        this.readByteBuffer = null;
+        this.readPending = false; //必须放最后
+    }
+
+    public void doWrite() {
+        try {
+            final boolean invokeDirect = this.ioThread.inSameThread();
+            int totalCount = 0;
+            boolean hasRemain = true;
+            while (invokeDirect && hasRemain) {
+                int writeCount;
+                if (writeByteBuffer != null) {
+                    writeCount = channel.write(writeByteBuffer);
+                    hasRemain = writeByteBuffer.hasRemaining();
+                } else {
+                    writeCount = (int) channel.write(writeByteBuffers, writeOffset, writeLength);
+                    boolean remain = false;
+                    for (int i = writeByteBuffers.length - 1; i >= writeOffset; i--) {
+                        if (writeByteBuffers[i].hasRemaining()) {
+                            remain = true;
+                            break;
+                        }
+                    }
+                    hasRemain = remain;
+                }
+                if (writeCount <= 0) {
+                    if (totalCount == 0) totalCount = writeCount;
+                    break;
+                }
+                totalCount += writeCount;
+            }
+
+            if (totalCount > 0 || !hasRemain) {
+                if (writeKey != null) writeKey.interestOps(writeKey.interestOps() & ~SelectionKey.OP_WRITE);
+                CompletionHandler<Integer, Object> handler = this.writeCompletionHandler;
+                Object attach = this.writeAttachment;
+                clearWrite();
+                if (handler != null) {
+                    if (this.workExecutor == null) {
+                        handler.completed(totalCount, attach);
+                    } else {
+                        final int totalCount0 = totalCount;
+                        this.workExecutor.execute(() -> handler.completed(totalCount0, attach));
+                    }
+                }
+            } else if (writeKey == null) {
+                ioThread.register(selector -> {
+                    try {
+                        writeKey = channel.register(selector, SelectionKey.OP_WRITE);
+                        writeKey.attach(this);
+                    } catch (ClosedChannelException e) {
+                        CompletionHandler<Integer, Object> handler = this.writeCompletionHandler;
+                        Object attach = this.writeAttachment;
+                        clearWrite();
+                        if (handler != null) {
+                            if (this.workExecutor == null) {
+                                handler.failed(e, attach);
+                            } else {
+                                this.workExecutor.execute(() -> handler.failed(e, attach));
+                            }
+                        }
+                    }
+                });
+            } else {
+                ioGroup.interestOpsOr(ioThread, writeKey, SelectionKey.OP_WRITE);
+            }
+        } catch (IOException e) {
+            CompletionHandler<Integer, Object> handler = this.writeCompletionHandler;
+            Object attach = this.writeAttachment;
+            clearWrite();
+            if (handler != null) {
+                if (this.workExecutor == null) {
+                    handler.failed(e, attach);
+                } else {
+                    this.workExecutor.execute(() -> handler.failed(e, attach));
+                }
+            }
+        }
+    }
+
+    private void clearWrite() {
+        this.writeCompletionHandler = null;
+        this.writeAttachment = null;
+        this.writeByteBuffer = null;
+        this.writeByteBuffers = null;
+        this.writeOffset = 0;
+        this.writeLength = 0;
+        this.writePending = false; //必须放最后
+    }
+
+    @Override
+    public final void close() throws IOException {
+        super.close();
+        if (this.connectKey != null) this.connectKey.cancel();
+        if (this.readKey != null) this.readKey.cancel();
+        if (this.writeKey != null) this.writeKey.cancel();
+        channel.close();
+    }
+}

src/org/redkale/net/TcpNioProtocolServer.java

@@ -0,0 +1,146 @@
+/*
+ * To change this license header, choose License Headers in Project Properties.
+ * To change this template file, choose Tools | Templates
+ * and open the template in the editor.
+ */
+package org.redkale.net;
+
+import java.io.IOException;
+import java.net.*;
+import java.nio.ByteBuffer;
+import java.nio.channels.*;
+import java.util.*;
+import java.util.concurrent.atomic.AtomicLong;
+import org.redkale.net.*;
+import org.redkale.net.nio.*;
+import org.redkale.util.*;
+
+/**
+ *
+ * <p>
+ * 详情见: https://redkale.org
+ *
+ * @author zhangjx
+ *
+ * @since 2.1.0
+ */
+public class TcpNioProtocolServer extends ProtocolServer {
+
+    private ObjectPool<ByteBuffer> bufferPool;
+
+    private ObjectPool<Response> responsePool;
+
+    private ServerSocketChannel serverChannel;
+
+    private Selector selector;
+
+    private NioThreadGroup ioGroup;
+
+    private Thread acceptThread;
+
+    private boolean closed;
+
+    public TcpNioProtocolServer(Context context) {
+        super(context);
+    }
+
+    @Override
+    public void open(AnyValue config) throws IOException {
+        this.serverChannel = ServerSocketChannel.open();
+        this.serverChannel.configureBlocking(false);
+        this.selector = Selector.open();
+        final Set<SocketOption<?>> options = this.serverChannel.supportedOptions();
+        if (options.contains(StandardSocketOptions.TCP_NODELAY)) {
+            this.serverChannel.setOption(StandardSocketOptions.TCP_NODELAY, true);
+        }
+        if (options.contains(StandardSocketOptions.SO_KEEPALIVE)) {
+            this.serverChannel.setOption(StandardSocketOptions.SO_KEEPALIVE, true);
+        }
+        if (options.contains(StandardSocketOptions.SO_REUSEADDR)) {
+            this.serverChannel.setOption(StandardSocketOptions.SO_REUSEADDR, true);
+        }
+        if (options.contains(StandardSocketOptions.SO_RCVBUF)) {
+            this.serverChannel.setOption(StandardSocketOptions.SO_RCVBUF, 16 * 1024);
+        }
+        if (options.contains(StandardSocketOptions.SO_SNDBUF)) {
+            this.serverChannel.setOption(StandardSocketOptions.SO_SNDBUF, 16 * 1024);
+        }
+    }
+
+    @Override
+    public void bind(SocketAddress local, int backlog) throws IOException {
+        this.serverChannel.bind(local, backlog);
+    }
+
+    @Override
+    public <T> Set<SocketOption<?>> supportedOptions() {
+        return this.serverChannel.supportedOptions();
+    }
+
+    @Override
+    public <T> void setOption(SocketOption<T> name, T value) throws IOException {
+        this.serverChannel.setOption(name, value);
+    }
+
+    @Override
+    public void accept(Server server) throws IOException {
+        this.serverChannel.register(this.selector, SelectionKey.OP_ACCEPT);
+        
+        AtomicLong createBufferCounter = new AtomicLong();
+        AtomicLong cycleBufferCounter = new AtomicLong();
+        this.bufferPool = server.createBufferPool(createBufferCounter, cycleBufferCounter, server.bufferPoolSize);
+        AtomicLong createResponseCounter = new AtomicLong();
+        AtomicLong cycleResponseCounter = new AtomicLong();
+        this.responsePool = server.createResponsePool(createResponseCounter, cycleResponseCounter, server.responsePoolSize);
+        this.responsePool.setCreator(server.createResponseCreator(bufferPool, responsePool));
+        
+        this.ioGroup = new NioThreadGroup(Runtime.getRuntime().availableProcessors(), context.executor, bufferPool);
+        this.ioGroup.start();
+        
+        this.acceptThread = new Thread() {
+            @Override
+            public void run() {
+                while (!closed) {
+                    try {
+                        int count = selector.select();
+                        if (count == 0) continue;
+                        Set<SelectionKey> keys = selector.selectedKeys();
+                        Iterator<SelectionKey> it = keys.iterator();
+                        while (it.hasNext()) {
+                            SelectionKey key = it.next();
+                            it.remove();
+                            if (key.isAcceptable()) accept(key);
+                        }
+                    } catch (Throwable t) {
+                        t.printStackTrace();
+                    }
+                }
+            }
+        };
+        this.acceptThread.start();
+    }
+
+    private void accept(SelectionKey key) throws IOException {
+        SocketChannel channel = this.serverChannel.accept();
+        channel.configureBlocking(false);
+        channel.setOption(StandardSocketOptions.TCP_NODELAY, true);
+        channel.setOption(StandardSocketOptions.SO_KEEPALIVE, true);
+        channel.setOption(StandardSocketOptions.SO_REUSEADDR, true);
+        channel.setOption(StandardSocketOptions.SO_RCVBUF, 16 * 1024);
+        channel.setOption(StandardSocketOptions.SO_SNDBUF, 16 * 1024);
+        NioThread ioThread = ioGroup.nextThread();
+        AsyncConnection conn = new TcpNioAsyncConnection(ioGroup, ioThread, context.executor, bufferPool, channel, context.getSSLContext(), null, livingCounter, closedCounter);
+        new PrepareRunner(context, responsePool, conn, null, null).run();
+    }
+
+    @Override
+    public void close() throws IOException {
+        if (this.closed) return;
+        this.closed = true;
+        this.selector.wakeup();
+        this.ioGroup.close();
+        this.serverChannel.close();
+        this.selector.close();
+    }
+
+}

src/org/redkale/net/Transport.java

@@ -38,8 +38,6 @@ public final class Transport {
 
     protected final String name; //即<group>的name属性
 
-    protected final String subprotocol; //即<group>的subprotocol属性
-
     protected final boolean tcp;
 
     protected final String protocol;
@@ -58,18 +56,19 @@ public final class Transport {
     //负载均衡策略
     protected final TransportStrategy strategy;
 
-    protected Transport(String name, String subprotocol, TransportFactory factory, final ObjectPool<ByteBuffer> transportBufferPool,
+    //连接上限， 为null表示无限制
+    protected Semaphore semaphore;
+
+    protected Transport(String name, TransportFactory factory, final ObjectPool<ByteBuffer> transportBufferPool,
         final AsynchronousChannelGroup transportChannelGroup, final SSLContext sslContext, final InetSocketAddress clientAddress,
         final Collection<InetSocketAddress> addresses, final TransportStrategy strategy) {
-        this(name, DEFAULT_PROTOCOL, subprotocol, factory, transportBufferPool, transportChannelGroup, sslContext, clientAddress, addresses, strategy);
+        this(name, DEFAULT_PROTOCOL, factory, transportBufferPool, transportChannelGroup, sslContext, clientAddress, addresses, strategy);
     }
 
-    protected Transport(String name, String protocol, String subprotocol,
+    protected Transport(String name, String protocol, final TransportFactory factory, final ObjectPool<ByteBuffer> transportBufferPool,
-        final TransportFactory factory, final ObjectPool<ByteBuffer> transportBufferPool,
         final AsynchronousChannelGroup transportChannelGroup, final SSLContext sslContext, final InetSocketAddress clientAddress,
         final Collection<InetSocketAddress> addresses, final TransportStrategy strategy) {
         this.name = name;
-        this.subprotocol = subprotocol == null ? "" : subprotocol.trim();
         this.protocol = protocol;
         this.factory = factory;
         factory.transportReferences.add(new WeakReference<>(this));
@@ -82,9 +81,28 @@ public final class Transport {
         updateRemoteAddresses(addresses);
     }
 
+    public Semaphore getSemaphore() {
+        return semaphore;
+    }
+
+    public void setSemaphore(Semaphore semaphore) {
+        this.semaphore = semaphore;
+    }
+
     public final InetSocketAddress[] updateRemoteAddresses(final Collection<InetSocketAddress> addresses) {
         final TransportNode[] oldNodes = this.transportNodes;
         synchronized (this) {
+            boolean same = false;
+            if (this.transportNodes != null && addresses != null && this.transportNodes.length == addresses.size()) {
+                same = true;
+                for (TransportNode node : this.transportNodes) {
+                    if (!addresses.contains(node.getAddress())) {
+                        same = false;
+                        break;
+                    }
+                }
+            }
+            if (!same) {
                 List<TransportNode> list = new ArrayList<>();
                 if (addresses != null) {
                     for (InetSocketAddress addr : addresses) {
@@ -103,6 +121,7 @@ public final class Transport {
                 }
                 this.transportNodes = list.toArray(new TransportNode[list.size()]);
             }
+        }
         InetSocketAddress[] rs = new InetSocketAddress[oldNodes.length];
         for (int i = 0; i < rs.length; i++) {
             rs[i] = oldNodes[i].getAddress();
@@ -138,10 +157,6 @@ public final class Transport {
         return name;
     }
 
-    public String getSubprotocol() {
-        return subprotocol;
-    }
-
     public void close() {
         TransportNode[] nodes = this.transportNodes;
         if (nodes == null) return;
@@ -198,10 +213,52 @@ public final class Transport {
         return group;
     }
 
+    public String getProtocol() {
+        return protocol;
+    }
+
     public boolean isTCP() {
         return tcp;
     }
 
+    protected CompletableFuture<AsyncConnection> pollAsync(TransportNode node, SocketAddress addr, Supplier<CompletableFuture<AsyncConnection>> func, final int count) {
+        if (count >= 5) {
+            CompletableFuture<AsyncConnection> future = new CompletableFuture<>();
+            future.completeExceptionally(new RuntimeException("create AsyncConnection error"));
+            return future;
+        }
+        final BlockingQueue<AsyncConnection> queue = node.conns;
+        if (!queue.isEmpty()) {
+            AsyncConnection conn;
+            while ((conn = queue.poll()) != null) {
+                if (conn.isOpen()) {
+                    return CompletableFuture.completedFuture(conn);
+                } else {
+                    conn.dispose();
+                }
+            }
+        }
+        if (semaphore != null && !semaphore.tryAcquire()) {
+            return CompletableFuture.supplyAsync(() -> {
+                try {
+                    return queue.poll(1, TimeUnit.SECONDS);
+                } catch (Exception t) {
+                    return null;
+                }
+            }, factory.executor).thenCompose((conn2) -> {
+                if (conn2 != null && conn2.isOpen()) {
+                    return CompletableFuture.completedFuture(conn2);
+                }
+                return pollAsync(node, addr, func, count + 1);
+            });
+        }
+        return func.get().thenApply(conn -> {
+            if (conn != null && semaphore != null) conn.beforeCloseListener((c) -> semaphore.release());
+            return conn;
+        });
+
+    }
+
     public CompletableFuture<AsyncConnection> pollConnection(SocketAddress addr0) {
         if (this.strategy != null) return strategy.pollConnection(addr0, this);
         final TransportNode[] nodes = this.transportNodes;
@@ -222,18 +279,7 @@ public final class Transport {
                 if (node == null) {
                     return AsyncConnection.createTCP(bufferPool, group, sslContext, addr, factory.readTimeoutSeconds, factory.writeTimeoutSeconds);
                 }
-                final BlockingQueue<AsyncConnection> queue = node.conns;
+                return pollAsync(node, addr, () -> AsyncConnection.createTCP(bufferPool, group, sslContext, addr, factory.readTimeoutSeconds, factory.writeTimeoutSeconds), 1);
-                if (!queue.isEmpty()) {
-                    AsyncConnection conn;
-                    while ((conn = queue.poll()) != null) {
-                        if (conn.isOpen()) {
-                            return CompletableFuture.completedFuture(conn);
-                        } else {
-                            conn.dispose();
-                        }
-                    }
-                }
-                return AsyncConnection.createTCP(bufferPool, group, sslContext, addr, factory.readTimeoutSeconds, factory.writeTimeoutSeconds);
             }
 
             //---------------------随机取地址------------------------
@@ -257,11 +303,18 @@ public final class Transport {
                         }
                     }
                 }
+                return pollAsync(one, one.getAddress(), () -> {
                     CompletableFuture future = new CompletableFuture();
-                final AsynchronousSocketChannel channel = AsynchronousSocketChannel.open(group);
+                    AsynchronousSocketChannel channel0 = null;
-                channel.setOption(StandardSocketOptions.TCP_NODELAY, true);
+                    try {
-                channel.setOption(StandardSocketOptions.SO_KEEPALIVE, true);
+                        channel0 = AsynchronousSocketChannel.open(group);
-                channel.setOption(StandardSocketOptions.SO_REUSEADDR, true);
+                        channel0.setOption(StandardSocketOptions.TCP_NODELAY, true);
+                        channel0.setOption(StandardSocketOptions.SO_KEEPALIVE, true);
+                        channel0.setOption(StandardSocketOptions.SO_REUSEADDR, true);
+                    } catch (Exception ex) {
+                        ex.printStackTrace();
+                    }
+                    final AsynchronousSocketChannel channel = channel0;
                     channel.connect(one.address, one, new CompletionHandler<Void, TransportNode>() {
                         @Override
                         public void completed(Void result, TransportNode attachment) {
@@ -296,6 +349,7 @@ public final class Transport {
                         }
                     });
                     return future;
+                }, 1);
             }
             return pollConnection0(nodes, null, now);
         } catch (Exception ex) {

src/org/redkale/net/TransportFactory.java

@@ -21,8 +21,8 @@ import org.redkale.service.Service;
 import org.redkale.util.*;
 
 /**
- * System.getProperty("net.transport.pinginterval", "30") 心跳周期，默认30秒
+ * System.getProperty("net.transport.ping.interval", "30") 心跳周期，默认30秒
- * System.getProperty("net.transport.checkinterval", "30") 检查不可用地址周期，默认30秒
+ * System.getProperty("net.transport.check.interval", "30") 检查不可用地址周期，默认30秒
  *
  * <p>
  * 详情见: https://redkale.org
@@ -65,10 +65,10 @@ public class TransportFactory {
     protected final List<WeakReference<Transport>> transportReferences = new CopyOnWriteArrayList<>();
 
     //连接池大小
-    protected int poolmaxconns = Integer.getInteger("net.transport.poolmaxconns", Math.max(100, Runtime.getRuntime().availableProcessors() * 16)); //最少是wsthreads的两倍
+    protected int poolmaxconns = Integer.getInteger("net.transport.pool.maxconns", Math.max(100, Runtime.getRuntime().availableProcessors() * 16)); //最少是wsthreads的两倍
 
     //检查不可用地址周期， 单位：秒
-    protected int checkinterval = Integer.getInteger("net.transport.checkinterval", 30);
+    protected int checkinterval = Integer.getInteger("net.transport.check.interval", 30);
 
     //心跳周期， 单位：秒
     protected int pinginterval;
@@ -203,16 +203,11 @@ public class TransportFactory {
     }
 
     public Transport createTransportTCP(String name, final InetSocketAddress clientAddress, final Collection<InetSocketAddress> addresses) {
-        return new Transport(name, "TCP", "", this, this.bufferPool, this.channelGroup, this.sslContext, clientAddress, addresses, strategy);
+        return new Transport(name, "TCP", this, this.bufferPool, this.channelGroup, this.sslContext, clientAddress, addresses, strategy);
     }
 
     public Transport createTransport(String name, String protocol, final InetSocketAddress clientAddress, final Collection<InetSocketAddress> addresses) {
-        return new Transport(name, protocol, "", this, this.bufferPool, this.channelGroup, this.sslContext, clientAddress, addresses, strategy);
+        return new Transport(name, protocol, this, this.bufferPool, this.channelGroup, this.sslContext, clientAddress, addresses, strategy);
-    }
-
-    public Transport createTransport(String name, String protocol, String subprotocol,
-        final InetSocketAddress clientAddress, final Collection<InetSocketAddress> addresses) {
-        return new Transport(name, protocol, subprotocol, this, this.bufferPool, this.channelGroup, this.sslContext, clientAddress, addresses, strategy);
     }
 
     public String findGroupName(InetSocketAddress addr) {
@@ -251,7 +246,6 @@ public class TransportFactory {
         if (!checkName(info.name)) throw new RuntimeException("Transport.group.name only 0-9 a-z A-Z _ cannot begin 0-9");
         TransportGroupInfo old = groupInfos.get(info.name);
         if (old != null && !old.protocol.equals(info.protocol)) throw new RuntimeException("Transport.group.name repeat but protocol is different");
-        if (old != null && !old.subprotocol.equals(info.subprotocol)) throw new RuntimeException("Transport.group.name repeat but subprotocol is different");
         for (InetSocketAddress addr : info.addresses) {
             if (!groupAddrs.getOrDefault(addr, info.name).equals(info.name)) throw new RuntimeException(addr + " repeat but different group.name");
         }
@@ -266,23 +260,7 @@ public class TransportFactory {
         return true;
     }
 
-    public Transport loadSameGroupTransport(InetSocketAddress sncpAddress) {
+    public Transport loadTransport(InetSocketAddress sncpAddress, final Set<String> groups) {
-        return loadTransport(groupAddrs.get(sncpAddress), sncpAddress);
-    }
-
-    public Transport[] loadDiffGroupTransports(InetSocketAddress sncpAddress, final Set<String> diffGroups) {
-        if (diffGroups == null) return null;
-        final String sncpGroup = groupAddrs.get(sncpAddress);
-        final List<Transport> transports = new ArrayList<>();
-        for (String group : diffGroups) {
-            if (sncpGroup == null || !sncpGroup.equals(group)) {
-                transports.add(loadTransport(group, sncpAddress));
-            }
-        }
-        return transports.toArray(new Transport[transports.size()]);
-    }
-
-    public Transport loadRemoteTransport(InetSocketAddress sncpAddress, final Set<String> groups) {
         if (groups == null) return null;
         Set<InetSocketAddress> addresses = new HashSet<>();
         TransportGroupInfo info = null;
@@ -293,14 +271,7 @@ public class TransportFactory {
         }
         if (info == null) info = new TransportGroupInfo("TCP");
         if (sncpAddress != null) addresses.remove(sncpAddress);
-        return new Transport(groups.stream().sorted().collect(Collectors.joining(";")), info.protocol, info.subprotocol, this, this.bufferPool, this.channelGroup, this.sslContext, sncpAddress, addresses, this.strategy);
+        return new Transport(groups.stream().sorted().collect(Collectors.joining(";")), info.protocol, this, this.bufferPool, this.channelGroup, this.sslContext, sncpAddress, addresses, this.strategy);
-    }
-
-    private Transport loadTransport(final String groupName, InetSocketAddress sncpAddress) {
-        if (groupName == null) return null;
-        TransportGroupInfo info = groupInfos.get(groupName);
-        if (info == null) return null;
-        return new Transport(groupName, info.protocol, info.subprotocol, this, this.bufferPool, this.channelGroup, this.sslContext, sncpAddress, info.addresses, this.strategy);
     }
 
     public ExecutorService getExecutor() {

src/org/redkale/net/TransportGroupInfo.java

@@ -24,30 +24,27 @@ public class TransportGroupInfo {
 
     protected String protocol; //协议 取值范围:  TCP、UDP
 
-    protected String subprotocol; //子协议，预留使用
-
     protected Set<InetSocketAddress> addresses; //地址列表， 对应 resources-&#62;group-&#62;node节点信息
 
     public TransportGroupInfo() {
     }
 
     public TransportGroupInfo(String name, InetSocketAddress... addrs) {
-        this(name, "TCP", "", Utility.ofSet(addrs));
+        this(name, "TCP", Utility.ofSet(addrs));
     }
 
     public TransportGroupInfo(String name, Set<InetSocketAddress> addrs) {
-        this(name, "TCP", "", addrs);
+        this(name, "TCP", addrs);
     }
 
-    public TransportGroupInfo(String name, String protocol, String subprotocol, InetSocketAddress... addrs) {
+    public TransportGroupInfo(String name, String protocol, InetSocketAddress... addrs) {
-        this(name, protocol, subprotocol, Utility.ofSet(addrs));
+        this(name, protocol, Utility.ofSet(addrs));
     }
 
-    public TransportGroupInfo(String name, String protocol, String subprotocol, Set<InetSocketAddress> addrs) {
+    public TransportGroupInfo(String name, String protocol, Set<InetSocketAddress> addrs) {
         Objects.requireNonNull(name, "Transport.group.name can not null");
         this.name = name;
         this.protocol = protocol == null ? "TCP" : protocol;
-        this.subprotocol = subprotocol == null ? "" : subprotocol;
         this.addresses = addrs;
     }
 
@@ -68,15 +65,6 @@ public class TransportGroupInfo {
         this.protocol = protocol == null ? "TCP" : protocol;
     }
 
-    public String getSubprotocol() {
-        return subprotocol;
-    }
-
-    public void setSubprotocol(String subprotocol) {
-        this.subprotocol = subprotocol == null ? "" : subprotocol;
-        this.subprotocol = subprotocol;
-    }
-
     public Set<InetSocketAddress> getAddresses() {
         return addresses;
     }