enjoy 4.4 release ^_^

pom.xml

@@ -4,7 +4,7 @@
 	
 	<groupId>com.jfinal</groupId>
 	<artifactId>enjoy</artifactId>
-	<version>4.4-SNAPSHOT</version>
+	<version>4.4</version>
 	<packaging>jar</packaging>
 	
 	

src/main/java/com/jfinal/kit/StrKit.java

@@ -102,6 +102,10 @@ public class StrKit {
 		return true;
 	}
 	
+	public static String defaultIfBlank(String str, String defaultValue) {
+		return isBlank(str) ? defaultValue : str;
+	}
+	
 	public static String toCamelCase(String stringWithUnderline) {
 		if (stringWithUnderline.indexOf('_') == -1) {
 			return stringWithUnderline;
@@ -145,6 +149,17 @@ public class StrKit {
 		return sb.toString();
 	}
 	
+	public static String join(java.util.List<String> list, String separator) {
+		StringBuilder sb = new StringBuilder();
+		for (int i=0, len=list.size(); i<len; i++) {
+			if (i > 0) {
+				sb.append(separator);
+			}
+			sb.append(list.get(i));
+		}
+		return sb.toString();
+	}
+	
 	public static boolean slowEquals(String a, String b) {
 		byte[] aBytes = (a != null ? a.getBytes() : null);
 		byte[] bBytes = (b != null ? b.getBytes() : null);

src/main/java/com/jfinal/template/Engine.java

@@ -293,23 +293,31 @@ public class Engine {
 	}
 	
 	/**
-	 * Add directive
+	 * 添加自定义指令
+	 * 
+	 * 建议添加自定义指令时明确指定 keepLineBlank 变量值，其规则如下：
+	 *   1：keepLineBlank 为 true 时， 该指令所在行的前后空白字符以及末尾字符 '\n' 将会被保留
+	 *      一般用于具有输出值的指令，例如 #date、#para 等指令
+	 *    
+	 *   2：keepLineBlank 为 false 时，该指令所在行的前后空白字符以及末尾字符 '\n' 将会被删除
+	 *      一般用于没有输出值的指令，例如 #for、#if、#else、#end 这种性质的指令
+	 * 
 	 * <pre>
-	 * 示例：
+	 * 	示例：
-	 * addDirective("now", NowDirective.class)
+	 * 	addDirective("now", NowDirective.class, true)
 	 * </pre>
 	 */
-	public Engine addDirective(String directiveName, Class<? extends Directive> directiveClass) {
+	public Engine addDirective(String directiveName, Class<? extends Directive> directiveClass, boolean keepLineBlank) {
-		config.addDirective(directiveName, directiveClass);
+		config.addDirective(directiveName, directiveClass, keepLineBlank);
 		return this;
 	}
 	
 	/**
-	 * 该方法已被 addDirective(String, Class<? extends Directive>) 所代替
+	 * 添加自定义指令，keepLineBlank 使用默认值
 	 */
-	@Deprecated
+	public Engine addDirective(String directiveName, Class<? extends Directive> directiveClass) {
-	public Engine addDirective(String directiveName, Directive directive) {
+		config.addDirective(directiveName, directiveClass);
-		return addDirective(directiveName, directive.getClass());
+		return this;
 	}
 	
 	/**
@@ -478,16 +486,24 @@ public class Engine {
 	}
 	
 	/**
-	 * Enjoy 模板引擎对 UTF-8 的 encoding 做过性能优化，某些偏门字符在
+	 * Enjoy 模板引擎对 UTF-8 的 encoding 做过性能优化，某些罕见字符
-	 * 被编码为 UTF-8 时会出现异常，此时可以通过继承扩展 EncoderFactory
+	 * 无法被编码，可以配置为 JdkEncoderFactory 解决问题:
-	 * 来解决编码异常，具体用法参考：
+	 * 		engine.setEncoderFactory(new JdkEncoderFactory());
-	 *     http://www.jfinal.com/feedback/5340
 	 */
 	public Engine setEncoderFactory(EncoderFactory encoderFactory) {
 		config.setEncoderFactory(encoderFactory);
 		return this;
 	}
 	
+	/**
+	 * 配置为 JdkEncoderFactory，支持 utf8mb4，支持 emoji 表情字符，
+	 * 支持各种罕见字符编码
+	 */
+	public Engine setToJdkEncoderFactory() {
+		config.setEncoderFactory(new com.jfinal.template.io.JdkEncoderFactory());
+		return this;
+	}
+	
 	public Engine setWriterBufferSize(int bufferSize) {
 		config.setWriterBufferSize(bufferSize);
 		return this;

src/main/java/com/jfinal/template/EngineConfig.java

@@ -19,9 +19,11 @@ package com.jfinal.template;
 import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.Set;
 import com.jfinal.kit.StrKit;
 import com.jfinal.template.expr.ast.ExprList;
 import com.jfinal.template.expr.ast.SharedMethodKit;
@@ -59,6 +61,9 @@ public class EngineConfig {
 	private Map<String, Class<? extends Directive>> directiveMap = new HashMap<String, Class<? extends Directive>>(64, 0.5F);
 	private SharedMethodKit sharedMethodKit = new SharedMethodKit();
 	
+	// 保留指令所在行空白字符的指令
+	private Set<String> keepLineBlankDirectives = new HashSet<>();
+	
 	private boolean devMode = false;
 	private boolean reloadModifiedSharedFunctionInDevMode = true;
 	private String baseTemplatePath = null;
@@ -66,14 +71,19 @@ public class EngineConfig {
 	private String datePattern = "yyyy-MM-dd HH:mm";
 	
 	public EngineConfig() {
+		// 内置指令 #() 与 #include() 需要配置，保留指令所在行前后空白字符以及行尾换行字符 '\n'
+		setKeepLineBlank("output", true);
+		setKeepLineBlank("include", true);
+		
 		// Add official directive of Template Engine
-		addDirective("render", RenderDirective.class);
+		addDirective("render", RenderDirective.class, true);
-		addDirective("date", DateDirective.class);
+		addDirective("date", DateDirective.class, true);
-		addDirective("escape", EscapeDirective.class);
+		addDirective("escape", EscapeDirective.class, true);
-		addDirective("string", StringDirective.class);
+		addDirective("random", RandomDirective.class, true);
-		addDirective("random", RandomDirective.class);
+		addDirective("number", NumberDirective.class, true);
-		addDirective("number", NumberDirective.class);
+		
-		addDirective("call", CallDirective.class);
+		addDirective("call", CallDirective.class, false);
+		addDirective("string", StringDirective.class, false);
 		
 		// Add official shared method of Template Engine
 		addSharedMethod(new SharedMethodLib());
@@ -318,12 +328,7 @@ public class EngineConfig {
 		this.reloadModifiedSharedFunctionInDevMode = reloadModifiedSharedFunctionInDevMode;
 	}
 	
-	@Deprecated
+	public synchronized void addDirective(String directiveName, Class<? extends Directive> directiveClass, boolean keepLineBlank) {
-	public void addDirective(String directiveName, Directive directive) {
-		addDirective(directiveName, directive.getClass());
-	}
-	
-	public synchronized void addDirective(String directiveName, Class<? extends Directive> directiveClass) {
 		if (StrKit.isBlank(directiveName)) {
 			throw new IllegalArgumentException("directive name can not be blank");
 		}
@@ -333,7 +338,15 @@ public class EngineConfig {
 		if (directiveMap.containsKey(directiveName)) {
 			throw new IllegalArgumentException("directive already exists : " + directiveName);
 		}
+		
 		directiveMap.put(directiveName, directiveClass);
+		if (keepLineBlank) {
+			keepLineBlankDirectives.add(directiveName);
+		}
+	}
+	
+	public void addDirective(String directiveName, Class<? extends Directive> directiveClass) {
+		addDirective(directiveName, directiveClass, false);
 	}
 	
 	public Class<? extends Directive> getDirective(String directiveName) {
@@ -342,6 +355,19 @@ public class EngineConfig {
 	
 	public void removeDirective(String directiveName) {
 		directiveMap.remove(directiveName);
+		keepLineBlankDirectives.remove(directiveName);
+	}
+	
+	public void setKeepLineBlank(String directiveName, boolean keepLineBlank) {
+		if (keepLineBlank) {
+			keepLineBlankDirectives.add(directiveName);
+		} else {
+			keepLineBlankDirectives.remove(directiveName);
+		}
+	}
+	
+	public Set<String> getKeepLineBlankDirectives() {
+		return keepLineBlankDirectives;
 	}
 	
 	/**

src/main/java/com/jfinal/template/ext/directive/EscapeDirective.java

@@ -63,12 +63,7 @@ public class EscapeDirective extends Directive {
 	}
 	
 	private void escape(String str, Writer w) throws IOException {
-		int len = str.length();
+		for (int i = 0, len = str.length(); i < len; i++) {
-		if (len == 0) {
-			return ;
-		}
-		
-		for (int i = 0; i < len; i++) {
 			char cur = str.charAt(i);
 			switch (cur) {
 			case '<':

src/main/java/com/jfinal/template/io/ByteWriter.java

@@ -50,15 +50,17 @@ public class ByteWriter extends Writer {
 	}
 	
 	public void write(String str, int offset, int len) throws IOException {
-		while (len > chars.length) {
+		int size, byteLen;
-			write(str, offset, chars.length);
+		while (len > 0) {
-			offset += chars.length;
+			size = (len > chars.length ? chars.length : len);
-			len -= chars.length;
-		}
 			
-		str.getChars(offset, offset + len, chars, 0);
+			str.getChars(offset, offset + size, chars, 0);
-		int byteLen = encoder.encode(chars, 0, len, bytes);
+			byteLen = encoder.encode(chars, 0, size, bytes);
-		out.write(bytes, 0, byteLen);
+			out.write(bytes, 0, byteLen);
+			
+			offset += size;
+			len -= size;
+		}
 	}
 	
 	public void write(String str) throws IOException {
@@ -66,15 +68,17 @@ public class ByteWriter extends Writer {
 	}
 	
 	public void write(StringBuilder stringBuilder, int offset, int len) throws IOException {
-		while (len > chars.length) {
+		int size, byteLen;
-			write(stringBuilder, offset, chars.length);
+		while (len > 0) {
-			offset += chars.length;
+			size = (len > chars.length ? chars.length : len);
-			len -= chars.length;
-		}
 			
-		stringBuilder.getChars(offset, offset + len, chars, 0);
+			stringBuilder.getChars(offset, offset + size, chars, 0);
-		int byteLen = encoder.encode(chars, 0, len, bytes);
+			byteLen = encoder.encode(chars, 0, size, bytes);
-		out.write(bytes, 0, byteLen);
+			out.write(bytes, 0, byteLen);
+			
+			offset += size;
+			len -= size;
+		}
 	}
 	
 	public void write(StringBuilder stringBuilder) throws IOException {

src/main/java/com/jfinal/template/io/CharWriter.java

@@ -44,14 +44,16 @@ public class CharWriter extends Writer {
 	}
 	
 	public void write(String str, int offset, int len) throws IOException {
-		while (len > chars.length) {
+		int size;
-			write(str, offset, chars.length);
+		while (len > 0) {
-			offset += chars.length;
+			size = (len > chars.length ? chars.length : len);
-			len -= chars.length;
-		}
 			
-		str.getChars(offset, offset + len, chars, 0);
+			str.getChars(offset, offset + size, chars, 0);
-		out.write(chars, 0, len);
+			out.write(chars, 0, size);
+			
+			offset += size;
+			len -= size;
+		}
 	}
 	
 	public void write(String str) throws IOException {
@@ -59,14 +61,16 @@ public class CharWriter extends Writer {
 	}
 	
 	public void write(StringBuilder stringBuilder, int offset, int len) throws IOException {
-		while (len > chars.length) {
+		int size;
-			write(stringBuilder, offset, chars.length);
+		while (len > 0) {
-			offset += chars.length;
+			size = (len > chars.length ? chars.length : len);
-			len -= chars.length;
-		}
 			
-		stringBuilder.getChars(offset, offset + len, chars, 0);
+			stringBuilder.getChars(offset, offset + size, chars, 0);
-		out.write(chars, 0, len);
+			out.write(chars, 0, size);
+			
+			offset += size;
+			len -= size;
+		}
 	}
 	
 	public void write(StringBuilder stringBuilder) throws IOException {

src/main/java/com/jfinal/template/io/DateFormats.java

@@ -25,7 +25,7 @@ import java.util.Map;
  */
 public class DateFormats {
 	
-	private Map<String, SimpleDateFormat> map = new HashMap<String, SimpleDateFormat>();
+	private Map<String, SimpleDateFormat> map = new HashMap<String, SimpleDateFormat>(16, 0.25F);
 	
 	public SimpleDateFormat getDateFormat(String datePattern) {
 		SimpleDateFormat ret = map.get(datePattern);

src/main/java/com/jfinal/template/io/JdkEncoderFactory.java

@@ -0,0 +1,38 @@
+/**
+ * Copyright (c) 2011-2019, James Zhan 詹波 (jfinal@126.com).
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.jfinal.template.io;
+
+/**
+ * JdkEncoderFactory
+ * 
+ * 支持 utf8mb4，支持 emoji 表情字符，支持各种罕见字符编码
+ * 
+ * <pre>
+ * 配置方法：
+ * engine.setToJdkEncoderFactory();
+ * </pre>
+ */
+public class JdkEncoderFactory extends EncoderFactory {
+	
+	@Override
+	public Encoder getEncoder() {
+		return new JdkEncoder(charset);
+	}
+}
+
+
+

src/main/java/com/jfinal/template/io/Utf8Encoder.java

@@ -16,7 +16,7 @@
 
 package com.jfinal.template.io;
 
-import java.nio.charset.MalformedInputException;
+// import java.nio.charset.MalformedInputException;
 
 /**
  * Utf8Encoder
@@ -62,12 +62,16 @@ public class Utf8Encoder extends Encoder {
                         if (Character.isLowSurrogate(d)) {
                             uc = Character.toCodePoint(c, d);
                         } else {
-                            throw new RuntimeException("encode UTF8 error", new MalformedInputException(1));
+                            // throw new RuntimeException("encode UTF8 error", new MalformedInputException(1));
+                            bytes[dp++] = (byte) '?';
+                            continue;
                         }
                     }
                 } else {
                     if (Character.isLowSurrogate(c)) {
-                        throw new RuntimeException("encode UTF8 error", new MalformedInputException(1));
+                        // throw new RuntimeException("encode UTF8 error", new MalformedInputException(1));
+                        bytes[dp++] = (byte) '?';
+                        continue;
                     } else {
                         uc = c;
                     }

src/main/java/com/jfinal/template/stat/Lexer.java

@@ -18,6 +18,7 @@ package com.jfinal.template.stat;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Set;
 
 /**
  * DKFF(Dynamic Key Feature Forward) Lexer
@@ -35,10 +36,14 @@ class Lexer {
 	int forwardRow = 1;
 	TextToken previousTextToken = null;
 	
-	List<Token> tokens = new ArrayList<Token>();
 	String fileName;
+	Set<String> keepLineBlankDirectives;
+	
+	List<Token> tokens = new ArrayList<Token>();
+	
+	public Lexer(StringBuilder content, String fileName, Set<String> keepLineBlankDirectives) {
+		this.keepLineBlankDirectives = keepLineBlankDirectives;
 		
-	public Lexer(StringBuilder content, String fileName) {
 		int len = content.length();
 		buf = new char[len + 1];
 		content.getChars(0, content.length(), buf, 0);
@@ -110,7 +115,7 @@ class Lexer {
 					para = scanPara("");
 					idToken = new Token(Symbol.OUTPUT, beginRow);
 					paraToken = new ParaToken(para, beginRow);
-					return addOutputToken(idToken, paraToken);
+					return addIdParaToken(idToken, paraToken);
 				}
 				if (CharTable.isLetter(peek())) {		// # id
 					state = 10;
@@ -472,31 +477,6 @@ class Lexer {
 		}
 	}
 	
-	// 输出指令不对前后空白与换行进行任何处理，直接调用 tokens.add(...)
-	boolean addOutputToken(Token idToken, Token paraToken) {
-		tokens.add(idToken);
-		tokens.add(paraToken);
-		previousTextToken = null;
-		return prepareNextScan(0);
-	}
-	
-	// 向前看后续是否跟随的是空白 + 换行或者是空白 + EOF，是则表示当前指令后续没有其它有用内容
-	boolean lookForwardLineFeedAndEof() {
-		int forwardBak = forward;
-		int forwardRowBak = forwardRow;
-		for (char c=peek(); true; c=next()) {
-			if (CharTable.isBlank(c)) {
-				continue ;
-			}
-			if (c == '\n' || c == EOF) {
-				return true;
-			}
-			forward = forwardBak;
-			forwardRow = forwardRowBak;
-			return false;
-		}
-	}
-	
 	/**
 	 * 带参指令处于独立行时删除前后空白字符，并且再删除一个后续的换行符
 	 * 处于独立行是指：向前看无有用内容，在前面情况成立的基础之上
@@ -509,32 +489,68 @@ class Lexer {
 		tokens.add(idToken);
 		tokens.add(paraToken);
 		
+		skipFollowingComment();
+		
+		// 保留指令所在行空白字符
+		// #define xxx() 模板函数名、#@xxx() 模板函数名，可以与指令同名，需要排除掉这三种 Symbol
+		if (keepLineBlankDirectives.contains(idToken.value())
+			&& idToken.symbol != Symbol.DEFINE
+			&& idToken.symbol != Symbol.CALL
+			&& idToken.symbol != Symbol.CALL_IF_DEFINED
+			) {
+			
+			prepareNextScan(0);
+		} else {
+			trimLineBlank();
+		}
+		
+		previousTextToken = null;
+		return true;
+	}
+	
+	// #set 这类指令，处在独立一行时，需要删除当前行的前后空白字符以及行尾字符 '\n'
+	void trimLineBlank() {
 		// if (lookForwardLineFeed() && (deletePreviousTextTokenBlankTails() || lexemeBegin == 0)) {
 		if (lookForwardLineFeedAndEof() && deletePreviousTextTokenBlankTails()) {
 			prepareNextScan(peek() != EOF ? 1 : 0);
 		} else {
 			prepareNextScan(0);
 		}
-		previousTextToken = null;
-		return true;
 	}
 	
-	// 处理前后空白的逻辑与 addIdParaToken() 基本一样，仅仅多了一个对于紧随空白的 next() 操作
+	// 无参指令无条件调用 trimLineBlank()
 	boolean addNoParaToken(Token noParaToken) {
 		tokens.add(noParaToken);
+		
+		skipFollowingComment();
+		
 		if (CharTable.isBlank(peek())) {
 			next();	// 无参指令之后紧随的一个空白字符仅为分隔符，不参与后续扫描
 		}
 		
-		if (lookForwardLineFeedAndEof() && deletePreviousTextTokenBlankTails()) {
+		trimLineBlank();
-			prepareNextScan(peek() != EOF ? 1 : 0);
+		
-		} else {
-			prepareNextScan(0);
-		}
 		previousTextToken = null;
 		return true;
 	}
 	
+	// 向前看后续是否跟随的是空白 + 换行或者是空白 + EOF，是则表示当前指令后续没有其它有用内容
+	boolean lookForwardLineFeedAndEof() {
+		int fp = forward;
+		for (char c=buf[fp]; true; c=buf[++fp]) {
+			if (CharTable.isBlank(c)) {
+				continue ;
+			}
+			
+			if (c == '\n' || c == EOF) {
+				forward = fp;
+				return true;
+			}
+			
+			return false;
+		}
+	}
+	
 	/**
 	 * 1：当前指令前方仍然是指令 (previousTextToken 为 null)，直接返回 true
 	 * 2：当前指令前方为 TextToken 时的处理逻辑与返回值完全依赖于 TextToken.deleteBlankTails()
@@ -543,6 +559,54 @@ class Lexer {
 		// return previousTextToken != null ? previousTextToken.deleteBlankTails() : false;
 		return previousTextToken == null || previousTextToken.deleteBlankTails();
 	}
+	
+	/**
+	 * 跳过指令后方跟随的注释，以便正确处理各类换行逻辑
+	 */
+	void skipFollowingComment() {
+		int fp = forward;
+		for (char c=buf[fp]; true; c=buf[++fp]) {
+			if (CharTable.isBlank(c)) {
+				continue ;
+			}
+			
+			// 勿使用 next()
+			if (c == '#') {
+				if (buf[fp + 1] == '#' && buf[fp + 2] == '#') {
+					forward = fp;
+					skipFollowingSingleLineComment();
+				} else if (buf[fp + 1] == '-' && buf[fp + 2] == '-') {
+					forward = fp;
+					skipFollowingMultiLineComment();
+				}
+			}
+			
+			return ;
+		}
+	}
+	
+	void skipFollowingSingleLineComment() {
+		forward = forward + 3;
+		for (char c=peek(); true; c=next()) {
+			if (c == '\n' || c == EOF) {
+				break ;
+			}
+		}
+	}
+	
+	void skipFollowingMultiLineComment() {
+		forward = forward + 3;
+		for (char c=peek(); true; c=next()) {
+			if (c == '-' && buf[forward + 1] == '-' && buf[forward + 2] == '#') {
+				forward = forward + 3;
+				break ;
+			}
+			
+			if (c == EOF) {
+				throw new ParseException("The multiline comment start block \"#--\" can not match the end block: \"--#\"", new Location(fileName, beginRow));
+			}
+		}
+	}
 }
 
 

src/main/java/com/jfinal/template/stat/Parser.java

@@ -71,7 +71,7 @@ public class Parser {
 	}
 	
 	public StatList parse() {
-		tokenList = new Lexer(content, fileName).scan();
+		tokenList = new Lexer(content, fileName, env.getEngineConfig().getKeepLineBlankDirectives()).scan();
 		tokenList.add(EOF);
 		StatList statList = statList();
 		if (peek() != EOF) {
@@ -207,11 +207,11 @@ public class Parser {
 				matchEnd(name);
 			}
 			return ret;
+		case EOF:
 		case PARA:
 		case ELSEIF:
 		case ELSE:
 		case END:
-		case EOF:
 		case CASE:
 		case DEFAULT:
 			return null;

src/main/java/com/jfinal/template/stat/Scope.java

@@ -89,7 +89,8 @@ public class Scope {
 	 * 自内向外在作用域栈中查找变量，返回最先找到的变量
 	 */
 	public Object get(Object key) {
-		for (Scope cur=this; cur!=null; cur=cur.parent) {
+		Scope cur = this;
+		do {
 //			if (cur.data != null && cur.data.containsKey(key)) {
 //				return cur.data.get(key);
 //			}
@@ -104,7 +105,10 @@ public class Scope {
 					return null;
 				}
 			}
-		}
+			
+			cur = cur.parent;
+		} while (cur != null);
+		
 		// return null;
 		return sharedObjectMap != null ? sharedObjectMap.get(key) : null;
 	}

src/main/java/com/jfinal/template/stat/TextToken.java

@@ -63,6 +63,9 @@ class TextToken extends Token {
 		}
 		
 		// 两个指令之间全是空白字符， 设置其长度为 0，为 Parser 过滤内容为空的 Text 节点做准备
+		// 典型测试用例：两个带有前导空格，并且都在独立一行的 #set(...) 指令，前一个 #set 指令
+		//             虽然是 '\n' 结尾，但已在 Lexer 中被 prepareNextScan(...) 删掉
+		// 另一典型用例：#date() #date()，可通过配置 keepLineBlank 为 true 保留指令间的空白字符
 		text.setLength(0);
 		return true;		// 当两指令之间全为空白字符时，告知调用方需要吃掉行尾的 '\n'
 	}