lxy/meta-kit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

'修改qtask相关'

Browse Source
This commit is contained in:
lxyer
2019-04-22 19:01:59 +08:00
parent ebbd490913
commit e5a4451a7e
23 changed files with 236 additions and 261 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/net/tccn/base/MetaKit.java
View File

@@ -471,6 +471,10 @@ public class MetaKit {
return new DbKit(dbAccount.get());
}
public static DbKit getDbKit(String dbPlatId, String catalog) {
Optional<DbAccount> dbAccount = dbPlats.stream().filter(x -> x.getKey().equals(dbPlatId)).findAny();
return new DbKit(dbAccount.get(), catalog);
}
public static DbAccount getDbPlat(String dbPlatId) {
Optional<DbAccount> dbAccount = dbPlats.stream().filter(x -> x.getKey().equals(dbPlatId)).findFirst();

7
src/main/java/net/tccn/dbq/Field.java
View File

@@ -67,8 +67,9 @@ public class Field {
//------------------------------------
public enum InType {
SELECT_EXT("select_ext"),
INPUT_DT("input_dt");
SELECT_EXT("SELECT_EXT"),
INPUT_DT("INPUT_DT"),
FMT_FUN("FMT_FUN");
String name;
@@ -86,6 +87,8 @@ public class Field {
return name + "|" + inExt;
} else if (InType.INPUT_DT.name.equalsIgnoreCase(inType)) {
return name + "=dt";
} else if (InType.FMT_FUN.name.equalsIgnoreCase(inType)) {
return name + "=" + inExt;
}
return name;

10
src/main/java/net/tccn/dbq/jdbc/api/DbKit.java
View File

@@ -9,6 +9,7 @@ public class DbKit implements DbSource{
private DbAccount dbAccount;
private DbSource dbSource;
private String catalog;
public DbKit(DbAccount dbAccount) {
this.dbAccount = dbAccount;
@@ -18,6 +19,15 @@ public class DbKit implements DbSource{
throw new IllegalArgumentException(String.format("创建DbKit失败：数据库类型[cate:%s]未知", dbAccount.getCate()));
}
}
public DbKit(DbAccount dbAccount, String catalog) {
this.dbAccount = dbAccount;
this.catalog = catalog;
if ("mysql".equalsIgnoreCase(dbAccount.getCate())) {
dbSource = new DbSourceMysql(dbAccount, catalog);
} else {
throw new IllegalArgumentException(String.format("创建DbKit失败：数据库类型[cate:%s]未知", dbAccount.getCate()));
}
}
@Override

16
src/main/java/net/tccn/dbq/jdbc/api/DbSourceMysql.java
View File

@@ -22,11 +22,17 @@ public class DbSourceMysql implements DbSource {
private String accountKey;
private DbAccount dbAccount;
private String catalog;
public DbSourceMysql(DbAccount dbAccount) {
this.dbAccount = dbAccount;
this.accountKey = dbAccount.accountKey();
}
public DbSourceMysql(DbAccount dbAccount, String catalog) {
this.dbAccount = dbAccount;
this.catalog = catalog;
this.accountKey = dbAccount.accountKey();
}
@Override
public <T> List<T> findList(String sql, Class<T> type) {
@@ -136,7 +142,15 @@ public class DbSourceMysql implements DbSource {
}
private Connection connection() {
return connection(0);
Connection connection = connection(0);
if (connection != null && catalog != null) {
try {
connection.setCatalog(catalog); //还回连接的时候是否需要重置catalog？ 后续观察
} catch (SQLException e) {
e.printStackTrace();
}
}
return connection;
}
private Connection connection(int n) {
LinkedBlockingQueue<Connection> queue = conns.getOrDefault(accountKey, new LinkedBlockingQueue<>(15));

14
src/main/java/net/tccn/oth/QtaskTest.java Normal file
View File

@@ -0,0 +1,14 @@
package net.tccn.oth;
import net.tccn.base.Kv;
/**
* @author: liangxianyou
*/
public class QtaskTest {
public Kv abx(Kv kv) {
return Kv.of("hello", "world").putAll(kv);
}
}

2
src/main/java/net/tccn/qtask/QRuner.java
View File

@@ -13,7 +13,7 @@ public class QRuner {
switch (cate.toLowerCase()) {
case "mysql":
return new QTaskMysql(task).execute();
case "method":
case "localapi":
return new QTaskMethod(task).execute();
case "http":
return new QTaskHttp(task).execute();

11
src/main/java/net/tccn/qtask/Task.java
View File

@@ -16,7 +16,8 @@ public class Task {
private String name; // 任务名，同一系统唯一
private String dbPlatId;// 数据源id
private String dbPlatId; // 数据源id
private String catalog; // 数据库 database
private String content;
private String comment;
@@ -41,6 +42,14 @@ public class Task {
this.dbPlatId = dbPlatId;
}
public String getCatalog() {
return catalog;
}
public void setCatalog(String catalog) {
this.catalog = catalog;
}
public String getContent() {
return content;
}

22
src/main/java/net/tccn/qtask/TaskEntity.java
View File

@@ -1,6 +1,5 @@
package net.tccn.qtask;
import net.tccn.base.Kv;
import net.tccn.base.arango.Doc;
import javax.persistence.Table;
@@ -8,17 +7,18 @@ import javax.persistence.Table;
/**
* Created by liangxianyou at 2019/4/20 20:04.
*/
@Table(name = "qtask", catalog = "db_dev")
@Table(name = "qtask", catalog = "db_demo")
public class TaskEntity extends Doc<TaskEntity> {
public static TaskEntity dao = dao(TaskEntity.class);
private String name; // 任务名称
private String name; // 任务标识码
private String comment; // 任务名称
//private String cate; // 任务类型
private String dbPlatId; // 数据平台id
private String comment; // 任务名称
private String catalog; //
private String content; //任务内容
private Kv<String, String> para; //任务参数
private String para; //任务参数
private String sysPlatId; // 平台id
// ---------------------
@@ -38,6 +38,14 @@ public class TaskEntity extends Doc<TaskEntity> {
this.dbPlatId = dbPlatId;
}
public String getCatalog() {
return catalog;
}
public void setCatalog(String catalog) {
this.catalog = catalog;
}
public String getComment() {
return comment;
}
@@ -54,11 +62,11 @@ public class TaskEntity extends Doc<TaskEntity> {
this.content = content;
}
public Kv<String, String> getPara() {
public String getPara() {
return para;
}
public void setPara(Kv<String, String> para) {
public void setPara(String para) {
this.para = para;
}

25
src/main/java/net/tccn/qtask/TaskKit.java
View File

@@ -1,9 +1,11 @@
package net.tccn.qtask;
import com.google.gson.Gson;
import net.tccn.base.Kv;
import net.tccn.base.MetaKit;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
@@ -19,14 +21,28 @@ public class TaskKit {
public static Task buildTask(String name, String platToken, Kv para) {
TaskEntity taskEntity = getTaskEntity(name, platToken);
return buildTask(taskEntity, para);
}
public static Task buildTask(TaskEntity taskEntity, Kv para) {
Task task = new Task();
task.setName(taskEntity.getName());
task.setComment(taskEntity.getComment());
task.setContent(taskEntity.getContent());
task.setDbPlatId(taskEntity.getDbPlatId());
task.setPara(taskEntity.getPara().putAll(para));
task.setDbAccount(MetaKit.getDbPlat(taskEntity.getDbPlatId()));
task.setCatalog(taskEntity.getCatalog());
Kv _para = Kv.of().putAll(para);
if (taskEntity.getPara() != null) {
try {
Map map = new Gson().fromJson(taskEntity.getPara(), Map.class);
map.forEach((k,v) -> _para.put(k, v));
} catch (Exception e) {
new IllegalArgumentException(String.format("fromJson error:[%s]",taskEntity.getPara()), e);
}
}
task.setPara(_para);
task.setDbAccount(MetaKit.getDbPlat(taskEntity.getDbPlatId()));
return task;
}
@@ -44,4 +60,9 @@ public class TaskKit {
Task task = buildTask(name, platToken, para);
return QRuner.query(task);
}
public static Object taskRun(TaskEntity entity) {
Task task = buildTask(entity, Kv.of());
return QRuner.query(task);
}
}

5
src/main/java/net/tccn/qtask/impl/QTaskAbs.java
View File

@@ -17,11 +17,10 @@ public abstract class QTaskAbs implements QTask {
}
public Kv getPara() {
return getTask().getPara();
return task.getPara();
}
public String getContent() {
return getTask().getContent();
return task.getContent();
}
}

7
src/main/java/net/tccn/qtask/impl/QTaskMysql.java
View File

@@ -19,19 +19,20 @@ public class QTaskMysql extends QTaskAbs implements QTask {
public static Engine engine = Engine.create("sqlTpl");
private static MysqlDialect dialect = new MysqlDialect();
protected DbKit dbKit;
static {
engine.setDevMode(true);
}
public QTaskMysql(Task task) {
super(task);
this.dbKit = MetaKit.getDbKit(task.getDbPlatId(), task.getCatalog());
}
@Override
public Object execute() {
DbKit dbKit = MetaKit.getDbKit(getTask().getDbPlatId());
Template tpl = engine.getTemplateByString(task.getContent());
String sql = tpl.renderToString(getTask().getPara()).replaceAll("[\\s]+", " ");

1
src/main/java/net/tccn/service/BaseService.java
View File

@@ -107,6 +107,7 @@ public class BaseService implements Service {
return prop.getProperty(k);
}
@RestMapping(ignore = true)
public String platId(String token) {
return MetaKit.getPlatId(token);
}

6
src/main/java/net/tccn/service/MetadataService.java
View File

@@ -16,7 +16,6 @@ import org.redkale.net.http.RestService;
import javax.annotation.Resource;
import java.util.List;
import java.util.Map;
import java.util.logging.Level;
import java.util.stream.Collectors;
/**
@@ -31,11 +30,6 @@ public class MetadataService extends BaseService { //arango
public MetaTable findMetaTable(String name, String token) {
return getT("meta_cols_" + name + token, MetaTable.class, () -> {
SysPlat sysPlat = qtaskService.getSysPlat(token);
if (sysPlat == null) {
logger.log(Level.INFO, "平台信息未知");
throw new IllegalArgumentException("平台信息未知");
}
return MetaKit.getMetaTables()
.stream()

113
src/main/java/net/tccn/service/QtaskService.java
View File

@@ -2,119 +2,32 @@ package net.tccn.service;
import net.tccn.base.JBean;
import net.tccn.base.Kv;
import net.tccn.dbq.jdbc.api.DbAccount;
import net.tccn.dbq.qtask.Qtask;
import net.tccn.plat.SysPlat;
import net.tccn.qtask.TaskKit;
import org.redkale.net.http.RestMapping;
import org.redkale.net.http.RestParam;
import org.redkale.net.http.RestService;
import java.sql.SQLException;
import java.util.function.Predicate;
import java.util.Map;
/**
* @author: liangxianyou at 2018/11/13 18:14.
*/
@RestService(name = "qtask", automapping = true, comment = "qtask查询服务")
@RestService(automapping = true, comment = "qtask查询服务")
public class QtaskService extends BaseService {
private Predicate<String> checkObj = (s) -> s != null && s.startsWith("{") && s.endsWith("}");
// 调用示例: http://qtask_service_addr_xxxxxx/qtask/call?name=abxx&platToken=3421432&para={h:1}
@RestMapping(name = "call", auth = false)
public JBean call(String name, Map<String, String> para, @RestParam(name = "platToken") String token) {
JBean jBean = new JBean();
public SysPlat getSysPlat(String token) {
return getT(token, SysPlat.class, () -> SysPlat.dao.findFirst(new SysPlat(token)));
}
Kv kv = Kv.of();
if (para != null) {
para.forEach((k, v) -> kv.put(k, v));
}
///qtask/query?queryId=platform.list&para={id:211}
@RestMapping(name = "call", comment = "qtask查询")
public JBean call(String queryId, String para, @RestParam(name = "platToken") String token) {
JBean jBean = JBean.by(0, "");
do {
SysPlat sysPlat = getSysPlat(token);
if (sysPlat == null) {
jBean.set(-1, "平台信息未知"); break;
}
Qtask qtask = getQTask(queryId, sysPlat.getKey());
if (qtask == null) {
jBean.set(-1, "未知任务"); break;
}
if (!checkObj.test(para)) {
para = qtask.getPara();
}
Kv kv = null;
if (checkObj.test(para)) {
kv = BaseService.gson.fromJson(para, Kv.class);
}
try {
jBean.setBody(run(qtask, kv));
} catch (SQLException e) {
e.printStackTrace();
return jBean.set(-1, "数据查询失败", e.getMessage());
}
} while (false);
jBean.setBody(TaskKit.taskRun(name, token, kv));
return jBean;
}
@RestMapping(name = "test", comment = "qtask调试")
public JBean test(Qtask qtask) {
JBean jBean = JBean.by(0, "");
do {
//check data
if (isEmpty.test(qtask.getSql())) {
jBean.set(-1, "查询失败，请设置正确的【SQL】"); break;
}
if (isEmpty.test(qtask.getPlatId()) || isEmpty.test(qtask.getCatalog())) {
jBean.set(-1, "查询失败，请设置正确的【数据平台】"); break;
}
DbAccount dbPlat = DbAccount.dao.findByKey(qtask.getPlatId());//arangoSource.findById("db_plat/" + qTask.getPlatId(), DbAccount.class);
if (dbPlat == null) {
jBean.set(-1, "查询失败，请设置正确的【数据平台】"); break;
}
Kv kv = null;
if (checkObj.test(qtask.getPara())) {
try {
kv = BaseService.gson.fromJson(qtask.getPara(), Kv.class);
} catch (Exception e) {
jBean.set(-1, "查询失败，请设置正确的【查询参数】"); break;
}
}
try {
jBean.setBody(run(qtask, kv));
} catch (SQLException e) {
e.printStackTrace();
jBean.set(-1, e.getMessage());
}
} while (false);
return jBean;
}
// /qtask/query?queryId=platform.list&para={id:211}
@RestMapping(ignore = true, comment = "qtask查询数据")
private Object run(Qtask qTask, Kv kv) throws SQLException {
return null;
}
//----------- Qtask ---------------
@RestMapping(name = "qsave", comment = "qtask保存")
public JBean qtaskSave(Qtask qtask) {
JBean jBean = JBean.by(0, "");
return jBean;
}
@RestMapping(ignore = true, comment = "qtask获取")
public Qtask getQTask(String queryId, String sysPlatId) {
Qtask qtask = null;
return qtask;
}
//-------------------------------------------------------
}

18
src/main/java/net/tccn/service/_FileService.java
View File

@@ -10,7 +10,6 @@ import org.redkale.net.http.RestMapping;
import org.redkale.net.http.RestParam;
import org.redkale.net.http.RestService;
import javax.annotation.Resource;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,9 +27,6 @@ public class _FileService extends BaseService {
private final static String[] FIELDS = {"field", "cate", "must", "remark1", "remark2", "tag", "selects", "column", "filter", "ck", "edit"};
@Resource
private QtaskService qtaskService;
@RestMapping(name = "sheets", comment = "得到所有的sheetName")
public List<String> sheets(String filePath) {
@@ -50,10 +46,14 @@ public class _FileService extends BaseService {
}).collect(Collectors.toList());
}
public SysPlat getSysPlat(String token) {
return getT(token, SysPlat.class, () -> SysPlat.dao.findFirst(new SysPlat(token)));
}
@RestMapping(name = "data", comment = "得到文件数据")
public JBean data(String filePath, @RestParam(name = "platToken") String token) {
JBean jBean = new JBean();
SysPlat sysPlat = qtaskService.getSysPlat(token);
//SysPlat sysPlat = getSysPlat(token);
File file = new File(webroot, filePath);
if (file.exists()) {
try {
@@ -73,7 +73,7 @@ public class _FileService extends BaseService {
buf.append("'").append(x.getName()).append("',");
});
buf.deleteCharAt(buf.length() - 1);
buf.append("] and d.sysPlatId=='" + sysPlat.getKey() + "'\n" +
buf.append("] and d.sysPlatId=='" + platId(token) + "'\n" +
" return d.name");
List<String> hv = MetaTable.dao.find(buf.toString(), String.class);
@@ -100,7 +100,6 @@ public class _FileService extends BaseService {
@RestMapping(name = "sheet_data", comment = "得到sheet数据")
public JBean sheetData(String filePath, String sheetName, @RestParam(name = "platToken") String token) {
JBean jBean = new JBean();
SysPlat sysPlat = qtaskService.getSysPlat(token);
File file = new File(webroot, filePath);
try {
List<Map> list = ExcelKit.readExcel(file, FIELDS, sheetName);
@@ -120,7 +119,6 @@ public class _FileService extends BaseService {
@RestParam(name = "filePath", comment = "文件路径") String filePath,
@RestParam(name = "platToken") String token) {
JBean jBean = new JBean();
SysPlat sysPlat = qtaskService.getSysPlat(token);
File file = new File(webroot, filePath);
String[] fields = {"field", "cate", "must", "remark1", "remark2", "tag", "selects", "column", "filter", "ck", "edit"};
@@ -132,12 +130,12 @@ public class _FileService extends BaseService {
for (String sn : sheetArr) {
if (table.getName().equals(sn)) { //在选中列表中
MetaTable metaTable = new MetaTable();
metaTable.setSysPlatId(sysPlat.getKey());
metaTable.setSysPlatId(platId(token));
metaTable.setName(table.getName());
MetaTable _metaTable = MetaTable.dao.findFirst(metaTable);
if (_metaTable == null) {//库里没有数据保存
table.setSysPlatId(sysPlat.getKey());
table.setSysPlatId(platId(token));
table.save();
}
break;

42
src/main/java/net/tccn/service/_QtaskService.java
View File

@@ -4,6 +4,7 @@ import net.tccn.base.JBean;
import net.tccn.base.MetaKit;
import net.tccn.base.PageBean;
import net.tccn.qtask.TaskEntity;
import net.tccn.qtask.TaskKit;
import org.redkale.net.http.RestParam;
import org.redkale.net.http.RestService;
import org.redkale.source.Flipper;
@@ -14,6 +15,10 @@ public class _QtaskService extends BaseService{
@Comment("qtask列表")
public JBean list(TaskEntity task, Flipper flipper, @RestParam(name = "platToken") String token) {
if (task == null) {
task = new TaskEntity();
}
task.setSysPlatId(platId(token));
PageBean<TaskEntity> page = TaskEntity.dao.findPage(task, flipper);
@@ -24,14 +29,22 @@ public class _QtaskService extends BaseService{
public JBean save(TaskEntity task, @RestParam(name = "platToken") String token) {
JBean jBean = new JBean();
// 同平台name 唯一校验
TaskEntity bean = new TaskEntity();
bean.setSysPlatId(platId(token));
bean.setName(task.getName());
long count = bean.count();
if (count > 0) {
jBean.set(-1, "任务名称占用");
} else {
do {
if (isEmpty.test(task.getName())) {
jBean.set(-1, "任务标识码不能为空");
break;
}
// 同平台name 唯一校验
TaskEntity bean = new TaskEntity();
bean.setSysPlatId(platId(token));
bean.setName(task.getName());
TaskEntity entity = TaskEntity.dao.findFirst(bean);
if (entity != null && !entity.getKey().equals(task.getKey())) {
jBean.set(-1, "任务标识码被占用");
break;
}
if (task.getKey() != null) {
task.update();
@@ -40,9 +53,20 @@ public class _QtaskService extends BaseService{
task.save();
}
MetaKit.reload(task);
}
} while (false);
return jBean;
}
@Comment("debug调试接口")
public JBean debug(TaskEntity task, @RestParam(name = "platToken") String token) {
JBean jBean = new JBean();
Object res = TaskKit.taskRun(task);
jBean.setBody(res);
return jBean;
}
}

13
src/test/java/RunTest.java
View File

@@ -8,8 +8,8 @@ import net.tccn.dbq.jdbc.api.DbAccount;
import net.tccn.dbq.jdbc.api.DbKit;
import net.tccn.dbq.parser.ParseMysql;
import net.tccn.meta.MetaTable;
import net.tccn.qtask.QRuner;
import net.tccn.qtask.Task;
import net.tccn.qtask.TaskEntity;
import net.tccn.qtask.TaskKit;
import net.tccn.user.User;
import org.junit.Test;
@@ -194,7 +194,7 @@ public class RunTest<T> {
TplKit tplKit = TplKit.use();
@Test
// @Test
public void buildMethod() {
tplKit.addTpl(new File(FileKit.rootPath(), "/tpl/_t.tpl"));
@@ -223,6 +223,13 @@ public class RunTest<T> {
return str.substring(0, 1).toUpperCase() + str.substring(1);
}
@Test
public void taskRunTest() {
TaskEntity entity = TaskEntity.dao.findByKey("23074420");
if (entity != null) {
System.out.printf("------------------------%n%s%n------------------------%n", new Gson().toJson(TaskKit.taskRun(entity)));
}
}
}