lxy/meta-kit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
lxyer
2019-03-18 17:10:52 +08:00
parent e057d613b9
commit 7ef48dfc3e
37 changed files with 827 additions and 258 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/net/tccn/dbq/fbean/FBean.java
View File

@@ -49,6 +49,7 @@ public class FBean {
//-----------------------------------
//组装查询sql，[list, count]
@Deprecated
public String[] buildSqlFilter() {
//where 1=1 and x=1 and y=1 order by x,y limit 1,3;
String where = Filter.filter(filters);
@@ -63,7 +64,7 @@ public class FBean {
return new String[]{list, count};
}
@Deprecated
public void setLimit(int pn, int ps) {
limit.setPn(pn);
limit.setPs(ps);

21
src/main/java/net/tccn/dbq/fbean/ParseArango.java
View File

@@ -1,29 +1,12 @@
package net.tccn.dbq.fbean;
import net.tccn.meta.MetaService;
import java.util.Map;
/**
* Created by liangxianyou at 2018/12/24 15:49.
*/
public class ParseArango implements Parser {
@Override
public String parse(FBean fBean) {
return null;
}
@Override
public String parse(Filter filter) {
return null;
}
@Override
public String parseSave(MetaService ms, Map data) {
return "";
public String[] parse(FBean fBean) {
return new String[0];
}
}

17
src/main/java/net/tccn/dbq/fbean/ParseEs.java
View File

@@ -1,26 +1,13 @@
package net.tccn.dbq.fbean;
import net.tccn.meta.MetaService;
import java.util.Map;
/**
* Created by liangxianyou at 2018/12/24 15:49.
*/
public class ParseEs implements Parser {
@Override
public String parse(FBean fBean) {
return null;
}
@Override
public String parse(Filter filter) {
return null;
}
@Override
public String parseSave(MetaService ms, Map data) {
return "";
public String[] parse(FBean fBean) {
return new String[0];
}
}

182
src/main/java/net/tccn/dbq/fbean/ParseMysql.java
View File

@@ -1,5 +1,6 @@
package net.tccn.dbq.fbean;
import net.tccn.base.Kv;
import net.tccn.meta.MetaKit;
import net.tccn.meta.MetaLink;
import net.tccn.meta.MetaService;
@@ -8,9 +9,17 @@ import net.tccn.meta.MetaTable;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import static java.util.Arrays.asList;
/**
* 查询的数据是否同库，
* 是：支持副表过滤
* 否：只支持主表过滤
* Created by liangxianyou at 2018/12/24 15:49.
*/
public class ParseMysql implements Parser {
@@ -29,75 +38,154 @@ public class ParseMysql implements Parser {
return false;
};
BiFunction<Filter, String, String> buildFilter = (f, cate) -> {
return null;
};
@Deprecated
Function<FBean, Kv<String, MetaTable>> tablesFun = fbean -> {
MetaService metaService = MetaKit.metaService(fbean.getName());
List<Filter> filters = fbean.getFilters();
List<String> shows = metaService.getShows();
Kv tables = Kv.of();
shows.forEach(x-> {
String table = x.split("[.]")[0];
tables.set(table, MetaKit.getMetaTableByAlias(table));
});
filters.forEach(f -> {
String table = f.getCol().split("[.]")[0];
if (!tables.containsKey(table)) {
tables.set(table, MetaKit.getMetaTableByAlias(table));
}
});
metaService.getShows();
//查询用的过滤条件
Map<String, List<Filter>> filterMap = filters.stream().collect(Collectors.groupingBy(x -> x.getCol().split("[.]")[0]));
//要展示的数据
Map<String, List<String>> showMap = shows.stream().collect(Collectors.groupingBy(x -> x.split("[.]")[0]));
StringBuffer bufWhere = new StringBuffer();
filterMap.getOrDefault(metaService.getTable(), asList()).forEach(f -> {
bufWhere.append(f.getCol()).append(f.getType()).append(f.getValue());
});
//select * from tableA a left join tableB b on a.xx = b.yy where b.zz = xxx;
//where tableB where b.
//是否跨库
return tables;
};
Predicate<Kv<String, MetaTable>> sameDbFun = (kv) -> {
String dbPlatId = null;
for (MetaTable table : kv.values()) {
if (dbPlatId == null) {
dbPlatId = table.getDbPlatId();
} else if (!dbPlatId.equals(table.getDbPlatId())) {
return false;
}
}
return true;
};
@Override
public String parse(FBean fBean) {
public String[] parse(FBean fBean) {
MetaService metaService = MetaKit.metaService(fBean.getName());
//基础属性
MetaTable metaTable = metaService.getMetaTable();//基础元数据
List<MetaLink> linkTables = metaService.getMetaLinks();//表关联条件
Map<String, MetaTable> attachTable = metaService.getTables();//用到的关联表
Kv<String, MetaTable> tables = tablesFun.apply(fBean);//所有的关联表信息
MetaTable metaTable = tables.get(metaService.getTable());//基础元数据
List<String> shows = metaService.getShows();//查询的属性
List<String> _filters = fBean.getFilters().stream().map(Filter::getCol).collect(Collectors.toList());
List<MetaLink> links = MetaKit.getMetaLinks(metaService.getTable(), shows, _filters);
//查询条件
List<Filter> filters = fBean.getFilters();
Limit limit = fBean.getLimit();
List<Order> orders = fBean.getOrders();
StringBuilder buf = new StringBuilder();
//select
buf.append("select ");
if (!isEmpty.test(shows)) {
shows.forEach(x -> {
//buf.append("`").append("x").append("`").append(",");
buf.append(x).append(",");
});
buf.deleteCharAt(buf.length() - 1);
} else {
buf.append("*");
}
//Map<String, List<Filter>> filterMap = filters.stream().collect(Collectors.groupingBy(x -> x.getCol().split("[.]")[0]));
//Map<String, List<String>> showMap = shows.stream().collect(Collectors.groupingBy(x -> x.split("[.]")[0]));
//from
buf.append(" from ").append("`").append(metaTable.getName()).append("` ").append(metaTable.getAlias());
//left join
if (!isEmpty.test(linkTables)) {
linkTables.forEach(x -> {
MetaTable leftTable = attachTable.get(metaTable.getAlias().equals(x.getTables()[0]) ? x.getTables()[1] : x.getTables()[0]);
//判断是否为同库
if (sameDbFun.test(tables)) {
// where 1=1 and xx=xx
StringBuffer bufWhere = new StringBuffer();
if (!isEmpty.test(filters)) {
bufWhere.append(Filter.filter(filters, DbType.MYSQL));
}
buf.append(" left join ").append(leftTable.getName()).append(" ").append(leftTable.getAlias()).append(" on ");
int tag = buf.length();
x.getLink().forEach((k,v) -> {
if (buf.length() > tag) {
buf.append(" and ");
}
buf.append(k).append("=").append(v);
//select a.x, b.y, c.z
StringBuffer bufSelect = new StringBuffer();
bufSelect.append("select ");
if (!isEmpty.test(shows)) {
shows.forEach(x -> {
//buf.append("`").append("x").append("`").append(",");
bufSelect.append(x).append(",");
});
});
bufSelect.deleteCharAt(bufSelect.length() - 1);
} else {
bufSelect.append("*");
}
//from
StringBuilder bufFrom = new StringBuilder();
bufFrom.append(" from ").append(metaTable.getCatalog()).append(".`").append(metaTable.getName()).append("` ").append(metaTable.getAlias());
//left join
if (!isEmpty.test(links)) {
links.forEach(x -> {
MetaTable rightTable = tables.get(metaTable.getAlias().equals(x.getTables()[0]) ? x.getTables()[1] : x.getTables()[0]);
bufFrom.append(" left join ").append(rightTable.getCatalog()).append(".").append(rightTable.getName()).append(" ").append(rightTable.getAlias()).append(" on ");
int tag = bufFrom.length();
x.getLink().forEach((k,v) -> {
if (bufFrom.length() > tag) {
bufFrom.append(" and ");
}
bufFrom.append(k).append("=").append(v);
});
});
}
StringBuffer bufOth = new StringBuffer();
//order by
if (!isEmpty.test(orders)) {
bufOth.append(" ").append(Order.order(orders, DbType.MYSQL));
}
//limit
bufOth.append(" ").append(limit.limit());
return new String[]{
"select count(1) " + bufFrom + bufWhere,
"" + bufSelect + bufFrom + bufWhere + bufOth
};
}
//where
if (!isEmpty.test(filters)) {
buf.append(Filter.filter(filters, DbType.MYSQL));
}
//order by
if (!isEmpty.test(orders)) {
buf.append(" ").append(Order.order(orders, DbType.MYSQL));
}
//limit
buf.append(" ").append(limit.limit());
return buf.toString();
return null;
}
@Override
/*@Override
public String parse(Filter filter) {
return null;
}
@Override
public String parseSave(MetaService ms, Map data) {
MetaTable metaTable = MetaKit.getMetaTableByAlias(ms.getName());//基础元数据
String sqlTpl = "insert %s (%s) value (%s)";
StringBuilder bufK = new StringBuilder();
StringBuilder bufV = new StringBuilder();
@@ -113,6 +201,6 @@ public class ParseMysql implements Parser {
});
bufK.deleteCharAt(bufK.length() - 1);
bufV.deleteCharAt(bufV.length() - 1);
return String.format(sqlTpl, ms.getMetaTable().getName(), bufK.toString(), bufV.toString());
}
return String.format(sqlTpl, metaTable.getName(), bufK.toString(), bufV.toString());
}*/
}

9
src/main/java/net/tccn/dbq/fbean/Parser.java
View File

@@ -10,22 +10,23 @@ import java.util.Map;
public interface Parser {
/**
* 组装完整的查询条件
* 组装完整分页查询
* @param fBean
* @return [countSql, listSql]
*/
String parse(FBean fBean);
String[] parse(FBean fBean);
/**
* 解析一个查询条件
* @param filter
*/
String parse(Filter filter);
//String parse(Filter filter);
/**
* 解析入库语句
* @param data
* @return
*/
String parseSave(MetaService ms, Map data);
//String parseSave(MetaService ms, Map data);
}

13
src/main/java/net/tccn/dbq/jdbc/JdbcAccount.java
View File

@@ -10,10 +10,11 @@ public class JdbcAccount {
private String url;
private String user;
private String pwd;
private String cate;//数据库类型
private Integer connectMax = 5;//默认最大连接数5
public JdbcAccount() {
}
/*public JdbcAccount() {
}*/
public JdbcAccount(String url, String user, String pwd) {
this.url = url;
@@ -45,6 +46,14 @@ public class JdbcAccount {
this.pwd = pwd;
}
public String getCate() {
return cate;
}
public void setCate(String cate) {
this.cate = cate;
}
public Integer getConnectMax() {
return connectMax;
}

17
src/main/java/net/tccn/dbq/jdbc/JdbcService.java
View File

@@ -1,13 +1,9 @@
package net.tccn.dbq.jdbc;
import net.tccn.dbq.table.Table;
import org.redkale.util.Comment;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -63,6 +59,9 @@ public class JdbcService {
connection = DriverManager.getConnection(account.getUrl(), account.getUser(), account.getPwd());
jdbcPool.put(account, connection);
}
int i = 8000 * 12;//96000;
return connection;
}
@@ -145,7 +144,11 @@ public class JdbcService {
System.out.println(connection);
String sql = "select * from basic_concat";
String sql = "select * from basic_concat limit 10";
List<Map> list = JdbcService.findList(connection, sql);
System.out.println(list);
}
}

17
src/main/java/net/tccn/dbq/jdbc/JdbcSource.java
View File

@@ -1,21 +1,9 @@
package net.tccn.dbq.jdbc;
import net.tccn.dbq.qtask.Qtask;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.sql.*;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.LinkedBlockingDeque;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
@@ -26,7 +14,6 @@ import java.util.function.Function;
public class JdbcSource {
private JdbcAccount account;
private Queue<Qtask> queue = new LinkedBlockingDeque();
private static ConcurrentHashMap<String, JdbcSource> sources = new ConcurrentHashMap<>();
private List<Connection> connections = new ArrayList<>();
private AtomicInteger connectNum = new AtomicInteger();

25
src/main/java/net/tccn/dbq/DbPlat.java → src/main/java/net/tccn/dbq/jdbc/api/DbAccount.java
View File

@@ -1,21 +1,19 @@
package net.tccn.dbq;
package net.tccn.dbq.jdbc.api;
import net.tccn.base.arango.Doc;
import net.tccn.dbq.jdbc.JdbcAccount;
import org.redkale.convert.json.JsonConvert;
import javax.persistence.Table;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
/**
* 数据库平台
* @author: liangxianyou at 2018/11/14 12:58.
*/
@SuppressWarnings("Duplicates")
@Table(name = "db_plat", catalog = "db_dev")
public class DbPlat extends Doc<DbPlat> {
public static DbPlat dao = dao(DbPlat.class);
public class DbAccount extends Doc<DbAccount> {
public static DbAccount dao = dao(DbAccount.class);
private String name; //名称
private String cate; //类型 mysql|ArangoDb
@@ -25,7 +23,7 @@ public class DbPlat extends Doc<DbPlat> {
private String pwd; //密码
private List<String> catalogs; //库
private JdbcAccount jdbcAccount;
//private JdbcAccount jdbcAccount;
//------------- setter/getter ---------------
public String getName() {
return name;
@@ -89,10 +87,21 @@ public class DbPlat extends Doc<DbPlat> {
}
//-------------------------------------------------------------------------
protected Connection connection() throws SQLException {
/*protected Connection connection() throws SQLException {
if (jdbcAccount == null) {
jdbcAccount = new JdbcAccount(url, user, pwd);
}
return jdbcAccount.getConnection();
}*/
public String accountKey() {
int start = url.indexOf("//") + 2;
int end = url.indexOf("/", start);
int endDef = url.indexOf("?", end);
if (endDef == -1) {
endDef = url.length();
}
String host = url.substring(start, end == -1 ? url.length() : end);
return user + "@" + host;
}
}

57
src/main/java/net/tccn/dbq/jdbc/api/DbKit.java Normal file
View File

@@ -0,0 +1,57 @@
package net.tccn.dbq.jdbc.api;
import java.util.List;
/**
* Created by liangxianyou at 2019/3/12 14:11.
*/
public class DbKit implements DbSource{
private DbAccount dbAccount;
private DbSource dbSource;
public DbKit(DbAccount dbAccount) {
this.dbAccount = dbAccount;
if ("mysql".equalsIgnoreCase(dbAccount.getCate())) {
dbSource = new DbSourceMysql(dbAccount);
} else {
throw new IllegalArgumentException(String.format("创建DbKit失败：数据库类型[cate:%s]未知", dbAccount.getCate()));
}
}
@Override
public <T> List<T> findList(String sql, Class<T> type) {
return dbSource.findList(sql, type);
}
@Override
public <T> T findfirst(String sql, Class<T> type) {
return dbSource.findfirst(sql, type);
}
@Override
public <T> T queryColumn(String sql, Class<T> type) {
return dbSource.queryColumn(sql, type);
}
@Override
public <T> T queryInt(String sql) {
return null;
}
@Override
public <T> T queryLong(String sql) {
return null;
}
@Override
public <T> T queryDouble(String sql) {
return null;
}
@Override
public <T> T queryDate(String sql) {
return null;
}
}

33
src/main/java/net/tccn/dbq/jdbc/api/DbSource.java Normal file
View File

@@ -0,0 +1,33 @@
package net.tccn.dbq.jdbc.api;
import java.util.Date;
import java.util.List;
/**
* Created by liangxianyou at 2019/3/12 14:07.
*/
public interface DbSource {
<T> List<T> findList(String sql, Class<T> type);
<T> T findfirst(String sql, Class<T> type);
<T> T queryColumn(String sql, Class<T> type);
default <T> void save(String tableName, T t) {}
default <T> void update(String tableName, T t) {}
default int queryInt(String sql) {
return queryColumn(sql, int.class);
}
default long queryLong(String sql) {
return queryColumn(sql, long.class);
}
default double queryDouble(String sql) {
return queryColumn(sql, double.class);
}
default Date queryDate(String sql) {
return queryColumn(sql, Date.class);
}
}

167
src/main/java/net/tccn/dbq/jdbc/api/DbSourceMysql.java Normal file
View File

@@ -0,0 +1,167 @@
package net.tccn.dbq.jdbc.api;
import net.tccn.base.Kv;
import java.sql.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReferenceArray;
/**
* Created by liangxianyou at 2019/3/12 14:20.
*/
@SuppressWarnings("Duplicates")
public class DbSourceMysql implements DbSource {
private static ConcurrentHashMap<String, AtomicReferenceArray<Connection>> conns = new ConcurrentHashMap<>();
private static ConcurrentHashMap<String, AtomicInteger> counter = new ConcurrentHashMap<>();
private String accountKey;
private DbAccount dbAccount;
public DbSourceMysql(DbAccount dbAccount) {
this.dbAccount = dbAccount;
this.accountKey = dbAccount.accountKey();
}
@Override
public <T> List<T> findList(String sql, Class<T> type) {
Connection connection = connection();
try (
PreparedStatement ps = connection.prepareStatement(sql);
ResultSet rs = ps.executeQuery()) {
List list = new ArrayList();
while (rs.next()) {
ResultSetMetaData metaData = rs.getMetaData();
int count = metaData.getColumnCount();
Map row = new HashMap();
for (int i = 1; i <= count; i++) {
String columnTypeName = metaData.getColumnTypeName(i);
String columnName = metaData.getColumnName(i);
row.put(columnName, null);
if (rs.getObject(i) != null) {
switch (columnTypeName) {
case "DATETIME":
case "TIMESTAMP":
case "DATE":
row.put(columnName, rs.getTimestamp(i).getTime()); break;
default:
row.put(columnName, rs.getObject(i));
}
}
}
list.add(Map.class == type ? row : Kv.toBean(row, type));
}
return list;
} catch (SQLException e) {
e.printStackTrace();
return null;
} finally {
release(connection);
}
}
@Override
public <T> T findfirst(String sql, Class<T> type) {
List<T> list = findList(sql, type);
return list.size() > 0 ? list.get(0) : null;
}
@Override
public <T> T queryColumn(String sql, Class<T> type) {
Connection connection = connection();
try (
PreparedStatement ps = connection.prepareStatement(sql);
ResultSet rs = ps.executeQuery()
) {
Object v = null;
while (rs.next()) {
ResultSetMetaData metaData = rs.getMetaData();
int count = metaData.getColumnCount();
for (int i = 1; i <= count; i++) {
String columnTypeName = metaData.getColumnTypeName(i);
if (rs.getObject(i) != null) {
switch (columnTypeName) {
case "DATETIME":
case "TIMESTAMP":
case "DATE":
v = rs.getTimestamp(i).getTime(); break;
default:
v = rs.getObject(i);
}
}
}
}
return Kv.toAs(v, type);
} catch (SQLException e) {
e.printStackTrace();
return null;
} finally {
release(connection);
}
}
private Connection connection() {
return connection(0);
}
private Connection connection(int n) {
AtomicReferenceArray<Connection> arr = conns.getOrDefault(accountKey, new AtomicReferenceArray<>(15));
Connection connection = null;
AtomicInteger num = counter.getOrDefault(accountKey, new AtomicInteger(0));
for (int i = 0; num.get() > 0 && i < arr.length() && connection == null; i++) {
try {
connection = arr.getAndUpdate(i, null);
} catch (Exception e) {
System.out.println("getAndUpdate exception");
}
}
if (connection == null) {
try {
if (num.get() < 15) {
connection = DriverManager.getConnection(dbAccount.getUrl(), dbAccount.getUser(), dbAccount.getPwd());
num.getAndIncrement();
} else {
//连接被全部使用中，等待1s后再次获取连接,直到得到连接 ！！！
Thread.sleep(1000);
if (++n > 3)
connection(n);
}
} catch (SQLException e) {
e.printStackTrace();
throw new IllegalArgumentException("获取数据库连接失败");
} catch (InterruptedException e) {
e.printStackTrace();
}
}
return connection;
}
private void release(Connection connection) {
AtomicReferenceArray<Connection> arr = conns.getOrDefault(accountKey, new AtomicReferenceArray<>(15));
int i = 0;
boolean bool = false;
while (i < arr.length() && !bool){
bool = arr.compareAndSet(i++, null, connection);
}
//如果没成功释放，关系连接
if (!bool && connection != null) {
try {
connection.close();
} catch (SQLException e) {
e.printStackTrace();
}
}
}
}

2
src/main/java/net/tccn/dbq/jdbc/Column.java → src/main/java/net/tccn/dbq/table/Column.java
View File

@@ -1,4 +1,4 @@
package net.tccn.dbq.jdbc;
package net.tccn.dbq.table;
/**
* 数据库表的列

2
src/main/java/net/tccn/dbq/jdbc/Table.java → src/main/java/net/tccn/dbq/table/Table.java
View File

@@ -1,4 +1,4 @@
package net.tccn.dbq.jdbc;
package net.tccn.dbq.table;
import java.util.List;