This commit is contained in:
redkale
2024-10-02 00:41:08 +08:00
parent cd15a1a586
commit 692f0fa8b1
12 changed files with 257 additions and 211 deletions

View File

@@ -62,7 +62,7 @@ public final class EnMember<W extends Writer, T, F> {
// 主要给protobuf使用 tag的大小
int tagSize;
public EnMember(Attribute<T, F> attribute, int tag, Encodeable<W, F> encoder) {
public EnMember(Attribute<T, F> attribute, Encodeable<W, F> encoder) {
this.attribute = attribute;
this.encoder = encoder;
Class t = attribute.type();

View File

@@ -162,4 +162,21 @@ public class MapDecoder<R extends Reader, K, V> implements Decodeable<R, Map<K,
public Decodeable<R, V> getValueDecoder() {
return valueDecoder;
}
// ---------------------------------------------------------------------------------
protected void setTag(DeMember member, int tag) {
member.tag = tag;
}
protected void setTagSize(DeMember member, int tagSize) {
member.tagSize = tagSize;
}
protected void setIndex(DeMember member, int index) {
member.index = index;
}
protected void setPosition(DeMember member, int position) {
member.position = position;
}
}

View File

@@ -140,4 +140,21 @@ public class MapEncoder<W extends Writer, K, V> implements Encodeable<W, Map<K,
public Encodeable<W, V> getValueEncoder() {
return valueEncoder;
}
// ---------------------------------------------------------------------------------
protected void setTag(EnMember member, int tag) {
member.tag = tag;
}
protected void setTagSize(EnMember member, int tagSize) {
member.tagSize = tagSize;
}
protected void setIndex(EnMember member, int index) {
member.index = index;
}
protected void setPosition(EnMember member, int position) {
member.position = position;
}
}

View File

@@ -205,7 +205,7 @@ public abstract class Writer {
*
* @param member 字段
*/
public final void writeField(final EnMember member) {
public void writeField(final EnMember member) {
Attribute attr = member.getAttribute();
this.writeField(member, attr.field(), attr.genericType(), member.getPosition());
}

View File

@@ -33,13 +33,8 @@ public class ProtobufArrayEncoder<T> extends ArrayEncoder<ProtobufWriter, T>
return;
}
Encodeable itemEncoder = this.componentEncoder;
T[] array = value;
// if (componentSizeRequired) {
// int tagSize = ProtobufFactory.computeSInt32SizeNoTag(member.getTag());
// out.writeLength(computeSize(tagSize, value));
// }
out.writeArrayB(array.length, itemEncoder, array);
for (T item : array) {
out.writeArrayB(value.length, itemEncoder, value);
for (T item : value) {
out.writeField(member);
if (item == null) {
out.writeLength(0);
@@ -48,7 +43,6 @@ public class ProtobufArrayEncoder<T> extends ArrayEncoder<ProtobufWriter, T>
} else {
ProtobufWriter tmp = out.pollChild();
itemEncoder.convertTo(tmp, item);
out.writeTuple(tmp);
out.offerChild(tmp);
}
}

View File

@@ -35,6 +35,31 @@ public class ProtobufByteBufferWriter extends ProtobufWriter {
return false;
}
@Override
public final ProtobufWriter pollChild() {
ProtobufWriter rs = super.pollChild();
this.delegate = null;
this.child = null;
rs.parent = null;
return rs;
}
@Override
public final void offerChild(ProtobufWriter child) {
int total = child.length();
ProtobufWriter next = child;
while ((next = next.child) != null) {
total += next.length();
}
writeLength(total);
writeTo(child.content(), 0, child.length());
next = child;
while ((next = next.child) != null) {
writeTo(next.content(), 0, next.length());
}
offerPool(child);
}
@Override
public ByteBuffer[] toBuffers() {
if (buffers == null) {

View File

@@ -8,6 +8,7 @@ package org.redkale.convert.pb;
import java.lang.reflect.Type;
import java.util.Collection;
import org.redkale.convert.*;
import org.redkale.util.Utility;
/**
* @author zhangjx
@@ -28,14 +29,10 @@ public class ProtobufCollectionEncoder<T> extends CollectionEncoder<ProtobufWrit
@Override
public void convertTo(ProtobufWriter out, EnMember member, Collection<T> value) {
this.checkInited();
if (value == null || value.isEmpty()) {
if (Utility.isEmpty(value)) {
return;
}
ProtobufEncodeable itemEncoder = (ProtobufEncodeable) this.componentEncoder;
// if (componentSizeRequired) {
// int tagSize = ProtobufFactory.computeSInt32SizeNoTag(member.getTag());
// out.writeLength(computeSize(tagSize, value));
// }
out.writeArrayB(value.size(), itemEncoder, value);
for (T item : value) {
out.writeField(member);
@@ -46,7 +43,6 @@ public class ProtobufCollectionEncoder<T> extends CollectionEncoder<ProtobufWrit
} else {
ProtobufWriter tmp = out.pollChild();
itemEncoder.convertTo(tmp, item);
out.writeTuple(tmp);
out.offerChild(tmp);
}
}

View File

@@ -8,6 +8,7 @@ package org.redkale.convert.pb;
import java.lang.reflect.Type;
import java.util.Map;
import org.redkale.convert.*;
import static org.redkale.convert.pb.ProtobufMapEncoder.createAttribute;
/**
* @author zhangjx
@@ -19,9 +20,19 @@ public class ProtobufMapDecoder<K, V> extends MapDecoder<ProtobufReader, K, V>
protected final boolean enumtostring;
protected final DeMember keyMember;
protected final DeMember valueMember;
public ProtobufMapDecoder(ConvertFactory factory, Type type) {
super(factory, type);
this.enumtostring = ((ProtobufFactory) factory).enumtostring;
int keyTag = ProtobufFactory.getTag(1, ((ProtobufEncodeable) keyDecoder).typeEnum());
int valTag = ProtobufFactory.getTag(2, ((ProtobufEncodeable) valueDecoder).typeEnum());
this.keyMember = new DeMember(createAttribute("key", keyDecoder.getType()), keyTag, keyDecoder);
this.valueMember = new DeMember(createAttribute("value", valueDecoder.getType()), valTag, valueDecoder);
setTagSize(keyMember, ProtobufFactory.computeSInt32SizeNoTag(keyMember.getTag()));
setTagSize(valueMember, ProtobufFactory.computeSInt32SizeNoTag(valueMember.getTag()));
}
@Override
@@ -35,9 +46,9 @@ public class ProtobufMapDecoder<K, V> extends MapDecoder<ProtobufReader, K, V>
while (in.hasNext()) {
int contentLen = in.readRawVarint32();
in.limit(in.position() + contentLen + 1);
in.readTag();
in.readTag(); // key tag
K key = kdecoder.convertFrom(in);
in.readTag();
in.readTag(); // value tag
V value = vdecoder.convertFrom(in);
result.put(key, value);
in.limit(limit);

View File

@@ -9,6 +9,8 @@ import java.lang.reflect.Type;
import java.util.*;
import java.util.function.BiFunction;
import org.redkale.convert.*;
import org.redkale.util.Attribute;
import org.redkale.util.TypeToken;
import org.redkale.util.Utility;
/**
@@ -19,13 +21,18 @@ import org.redkale.util.Utility;
public class ProtobufMapEncoder<K, V> extends MapEncoder<ProtobufWriter, K, V>
implements ProtobufEncodeable<ProtobufWriter, Map<K, V>> {
private final int keyTag;
private final int valTag;
private final EnMember keyMember;
private final EnMember valueMember;
public ProtobufMapEncoder(ConvertFactory factory, Type type) {
super(factory, type);
this.keyTag = ProtobufFactory.getTag(1, ((ProtobufEncodeable) keyEncoder).typeEnum());
this.valTag = ProtobufFactory.getTag(2, ((ProtobufEncodeable) valueEncoder).typeEnum());
this.keyMember = new EnMember(createAttribute("key", keyEncoder.getType()), keyEncoder);
this.valueMember = new EnMember(createAttribute("value", valueEncoder.getType()), valueEncoder);
setTag(keyMember, ProtobufFactory.getTag(1, ((ProtobufEncodeable) keyEncoder).typeEnum()));
setTag(valueMember, ProtobufFactory.getTag(2, ((ProtobufEncodeable) valueEncoder).typeEnum()));
setTagSize(keyMember, ProtobufFactory.computeSInt32SizeNoTag(keyMember.getTag()));
setTagSize(valueMember, ProtobufFactory.computeSInt32SizeNoTag(valueMember.getTag()));
}
@Override
@@ -38,22 +45,29 @@ public class ProtobufMapEncoder<K, V> extends MapEncoder<ProtobufWriter, K, V>
}
Set<String> ignoreColumns = this.ignoreMapColumns;
BiFunction<K, V, V> mapFieldFunc = out.mapFieldFunc();
Encodeable kencoder = this.keyEncoder;
Encodeable vencoder = this.valueEncoder;
ProtobufEncodeable kencoder = (ProtobufEncodeable) this.keyEncoder;
ProtobufEncodeable vencoder = (ProtobufEncodeable) this.valueEncoder;
boolean keySimpled = kencoder instanceof SimpledCoder;
boolean valSimpled = vencoder instanceof SimpledCoder;
out.writeMapB(values.size(), kencoder, vencoder, value);
values.forEach((key, val) -> {
values.forEach((key, val0) -> {
if (ignoreColumns == null || !ignoreColumns.contains(key)) {
V v = mapFieldFunc == null ? val : mapFieldFunc.apply(key, val);
if (v != null) {
V val = mapFieldFunc == null ? val0 : mapFieldFunc.apply(key, val0);
if (val != null) {
out.writeField(member);
ProtobufWriter tmp = out.pollChild();
tmp.writeTag(keyTag);
kencoder.convertTo(tmp, key);
tmp.writeTag(valTag);
vencoder.convertTo(tmp, v);
out.writeTuple(tmp);
if (keySimpled) {
tmp.writeField(keyMember);
kencoder.convertTo(tmp, key);
} else {
kencoder.convertTo(tmp, keyMember, key);
}
if (valSimpled) {
tmp.writeField(valueMember);
vencoder.convertTo(tmp, val);
} else {
vencoder.convertTo(tmp, valueMember, val);
}
out.offerChild(tmp);
}
}
@@ -78,4 +92,8 @@ public class ProtobufMapEncoder<K, V> extends MapEncoder<ProtobufWriter, K, V>
public final ProtobufTypeEnum typeEnum() {
return ProtobufTypeEnum.BYTES;
}
static Attribute createAttribute(String field, Type type) {
return Attribute.create(Map.class, field, TypeToken.typeToClass(type), type, null, null, null);
}
}

View File

@@ -69,13 +69,7 @@ public class ProtobufObjectEncoder<T> extends ObjectEncoder<ProtobufWriter, T>
@ClassDepends
protected ProtobufWriter objectWriter(ProtobufWriter out, EnMember parentMember, T value) {
// if (parentMember != null) {
// out.writeLength(computeSize(out, parentMember.getTagSize(), value));
// }
if (out.length() > out.initOffset) {
return out.pollChild().configParentFunc(out);
}
return out;
return parentMember != null ? out.pollChild() : out;
}
@ClassDepends

View File

@@ -43,7 +43,6 @@ public class ProtobufStreamEncoder<T> extends StreamEncoder<ProtobufWriter, T>
} else {
ProtobufWriter tmp = out.pollChild();
itemEncoder.convertTo(tmp, item);
out.writeTuple(tmp);
out.offerChild(tmp);
}
}

View File

@@ -23,7 +23,7 @@ public class ProtobufWriter extends Writer implements ByteTuple {
"redkale.convert.protobuf.writer.buffer.defsize",
Integer.getInteger("redkale.convert.writer.buffer.defsize", 1024));
private static final int CHILD_SIZE = 8;
private static final int CHILD_SIZE = 32;
protected static final byte[] EMPTY_BYTES = new byte[0];
protected static final int TENTHOUSAND_MAX = 10001;
@@ -62,33 +62,30 @@ public class ProtobufWriter extends Writer implements ByteTuple {
}
}
protected int initOffset;
protected int count;
protected boolean enumtostring;
protected ProtobufWriter parent;
private byte[] content;
protected ProtobufWriter child;
private List<ProtobufWriter> children;
// 链表结构
protected ProtobufWriter delegate;
private byte[] content;
private ArrayDeque<ProtobufWriter> pool;
protected ProtobufWriter(ProtobufWriter parent) {
this();
this.parent = parent;
if (parent != null) {
this.features = parent.features;
this.enumtostring = parent.enumtostring;
}
}
protected ProtobufWriter(byte[] bs) {
this.content = bs;
}
private ProtobufWriter(byte[] bs, int count) {
this.content = bs;
this.count = count;
}
public ProtobufWriter() {
this(DEFAULT_SIZE);
}
@@ -97,10 +94,9 @@ public class ProtobufWriter extends Writer implements ByteTuple {
this.content = new byte[Math.max(size, DEFAULT_SIZE)];
}
public ProtobufWriter(ByteArray array) {
this.content = array.content();
this.initOffset = array.offset();
this.count = array.length();
public ProtobufWriter(ByteTuple tuple) {
this.content = tuple.content();
this.count = tuple.length();
}
@Override
@@ -110,23 +106,10 @@ public class ProtobufWriter extends Writer implements ByteTuple {
}
protected final ProtobufWriter configWrite() {
this.initOffset = this.count;
return this;
}
protected final ProtobufWriter configParentFunc(ProtobufWriter parent) {
this.parent = parent;
if (parent != null) {
this.features = parent.features;
this.enumtostring = parent.enumtostring;
}
return this;
}
protected final ProtobufWriter configFieldFunc(ProtobufWriter out) {
if (out == null) {
return this;
}
this.mapFieldFunc = out.mapFieldFunc;
this.objFieldFunc = out.objFieldFunc;
this.objExtFunc = out.objExtFunc;
@@ -146,27 +129,39 @@ public class ProtobufWriter extends Writer implements ByteTuple {
@Override
protected boolean recycle() {
super.recycle();
if (this.delegate != null && this.pool != null) {
List<ProtobufWriter> list = new ArrayList<>();
ProtobufWriter next = this;
while ((next = next.child) != null) {
list.add(next);
}
for (ProtobufWriter item : list) {
offerPool(item);
}
}
this.parent = null;
this.child = null;
this.delegate = null;
this.mapFieldFunc = null;
this.objFieldFunc = null;
this.objExtFunc = null;
this.features = 0;
this.enumtostring = false;
this.count = 0;
this.initOffset = 0;
if (this.content.length > DEFAULT_SIZE) {
this.content = new byte[DEFAULT_SIZE];
}
if (this.children != null) {
for (ProtobufWriter child : children) {
offerChild2(child);
}
this.children = null;
}
return true;
}
public final ProtobufWriter pollChild() {
protected final void offerPool(ProtobufWriter item) {
if (this.pool != null && this.pool.size() < CHILD_SIZE) {
item.recycle();
this.pool.offer(item);
}
}
public ProtobufWriter pollChild() {
Queue<ProtobufWriter> queue = this.pool;
if (queue == null) {
this.pool = new ArrayDeque<>(CHILD_SIZE);
@@ -176,76 +171,43 @@ public class ProtobufWriter extends Writer implements ByteTuple {
if (result == null) {
result = new ProtobufWriter();
}
return result.configFieldFunc(this);
if (delegate == null) {
result.parent = this;
this.child = result;
delegate = result;
} else {
result.parent = delegate;
delegate.child = result;
delegate = result;
}
result.configFieldFunc(result.parent);
return result;
}
public void offerChild(final ProtobufWriter child) {
Queue<ProtobufWriter> queue = this.pool;
if (child != null && queue != null && queue.size() < CHILD_SIZE) {
child.recycle();
queue.offer(child);
}
}
public final ProtobufWriter createChild2() {
Queue<ProtobufWriter> queue = this.pool;
if (queue == null) {
this.pool = new ArrayDeque<>(CHILD_SIZE);
queue = this.pool;
}
ProtobufWriter result = queue.poll();
if (result == null) {
result = new ProtobufWriter();
}
if (this.children == null) {
this.children = new ArrayList<>();
}
this.children.add(result);
return result.configFieldFunc(this);
}
private void offerChild2(final ProtobufWriter child) {
Queue<ProtobufWriter> queue = this.pool;
if (child != null && queue != null && queue.size() < CHILD_SIZE) {
child.recycle();
queue.offer(child);
public void offerChild(ProtobufWriter child) {
if (child != null) {
int len = child.length();
ProtobufWriter next = child;
while ((next = next.child) != null) {
len += next.length();
}
child.parent.writeSelfLength(len);
}
}
@Override
public final int length() {
int total = count;
if (children != null) {
for (ProtobufWriter child : children) {
int len = child.length();
total += ProtobufFactory.computeSInt32SizeNoTag(len) + len;
}
}
return total;
return count;
}
@Override
public byte[] content() {
if (children != null) {
byte[] data = new byte[length()];
System.arraycopy(content, 0, data, 0, count);
int pos = count;
Utility.println("自身对象: ", content, 0, count);
for (ProtobufWriter child : children) {
int len = child.length();
data[pos++] = (byte) len;
Utility.println("子对象: ", child.content(), 0, child.length());
System.arraycopy(child.content(), 0, data, pos, len);
pos += len;
}
return data;
}
return content;
}
@Override
public final int offset() {
return initOffset;
return 0;
}
/**
@@ -267,12 +229,26 @@ public class ProtobufWriter extends Writer implements ByteTuple {
@Override
public byte[] toArray() {
if (children != null) {
return content();
if (delegate == null) {
byte[] copy = new byte[count];
System.arraycopy(content, 0, copy, 0, count);
return copy;
} else {
int total = count;
ProtobufWriter next = this;
while ((next = next.child) != null) {
total += next.length();
}
byte[] data = new byte[total];
System.arraycopy(content, 0, data, 0, count);
next = this;
int pos = count;
while ((next = next.child) != null) {
System.arraycopy(next.content(), 0, data, pos, next.length());
pos += next.length();
}
return data;
}
byte[] copy = new byte[count];
System.arraycopy(content, 0, copy, 0, count);
return copy;
}
public ProtobufWriter enumtostring(boolean enumtostring) {
@@ -291,8 +267,12 @@ public class ProtobufWriter extends Writer implements ByteTuple {
}
public void writeTo(final byte ch) {
expand(1);
content[count++] = ch;
if (delegate == null) {
expand(1);
content[count++] = ch;
} else {
delegate.writeTo(ch);
}
}
public final void writeTo(final byte... chs) {
@@ -300,14 +280,18 @@ public class ProtobufWriter extends Writer implements ByteTuple {
}
public void writeTo(final byte[] chs, final int start, final int len) {
expand(len);
System.arraycopy(chs, start, content, count, len);
count += len;
if (delegate == null) {
expand(len);
System.arraycopy(chs, start, content, count, len);
count += len;
} else {
delegate.writeTo(chs, start, len);
}
}
public ProtobufWriter clear() {
this.count = 0;
this.initOffset = 0;
this.delegate = null;
return this;
}
@@ -345,15 +329,7 @@ public class ProtobufWriter extends Writer implements ByteTuple {
@Override
public final void writeObjectE(Object obj) {
if (parent != null) {
parent.writeTuple(this);
}
}
protected final void writeTuple(ByteTuple tuple) {
int len = tuple.length();
writeLength(len);
writeTo(tuple.content(), tuple.offset(), len);
// do nothing
}
@Override
@@ -386,6 +362,15 @@ public class ProtobufWriter extends Writer implements ByteTuple {
// do nothing
}
/**
* 输出一个字段名
*
* @param member 字段
*/
public final void writeField(final EnMember member) {
writeTag(member.getTag());
}
// 被ObjectEncoder调用
@Override
public final void writeField(EnMember member, String fieldName, Type fieldType, int fieldPos) {
@@ -1189,8 +1174,9 @@ public class ProtobufWriter extends Writer implements ByteTuple {
}
}
@Override
@ClassDepends // objExtFunc扩展字段时member=null
public final void writeObjectField2(@Nullable EnMember member, Object obj) {
public final void writeObjectField(@Nullable EnMember member, Object obj) {
Object value;
if (objFieldFunc == null) {
value = member.getFieldValue(obj);
@@ -1201,41 +1187,11 @@ public class ProtobufWriter extends Writer implements ByteTuple {
return;
}
ProtobufEncodeable encoder = (ProtobufEncodeable) member.getEncoder();
if (encoder.requireSize()) {
writeLength(encoder.computeSize(this, member.getTagSize(), value));
}
encoder.convertTo(this, member, value);
}
@Override
@ClassDepends // objExtFunc扩展字段时member=null
public final void writeObjectField(final EnMember member, Object obj) {
Object value;
if (objFieldFunc == null) {
value = member.getFieldValue(obj);
} else {
value = objFieldFunc.apply(member.getAttribute(), obj);
}
if (value == null) {
return;
}
Encodeable encoder = member.getEncoder();
if (encoder instanceof MapEncoder) {
if (!((Map) value).isEmpty()) {
((MapEncoder) encoder).convertTo(this, member, (Map) value);
}
} else if (encoder instanceof ProtobufArrayEncoder) {
ProtobufArrayEncoder arrayEncoder = (ProtobufArrayEncoder) encoder;
arrayEncoder.convertTo(this, member, (Object[]) value);
} else if (encoder instanceof ProtobufCollectionEncoder) {
ProtobufCollectionEncoder collectionEncoder = (ProtobufCollectionEncoder) encoder;
collectionEncoder.convertTo(this, member, (Collection) value);
} else if (encoder instanceof ProtobufStreamEncoder) {
ProtobufStreamEncoder streamEncoder = (ProtobufStreamEncoder) encoder;
streamEncoder.convertTo(this, member, (Stream) value);
} else {
if (encoder instanceof SimpledCoder) {
this.writeField(member);
encoder.convertTo(this, value);
} else {
encoder.convertTo(this, member, value);
}
}
@@ -1256,6 +1212,17 @@ public class ProtobufWriter extends Writer implements ByteTuple {
}
}
protected final void writeSelfLength(int value) {
ProtobufWriter old = this.delegate;
this.delegate = null;
if (value < 128) {
writeTo((byte) value);
} else {
writeUInt32(value);
}
this.delegate = old;
}
protected void writeUInt32(int value) {
if (value >= 0 && value < TENTHOUSAND_MAX) {
writeTo(TENTHOUSAND_UINT_BYTES[value]);
@@ -1264,18 +1231,22 @@ public class ProtobufWriter extends Writer implements ByteTuple {
writeTo(TENTHOUSAND_UINT_BYTES2[-value]);
return;
}
expand(5);
int curr = this.count;
byte[] data = this.content;
while (true) {
if ((value & ~0x7F) == 0) {
data[curr++] = (byte) value;
this.count = curr;
return;
} else {
data[curr++] = (byte) ((value & 0x7F) | 0x80);
value >>>= 7;
if (delegate == null) {
expand(5);
int curr = this.count;
byte[] data = this.content;
while (true) {
if ((value & ~0x7F) == 0) {
data[curr++] = (byte) value;
this.count = curr;
return;
} else {
data[curr++] = (byte) ((value & 0x7F) | 0x80);
value >>>= 7;
}
}
} else {
delegate.writeUInt32(value);
}
}
@@ -1287,18 +1258,22 @@ public class ProtobufWriter extends Writer implements ByteTuple {
writeTo(TENTHOUSAND_UINT_BYTES2[(int) -value]);
return;
}
expand(10);
int curr = this.count;
byte[] data = this.content;
while (true) {
if ((value & ~0x7FL) == 0) {
data[curr++] = (byte) value;
this.count = curr;
return;
} else {
data[curr++] = (byte) (((int) value & 0x7F) | 0x80);
value >>>= 7;
if (delegate == null) {
expand(10);
int curr = this.count;
byte[] data = this.content;
while (true) {
if ((value & ~0x7FL) == 0) {
data[curr++] = (byte) value;
this.count = curr;
return;
} else {
data[curr++] = (byte) (((int) value & 0x7F) | 0x80);
value >>>= 7;
}
}
} else {
delegate.writeUInt64(value);
}
}