diff --git a/.idea/compiler.xml b/.idea/compiler.xml index 773bad4..783803f 100644 --- a/.idea/compiler.xml +++ b/.idea/compiler.xml @@ -1,10 +1,7 @@ - - - - + diff --git a/.idea/jarRepositories.xml b/.idea/jarRepositories.xml new file mode 100644 index 0000000..efa4625 --- /dev/null +++ b/.idea/jarRepositories.xml @@ -0,0 +1,20 @@ + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index 2f00e79..22f380b 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -4,6 +4,9 @@ + + + diff --git a/.idea/modules.xml b/.idea/modules.xml index ad80793..3986c39 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -3,11 +3,8 @@ - - - - - + + \ No newline at end of file diff --git a/.idea/modules/ETF-Java.iml b/.idea/modules/ETF-Java.iml index 71c1d11..47d9110 100644 --- a/.idea/modules/ETF-Java.iml +++ b/.idea/modules/ETF-Java.iml @@ -1,5 +1,6 @@ - + + diff --git a/.idea/modules/ETF-Java_main.iml b/.idea/modules/ETF-Java.main.iml similarity index 62% rename from .idea/modules/ETF-Java_main.iml rename to .idea/modules/ETF-Java.main.iml index 77aac3a..d79cc51 100644 --- a/.idea/modules/ETF-Java_main.iml +++ b/.idea/modules/ETF-Java.main.iml @@ -1,13 +1,14 @@ - - - + + + - + + \ No newline at end of file diff --git a/.idea/modules/ETF-Java.test.iml b/.idea/modules/ETF-Java.test.iml new file mode 100644 index 0000000..8744c33 --- /dev/null +++ b/.idea/modules/ETF-Java.test.iml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules/ETF-Java_jmh.iml b/.idea/modules/ETF-Java_jmh.iml index 7950568..fe6043c 100644 --- a/.idea/modules/ETF-Java_jmh.iml +++ b/.idea/modules/ETF-Java_jmh.iml @@ -1,6 +1,6 @@ - + diff --git a/.idea/modules/ETF-Java_test.iml b/.idea/modules/ETF-Java_test.iml deleted file mode 100644 index 72b272e..0000000 --- a/.idea/modules/ETF-Java_test.iml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/gui/gui.iml b/.idea/modules/gui/gui.iml deleted file mode 100644 index be7c72e..0000000 --- a/.idea/modules/gui/gui.iml +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/gui/gui_main.iml b/.idea/modules/gui/gui_main.iml deleted file mode 100644 index 07a155a..0000000 --- a/.idea/modules/gui/gui_main.iml +++ /dev/null @@ -1,67 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/.idea/modules/gui/gui_test.iml b/.idea/modules/gui/gui_test.iml deleted file mode 100644 index 0152b3a..0000000 --- a/.idea/modules/gui/gui_test.iml +++ /dev/null @@ -1,71 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/README.md b/README.md index 870a180..05abab0 100644 --- a/README.md +++ b/README.md @@ -2,36 +2,15 @@ A Java parser/writer for the ETF format. This api aims to provide a set of high-speed abstractions for ETF. -Additionally, it supports multiple subsets of ETF: standard ETF, BERT-rpc -and Hammer & Chisel's (Discord's) Loqui. + +Currently, its primary goal is to support the subset of ETF utilized by Discord (see: [Loqui](https://github.com/discord/loqui) and [erlpack](https://github.com/discord/erlpack)) +So the you should consider this the "minimum viable product" of an ETF parser for use with discord. ## Using ETF-Java **TODO** -## A note about performance -While this project attempts to be as well optimized as possible, the JVM -creates a limit to how fast this can be just due to how it manages memory. - -Performance can be improved via JNI or usage of the `Unsafe` class -but these have issues which has kept me using pure java. - -Regarding JNI: I do not know enough C/C++ to properly optimize and there -would still be a cost for interfacing natives with java (meaning that for -smaller ETF data there won't be much of a benefit). - -Regarding `Unsafe`: Unsafe is *unsafe*. Meaning it is not guaranteed to -work across java versions and jvm implementations. So despite its -significant performance benefits, it's impossible to be sure that it will -always work how I expect it to. - -## Caveats -Due to java not implementing proper primitive unsigned data types, I have -been forced to use much larger data types than should be necessary to -prevent potential underflows (ex. `long` instead of `int` or `BigInteger` -instead of `long`). This causes some strange internal data type handling -and some slight overhead. - ## Helpful Resources * [ETF Documentation](http://erlang.org/doc/apps/erts/erl_ext_dist.html) * [BERT Documentation](http://bert-rpc.org/) -* [Loqui](https://github.com/hammerandchisel/loqui) \ No newline at end of file +* [Loqui](https://github.com/discord/loqui) +* [erlpack](https://github.com/discord/erlpack) \ No newline at end of file diff --git a/build.gradle b/build.gradle index f98d76b..1d3bd71 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ group 'com.austinv11.etf' -version '1.0.0-SNAPSHOT' +version '2.0.0-SNAPSHOT' buildscript { repositories { @@ -8,13 +8,9 @@ buildscript { url "https://plugins.gradle.org/m2/" } } - dependencies { - classpath "me.champeau.gradle:jmh-gradle-plugin:0.3.1" - } } apply plugin: 'java' -apply plugin: "me.champeau.gradle.jmh" sourceCompatibility = 1.8 @@ -23,11 +19,7 @@ repositories { } dependencies { - testCompile group: 'junit', name: 'junit', version: '4.11' - testCompile 'org.json:json:20160810' - testCompile 'com.google.code.gson:gson:2.8.0' + compile 'io.netty:netty-buffer:4.1.50.Final' - jmh 'org.json:json:20160810' - jmh 'com.google.code.gson:gson:2.8.0' - jmh 'com.fasterxml.jackson.module:jackson-module-afterburner:2.8.7' + testCompile group: 'junit', name: 'junit', version: '4.11' } diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 2c5777a..11f5fc0 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Sun Mar 12 23:31:05 EDT 2017 +#Fri Jun 05 18:18:49 EDT 2020 +distributionUrl=https\://services.gradle.org/distributions/gradle-6.5-all.zip distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-3.1-bin.zip +zipStoreBase=GRADLE_USER_HOME diff --git a/settings.gradle b/settings.gradle index ae3223a..f455a07 100644 --- a/settings.gradle +++ b/settings.gradle @@ -1,4 +1 @@ -rootProject.name = 'ETF-Java' -include 'gui' -include 'gui' - +rootProject.name = 'ETF-Java' \ No newline at end of file diff --git a/src/jmh/java/com/austinv11/etf/benchmarks/Benchmarks.java b/src/jmh/java/com/austinv11/etf/benchmarks/Benchmarks.java deleted file mode 100644 index 3e80733..0000000 --- a/src/jmh/java/com/austinv11/etf/benchmarks/Benchmarks.java +++ /dev/null @@ -1,37 +0,0 @@ -package com.austinv11.etf.benchmarks; - -import com.austinv11.etf.ETFConfig; -import org.openjdk.jmh.annotations.*; - -import java.util.concurrent.TimeUnit; - -public class Benchmarks { - - @State(Scope.Benchmark) - public static class Context { - - ETFConfig config; - - @Setup(Level.Trial) - public void init() { - config = new ETFConfig().setBert(false).setCompression(false) - .setIncludeDistributionHeader(false).setIncludeHeader(false).setLoqui(true).setVersion(131); - } - - @TearDown(Level.Trial) - public void clean() { - config = null; - } - } - - public static class TestClass { - - } - - @Benchmark - @BenchmarkMode(Mode.All) - @OutputTimeUnit(TimeUnit.NANOSECONDS) - public void etf(Context context) { - //TODO - } -} diff --git a/src/main/java/com/austinv11/etf/ETFConfig.java b/src/main/java/com/austinv11/etf/ETFConfig.java deleted file mode 100644 index 49a2f70..0000000 --- a/src/main/java/com/austinv11/etf/ETFConfig.java +++ /dev/null @@ -1,197 +0,0 @@ -package com.austinv11.etf; - -import com.austinv11.etf.parsing.ETFParser; -import com.austinv11.etf.util.ETFConstants; -import com.austinv11.etf.util.Mapper; -import com.austinv11.etf.writing.ETFWriter; - -/** - * This provides a clean way to configure etf handlers. - */ -public class ETFConfig { - - private boolean bert = false; - private int version = ETFConstants.VERSION; - private boolean includeHeader = true; - private boolean includeDistributionHeader = false; - private boolean loqui = false; - private boolean compress = false; - - /** - * This returns whether this supports BERT. - * - * @return True if BERT is configured, false if otherwise. - */ - public boolean isBert() { - return bert; - } - - /** - * This sets whether this handles BERT. - * - * @param bert Whether to support BERT. - * @return The current config instance (for chaining). - * - * @see com.austinv11.etf.util.BertCompatible - * @see com.austinv11.etf.util.BertType - */ - public ETFConfig setBert(boolean bert) { - this.bert = bert; - return this; - } - - /** - * This gets the version of ETF this is handling. - * - * @return The version. - */ - public int getVersion() { - return version; - } - - /** - * This sets the specific version of etf this is handling. - * WARNING: No matter what version you set, the internal handlers will only respect {@link ETFConstants#VERSION}'s - * spec. - * - * @param version The version integer. - * @return The current config instance (for chaining). - */ - public ETFConfig setVersion(int version) { - this.version = version; - return this; - } - - /** - * This returns whether headers are included. - * NOTE This is NOT a distribution header setting. - * - * @return True if including a header. - */ - public boolean isIncludingHeader() { - return includeHeader; - } - - /** - * Sets whether this will include a header. - * NOTE This is NOT a distribution header setting. - * - * @param includeHeader Set to true to include a header. - * @return The current config instance (for chaining). - */ - public ETFConfig setIncludeHeader(boolean includeHeader) { - this.includeHeader = includeHeader; - return this; - } - - /** - * This returns whether this config supports the Loqui - * (Discord's) transport protocol. - * - * @return True if loqui handling is enabled, false if otherwise. - */ - public boolean isLoqui() { - return loqui; - } - - /** - * Sets whether this will have special handling for Loqui, - * Discord's transport protocol. - * - * @param loqui Set to true to handle loqui types, false to ignore them. - * @return The current config instance (for chaining). - */ - public ETFConfig setLoqui(boolean loqui) { - this.loqui = loqui; - return this; - } - - /** - * This returns whether distribution headers are written. - * - * @return True when enabled, false when otherwise. - */ - public boolean isIncludingDistributionHeader() { - return includeDistributionHeader; //TODO - } - - /** - * This sets whether distribution headers are written. - * - * @param includeDistributionHeader Set to true to write distribution headers, false to not. - * @return The current config instance (for chaining). - */ - public ETFConfig setIncludeDistributionHeader(boolean includeDistributionHeader) { - this.includeDistributionHeader = includeDistributionHeader; - return this; - } - - /** - * This gets whether ETF should be compressed when written. - * - * @return True when etf is compressed when written. - */ - public boolean isCompressing() { - return compress; - } - - /** - * This sets whether ETF should be compressed when written. - * - * @param compress Set to true to compress, false to not. - * @return The current config instance (for chaining). - */ - public ETFConfig setCompression(boolean compress) { - this.compress = compress; - return this; - } - - /** - * This creates a new parser using the set configuration. - * - * @param data The data to parse. - * @return The new parser instance. - */ - public ETFParser createParser(byte[] data) { - return createParser(data, false); - } - - /** - * This creates a new parser using the set configuration. - * - * @param data The data to parse. - * @param partial Whether the data should be treated as partial (meaning no headers). - * @return The new parser instance. - */ - public ETFParser createParser(byte[] data, boolean partial) { - return new ETFParser(data, this, partial); - } - - /** - * This creates a new writer using the set configuration. - * - * @return The new writer instance. - */ - public ETFWriter createWriter() { - return createWriter(false); - } - - /** - * This creates a new writer using the set configuration. - * - * @param partial Whether the data should be treated as partial (meaning no headers). - * @return The new writer instance. - */ - public ETFWriter createWriter(boolean partial) { - return new ETFWriter(this, partial); - } - - /** - * This creates a new mapper using this configuration. - * - * @return The new mapper instance. - */ - public Mapper createMapper() { - return new Mapper(this); - } -} diff --git a/src/main/java/com/austinv11/etf/EtfBuffer.java b/src/main/java/com/austinv11/etf/EtfBuffer.java new file mode 100644 index 0000000..07044e2 --- /dev/null +++ b/src/main/java/com/austinv11/etf/EtfBuffer.java @@ -0,0 +1,509 @@ +package com.austinv11.etf; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.Unpooled; + +import java.io.Closeable; +import java.nio.charset.StandardCharsets; + +import static com.austinv11.etf.Terms.*; + +/** + * A class which manages a buffer of ETF data. + */ +public class EtfBuffer implements Closeable { + + private final ByteBuf buffer; + + private EtfBuffer(ByteBuf buffer) { + buffer.retain(); + this.buffer = buffer; + } + + /** + * Builds a heap-bound ETF buffer. + * @return The buffer. + */ + public static EtfBuffer onHeapBuffer() { + return new EtfBuffer(Unpooled.buffer()); + } + + /** + * Builds a heap-less ETF buffer. + * @return The buffer. + */ + public static EtfBuffer directBuffer() { + return new EtfBuffer(Unpooled.directBuffer()); + } + + /** + * Wraps a byte buffer for reading only. + * @param bb The byte buffer to wrap. + * @return The buffer. + */ + public static EtfBuffer wrapForRead(ByteBuf bb) { + return new EtfBuffer(bb); + } + + /** + * Returns a read-only view over the internal buffer. + * @return The buffer. + */ + public ByteBuf write() { + return buffer.asReadOnly(); + } + + /** + * Starts writing to the buffer. This isn't explicitly required, but ETF parsers may not understand the output + * unless you explicitly start writing (only once!). + * @return The buffer for chaining. + */ + public EtfBuffer startWrite() { + buffer.writeByte(EtfConstants.VERSION); + return this; + } + + /** + * Starts reading from the buffer. + * @return The buffer for reading. + */ + public EtfBuffer startRead() { + assert buffer.readByte() == EtfConstants.VERSION; + return this; + } + + /** + * Release the resources being used. + * + * @return The buffer for chaining. + */ + public EtfBuffer endWrite() { + close(); + return this; + } + + /** + * Release the resources being used. + * + * @return The buffer for chaining. + */ + public EtfBuffer endRead() { + close(); + return this; + } + + @Override + public void close() { + buffer.release(); + } + + private EtfBuffer write(byte... bytes) { + for (byte b : bytes) { + buffer.writeByte(b); + } + return this; + } + + private EtfBuffer write(int... bytes) { + for (int b : bytes) { + buffer.writeByte(b); + } + return this; + } + + private void requireType(byte type) { + if (peek() != type) + throw new EtfException("Unexpected type " + type); + buffer.readByte(); + } + + private byte requireEitherType(byte... types) { + byte peeked = peek(); + for (byte t : types) { + if (t == peeked) { + return buffer.readByte(); + } + } + throw new EtfException("Unexpected type " + peeked); + } + + /** + * Peeks at the next byte. + * @return The next byte. + */ + public byte peek() { + int i = buffer.readerIndex(); + return buffer.getByte(i); + } + + /** + * Writes a small integer (unsigned byte). + * @return The buffer for chaining. + */ + public EtfBuffer writeSmallInt(byte b) { + return write(SMALL_INTEGER_EXT, b); + } + + /** + * Reads a small integer (unsigned byte). + * @return The integer. + */ + public byte readSmallInt() { + requireType(SMALL_INTEGER_EXT); + return buffer.readByte(); + } + + /** + * Writes a signed integer. + * @param i The integer. + * @return The buffer for chaining. + */ + public EtfBuffer writeInt(int i) { + return write(INTEGER_EXT, ((i >>> 24) & 0xff), + ((i >>> 16) & 0xff), + ((i >>> 8) & 0xff), + (i & 0xff)); + } + + /** + * Reads a signed integer. + * @return The integer. + */ + public int readInt() { + requireType(INTEGER_EXT); + return (buffer.readByte() << 24) + | (buffer.readByte() << 16) + | (buffer.readByte() << 8) + | buffer.readByte(); + } + + /** + * Indicates that a tuple will be written. + * @param count The number of tuple entries (as an unsigned integer). This indicates how many write operations + * will be counted as part of the tuple. + * @return The buffer for chaining. + */ + public EtfBuffer startWriteTuple(int count) { + return startWriteTuple(count, count < 256); + } + + /** + * Starts reading a tuple. + * @return The number of elements in the tuple. + */ + public int startReadTuple() { + byte type = requireEitherType(SMALL_TUPLE_EXT, LARGE_TUPLE_EXT); + if (type == SMALL_TUPLE_EXT) { + return buffer.readUnsignedByte(); + } else { + return (buffer.readByte() << 24) + | (buffer.readByte() << 16) + | (buffer.readByte() << 8) + | buffer.readByte(); + } + } + + /** + * Indicates that a tuple will be written. + * @param count The number of tuple entries (as an unsigned integer). This indicates how many write operations + * will be counted as part of the tuple. + * @param small Whether this should be explicitly set as a small tuple or large tuple. + * @return The buffer for chaining. + */ + public EtfBuffer startWriteTuple(int count, boolean small) { + if (small) { + return write(SMALL_TUPLE_EXT, count); + } else { + return write(LARGE_TUPLE_EXT, ((count >>> 24) & 0xff), + ((count >>> 16) & 0xff), + ((count >>> 8) & 0xff), + (count & 0xff)); + } + } + + /** + * Indicates that a map will be written. + * @param count The number of K-V pairs (as an unsigned integer). This indicates the number of write opertions times + * 2 that will be counted as part of the map. The following elements are written as K1, V1, ..., Kn, Vn where n is + * the count. + * @return The buffer for chaining. + */ + public EtfBuffer startWriteMap(int count) { + return write(MAP_EXT, ((count >>> 24) & 0xff), + ((count >>> 16) & 0xff), + ((count >>> 8) & 0xff), + (count & 0xff)); + } + + /** + * Starts reading a map. + * @return The number of K-V pairs in the map. + */ + public int startReadMap() { + requireType(MAP_EXT); + return (buffer.readByte() << 24) + | (buffer.readByte() << 16) + | (buffer.readByte() << 8) + | buffer.readByte(); + } + + /** + * Writes nil. + * + * @return The buffer for chaining. + */ + public EtfBuffer writeNil() { + return write(NIL_EXT); + } + + /** + * Reads nil. + */ + public void readNil() { + requireType(NIL_EXT); + } + + /** + * Writes a string of bytes. + * @param bytes The bytes. + * @return The buffer for chaining. + */ + public EtfBuffer writeString(byte[] bytes) { + write(STRING_EXT, + ((bytes.length >>> 8) & 0xff), + (bytes.length & 0xff)); + return write(bytes); + } + + /** + * Reads a string of bytes. + * @return The bytes. + */ + public byte[] readString() { + requireType(STRING_EXT); + int count = (buffer.readByte() << 8) + | buffer.readByte(); + byte[] bytes = new byte[count]; + buffer.readBytes(bytes); + return bytes; + } + + /** + * Starts writing a list. NOTE: A list in ETF is terminated with a tail value (so the list contains count + 1 + * elements in total), this tail is usually nil (proper list). + * @param count The number of elements of the list (unsigned). + * @return The buffer for chaining. + * + * @see #writeNil() + */ + public EtfBuffer startWriteList(int count) { + return write(LIST_EXT, ((count >>> 24) & 0xff), + ((count >>> 16) & 0xff), + ((count >>> 8) & 0xff), + (count & 0xff)); + } + + /** + * Starts reading a list. NOTE: A list in ETF is terminated with a tail value (so the list contains count + 1 + * elements in total), this tail is usually nil (proper list). + * @return The number of elements (excluding the tail). + */ + public int startReadList() { + requireType(LIST_EXT); + return (buffer.readByte() << 24) + | (buffer.readByte() << 16) + | (buffer.readByte() << 8) + | buffer.readByte(); + } + + /** + * Writes a floating point number. + * + * @param f The number. + * @return The buffer for chaining. + */ + public EtfBuffer writeFloat(double f) { + long longBits = Double.doubleToLongBits(f); + return write(NEW_FLOAT_EXT, + (byte) ((longBits >>> 56) & 0xff), (byte) ((longBits >>> 48) & 0xff), + (byte) ((longBits >>> 40) & 0xff), (byte) ((longBits >>> 32) & 0xff), + (byte) ((longBits >>> 24) & 0xff), (byte) ((longBits >>> 16) & 0xff), + (byte) ((longBits >>> 8) & 0xff), (byte) (longBits & 0xff)); + } + + /** + * Reads a floating point number. + * @return The number. + */ + public double readFloat() { + requireType(NEW_FLOAT_EXT); + long bits = ((long) buffer.readByte() << 56) + | ((long) buffer.readByte() << 48) + | ((long) buffer.readByte() << 40) + | ((long) buffer.readByte() << 32) + | ((long) buffer.readByte() << 24) + | ((long) buffer.readByte() << 16) + | ((long) buffer.readByte() << 8) + | buffer.readByte(); + return Double.longBitsToDouble(bits); + } + + /** + * Write a utf-8 atom. + * @param s The string to write. + * @return The buffer for chaining. + */ + public EtfBuffer writeAtom(String s) { + byte[] bytes = s.getBytes(StandardCharsets.UTF_8); + return writeAtom(bytes, bytes.length < 256); + } + + /** + * Read a utf-8 atom. NOTE: In loqui (Discord), an atom of "nil" is a nil value and "true"/"false" atoms are + * booleans. + * @return The atom. + */ + public String readAtom() { + byte type = requireEitherType(SMALL_ATOM_UTF8_EXT, ATOM_UTF8_EXT); + + int length; + if (type == SMALL_TUPLE_EXT) { + length = buffer.readUnsignedByte(); + } else { + length = (buffer.readByte() << 8) + | buffer.readByte(); + } + + byte[] bytes = new byte[length]; + buffer.readBytes(bytes); + return new String(bytes, StandardCharsets.UTF_8); + } + + + /** + * Writes a utf-8 atom with explicit typing. + * @param s The string to write. + * @param small True if a small atom, false for a normal atom. + * @return The buffer for chaining. + */ + public EtfBuffer writeAtom(String s, boolean small) { + return writeAtom(s.getBytes(StandardCharsets.UTF_8), small); + } + + private EtfBuffer writeAtom(byte[] s, boolean small) { + if (small) { + write(SMALL_ATOM_UTF8_EXT, (s.length & 0xff)); + } else { + write(ATOM_UTF8_EXT, ((s.length >>> 8) & 0xff), (s.length & 0xff)); + } + + for (byte b : s) { + write(b); + } + return this; + } + + /** + * Writes a signed big integer. + * @param l The integer. + * @return The buffer for chaining. + */ + public EtfBuffer writeBig(long l) { + return writeBig(l, l >= 0); + } + + /** + * Writes an unsigned big integer. + * @param l The integer. + * @return The buffer for chaining. + */ + public EtfBuffer writeBigUnsigned(long l) { + return writeBig(l, true); // Write as positive + } + + // Implementation based on https://github.com/discord/erlpack/blob/master/cpp/encoder.h#L94 + private EtfBuffer writeBig(long l, boolean isPositive) { + // Longs only support 64bit nums so a SMALL_BIG_EXT should work + write(SMALL_BIG_EXT); + byte[] buf = new byte[2 + 8]; // max size of the buffer, 2 meta bytes + the 8 bytes for a long + char bytes = 0; + + while (l != 0) { + buf[2 + bytes] = (byte) (l & 0xFF); + l >>= 8; + bytes++; + } + + buf[0] = (byte) bytes; + buf[1] = (byte) (isPositive ? 1 : 0); + + buffer.writeBytes(buf, 0, 2 + bytes); + + return this; + } + + /** + * Reads a big integer. + * @return The integer. + */ + public long readBig() { + requireType(SMALL_BIG_EXT); + + short n = buffer.readUnsignedByte(); + byte sign = buffer.readByte(); + + long big = 0; + for (short i = 0; i < n; i++) { + big |= buffer.readByte() << 8 * i; + } + + if (big >= 0 && sign == 0) { + big *= -1L; + } + + return big; + } + + // Loqui (discord) specific differences + + /** + * Writes a boolean as a small atom. (Loqui specific). + * @param bool The boolean. + * @return The buffer for chaining. + */ + public EtfBuffer writeBoolean(boolean bool) { + return writeAtom(bool ? "true" : "false"); + } + + /** + * Reads a boolean from an atom. (Loqui specific). + * @return The boolean. + */ + public boolean readBoolean() { + String atom = readAtom(); + if ("true".equals(atom)) { + return true; + } else if ("false".equals(atom)) { + return false; + } else { + throw new EtfException("Atom is not a boolean!"); + } + } + + /** + * Writes "nil" as an atom. (Loqui specific encoding for nil). + * @return The buffer for chaining. + */ + public EtfBuffer writeLoquiNil() { + return writeAtom("nil"); + } + + /** + * Reads "nil" as an atom. (Loqui specific). + */ + public void readLoquiNil() { + if (!"nil".equals(readAtom())) { + throw new EtfException("Atom is not nil!"); + } + } +} diff --git a/src/main/java/com/austinv11/etf/EtfConstants.java b/src/main/java/com/austinv11/etf/EtfConstants.java new file mode 100644 index 0000000..2ad264b --- /dev/null +++ b/src/main/java/com/austinv11/etf/EtfConstants.java @@ -0,0 +1,5 @@ +package com.austinv11.etf; + +public class EtfConstants { + public final static byte VERSION = (byte) 131; //The spec version this supports (131) +} diff --git a/src/main/java/com/austinv11/etf/EtfException.java b/src/main/java/com/austinv11/etf/EtfException.java new file mode 100644 index 0000000..d655f6b --- /dev/null +++ b/src/main/java/com/austinv11/etf/EtfException.java @@ -0,0 +1,12 @@ +package com.austinv11.etf; + +public class EtfException extends RuntimeException { + + public EtfException() { + + } + + public EtfException(String message) { + super(message); + } +} diff --git a/src/main/java/com/austinv11/etf/common/TermTypes.java b/src/main/java/com/austinv11/etf/Terms.java similarity index 76% rename from src/main/java/com/austinv11/etf/common/TermTypes.java rename to src/main/java/com/austinv11/etf/Terms.java index cef4058..460c5c2 100644 --- a/src/main/java/com/austinv11/etf/common/TermTypes.java +++ b/src/main/java/com/austinv11/etf/Terms.java @@ -1,45 +1,29 @@ -package com.austinv11.etf.common; +package com.austinv11.etf; -import com.austinv11.etf.util.BertCompatible; - -/** - * This represents an erlang term. - */ -public class TermTypes { - - @BertCompatible +public final class Terms { public static final byte HEADER = 80; public static final byte DISTRIBUTION_HEADER = 68; public static final byte ATOM_CACHE_REF = 82; - @BertCompatible public static final byte SMALL_INTEGER_EXT = 97; - @BertCompatible public static final byte INTEGER_EXT = 98; - @BertCompatible + @Deprecated // deprecated by ETF so we are not using it public static final byte FLOAT_EXT = 99; - @BertCompatible + @Deprecated // deprecated for ATOM_UTF8 public static final byte ATOM_EXT = 100; public static final byte REFERENCE_EXT = 101; public static final byte PORT_EXT = 102; public static final byte PID_EXT = 103; - @BertCompatible public static final byte SMALL_TUPLE_EXT = 104; - @BertCompatible public static final byte LARGE_TUPLE_EXT = 105; public static final byte MAP_EXT = 116; - @BertCompatible public static final byte NIL_EXT = 106; - @BertCompatible public static final byte STRING_EXT = 107; - @BertCompatible public static final byte LIST_EXT = 108; - @BertCompatible public static final byte BINARY_EXT = 109; - @BertCompatible public static final byte SMALL_BIG_EXT = 110; - @BertCompatible public static final byte LARGE_BIG_EXT = 111; public static final byte NEW_REFERENCE_EXT = 114; + @Deprecated // deprecated for SMALL_ATOM_UTF8 public static final byte SMALL_ATOM_EXT = 115; public static final byte FUN_EXT = 117; public static final byte NEW_FUN_EXT = 112; diff --git a/src/main/java/com/austinv11/etf/erlang/DistributionHeader.java b/src/main/java/com/austinv11/etf/erlang/DistributionHeader.java deleted file mode 100644 index fea9330..0000000 --- a/src/main/java/com/austinv11/etf/erlang/DistributionHeader.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; - -//TODO implement -public class DistributionHeader implements ErlangObject { - - private final byte[] data; - - public DistributionHeader(byte[] data) { - this.data = data; - } - - @Override - public byte type() { - return TermTypes.DISTRIBUTION_HEADER; - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/ErlangList.java b/src/main/java/com/austinv11/etf/erlang/ErlangList.java deleted file mode 100644 index 59528ec..0000000 --- a/src/main/java/com/austinv11/etf/erlang/ErlangList.java +++ /dev/null @@ -1,248 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; -import com.austinv11.etf.util.BertCompatible; - -import java.math.BigInteger; -import java.util.AbstractList; - -/** - * This represents an immutable ETF list. - */ -@BertCompatible -public class ErlangList extends AbstractList implements ErlangObject { - - private final Object[] data; - private final Object tail; - - public ErlangList(Object[] data, Object tail) { - this.data = data; - this.tail = tail; - } - - @Override - public Object get(int index) { - Object obj; - if (index == data.length) - obj = tail; - else - obj = data[index]; - - if (obj instanceof byte[]) - return new String((byte[]) obj); - - return obj; - } - - @Override - public int size() { - return isProper() ? data.length : data.length+1; - } - - /** - * Gets the specified object as an integer. - * - * @param index The object's index. - * @return The object. - */ - public int getInt(int index) { - return (int) get(index); - } - - /** - * Gets the specified object as a short. - * - * @param index The object's index. - * @return The object. - */ - public short getShort(int index) { - return (short) get(index); - } - - /** - * Gets the specified object as a character. - * - * @param index The object's index. - * @return The object. - */ - public char getChar(int index) { - return (char) get(index); - } - - /** - * Gets the specified object as a byte. - * - * @param index The object's index. - * @return The object. - */ - public byte getByte(int index) { - return (byte) get(index); - } - - /** - * Gets the specified object as a float. - * - * @param index The object's index. - * @return The object. - */ - public float getFloat(int index) { - return (float) get(index); - } - - /** - * Gets the specified object as a long. - * - * @param index The object's index. - * @return The object. - */ - public long getLong(int index) { - return (long) get(index); - } - - /** - * Gets the specified object as a boolean. - * - * @param index The object's index. - * @return The object. - */ - public boolean getBoolean(int index) { - return (boolean) get(index); - } - - /** - * Gets the specified object as a nil. - * - * @param index The object's index. - * @return The object. - */ - public void getNil(int index) {} - - /** - * Gets the specified object as a string. - * - * @param index The object's index. - * @return The object. - */ - public String getString(int index) { - Object obj = get(index); - if (obj instanceof String) - return (String) obj; - else - return new String((byte[]) obj); - } - - /** - * Gets the specified object as a BigInteger. - * - * @param index The object's index. - * @return The object. - */ - public BigInteger getBigInteger(int index) { - return (BigInteger) get(index); - } - - /** - * Gets the specified object as an ErlangList. - * - * @param index The object's index. - * @return The object. - */ - public ErlangList getErlangList(int index) { - return (ErlangList) get(index); - } - - /** - * Gets the specified object as an ErlangMap. - * - * @param index The object's index. - * @return The object. - */ - public ErlangMap getErlangMap(int index) { - return (ErlangMap) get(index); - } - - /** - * Gets the specified object as a Fun. - * - * @param index The object's index. - * @return The object. - */ - public Fun getFun(int index) { - return (Fun) get(index); - } - - /** - * Gets the specified object as a PID. - * - * @param index The object's index. - * @return The object. - */ - public PID getPID(int index) { - return (PID) get(index); - } - - /** - * Gets the specified object as a Port. - * - * @param index The object's index. - * @return The object. - */ - public Port getPort(int index) { - return (Port) get(index); - } - - /** - * Gets the specified object as a Reference. - * - * @param index The object's index. - * @return The object. - */ - public Reference getReference(int index) { - return (Reference) get(index); - } - - /** - * Gets the specified object as a Tuple. - * - * @param index The object's index. - * @return The object. - */ - public Tuple getTuple(int index) { - return (Tuple) get(index); - } - - /** - * Gets the specified object as binary. - * - * @param index The object's index. - * @return The object. - */ - public byte[] getBinary(int index) { - return (byte[]) get(index); - } - - /** - * Gets if this list is proper (Tail is nil/null). - * - * @return True when proper, false when improper. - * @see #getTail() - */ - public boolean isProper() { - return tail == null; - } - - /** - * Gets the tail of this list. - * - * @return The tail (this is null when this is a proper list). - * @see #isProper() - */ - public Object getTail() { - return tail; - } - - @Override - public byte type() { - return TermTypes.LIST_EXT; - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/ErlangMap.java b/src/main/java/com/austinv11/etf/erlang/ErlangMap.java deleted file mode 100644 index a7f2707..0000000 --- a/src/main/java/com/austinv11/etf/erlang/ErlangMap.java +++ /dev/null @@ -1,232 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; -import com.austinv11.etf.util.BertCompatible; - -import java.math.BigInteger; -import java.util.AbstractMap; -import java.util.Map; -import java.util.Set; - -/** - * This represents an immutable ETF list. - */ -@BertCompatible -public class ErlangMap extends AbstractMap implements ErlangObject { - - private final Map data; - - public ErlangMap(Map data) { - this.data = data; - } - - @Override - public byte type() { - return TermTypes.MAP_EXT; - } - - @Override - public Set> entrySet() { - return data.entrySet(); - } - - @Override - public Object get(Object key) { - Object obj = data.get(key); - if (obj instanceof byte[]) - return new String((byte[]) obj); - return obj; - } - - /** - * Gets the specified object as an integer. - * - * @param key The object's key. - * @return The object. - */ - public int getInt(Object key) { - return (int) get(key); - } - - /** - * Gets the specified object as a short. - * - * @param key The object's key. - * @return The object. - */ - public short getShort(Object key) { - return (short) get(key); - } - - /** - * Gets the specified object as a character. - * - * @param key The object's key. - * @return The object. - */ - public char getChar(Object key) { - return (char) get(key); - } - - /** - * Gets the specified object as a byte. - * - * @param key The object's key. - * @return The object. - */ - public byte getByte(Object key) { - return (byte) get(key); - } - - /** - * Gets the specified object as a float. - * - * @param key The object's key. - * @return The object. - */ - public float getFloat(Object key) { - return (float) get(key); - } - - /** - * Gets the specified object as a long. - * - * @param key The object's key. - * @return The object. - */ - public long getLong(Object key) { - return (long) get(key); - } - - /** - * Gets the specified object as a boolean. - * - * @param key The object's key. - * @return The object. - */ - public boolean getBoolean(Object key) { - return (boolean) get(key); - } - - /** - * Gets the specified object as a nil. - * - * @param key The object's key. - * @return The object. - */ - public void getNil(Object key) {} - - /** - * Gets the specified object as a string. - * - * @param key The object's key. - * @return The object. - */ - public String getString(Object key) { - Object obj = get(key); - if (obj instanceof String) - return (String) obj; - else - return new String((byte[]) obj); - } - - /** - * Gets the specified object as a BigInteger. - * - * @param key The object's key. - * @return The object. - */ - public BigInteger getBigInteger(Object key) { - return (BigInteger) get(key); - } - - /** - * Gets the specified object as an ErlangList. - * - * @param key The object's key. - * @return The object. - */ - public ErlangList getErlangList(Object key) { - return (ErlangList) get(key); - } - - /** - * Gets the specified object as an ErlangMap. - * - * @param key The object's key. - * @return The object. - */ - public ErlangMap getErlangMap(Object key) { - return (ErlangMap) get(key); - } - - /** - * Gets the specified object as a Fun. - * - * @param key The object's key. - * @return The object. - */ - public Fun getFun(Object key) { - return (Fun) get(key); - } - - /** - * Gets the specified object as a PID. - * - * @param key The object's key. - * @return The object. - */ - public PID getPID(Object key) { - return (PID) get(key); - } - - /** - * Gets the specified object as a Port. - * - * @param key The object's key. - * @return The object. - */ - public Port getPort(Object key) { - return (Port) get(key); - } - - /** - * Gets the specified object as a Reference. - * - * @param key The object's key. - * @return The object. - */ - public Reference getReference(Object key) { - return (Reference) get(key); - } - - /** - * Gets the specified object as a Tuple. - * - * @param key The object's key. - * @return The object. - */ - public Tuple getTuple(Object key) { - return (Tuple) get(key); - } - - /** - * Gets the specified object as binary. - * - * @param key The object's key. - * @return The object. - */ - public byte[] getBinary(Object key) { - return (byte[]) get(key); - } - - @Override - public Object remove(Object key) { - throw new UnsupportedOperationException(); - } - - @Override - public void clear() { - throw new UnsupportedOperationException(); - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/ErlangObject.java b/src/main/java/com/austinv11/etf/erlang/ErlangObject.java deleted file mode 100644 index 3b1c66a..0000000 --- a/src/main/java/com/austinv11/etf/erlang/ErlangObject.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.austinv11.etf.erlang; - -/** - * This represents a generic erlang object. - */ -public interface ErlangObject { - - /** - * This gets the type number for this object. - * - * @return The type. - */ - byte type(); -} diff --git a/src/main/java/com/austinv11/etf/erlang/Fun.java b/src/main/java/com/austinv11/etf/erlang/Fun.java deleted file mode 100644 index 188fb6e..0000000 --- a/src/main/java/com/austinv11/etf/erlang/Fun.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; - -//TODO implement -public class Fun implements ErlangObject { - - private final byte[] data; - - public Fun(byte[] data) { - this.data = data; - } - - @Override - public byte type() { - return TermTypes.FUN_EXT; //TODO differentiate between new, old fun & exports - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/PID.java b/src/main/java/com/austinv11/etf/erlang/PID.java deleted file mode 100644 index 037d52c..0000000 --- a/src/main/java/com/austinv11/etf/erlang/PID.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; - -//TODO -public class PID implements ErlangObject { - - private final String atom; - private final int atomCacheRef; - private final int id; - private final int serial; - private final byte creation; - - public PID(String atom, int id, int serial, byte creation) { - this.atom = atom; - this.id = id; - this.serial = serial; - this.creation = creation; - this.atomCacheRef = -1; - } - - public PID(int atomCacheRef, int id, int serial, byte creation) { - this.atomCacheRef = atomCacheRef; - this.id = id; - this.serial = serial; - this.creation = creation; - this.atom = null; - } - - @Override - public byte type() { - return TermTypes.PID_EXT; - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/Port.java b/src/main/java/com/austinv11/etf/erlang/Port.java deleted file mode 100644 index b60a720..0000000 --- a/src/main/java/com/austinv11/etf/erlang/Port.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; - -public class Port implements ErlangObject { - - private final String atom; - private final int atomCacheRef; - private final int id; - private final byte creation; - - public Port(String atom, int id, byte creation) { - this.atom = atom; - this.id = id; - this.creation = creation; - this.atomCacheRef = -1; - } - - public Port(int atomCacheRef, int id, byte creation) { - this.atomCacheRef = atomCacheRef; - this.id = id; - this.creation = creation; - this.atom = null; - } - - @Override - public byte type() { - return TermTypes.PORT_EXT; - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/Reference.java b/src/main/java/com/austinv11/etf/erlang/Reference.java deleted file mode 100644 index f3c2812..0000000 --- a/src/main/java/com/austinv11/etf/erlang/Reference.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; - -//TODO -public class Reference implements ErlangObject { - - private final String atom; - private final int atomCacheRef; - private final long id; - private final byte creation; - private final long[] newID; - - public Reference(String atom, long id, byte creation) { - this.atom = atom; - this.id = id; - this.creation = creation; - this.atomCacheRef = -1; - this.newID = null; - } - - public Reference(int atomCacheRef, long id, byte creation) { - this.atomCacheRef = atomCacheRef; - this.id = id; - this.creation = creation; - this.atom = null; - this.newID = null; - } - - public Reference(String atom, byte creation, long[] newID) { - this.atom = atom; - this.id = -1; - this.creation = creation; - this.atomCacheRef = -1; - this.newID = newID; - } - - public Reference(int atomCacheRef, byte creation, long[] newID) { - this.atomCacheRef = atomCacheRef; - this.id = -1; - this.creation = creation; - this.atom = null; - this.newID = newID; - } - - public boolean isNew() { - return newID != null; - } - - @Override - public byte type() { - return isNew() ? TermTypes.NEW_REFERENCE_EXT : TermTypes.REFERENCE_EXT; - } -} diff --git a/src/main/java/com/austinv11/etf/erlang/Tuple.java b/src/main/java/com/austinv11/etf/erlang/Tuple.java deleted file mode 100644 index 2f7510a..0000000 --- a/src/main/java/com/austinv11/etf/erlang/Tuple.java +++ /dev/null @@ -1,251 +0,0 @@ -package com.austinv11.etf.erlang; - -import com.austinv11.etf.common.TermTypes; -import com.austinv11.etf.util.BertCompatible; - -import java.math.BigInteger; -import java.util.AbstractList; - -/** - * This represents an immutable ETF tuple. - * NOTE: This can also be a BERT "advanced" type if the first object is "bert". - */ -@BertCompatible -public class Tuple extends AbstractList implements ErlangObject { - - private final Object[] data; - - /** - * Creates a tuple with the provided objects. - * - * @param data The data. - * @return The tuple containing the data. - */ - public static Tuple of(Object... data) { - return new Tuple(data); - } - - public Tuple(Object[] data) { - this.data = data; - } - - /** - * Returns the array containing all the tuple's objects. - * - * @return The objects. - */ - public Object[] getAllObjects() { - return data; - } - - @Override - public Object get(int index) { - return data[index]; - } - - @Override - public int size() { - return data.length; - } - - /** - * Gets the specified object as an integer. - * - * @param index The object's index. - * @return The object. - */ - public int getInt(int index) { - return (int) data[index]; - } - - /** - * Gets the specified object as a short. - * - * @param index The object's index. - * @return The object. - */ - public short getShort(int index) { - return (short) data[index]; - } - - /** - * Gets the specified object as a character. - * - * @param index The object's index. - * @return The object. - */ - public char getChar(int index) { - return (char) data[index]; - } - - /** - * Gets the specified object as a byte. - * - * @param index The object's index. - * @return The object. - */ - public byte getByte(int index) { - return (byte) data[index]; - } - - /** - * Gets the specified object as a float. - * - * @param index The object's index. - * @return The object. - */ - public float getFloat(int index) { - return (float) data[index]; - } - - /** - * Gets the specified object as a long. - * - * @param index The object's index. - * @return The object. - */ - public long getLong(int index) { - return (long) data[index]; - } - - /** - * Gets the specified object as a boolean. - * - * @param index The object's index. - * @return The object. - */ - public boolean getBoolean(int index) { - return (boolean) data[index]; - } - - /** - * Gets the specified object as a nil. - * - * @param index The object's index. - * @return The object. - */ - public void getNil(int index) {} - - /** - * Gets the specified object as a string. - * - * @param index The object's index. - * @return The object. - */ - public String getString(int index) { - return (String) data[index]; - } - - /** - * Gets the specified object as a BigInteger. - * - * @param index The object's index. - * @return The object. - */ - public BigInteger getBigInteger(int index) { - return (BigInteger) data[index]; - } - - /** - * Gets the specified object as an ErlangList. - * - * @param index The object's index. - * @return The object. - */ - public ErlangList getErlangList(int index) { - return (ErlangList) data[index]; - } - - /** - * Gets the specified object as an ErlangMap. - * - * @param index The object's index. - * @return The object. - */ - public ErlangMap getErlangMap(int index) { - return (ErlangMap) data[index]; - } - - /** - * Gets the specified object as a Fun. - * - * @param index The object's index. - * @return The object. - */ - public Fun getFun(int index) { - return (Fun) data[index]; - } - - /** - * Gets the specified object as a PID. - * - * @param index The object's index. - * @return The object. - */ - public PID getPID(int index) { - return (PID) data[index]; - } - - /** - * Gets the specified object as a Port. - * - * @param index The object's index. - * @return The object. - */ - public Port getPort(int index) { - return (Port) data[index]; - } - - /** - * Gets the specified object as a Reference. - * - * @param index The object's index. - * @return The object. - */ - public Reference getReference(int index) { - return (Reference) data[index]; - } - - /** - * Gets the specified object as a Tuple. - * - * @param index The object's index. - * @return The object. - */ - public Tuple getTuple(int index) { - return (Tuple) data[index]; - } - - /** - * Gets the specified object as binary. - * - * @param index The object's index. - * @return The object. - */ - public byte[] getBinary(int index) { - return (byte[]) data[index]; - } - - /** - * This returns true if this is considered a "small" tuple by erlang. - * - * @return True when small, false when large. - */ - public boolean isSmall() { - return data.length <= 255; - } - - /** - * Checks if this is a bert object. - * - * @return True if a bert object, false if otherwise. - */ - public boolean isBertObject() { - return data.length > 0 && data[0] instanceof String && data[0].equals("bert"); - } - - @Override - public byte type() { - return isSmall() ? TermTypes.SMALL_TUPLE_EXT : TermTypes.LARGE_TUPLE_EXT; - } -} diff --git a/src/main/java/com/austinv11/etf/parsing/ETFParser.java b/src/main/java/com/austinv11/etf/parsing/ETFParser.java deleted file mode 100644 index 8fd362e..0000000 --- a/src/main/java/com/austinv11/etf/parsing/ETFParser.java +++ /dev/null @@ -1,991 +0,0 @@ -package com.austinv11.etf.parsing; - -import com.austinv11.etf.ETFConfig; -import com.austinv11.etf.erlang.*; -import com.austinv11.etf.util.BertCompatible; -import com.austinv11.etf.util.ETFException; - -import java.io.ByteArrayOutputStream; -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.*; -import java.util.zip.Inflater; - -import static com.austinv11.etf.common.TermTypes.*; - -/** - * This represents a parser for parsing data from an etf object. - */ -public class ETFParser { - - private final byte[] data; - private int offset = 0; - private final int expectedVersion; - private final boolean bert; - private final boolean loqui; - - public ETFParser(byte[] data, ETFConfig config) { - this(data, config, false); - } - - public ETFParser(byte[] data, ETFConfig config, boolean partial) { - this.expectedVersion = config.getVersion(); - this.bert = config.isBert(); - this.loqui = config.isLoqui(); - - int initialOffset = 0; - if (Byte.toUnsignedInt(data[initialOffset]) == expectedVersion) //Skip the version number - initialOffset++; - - if (!partial && config.isIncludingHeader()) { - if (data[initialOffset] != HEADER) - throw new ETFException("Missing header! Is this data malformed?").withData(data, initialOffset); - initialOffset++; - - long uncompressedSize = wrap(data, initialOffset, 4).getInt(); - initialOffset += 4; - - Inflater inflater = new Inflater(); - inflater.setInput(Arrays.copyOfRange(data, initialOffset, data.length)); - - try { - ByteArrayOutputStream outputStream = new ByteArrayOutputStream(data.length); - byte[] buffer = new byte[1024]; - while (inflater.getBytesWritten() != uncompressedSize) { - int count = inflater.inflate(buffer); - outputStream.write(buffer, 0, count); - } - inflater.end(); - outputStream.close(); - this.data = outputStream.toByteArray(); - } catch (Exception e) { - throw new ETFException(e).withData(data, initialOffset); - } - } else { - this.data = data; - } - } - - private static ByteBuffer wrap(byte[] array) { - return ByteBuffer.wrap(array).order(ByteOrder.BIG_ENDIAN); - } - - private static ByteBuffer wrap(byte[] array, int offset, int length) { - return ByteBuffer.wrap(array, offset, length).order(ByteOrder.BIG_ENDIAN); - } - - private void skipVersion() { - if (Byte.toUnsignedInt(data[offset]) == expectedVersion) - offset++; - } - - /** - * This gets the number of bytes in the uncompressed term data. - * - * @return The number of bytes. - */ - public int getSize() { - return data.length; - } - - /** - * Gets the version of etf this is using. - * - * @return The version. - */ - public int getVersion() { - return expectedVersion; - } - - /** - * Checks if this parser is BERT compatible. - * - * @return True when BERT compatible, false if otherwise. - */ - public boolean isBert() { - return bert; - } - - /** - * This gets the raw term data (excluding the initial distribution header. - * - * @return The raw data. - */ - public byte[] getRawData() { - return data; - } - - /** - * This gets the current position the parser is at in the raw data array. - * - * @return The current array offset. - */ - public int getPosition() { - return offset; - } - - /** - * This checks if there is no more data to read. - * - * @return True when there is no more data to read, false when otherwise. - */ - public boolean isFinished() { - return offset >= data.length; - } - - private void checkPreconditions() throws ETFException { - checkPreconditions(null); - } - - private void checkPreconditions(byte type) throws ETFException { - checkPreconditions(type, null); - } - - private void checkPreconditions(Boolean bertStatus) throws ETFException { - checkPreconditions(-1, bertStatus); - } - - private void checkPreconditions(int type, Boolean bertStatus) throws ETFException { - if (bertStatus != null) { //bert status is relevant - if (bertStatus != isBert()) - throw new ETFException("BERT vs ETF spec mismatch"); - } - - if (isFinished()) { - throw new ETFException("No more data to read!").withData(data, offset); - } - - skipVersion(); - - if (type != -1) { - if (type != peek()) { - throw new ETFException("ETF Term type mismatch!").withData(data, offset); - } else { - offset++; - } - } - } - - /** - * This peeks at the type of the next term. - * - * @return The type of the next term. - * - * @see com.austinv11.etf.common.TermTypes - */ - public byte peek() { - checkPreconditions(); - - return data[offset]; - } - - /** - * This gets the next distribution header. - * - * @return The next header. - */ - public DistributionHeader nextDistributionHeader() { //TODO? - throw new UnsupportedOperationException("Not implemented"); - } - - /** - * This gets an index referring to an atom cache reference in the distribution header. - * - * @return The index. - * - * @see #nextDistributionHeader() - */ - public short nextAtomCacheIndex() { - checkPreconditions(ATOM_CACHE_REF, false); - - return (short) Byte.toUnsignedInt(data[offset++]); - } - - /** - * This gets the next small integer (unsigned 8 bit int). - * - * @return The int. - */ - @BertCompatible - public int nextSmallInt() { - checkPreconditions(SMALL_INTEGER_EXT); - - return Byte.toUnsignedInt(data[offset++]); - } - - /** - * This gets the next large integer (signed 32 bit). - * - * @return The int. - */ - @BertCompatible - public int nextLargeInt() { - checkPreconditions(INTEGER_EXT); - - int integer = wrap(data, offset, 4).getInt(); - - offset += 4; - - return integer; - } - - /** - * This gets the next large or small integer. - * - * @return The int. - */ - @BertCompatible - public int nextInt() { - byte type = peek(); - - if (type == SMALL_INTEGER_EXT) { - return nextSmallInt(); - } else { - return nextLargeInt(); - } - } - - /** - * This gets the next old formatted float. - * - * @return The float. - */ - @BertCompatible - public double nextOldFloat() { - checkPreconditions(FLOAT_EXT); - - return Float.parseFloat(new String(Arrays.copyOfRange(data, offset, (offset += 31)))); - } - - /** - * This gets the next new formatted float. - * - * @return The float. - */ - public double nextNewFloat() { - checkPreconditions(NEW_FLOAT_EXT, false); - - double num = wrap(data, offset, 8).getDouble(); - - offset += 8; - - return num; - } - - /** - * This gets the next new or old float. - * - * @return The float. - */ - public double nextFloat() { - byte version = peek(); - - if (version == FLOAT_EXT) { - return nextOldFloat(); - } else { - return nextNewFloat(); - } - } - - /** - * Gets the next large Latin-1 encoded atom. - * - * @return The atom name. - */ - @BertCompatible - public String nextLargeAtom() { - checkPreconditions(ATOM_EXT); - - char len = wrap(data, offset, 2).getChar(); //Because we don't have unsigned shorts - offset += 2; - - try { - return new String(Arrays.copyOfRange(data, offset, (offset += len)), "ISO-8859-1" /*Latin-1 charset*/); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e).withData(data, offset-len); - } - } - - /** - * Gets the next small Latin-1 encoded atom. - * - * @return The atom name. - */ - public String nextSmallAtom() { - checkPreconditions(SMALL_ATOM_EXT, false); - - int len = Byte.toUnsignedInt(data[offset++]); - - try { - return new String(Arrays.copyOfRange(data, offset, (offset += len)), "ISO-8859-1" /*Latin-1 charset*/); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e).withData(data, offset-len); - } - } - - /** - * Gets the next large UTF-8 encoded atom. - * - * @return The atom name. - */ - public String nextLargeUTF8Atom() { - checkPreconditions(ATOM_UTF8_EXT, false); - - char len = wrap(data, offset, 2).getChar(); //Because we don't have unsigned shorts - offset += 2; - - try { - return new String(Arrays.copyOfRange(data, offset, (offset += len)), "UTF8"); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e).withData(data, offset-len); - } - } - - /** - * Gets the next small UTF-8 encoded atom. - * - * @return The atom name. - */ - public String nextSmallUTF8Atom() { - checkPreconditions(SMALL_ATOM_UTF8_EXT, false); - - int len = Byte.toUnsignedInt(data[offset++]); - - try { - return new String(Arrays.copyOfRange(data, offset, (offset += len)), "UTF8"); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e).withData(data, offset-len); - } - } - - /** - * Gets the next atom (small or large and latin-1 or utf-8). - * - * @return The atom name. - */ - public String nextAtom() { - byte type = peek(); - - if (type == SMALL_ATOM_EXT) { - return nextSmallAtom(); - } else if (type == ATOM_EXT) { - return nextLargeAtom(); - } else if (type == ATOM_UTF8_EXT) { - return nextLargeUTF8Atom(); - } else { - return nextSmallUTF8Atom(); - } - } - - /** - * Gets the next atom (small or large and latin-1 or utf-8) as an enum. - * - * @param enumClass The enum this atom represents. - * @return The atom. - */ - public > T nextAtom(Class enumClass) { - return Enum.valueOf(enumClass, nextAtom()); - } - - /** - * Gets the next boolean (expects the boolean to be encoded as an atom!), - * - * @return The next boolean encoded as an atom. - */ - public boolean nextBoolean() { - if (!loqui) - throw new ETFException("Loqui booleans not supported!"); - - String atom; - - if (peek() == SMALL_ATOM_EXT) - atom = nextSmallAtom(); - else - atom = nextSmallUTF8Atom(); - - return Boolean.parseBoolean(atom); - } - - /** - * This gets the next binary representation of a list or term. - * - * @return The binary data. - */ - @BertCompatible - public byte[] nextBinary() { - checkPreconditions(BINARY_EXT); - - long len = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - return Arrays.copyOfRange(data, offset, (offset += len)); - } - - /** - * This gets the next bitstring. - * - * @return The binary data. - */ - public String nextBitBinary() { - checkPreconditions(BIT_BINARY_EXT); - - long len = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - byte bits = data[offset++]; - - byte[] bytes = new byte[(int) len]; - for (int i = 0; i < len; i+=4) { - byte val = data[offset++]; - - if (i == len-1) //Tail - val >>>= 8-bits; //bits = # of significant bits from 1-8, so we remove the insignificant ones - - bytes[i] = val; - } - - return new String(Arrays.copyOfRange(bytes, 0, (int) len)); - } - - /** - * Gets the next "string". NOTE: Erlang doesn't natively support strings, strings are actually just unsigned byte - * lists (or char list in java). So the string might be nonsensical. - * - * @return The string. - * - * @see String#toCharArray() - */ - @BertCompatible - public String nextErlangString() { - checkPreconditions(STRING_EXT); - - char len = wrap(data, offset, 2).getChar(); //Because we don't have unsigned shorts - offset += 2; - - return new String(Arrays.copyOfRange(data, offset, (offset += len))); - } - - /** - * This gets the next atom or string. - * - * @return The atom or string. - */ - public String nextString() { - byte type = peek(); - - if (type == STRING_EXT) { - return nextErlangString(); - } else if (type == BIT_BINARY_EXT) { - return nextBitBinary(); - } else if (type == BINARY_EXT) { - return new String(nextBinary()); - } else { - return nextAtom(); - } - } - - private Node nextNode() { - int type = peek(); - - String atom = null; - int index = -1; - - if (type == ATOM_EXT) { //Only supports standard atoms + atom index - atom = nextLargeAtom(); - } else if (type == SMALL_ATOM_EXT) { - atom = nextSmallAtom(); - } else { - index = nextAtomCacheIndex(); - } - - if (index != -1) { - return new Node(index); - } else { - return new Node(atom); - } - } - - /** - * Gets the next port object. - * - * @return The port object. - */ - public Port nextPort() { //Pretty much identical to #nextReference - checkPreconditions(PORT_EXT, false); - - Node node = nextNode(); - - int id = wrap(data, offset, 4).getInt(); - offset += 4; - - byte creation = data[offset++]; - - if (node.isRef()) { - return new Port(node.ref, id, creation); - } else { - return new Port(node.atom, id, creation); - } - } - - /** - * Gets the next pid object. - * - * @return The pid object. - */ - public PID nextPID() { - checkPreconditions(PID_EXT, false); - - Node node = nextNode(); - - int id = wrap(data, offset, 4).getInt(); - offset += 4; - - int serial = wrap(data, offset, 4).getInt(); - offset += 4; - - byte creation = data[offset++]; - - if (node.isRef()) { - return new PID(node.ref, id, serial, creation); - } else { - return new PID(node.atom, id, serial, creation); - } - } - - private Tuple findTuple(long arity) { - Object[] data = new Object[(int)arity]; - for (int i = 0; i < arity; i++) { - data[i] = next(); - } - - return new Tuple(data); - } - - /** - * Gets the next small tuple. - * - * @return The tuple. - */ - public Tuple nextSmallTuple() { - checkPreconditions(SMALL_TUPLE_EXT); - - return findTuple(Byte.toUnsignedInt(data[offset++])); - } - - /** - * Gets the next large tuple. - * - * @return The tuple. - */ - @BertCompatible - public Tuple nextLargeTuple() { - checkPreconditions(LARGE_TUPLE_EXT); - - long arity = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - Tuple tuple = findTuple(arity); - - return tuple; - } - - /** - * Gets the next small or large tuple. - * - * @return The tuple. - */ - @BertCompatible - public Tuple nextTuple() { - byte type = peek(); - - if (type == SMALL_TUPLE_EXT) { - return nextSmallTuple(); - } else { - return nextLargeTuple(); - } - } - - /** - * Gets the next map. - * - * @return The map. - */ - @BertCompatible - public ErlangMap nextMap() { - checkPreconditions(MAP_EXT); - - long arity = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - Map map = new HashMap<>(); - for (long i = 0; i < arity; i++) { - map.put(next(), next()); - } - - return new ErlangMap(map); - } - - /** - * Checks if the next term is nil. - * - * @return True if the next term is nil, false if otherwise. - */ - @BertCompatible - public boolean isNil() { - if (peek() == NIL_EXT || !loqui) - return peek() == NIL_EXT; - else if (peek() == SMALL_ATOM_EXT || peek() == SMALL_ATOM_UTF8_EXT) {//Because Discord's api is annoying - int initialOffset = offset; - String atom = nextAtom(); - offset = initialOffset; //Reset offset because it wasn't nil! - return atom.equals("nil"); - } else { - return false; - } - } - - /** - * Gets the next nil. - */ - @BertCompatible - public void nextNil() { - if (peek() == NIL_EXT || !loqui) - checkPreconditions(NIL_EXT); //Offset should be incremented here - else if (peek() == SMALL_ATOM_EXT || peek() == SMALL_ATOM_UTF8_EXT) {//Because Discord's api is annoying - int initialOffset = offset; - if (!nextAtom().equals("nil")) - offset = initialOffset; //Reset offset because it wasn't nil! - } - } - - /** - * Gets the next proper or improper list. - * - * @return The list. - */ - @BertCompatible - public ErlangList nextList() { - checkPreconditions(LIST_EXT); - - long len = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - Object[] list = new Object[(int) len]; - for (int i = 0; i < len; i++) { - list[i] = next(); - } - - Object tail; - if (isNil()) { //Proper list - nextNil(); - tail = null; - } else { - tail = next(); - } - - return new ErlangList(list, tail); - } - - private long nextBig(long len) { - int sign = Byte.toUnsignedInt(data[offset++]); - - long total = 0L; - //Sorry for this algorithm but its what the docs say to do - for (long i = 0; i < len; i++) { - total += Byte.toUnsignedInt(data[offset++]) * (long)Math.pow(256, i); - } - - if (sign == 0) { //Positive - if (total < 0) - total *= -1; - } else if (sign == 1) { //Negative - if (total > 0) - total *= -1; - } - - return total; - } - - /** - * Gets the next small big number. - * - * @return The small big number. - */ - @BertCompatible - public long nextSmallBig() { - checkPreconditions(SMALL_BIG_EXT); - - return nextBig(Byte.toUnsignedInt(data[offset++])); - } - - /** - * Gets the next large big number. - * - * @return The large big number. - */ - @BertCompatible - public long nextLargeBig() { - checkPreconditions(LARGE_BIG_EXT); - - long len = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - return nextBig(len); - } - - /** - * Gets the next big number. - * - * @return The big number. - */ - @BertCompatible - public long nextBigNumber() { - if (peek() == SMALL_BIG_EXT) { - return nextSmallBig(); - } else { - return nextLargeBig(); - } - } - - /** - * Gets the next old reference object. - * - * @return The old reference object. - */ - public Reference nextOldReference() { - checkPreconditions(REFERENCE_EXT, false); - - Node node = nextNode(); - - long id = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - - byte creation = data[offset++]; - - if (node.isRef()) { - return new Reference(node.ref, id, creation); - } else { - return new Reference(node.atom, id, creation); - } - } - - /** - * Gets the next new reference object. - * - * @return The new reference object. - */ - public Reference nextNewReference() { - checkPreconditions(NEW_REFERENCE_EXT, false); - - char len = wrap(data, offset, 2).getChar(); //Because we don't have unsigned shorts - offset += 2; - - Node node = nextNode(); - - byte creation = data[offset++]; - - long[] id = new long[len]; - for (char i = 0; i < len; i++) { - id[i] = Integer.toUnsignedLong(wrap(data, offset, 4).getInt()); - offset += 4; - } - - if (node.isRef()) { - return new Reference(node.ref, creation, id); - } else { - return new Reference(node.atom, creation, id); - } - } - - /** - * Gets the next reference object. - * - * @return The reference object. - */ - public Reference nextReference() { - if (peek() == REFERENCE_EXT) { - return nextOldReference(); - } else { - return nextNewReference(); - } - } - - /** - * Gets the next old function reference. - * - * @return The function object. - */ - public Fun nextOldFun() { - throw new UnsupportedOperationException("Not implemented"); //TODO? - } - - /** - * Gets the next new function reference. - * - * @return The function object. - */ - public Fun nextNewFun() { - throw new UnsupportedOperationException("Not implemented"); //TODO? - } - - /** - * Gets the next export function reference. - * - * @return The function object. - */ - public Fun nextExport() { - throw new UnsupportedOperationException("Not implemented"); //TODO? - } - - /** - * Gets the next old/new/export function reference. - * - * @return The function object. - */ - public Fun nextFun() { - byte type = peek(); - - if (type == FUN_EXT) { - return nextOldFun(); - } else if (type == NEW_FUN_EXT) { - return nextNewFun(); - } else { - return nextExport(); - } - } - - //TODO: Implement advanced BERT objs - - private Object handleWeirdAtoms(String atom) { //Because Discord's api is annoying - if (!loqui) - return atom; - - switch (atom) { - case "true": - case "false": - return Boolean.parseBoolean(atom); - case "nil": - return null; - default: - return atom; - } - } - - /** - * This gets the next generic term. - * - * @return The next term. - */ - @BertCompatible - public Object next() { - switch (peek()) { - case HEADER: - throw new ETFException("Nested header found! Is the data malformed?").withData(data, offset); - case DISTRIBUTION_HEADER: - return nextDistributionHeader(); - case ATOM_CACHE_REF: - return nextAtomCacheIndex(); - case SMALL_INTEGER_EXT: - return nextSmallInt(); - case INTEGER_EXT: - return nextLargeInt(); - case FLOAT_EXT: - return nextOldFloat(); - case ATOM_EXT: - return handleWeirdAtoms(nextLargeAtom()); - case REFERENCE_EXT: - return nextOldReference(); - case PORT_EXT: - return nextPort(); - case PID_EXT: - return nextPID(); - case SMALL_TUPLE_EXT: - return nextSmallTuple(); - case LARGE_TUPLE_EXT: - return nextLargeTuple(); - case MAP_EXT: - return nextMap(); - case NIL_EXT: - nextNil(); - return null; - case STRING_EXT: - return nextErlangString(); - case LIST_EXT: - return nextList(); - case BINARY_EXT: - return nextBinary(); - case SMALL_BIG_EXT: - return nextSmallBig(); - case LARGE_BIG_EXT: - return nextLargeBig(); - case NEW_REFERENCE_EXT: - return nextNewReference(); - case SMALL_ATOM_EXT: - return handleWeirdAtoms(nextSmallAtom()); - case FUN_EXT: - return nextOldFun(); - case NEW_FUN_EXT: - return nextNewFun(); - case EXPORT_EXT: - return nextExport(); - case BIT_BINARY_EXT: - return nextBitBinary(); - case NEW_FLOAT_EXT: - return nextNewFloat(); - case ATOM_UTF8_EXT: - return handleWeirdAtoms(nextLargeUTF8Atom()); - case SMALL_ATOM_UTF8_EXT: - return handleWeirdAtoms(nextSmallUTF8Atom()); - default: - throw new ETFException("Unidentified type " + peek() + " is the data malformed?").withData(data, offset); - } - } - - /** - * This reads all of the terms in the provided etf data from the current offset. - * - * @return The list of all remaining terms. - */ - public List readFully() { - List terms = new ArrayList<>(); - while (!isFinished()) { - terms.add(next()); - } - - return terms; - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder("<"); - for (int i = 0; i < data.length; i++) { - builder.append(data[i]); - if (i+1 != data.length) - builder.append(", "); - } - builder.append(">"); - - return builder.toString(); - } - - //Internal use only, we don't actually provide a Node object - private class Node { - final String atom; - final int ref; - - public Node(String atom) { - this.atom = atom; - this.ref = -1; - } - - public Node(int ref) { - this.ref = ref; - this.atom = null; - } - - boolean isRef() { - return ref != -1; - } - } -} diff --git a/src/main/java/com/austinv11/etf/util/BertCompatible.java b/src/main/java/com/austinv11/etf/util/BertCompatible.java deleted file mode 100644 index 308bb30..0000000 --- a/src/main/java/com/austinv11/etf/util/BertCompatible.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.austinv11.etf.util; - -/** - * This is simply a marker annotation for a type that is BERT draft compatible. - */ -public @interface BertCompatible {} diff --git a/src/main/java/com/austinv11/etf/util/BertType.java b/src/main/java/com/austinv11/etf/util/BertType.java deleted file mode 100644 index ce6c53d..0000000 --- a/src/main/java/com/austinv11/etf/util/BertType.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.austinv11.etf.util; - -/** - * This is simply a marker annotation for a type that is only available in the BERT draft. - */ -public @interface BertType {} diff --git a/src/main/java/com/austinv11/etf/util/ETFConstants.java b/src/main/java/com/austinv11/etf/util/ETFConstants.java deleted file mode 100644 index db29e51..0000000 --- a/src/main/java/com/austinv11/etf/util/ETFConstants.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.austinv11.etf.util; - -public class ETFConstants { - public final static int VERSION = 131; //The spec version this supports -} diff --git a/src/main/java/com/austinv11/etf/util/ETFException.java b/src/main/java/com/austinv11/etf/util/ETFException.java deleted file mode 100644 index feca0bd..0000000 --- a/src/main/java/com/austinv11/etf/util/ETFException.java +++ /dev/null @@ -1,65 +0,0 @@ -package com.austinv11.etf.util; - -import java.util.Arrays; - -/** - * This represents a generic exception thrown when parsing/writing etf. - */ -public class ETFException extends RuntimeException { - - private byte[] data = null; - private int position = -1; - - public ETFException() { - super(); - } - - public ETFException(String message) { - super(message); - } - - public ETFException(String message, Throwable cause) { - super(message, cause); - } - - public ETFException(Throwable cause) { - super(cause); - } - - public ETFException(String message, Throwable cause, boolean enableSuppression, boolean writableStackTrace) { - super(message, cause, enableSuppression, writableStackTrace); - } - - public ETFException withData(byte[] data) { - this.data = data; - return this; - } - - public ETFException withData(byte[] data, int position) { - this.data = data; - this.position = position; - return this; - } - - @Override - public String getMessage() { - String message = super.getMessage(); - - if (data != null) { - message += System.lineSeparator() + Arrays.toString(data); - } - - if (position != -1) { - message += "\n "; - for (int i = 0; i < position; i++) { - String byteStringVal = String.valueOf(data[i]); - for (int j = 0; j < byteStringVal.length(); j++) - message += " "; - message += " "; - } - message += "^"; - } - - return message; - } -} diff --git a/src/main/java/com/austinv11/etf/util/GetterMethod.java b/src/main/java/com/austinv11/etf/util/GetterMethod.java deleted file mode 100644 index c1083c5..0000000 --- a/src/main/java/com/austinv11/etf/util/GetterMethod.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.austinv11.etf.util; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Marker annotation for a field's getter method. - */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface GetterMethod { - - /** - * This field's name. - * - * @return The field's name. - */ - String value(); -} diff --git a/src/main/java/com/austinv11/etf/util/Mapper.java b/src/main/java/com/austinv11/etf/util/Mapper.java deleted file mode 100644 index 8f748c6..0000000 --- a/src/main/java/com/austinv11/etf/util/Mapper.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.austinv11.etf.util; - -import com.austinv11.etf.ETFConfig; -import com.austinv11.etf.common.TermTypes; -import com.austinv11.etf.erlang.ErlangList; -import com.austinv11.etf.erlang.ErlangMap; -import com.austinv11.etf.parsing.ETFParser; -import com.austinv11.etf.util.ReflectionUtils.PropertyManager; -import com.austinv11.etf.writing.ETFWriter; - -import java.lang.reflect.Array; -import java.util.List; -import java.util.Map; - -/** - * This represents a mapper which will handle object serialization/deserialization. - */ -public class Mapper { - - private ETFConfig config; - - public Mapper(ETFConfig config) { - this.config = config; - } - - public byte[] writeToMap(T obj) { - return config.createWriter().writeMap(obj).toBytes(); - } - - public byte[] write(T obj) { - ETFWriter writer = config.createWriter(); - ReflectionUtils.findProperties(obj, obj.getClass()).forEach(writer::write); - return writer.toBytes(); - } - - public T read(ErlangMap data, Class clazz) { - T instance = ReflectionUtils.createInstance(clazz); - List properties = ReflectionUtils.findProperties(instance, clazz); - for (PropertyManager property : properties) { - if (data.containsKey(property.getName())) { - Object obj = data.get(property.getName()); - if (obj instanceof ErlangMap) { - obj = read((ErlangMap) obj, property.getSetterType()); - } else if (obj instanceof ErlangList && property.getSetterType().isArray()) { - if (((ErlangList) obj).size() > 0) { - Object array = Array.newInstance(property.getSetterType().getComponentType(), ((ErlangList) obj).size()); - for (int i = 0; i < ((ErlangList) obj).size(); i++) { - Object obj1 = ((ErlangList) obj).get(i); - if (obj1 != null) - Array.set(array, i, obj1 instanceof ErlangMap ? read(data, property.getSetterType().getComponentType()) : obj1); - } - obj = array; - } else - obj = Array.newInstance(property.getSetterType().getComponentType(), 0); - } - property.setValue(obj); - } - } - return instance; - } - - public T read(byte[] data, Class clazz) { - ETFParser parser = config.createParser(data); - T instance = ReflectionUtils.createInstance(clazz); - List properties = ReflectionUtils.findProperties(instance, clazz); - if (parser.peek() == TermTypes.MAP_EXT) { - ErlangMap map = parser.nextMap(); - if (Map.class.isAssignableFrom(clazz)) { //User wants a map so lets give it to them - return (T) map; - } else { - return read(map, clazz); - } - } else { - for (PropertyManager property : properties) { - if (parser.isFinished()) - break; - - property.setValue(parser.next()); - } - return instance; - } - } -} diff --git a/src/main/java/com/austinv11/etf/util/ReflectionUtils.java b/src/main/java/com/austinv11/etf/util/ReflectionUtils.java deleted file mode 100644 index 64e3574..0000000 --- a/src/main/java/com/austinv11/etf/util/ReflectionUtils.java +++ /dev/null @@ -1,374 +0,0 @@ -package com.austinv11.etf.util; - -import java.lang.reflect.Array; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -/** - * Internal class for reflection utilities. - */ -public class ReflectionUtils { - - //Woo unsafe! Adds some extra speed when available. - private static final Object UNSAFE; - - static { - Object tempUnsafe = null; - try { - Field theUnsafe = sun.misc.Unsafe.class.getDeclaredField("theUnsafe"); - theUnsafe.setAccessible(true); - tempUnsafe = theUnsafe.get(null); - } catch (Throwable e) { - //Unsafe unavailable - } - UNSAFE = tempUnsafe; - } - - public static List findProperties(Object instance, Class clazz) { - if (clazz.isPrimitive()) - return new ArrayList<>(); - - List properties = new ArrayList<>(); - for (Field field : getAllFields(clazz)) { - if (!Modifier.isTransient(field.getModifiers())) { - properties.add(new PropertyManager(instance, field)); - } - } - return properties; - } - - public static List getAllFields(Class clazz) { - List fields = new ArrayList<>(); - Collections.addAll(fields, clazz.getDeclaredFields()); - Collections.addAll(fields, clazz.getFields()); - return fields; - } - - public static List getAllMethods(Class clazz) { - List methods = new ArrayList<>(); - Collections.addAll(methods, clazz.getDeclaredMethods()); - Collections.addAll(methods, clazz.getMethods()); - return methods; - } - - public static T createInstance(Class clazz) { - if (int.class.equals(clazz) || Integer.class.equals(clazz)) { - return (T)(Integer) 0; - } else if (long.class.equals(clazz) || Long.class.equals(clazz)) { - return (T)(Long) 0L; - } else if (double.class.equals(clazz) || Double.class.equals(clazz)) { - return (T)(Double) 0D; - } else if (void.class.equals(clazz) || Void.class.equals(clazz)) { - return null; - } else if (float.class.equals(clazz) || Float.class.equals(clazz)) { - return (T)(Float) 0F; - } else if (byte.class.equals(clazz) || Byte.class.equals(clazz)) { - return (T)(Byte)(byte) 0; - } else if (char.class.equals(clazz) || Character.class.equals(clazz)) { - return (T)(Character)(char) 0; - } else if (boolean.class.equals(clazz) || Boolean.class.equals(clazz)) { - return (T)(Boolean) false; - } else if (short.class.equals(clazz) || Short.class.equals(clazz)) { - return (T)(Short)(short) 0; - } - - if (clazz.isArray()) { - return (T) Array.newInstance(clazz.getComponentType(), 0); - } - - if (UNSAFE != null) { //Unsafe available, use it to instantiate the class - try { - return (T) ((sun.misc.Unsafe) UNSAFE).allocateInstance(clazz); - } catch (InstantiationException e) {} - } - - //Fallback to reflection - try { - return clazz.getConstructor().newInstance(); - } catch (Exception e) { - throw new ETFException(e); - } - } - - public static void setField(Object instance, Field field, Object value) throws IllegalAccessException { - if (UNSAFE != null) { - if (int.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putInt(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (int) value); - } else if (long.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putLong(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (long) value); - } else if (double.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putDouble(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (double) value); - } else if (void.class.equals(field.getType())) { - - } else if (float.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putFloat(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (float) value); - } else if (byte.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putByte(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (byte) value); - } else if (char.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putChar(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (char) value); - } else if (boolean.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putBoolean(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (boolean) value); - } else if (short.class.equals(field.getType())) { - ((sun.misc.Unsafe) UNSAFE).putShort(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), (short) value); - } else { - ((sun.misc.Unsafe) UNSAFE).putObject(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field), value); - } - } else { //Fallback if unsafe isn't available - field.setAccessible(true); - if (int.class.equals(field.getType())) { - field.setInt(instance, (int) value); - } else if (long.class.equals(field.getType())) { - field.setLong(instance, (long) value); - } else if (double.class.equals(field.getType())) { - field.setDouble(instance, (double) value); - } else if (void.class.equals(field.getType())) { - - } else if (float.class.equals(field.getType())) { - field.setFloat(instance, (float) value); - } else if (byte.class.equals(field.getType())) { - field.setByte(instance, (byte) value); - } else if (char.class.equals(field.getType())) { - field.setChar(instance, (char) value); - } else if (boolean.class.equals(field.getType())) { - field.setBoolean(instance, (boolean) value); - } else if (short.class.equals(field.getType())) { - field.setShort(instance, (short) value); - } else { - field.set(instance, value); - } - } - } - - public static Object getField(Object instance, Field field) throws IllegalAccessException { - if (UNSAFE != null) { - if (int.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getInt(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (long.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getLong(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (double.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getDouble(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (void.class.equals(field.getType())) { - return null; - } else if (float.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getFloat(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (byte.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getByte(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (char.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getChar(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (boolean.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getBoolean(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else if (short.class.equals(field.getType())) { - return ((sun.misc.Unsafe) UNSAFE).getShort(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } else { - return ((sun.misc.Unsafe) UNSAFE).getObject(instance, ((sun.misc.Unsafe) UNSAFE).objectFieldOffset(field)); - } - } else { //Fallback if unsafe isn't available - field.setAccessible(true); - if (int.class.equals(field.getType())) { - return field.getInt(instance); - } else if (long.class.equals(field.getType())) { - return field.getLong(instance); - } else if (double.class.equals(field.getType())) { - return field.getDouble(instance); - } else if (void.class.equals(field.getType())) { - return null; - } else if (float.class.equals(field.getType())) { - return field.getFloat(instance); - } else if (byte.class.equals(field.getType())) { - return field.getByte(instance); - } else if (char.class.equals(field.getType())) { - return field.getChar(instance); - } else if (boolean.class.equals(field.getType())) { - return field.getBoolean(instance); - } else if (short.class.equals(field.getType())) { - return field.getShort(instance); - } else { - return field.get(instance); - } - } - } - - public interface IPropertyAccessor { - - Object get(); - - Class getType(); - } - - public interface IPropertyMutator { - - void set(Object o); - - Class getType(); - } - - public static class FieldAccessorAndMutator implements IPropertyAccessor, IPropertyMutator { - - private final Object object; - private final Field field; - - public FieldAccessorAndMutator(Object object, Field field) { - this.object = object; - this.field = field; - this.field.setAccessible(true); - } - - @Override - public Object get() { - try { - return getField(object, field); - } catch (Throwable e) { - throw new ETFException("Cannot access " + field.toGenericString(), e); - } - } - - @Override - public void set(Object o) { - try { - setField(object, field, o); - } catch (Throwable e) { - throw new ETFException("Cannot modify " + field.toGenericString(), e); - } - } - - @Override - public Class getType() { - return field.getType(); - } - } - - public static class MethodAccessorAndMutator implements IPropertyAccessor, IPropertyMutator { - - private final Object object; - private final Method method; - - public MethodAccessorAndMutator(Object object, Method method) { - this.object = object; - this.method = method; - this.method.setAccessible(true); - } - - @Override - public Object get() { - try { - return method.invoke(object); - } catch (Exception e) { - throw new ETFException("Cannot invoke " + method.toGenericString(), e); - } - } - - @Override - public void set(Object o) { - try { - method.invoke(object, o); - } catch (Exception e) { - throw new ETFException("Cannot invoke " + method.toGenericString(), e); - } - } - - @Override - public Class getType() { - return method.getParameterCount() == 0 ? method.getReturnType() : method.getParameterTypes()[0]; - } - } - - public static class NOPAccessorAndMutator implements IPropertyAccessor, IPropertyMutator { - - public static final NOPAccessorAndMutator INSTANCE = new NOPAccessorAndMutator(); - - @Override - public Object get() { - return null; - } - - @Override - public void set(Object o) {} - - @Override - public Class getType() { - return Void.class; - } - } - - public static class PropertyManager { - - private final Object instance; - private final IPropertyMutator mutator; - private final IPropertyAccessor accessor; - private final String name; - - private static String capitalize(String s) { - return s.substring(0, 1).toUpperCase() + (s.length() > 1 ? s.substring(1) : ""); - } - - public PropertyManager(Object instance, Field field) { - this.instance = instance; - field.setAccessible(true); - IPropertyAccessor accessor = null; - boolean isFinal = Modifier.isFinal(field.getModifiers()); - IPropertyMutator mutator = isFinal ? NOPAccessorAndMutator.INSTANCE : null; - for (Method m : getAllMethods(instance.getClass())) { - if (mutator != null && accessor != null) - break; - - m.setAccessible(true); - - if (m.getName().equals(String.format("get%s", capitalize(field.getName())))) { - accessor = new MethodAccessorAndMutator(instance, m); - continue; - } else if (!isFinal - && m.getName().equals(String.format("set%s", capitalize(field.getName()))) - && m.getParameterCount() == 1 && m.getParameterTypes()[0].equals(field.getType())) { - mutator = new MethodAccessorAndMutator(instance, m); - continue; - } else if (m.getDeclaredAnnotation(GetterMethod.class) != null) { - if (m.getDeclaredAnnotation(GetterMethod.class).value().equals(field.getName())) { - accessor = new MethodAccessorAndMutator(instance, m); - continue; - } - } else if (!isFinal && m.getDeclaredAnnotation(SetterMethod.class) != null - && m.getParameterCount() == 1) { - if (m.getDeclaredAnnotation(SetterMethod.class).value().equals(field.getName())) { - mutator = new MethodAccessorAndMutator(instance, m); - continue; - } - } - } - - if (accessor == null) - accessor = new FieldAccessorAndMutator(instance, field); - - if (mutator == null) - mutator = new FieldAccessorAndMutator(instance, field); - - this.accessor = accessor; - this.mutator = mutator; - - this.name = field.getName(); - } - - public void setValue(Object value) { - mutator.set(value); - } - - public Object getValue() { - return accessor.get(); - } - - public String getName() { - return name; - } - - public Class getSetterType() { - return mutator.getType(); - } - - public Class getGetterType() { - return accessor.getType(); - } - } -} diff --git a/src/main/java/com/austinv11/etf/util/SetterMethod.java b/src/main/java/com/austinv11/etf/util/SetterMethod.java deleted file mode 100644 index c4965df..0000000 --- a/src/main/java/com/austinv11/etf/util/SetterMethod.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.austinv11.etf.util; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -/** - * Marker annotation for a field's setter method. - */ -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface SetterMethod { - - /** - * This field's name. - * - * @return The field's name. - */ - String value(); -} diff --git a/src/main/java/com/austinv11/etf/writing/ETFWriter.java b/src/main/java/com/austinv11/etf/writing/ETFWriter.java deleted file mode 100644 index 05e2c2c..0000000 --- a/src/main/java/com/austinv11/etf/writing/ETFWriter.java +++ /dev/null @@ -1,649 +0,0 @@ -package com.austinv11.etf.writing; - -import com.austinv11.etf.ETFConfig; -import com.austinv11.etf.erlang.*; -import com.austinv11.etf.util.ETFException; -import com.austinv11.etf.util.ReflectionUtils; - -import java.io.UnsupportedEncodingException; -import java.math.BigInteger; -import java.nio.ByteBuffer; -import java.util.*; - -import static com.austinv11.etf.common.TermTypes.*; - -/** - * This represents a writer for writing data to an etf object. - */ -public class ETFWriter { - - private byte[] data = new byte[64]; - private int offset = 0; - private final boolean bert; - private final byte version; - private final boolean includeHeader; - private final boolean includeDistributionHeader; - private final boolean loqui; - private final boolean compress; - - public ETFWriter(ETFConfig config) { - this(config, false); - } - - public ETFWriter(ETFConfig config, boolean partial) { - if (partial) { //These should never be true when partial - includeDistributionHeader = false; - includeHeader = false; - compress = false; //Can't compress without a header - } else { - includeDistributionHeader = config.isIncludingDistributionHeader(); - includeHeader = config.isIncludingHeader(); - compress = config.isCompressing(); - } - bert = config.isBert(); - version = (byte) config.getVersion(); - loqui = config.isLoqui(); - } - - private void writeToBuffer(byte... data) { - if (this.data.length - offset < data.length+1/*Ensure room for a version byte if necessary*/) { //We need to expand the buffer - this.data = Arrays.copyOf(this.data, this.data.length * 2); - } - - if (this.data[0] != version && !includeDistributionHeader) { - this.data[offset++] = version; - } - - for (byte b : data) { - this.data[offset++] = b; - } - } - - public ETFWriter writeAtomCacheIndex(short index) { - writeToBuffer(ATOM_CACHE_REF, (byte) index); - return this; - } - - public ETFWriter writeSmallInt(int integer) { - writeToBuffer(SMALL_INTEGER_EXT, (byte) (integer & 0xff)); - return this; - } - - public ETFWriter writeLargeInt(int integer) { - writeToBuffer(INTEGER_EXT, (byte) ((integer >>> 24) & 0xff), (byte) ((integer >>> 16) & 0xff), - (byte) ((integer >>> 8) & 0xff), (byte) (integer & 0xff)); - return this; - } - - public ETFWriter writeInt(int integer) { - if (Integer.BYTES * integer == 1) { - writeSmallInt((short) integer); - } else { - writeLargeInt(integer); - } - return this; - } - - public strictfp ETFWriter writeOldFloat(double num) { - writeToBuffer(FLOAT_EXT); - writeToBuffer(String.format("%.20f", num).getBytes()); - return this; - } - - public strictfp ETFWriter writeNewFloat(double num) { - long longBits = Double.doubleToLongBits(num); - writeToBuffer(NEW_FLOAT_EXT, - (byte) ((longBits >>> 56) & 0xff), (byte) ((longBits >>> 48) & 0xff), - (byte) ((longBits >>> 40) & 0xff), (byte) ((longBits >>> 32) & 0xff), - (byte) ((longBits >>> 24) & 0xff), (byte) ((longBits >>> 16) & 0xff), - (byte) ((longBits >>> 8) & 0xff), (byte) (longBits & 0xff)); - return this; - } - - public ETFWriter writeFloat(double num) { //TODO: Add header checks - return writeNewFloat(num); - } - - public ETFWriter writeLargeAtom(String atom) { - try { - byte[] bytes = atom.getBytes("ISO-8859-1" /*Latin-1 charset*/); - writeToBuffer(ATOM_EXT); - writeToBuffer((byte) ((bytes.length >>> 8) & 0xff), (byte) (bytes.length & 0xff)); //Length number - writeToBuffer(bytes); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e); - } - return this; - } - - public ETFWriter writeSmallAtom(String atom) { - try { - byte[] bytes = atom.getBytes("ISO-8859-1" /*Latin-1 charset*/); - writeToBuffer(SMALL_ATOM_EXT); - writeToBuffer((byte) (bytes.length & 0xff)); //Length number - writeToBuffer(bytes); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e); - } - return this; - } - - public ETFWriter writeLargeUTF8Atom(String atom) { - try { - byte[] bytes = atom.getBytes("UTF8"); - writeToBuffer(ATOM_EXT); - writeToBuffer((byte) ((bytes.length >>> 8) & 0xff), (byte) (bytes.length & 0xff)); //Length number - writeToBuffer(bytes); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e); - } - return this; - } - - public ETFWriter writeSmallUTF8Atom(String atom) { - try { - byte[] bytes = atom.getBytes("UTF8"); - writeToBuffer(SMALL_ATOM_EXT); - writeToBuffer((byte) (bytes.length & 0xff)); //Length number - writeToBuffer(bytes); - } catch (UnsupportedEncodingException e) { - throw new ETFException(e); - } - return this; - } - - public ETFWriter writeAtom(String atom) { - //TODO Header check for UTF8 - if (atom.length() > 256) - writeLargeAtom(atom); - else - writeSmallAtom(atom); - -// if (atom.length() > 256) -// writeLargeUTF8Atom(atom); -// else -// writeSmallUTF8Atom(atom); - return this; - } - - public ETFWriter writeBoolean(boolean bool) { - if (!loqui) - throw new ETFException("Loqui booleans not supported!"); - - return writeAtom(bool ? "true" : "false"); - } - - public ETFWriter writeBinary(String bin) { - return writeBinary(bin.getBytes()); - } - - public ETFWriter writeBinary(byte[] bin) { - writeToBuffer(BINARY_EXT); - writeToBuffer((byte) ((bin.length >>> 24) & 0xff), (byte) ((bin.length >>> 16) & 0xff), - (byte) ((bin.length >>> 8) & 0xFF), (byte) (bin.length & 0xff)); - writeToBuffer(bin); - return this; - } - - public ETFWriter writeBitString(String string) { - byte[] bytes = string.getBytes(); - writeToBuffer(BINARY_EXT); - writeToBuffer((byte) ((bytes.length >>> 24) & 0xff), (byte) ((bytes.length >>> 16) & 0xff), - (byte) ((bytes.length >>> 8) & 0xFF), (byte) (bytes.length & 0xff)); - int unsigned = Byte.toUnsignedInt(bytes[bytes.length-1]); - int i = 1; - while (i < unsigned) - i <<= 1; - writeToBuffer((byte) (i-1)); - writeToBuffer(bytes); - return this; - } - - public ETFWriter writeErlangString(String string) { - char[] chars = string.toCharArray(); - writeToBuffer(STRING_EXT); - writeToBuffer((byte) ((chars.length >>> 8) & 0xff), (byte) (chars.length & 0xff)); - for (char character : chars) - writeToBuffer((byte) character); - return this; - } - - public ETFWriter writePort(Port port) { - //TODO - return this; - } - - public ETFWriter writePID(PID pid) { - //TODO - return this; - } - - public ETFWriter writeSmallTuple(Collection tuple) { - int arity = (tuple.size() & 0xFF); - writeToBuffer(SMALL_TUPLE_EXT, (byte) arity); - Iterator iterator = tuple.iterator(); - for (int i = 0; i < arity; i++) - write(iterator.next()); - return this; - } - - public ETFWriter writeSmallTuple(T[] tuple) { - int arity = (tuple.length & 0xFF); - writeToBuffer(SMALL_TUPLE_EXT, (byte) arity); - for (int i = 0; i < arity; i++) - write(tuple[i]); - return this; - } - - //TODO primitive tuples - - public ETFWriter writeLargeTuple(Collection tuple) { - writeToBuffer(LARGE_TUPLE_EXT, (byte) ((tuple.size() >>> 24) & 0xFF), - (byte) ((tuple.size() >>> 16) & 0xFF), (byte) ((tuple.size() >>> 8) & 0xFF), - (byte) (tuple.size() & 0xFF)); - Iterator iterator = tuple.iterator(); - for (int i = 0; i < tuple.size(); i++) - write(iterator.next()); - return this; - } - - public ETFWriter writeLargeTuple(T[] tuple) { - writeToBuffer(LARGE_TUPLE_EXT, (byte) ((tuple.length >>> 24) & 0xFF), - (byte) ((tuple.length >>> 16) & 0xFF), (byte) ((tuple.length >>> 8) & 0xFF), - (byte) (tuple.length & 0xFF)); - for (int i = 0; i < tuple.length; i++) - write(tuple[i]); - return this; - } - - //TODO primitive tuples - - public ETFWriter writeTuple(Collection tuple) { - if (tuple.size() > 256) - writeSmallTuple(tuple); - else - writeLargeTuple(tuple); - return this; - } - - public ETFWriter writeTuple(T[] tuple) { - if (tuple.length > 256) - writeSmallTuple(tuple); - else - writeLargeTuple(tuple); - return this; - } - - //TODO primitive tuples - - public ETFWriter writeMap(Map map) { - writeToBuffer(MAP_EXT, (byte) ((map.size() >>> 24) & 0xFF), - (byte) ((map.size() >>> 16) & 0xFF), (byte) ((map.size() >>> 8) & 0xFF), - (byte) (map.size() & 0xFF)); - for (K key : map.keySet()) { - write(key); - write(map.get(key)); - } - return this; - } - - public ETFWriter writeMap(Object o) { - if (o instanceof Map) { - return writeMap((Map) o); - } else { - Map properties = new HashMap<>(); - for (ReflectionUtils.PropertyManager property : ReflectionUtils.findProperties(o, o.getClass())) { - if (property.getGetterType().isEnum()) - properties.put(property.getName(), Enum.valueOf((Class) property.getGetterType(), property.getValue().toString())); - else - properties.put(property.getName(), property.getValue()); - } - return writeMap(properties); - } - } - - public ETFWriter writeNil() { - return writeNil(false); - } - - public ETFWriter writeNil(boolean forceNonLoqui) { - if (loqui && !forceNonLoqui) - writeAtom("nil"); - else - writeToBuffer(NIL_EXT); - return this; - } - - public ETFWriter writeList(Collection list) { - writeToBuffer(LIST_EXT, (byte) ((list.size() >>> 24) & 0xFF), - (byte) ((list.size() >>> 16) & 0xFF), (byte) ((list.size() >>> 8) & 0xFF), - (byte) (list.size() & 0xFF)); - for (T obj : list) - write(obj); - writeNil(true); //The tail is nil so that this can be a proper list - return this; - } - - public ETFWriter writeList(T[] list) { - writeToBuffer(LIST_EXT, (byte) ((list.length >>> 24) & 0xFF), - (byte) ((list.length >>> 16) & 0xFF), (byte) ((list.length >>> 8) & 0xFF), - (byte) (list.length & 0xFF)); - for (T obj : list) - write(obj); - writeNil(true); //The tail is nil so that this can be a proper list - return this; - } - - //TODO primitive lists - - public ETFWriter writeSmallBig(BigInteger num) { - if (num.equals(BigInteger.ZERO)) { - writeToBuffer(SMALL_BIG_EXT, (byte) 0, (byte) 0); - } else { - byte signum = num.signum() == -1 ? (byte) 1 : (byte) 0; - num = num.abs(); - int n = (int) Math.ceil(num.bitLength()/8)+1; //Equivalent to Math.ceil(log256(num)) + 1 - byte[] bytes = new byte[n]; - writeToBuffer(SMALL_BIG_EXT, (byte) (n & 0xFF), signum); - n -= 1; - while (n >= 0) { - BigInteger[] res = num.divideAndRemainder(BigInteger.valueOf(256).pow(n)); - bytes[n] = res[0].byteValue(); //Quotient - num = res[1]; //Remainder - n--; - } - writeToBuffer(bytes); - } - return this; - } - - public ETFWriter writeSmallBig(long num, byte sign) { - if (num == 0) { - writeToBuffer(SMALL_BIG_EXT, (byte) 0, (byte) 0); - } else { - byte signum = sign < 0 ? (byte) 1 : (byte) 0; - num = Math.abs(num); - int n = (int) Math.ceil(Math.log(num)/Math.log(256))+1; //Equivalent to Math.ceil(log256(num)) + 1 - byte[] bytes = new byte[n]; - writeToBuffer(SMALL_BIG_EXT, (byte) (n & 0xFF), signum); - n -= 1; - while (n >= 0) { - long rem = num%(long) Math.pow(256, n); - long quo = num/(long) Math.pow(256, n); - bytes[n] = (byte) quo; - num = rem; - n--; - } - writeToBuffer(bytes); - } - return this; - } - - public ETFWriter writeLargeBig(BigInteger num) { - if (num.equals(BigInteger.ZERO)) { - writeToBuffer(LARGE_BIG_EXT, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0); - } else { - byte signum = num.signum() == -1 ? (byte) 1 : (byte) 0; - num = num.abs(); - int n = (int) Math.ceil(num.bitLength()/8)+1; //Equivalent to Math.ceil(log256(num)) + 1 - byte[] bytes = new byte[n]; - writeToBuffer(LARGE_BIG_EXT, (byte) ((n >>> 24) & 0xFF), (byte) ((n >>> 16) & 0xFF), - (byte) ((n >>> 8) & 0xFF), (byte) (n & 0xFF), signum); - n -= 1; - while (n >= 0) { - BigInteger[] res = num.divideAndRemainder(BigInteger.valueOf(256).pow(n)); - bytes[n] = res[0].byteValue(); //Quotient - num = res[1]; //Remainder - n--; - } - writeToBuffer(bytes); - } - return this; - } - - public ETFWriter writeLargeBig(long num, byte sign) { - if (num == 0) { - writeToBuffer(LARGE_BIG_EXT, (byte) 0, (byte) 0, (byte) 0, (byte) 0, (byte) 0); - } else { - byte signum = sign < 0 ? (byte) 1 : (byte) 0; - num = Math.abs(num); - int n = (int) Math.ceil(Math.log(num)/Math.log(256))+1; //Equivalent to Math.ceil(log256(num)) + 1 - byte[] bytes = new byte[n]; - writeToBuffer(LARGE_BIG_EXT, (byte) ((n >>> 24) & 0xFF), (byte) ((n >>> 16) & 0xFF), - (byte) ((n >>> 8) & 0xFF), (byte) (n & 0xFF), signum); - n -= 1; - while (n >= 0) { - long rem = num%(long) Math.pow(256, n); - long quo = num/(long) Math.pow(256, n); - bytes[n] = (byte) quo; - num = rem; - n--; - } - writeToBuffer(bytes); - } - return this; - } - - public ETFWriter writeBigNumber(BigInteger num) { - if ((int) Math.ceil(num.bitLength() / 8) + 1 > 256) //Equivalent to Math.ceil(log256(num)) + 1) - writeLargeBig(num); - else - writeSmallBig(num); - return this; - } - - public ETFWriter writeBigNumber(long num, byte sign) { - if ((int) Math.ceil(Math.log(num) / Math.log(256)) + 1 > 256) //Equivalent to Math.ceil(log256(num)) + 1) - writeLargeBig(num, sign); - else - writeSmallBig(num, sign); - return this; - } - - public ETFWriter writeBigNumber(long num) { - return writeBigNumber(num, (byte) 1); //TODO Maybe we shouldn't assume unsigned - } - - public ETFWriter writeOldReference(Reference reference) { - //TODO - return this; - } - - public ETFWriter writeNewReference(Reference reference) { - //TODO - return this; - } - - public ETFWriter writeReference(Reference reference) { - //TODO - return this; - } - - public ETFWriter writeOldFun(Fun fun) { - //TODO - return this; - } - - public ETFWriter writeNewFun(Fun fun) { - //TODO - return this; - } - - public ETFWriter writeExport(Fun fun) { - //TODO - return this; - } - - public ETFWriter writeFun(Fun fun) { - //TODO - return this; - } - - /** - * This writes a supported object to ETF. - * - * @param o The object to write. - * - * @throws com.austinv11.etf.util.ETFException When the object isn't supported. - */ - public ETFWriter write(Object o) { - if (o == null) { - writeNil(); - return this; - } else if (o instanceof Number) { - if (o instanceof BigInteger) { - writeBigNumber((BigInteger) o); - return this; - } else if (o instanceof Short || o instanceof Byte || o instanceof Integer) { - writeInt(((Number) o).intValue()); - return this; - } else if (o instanceof Long) { - writeBigNumber((long) o); - return this; - } else if (o instanceof Float || o instanceof Double) { - writeFloat(((Number) o).doubleValue()); - return this; - } - } else if (o instanceof Boolean) { - writeBoolean((Boolean) o); - return this; - } else if (o instanceof Character) { - writeAtom(o.toString()); - return this; - } else if (o instanceof ErlangObject) { - if (o instanceof DistributionHeader) { - //TODO - return this; - } else if (o instanceof ErlangList) { - writeList((ErlangList) o); - return this; - } else if (o instanceof ErlangMap) { - writeMap((ErlangMap) o); - return this; - } else if (o instanceof Fun) { - //TODO - return this; - } else if (o instanceof PID) { - //TODO - return this; - } else if (o instanceof Port) { - //TODO - return this; - } else if (o instanceof Reference) { - //TODO - return this; - } else if (o instanceof Tuple) { - writeTuple((Tuple) o); - return this; - } - } else if (o instanceof Map) { - writeMap((Map) o); - return this; - } else if (o instanceof Collection) { - if (o instanceof List) - writeList((List) o); - else - writeTuple((Collection) o); - return this; - } else if (o.getClass().isArray()) { - if (o instanceof byte[] || o instanceof Byte[]) { - if (o instanceof Byte[]) { - byte[] newArray = new byte[((Byte[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((Byte[]) o)[i]; - o = newArray; - } - writeBinary((byte[]) o); - return this; - } else if (o instanceof char[] || o instanceof Character[]) { - if (o instanceof Character[]) { - char[] newArray = new char[((Character[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((Character[]) o)[i]; - o = newArray; - } - writeBinary(new String((char[]) o)); //TODO should we optimize for other types? - return this; - } else { - if (o instanceof boolean[]) { - Boolean[] newArray = new Boolean[((boolean[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((boolean[]) o)[i]; - o = newArray; - } else if (o instanceof short[]) { - Short[] newArray = new Short[((short[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((short[]) o)[i]; - o = newArray; - } else if (o instanceof int[]) { - Integer[] newArray = new Integer[((int[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((int[]) o)[i]; - o = newArray; - } else if (o instanceof long[]) { - Long[] newArray = new Long[((long[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((long[]) o)[i]; - o = newArray; - } else if (o instanceof float[]) { - Float[] newArray = new Float[((float[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((float[]) o)[i]; - o = newArray; - } else if (o instanceof double[]) { - Double[] newArray = new Double[((double[]) o).length]; - for (int i = 0; i < newArray.length; i++) - newArray[i] = ((double[]) o)[i]; - o = newArray; - } - writeList((Object[]) o); - return this; - } - } else if (o instanceof String) { - if (!loqui || o.equals("true") || o.equals("false") || o.equals("nil")) - writeAtom((String) o); //TODO should we optimize for other types? - else - writeBinary((String) o); - return this; - } else if (o instanceof Enum) { - writeAtom(((Enum) o).name()); - return this; - } else { - writeMap(o); - return this; - } - - throw new ETFException("Unknown object type "+o.getClass()); - } - - /** - * This gets the current data in a byte array. - * - * @return The byte array representing this data. - */ - public byte[] toBytes() { - return Arrays.copyOfRange(data, 0, offset); - } - - /** - * Gives access to the direct buffer in the writer. - * - * @return The underlying buffer. - */ - public ByteBuffer toBuffer() { - return ByteBuffer.wrap(toBytes()); - } - - @Override - public String toString() { - StringBuilder builder = new StringBuilder("<"); - for (int i = 0; i < offset; i++) { - builder.append(data[i]); - if (i+1 != offset) - builder.append(", "); - } - builder.append(">"); - - return builder.toString(); - } -} diff --git a/src/test/java/com/austinv11/etf/test/ETFTester.java b/src/test/java/com/austinv11/etf/test/ETFTester.java deleted file mode 100644 index 86a6df8..0000000 --- a/src/test/java/com/austinv11/etf/test/ETFTester.java +++ /dev/null @@ -1,201 +0,0 @@ -package com.austinv11.etf.test; - -import com.austinv11.etf.ETFConfig; -import com.austinv11.etf.common.TermTypes; -import com.austinv11.etf.parsing.ETFParser; -import com.austinv11.etf.util.ETFConstants; -import com.google.gson.Gson; -import com.google.gson.JsonObject; -import org.json.JSONObject; -import org.junit.Assert; - -import java.io.*; -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; - -public class ETFTester { - - //These test cases were taken from: https://github.com/ccubed/Earl/blob/master/unit_tests.py - public static final char[] SMALL_INT = {131,97,10}; - public static final char[] BIG_INT = {131, 98, 0, 0, 4, 176}; - public static final char[] FLOAT = {131,70,64,9,33,250,252,139,0,122}; - public static final char[] MAP = {131,116,0,0,0,1,109,0,0,0,1,100,97,10}; - public static final char[] LIST = {131,108,0,0,0,3,97,1,97,2,97,3,106}; - public static final char[] NIL = {131, 106}; - public static final byte[] TEST = {-125, 116, 0, 0, 0, 4, 109, 0, 0, 0, 2, 111, 112, 98, 0, 0, 0, 2, 109, 0, 0, 0, 1, 115, 115, 3, 110, 105, 108, 109, 0, 0, 0, 1, 116, 115, 3, 110, 105, 108, 109, 0, 0, 0, 1, 100, 116, 0, 0, 0, 5, 109, 0, 0, 0, 8, 99, 111, 109, 112, 114, 101, 115, 115, 115, 4, 116, 114, 117, 101, 109, 0, 0, 0, 5, 115, 104, 97, 114, 100, 108, 0, 0, 0, 2, 98, 0, 0, 0, 0, 98, 0, 0, 0, 1, 106, 109, 0, 0, 0, 15, 108, 97, 114, 103, 101, 95, 116, 104, 114, 101, 115, 104, 111, 108, 100, 98, 0, 0, 0, -6, 109, 0, 0, 0, 10, 112, 114, 111, 112, 101, 114, 116, 105, 101, 115, 116, 0, 0, 0, 5, 109, 0, 0, 0, 7, 36, 100, 101, 118, 105, 99, 101, 109, 0, 0, 0, 9, 68, 105, 115, 99, 111, 114, 100, 52, 74, 109, 0, 0, 0, 17, 36, 114, 101, 102, 101, 114, 114, 105, 110, 103, 95, 100, 111, 109, 97, 105, 110, 109, 0, 0, 0, 0, 109, 0, 0, 0, 3, 36, 111, 115, 109, 0, 0, 0, 8, 77, 97, 99, 32, 79, 83, 32, 88, 109, 0, 0, 0, 8, 36, 98, 114, 111, 119, 115, 101, 114, 109, 0, 0, 0, 9, 68, 105, 115, 99, 111, 114, 100, 52, 74, 109, 0, 0, 0, 9, 36, 114, 101, 102, 101, 114, 114, 101, 114, 109, 0, 0, 0, 0, 109, 0, 0, 0, 5, 116, 111, 107, 101, 110, 109, 0, 0, 0, 63, 66, 111, 116, 32, 77, 84, 89, 119, 79, 68, 77, 52, 77, 122, 103, 120, 77, 122, 77, 49, 78, 106, 65, 53, 77, 122, 81, 48, 46, 67, 99, 56, 95, 54, 103, 46, 66, 104, 87, 102, 117, 85, 48, 101, 90, 109, 68, 98, 49, 74, 120, 87, 114, 83, 82, 102, 90, 114, 103, 83, 85, 82, 103}; - public static final ETFConfig CONFIG = new ETFConfig() - .setIncludeHeader(false) - .setCompression(false) - .setIncludeDistributionHeader(false) - .setBert(false) - .setVersion(ETFConstants.VERSION) - .setLoqui(true); - - private static void printParser(ETFParser parser) { - while (!parser.isFinished()) { - if (parser.peek() == TermTypes.MAP_EXT) { - Map map = parser.nextMap(); - map.forEach((k, v) -> { - printParser(CONFIG.createParser(CONFIG.createWriter(true).write(k).toBytes(), true)); - printParser(CONFIG.createParser(CONFIG.createWriter(true).write(v).toBytes(), true)); - }); - } else { - System.out.println(parser.peek()); - Object obj = parser.next(); - if (obj instanceof byte[]) { - System.out.println(new String((byte[]) obj)); - } else { - System.out.println(obj); - } - System.out.println(Arrays.toString(CONFIG.createWriter(true).write(obj).toBytes())); - } - } - } - - public static void main(String[] args) throws IOException, ClassNotFoundException { - ETFParser parser = CONFIG.setIncludeHeader(true).createParser(TEST, true); - printParser(parser); - - readEtf(ETFTester.class.getResourceAsStream("/test.etf")); - - readJson(ETFTester.class.getResourceAsStream("/test.json")); - - readGson(ETFTester.class.getResourceAsStream("/test.json")); - - testCase(SMALL_INT, 10, "small int"); - testCase(BIG_INT, 1200, "big int"); - testCase(FLOAT, 3.141592, "float"); - Map map = new HashMap<>(); - map.put("d", 10); - testCase(MAP, map, "map"); - testCase(LIST, Arrays.asList(1,2,3), "list"); - testCase(NIL, null, "nil"); - } - - private static byte[] charsToBytes(char[] chars) { - byte[] array = new byte[chars.length]; - for (int i = 0; i < chars.length; i++) - array[i] = (byte) chars[i]; - return array; - } - - private static void testCase(char[] etf, Object expected, String message) { - Object next = CONFIG.createParser(charsToBytes(etf), true).next(); - System.out.printf("Expected: %s, Parsed: %s%n", next == null ? "null" : expected.toString(), next == null ? "null" : next.toString()); - Assert.assertTrue(message, next == null ? expected == null : next.equals(expected)); - } - - private static void readEtf(InputStream is) throws IOException, ClassNotFoundException { - ByteArrayOutputStream buffer = new ByteArrayOutputStream(); - - int nRead; - byte[] data = new byte[16384]; - - while ((nRead = is.read(data, 0, data.length)) != -1) { - buffer.write(data, 0, nRead); - } - - buffer.flush(); - - byte[] dataArray = buffer.toByteArray(); - - System.out.println("ETF: "); - long init = System.currentTimeMillis(); - System.out.println(CONFIG.createParser(dataArray, true).nextMap().toString()); - System.out.println(System.currentTimeMillis() - init); - } - - private static void readJson(InputStream is2) { - BufferedReader br = null; - StringBuilder sb = new StringBuilder(); - - String line; - try { - - br = new BufferedReader(new InputStreamReader(is2)); - while ((line = br.readLine()) != null) { - sb.append(line); - } - - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (br != null) { - try { - br.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - System.out.println("org.json: "); - long init2 = System.currentTimeMillis(); - System.out.println(new JSONObject(sb.toString()).toMap().toString()); - System.out.println(System.currentTimeMillis() - init2); - } - - private static void readGson(InputStream inputStream) { - BufferedReader br = null; - StringBuilder sb = new StringBuilder(); - - String line; - try { - - br = new BufferedReader(new InputStreamReader(inputStream)); - while ((line = br.readLine()) != null) { - sb.append(line); - } - - } catch (IOException e) { - e.printStackTrace(); - } finally { - if (br != null) { - try { - br.close(); - } catch (IOException e) { - e.printStackTrace(); - } - } - } - - Gson gson = new Gson(); - - System.out.println("Gson (POGO): "); - long init3 = System.currentTimeMillis(); - TestObject object = gson.fromJson(sb.toString(), TestObject.class); - System.out.println(object.toString()); - System.out.println(System.currentTimeMillis() - init3); - - Gson gson2 = new Gson(); - System.out.println("Gson (manual parsing): "); - long init2 = System.currentTimeMillis(); - JsonObject jo = gson2.fromJson(sb.toString(), JsonObject.class); - System.out.println(jo.toString()); - System.out.println(System.currentTimeMillis() - init2); - } - - private static class TestObject { - - public int op; - public String s; - public String t; - public Event d; - - private static class Event { - public String[] _trace; - public int heartbeat_interval; - - @Override - public String toString() { - return String.format("{_trace=%s, heartbeat_interval=%d}", Arrays.toString(_trace), heartbeat_interval); - } - } - - @Override - public String toString() { - return String.format("{op=%d, s=%s, t=%s, d=%s}", op, s, t, d.toString()); - } - } -} diff --git a/src/test/resources/test.json b/src/test/resources/test.json deleted file mode 100644 index 31e1a7c..0000000 --- a/src/test/resources/test.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "s": null, - "t": null, - "op": 10, - "d": { - "_trace": [], - "heartbeat_interval": 41250 - } -} \ No newline at end of file