diff options
| author | ketsuban <ketsuban@192.168.3.110> | 2026-08-26 06:20:43 +0000 |
|---|---|---|
| committer | ketsuban <ketsuban@192.168.3.110> | 2026-08-26 06:20:43 +0000 |
| commit | 6d14259175d07eeee7e5e1bc9eecf838ff442fe4 (patch) | |
| tree | 23e166c9b44b1b4589e47fd88c0397bed44a560c /modules | |
| parent | 5f3d5537b4dd137c75f815ca03835d05a8cd68f0 (diff) | |
| download | kukuri-6d14259175d07eeee7e5e1bc9eecf838ff442fe4.tar.gz kukuri-6d14259175d07eeee7e5e1bc9eecf838ff442fe4.tar.bz2 kukuri-6d14259175d07eeee7e5e1bc9eecf838ff442fe4.zip | |
Vendor modified nabu core (MIT) as java-library; wire into content-store jvm
Diffstat (limited to 'modules')
40 files changed, 6683 insertions, 0 deletions
diff --git a/modules/orgflow-content-store/build.gradle.kts b/modules/orgflow-content-store/build.gradle.kts index e4a07d9..a6a9fdd 100644 --- a/modules/orgflow-content-store/build.gradle.kts +++ b/modules/orgflow-content-store/build.gradle.kts @@ -17,6 +17,9 @@ kotlin { commonMain.dependencies { implementation(libs.okio) } + jvmMain.dependencies { + implementation(project(":modules:vendor-nabu")) + } commonTest.dependencies { implementation(libs.kotlin.test) implementation(libs.coroutines.test) diff --git a/modules/orgflow-content-store/src/commonMain/kotlin/jp/orgflow/contentstore/ContentSplitter.kt b/modules/orgflow-content-store/src/commonMain/kotlin/jp/orgflow/contentstore/ContentSplitter.kt new file mode 100644 index 0000000..b3b0231 --- /dev/null +++ b/modules/orgflow-content-store/src/commonMain/kotlin/jp/orgflow/contentstore/ContentSplitter.kt @@ -0,0 +1,67 @@ +package jp.orgflow.contentstore + +// Variable-size (content-defined) chunking - full implementation per direction override. +// Rolling-hash CDC (BuzHash-family): boundary where (hash & avgMask) == 0, clamped to [min,max]. +// Deterministic on every platform: all arithmetic is fixed-width Long (mod 2^64). + +class ContentSplitter( + private val min: Int = DEFAULT_MIN, + private val avgMask: Int = DEFAULT_AVG_MASK, + private val window: Int = DEFAULT_WINDOW, + private val max: Int = DEFAULT_MAX, +) { + init { + require(min in 1 until max) { "min must be in [1, max)" } + require(window in 1..min) { "window must be <= min" } + require(max > min) { "max must be greater than min" } + } + + data class Params( + val min: Int = DEFAULT_MIN, + val avgMask: Int = DEFAULT_AVG_MASK, + val window: Int = DEFAULT_WINDOW, + val max: Int = DEFAULT_MAX, + ) + + fun split(data: ByteArray): List<ByteArray> { + if (data.isEmpty()) return emptyList() + val out = mutableListOf<ByteArray>() + var start = 0 + while (start < data.size) { + var end = cutAt(data, start) + if (end == start) end = start + 1 + out += data.copyOfRange(start, end) + start = end + } + return out + } + + private fun cutAt(data: ByteArray, start: Int): Int { + val hardEnd = minOf(start + max, data.size) + if (start + min >= hardEnd) return hardEnd + val win = IntArray(window) + var wp = 0 + var h = 0L + var power = 1L + repeat(window) { power *= PRIME } + var i = start + while (i < hardEnd) { + val b = data[i].toInt() and 0xff + val outgoing = win[wp] + win[wp] = b + wp = (wp + 1) % window + h = (h - outgoing * power) * PRIME + b + i++ + if (i - start >= min && (h and avgMask.toLong()) == 0L) return i + } + return hardEnd + } + + companion object { + const val DEFAULT_MIN = 2048 + const val DEFAULT_AVG_MASK = (1 shl 13) - 1 // ~8KiB average target + const val DEFAULT_WINDOW = 48 + const val DEFAULT_MAX = 65536 + private const val PRIME = 0x1000193L // FNV prime as roll base + } +} diff --git a/modules/vendor-nabu/LICENSE-nabu.txt b/modules/vendor-nabu/LICENSE-nabu.txt new file mode 100644 index 0000000..1b0e732 --- /dev/null +++ b/modules/vendor-nabu/LICENSE-nabu.txt @@ -0,0 +1,19 @@ +MIT Licence + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/modules/vendor-nabu/build.gradle.kts b/modules/vendor-nabu/build.gradle.kts new file mode 100644 index 0000000..067cdc3 --- /dev/null +++ b/modules/vendor-nabu/build.gradle.kts @@ -0,0 +1,20 @@ +// Vendored, modified subset of Peergos/nabu (MIT) โ content-addressing core only. +// Modifications: stripped networking (libp2p/bitswap/dht/kademlia), s3/auth/redis metadata, +// http/metrics; kept blockstore + cbor + util + io.ipfs cid/multihash/multibase micro-deps. +// See LICENSE-nabu.txt and MODIFICATIONS.md. Original: https://github.com/Peergos/nabu + +plugins { + `java-library` +} + +group = "net.kukuri.vendor" +version = "0.1.0-nabu-v0.9" + +java { + withSourcesJar() +} + +tasks.withType<JavaCompile>().configureEach { + options.encoding = "UTF-8" + options.release.set(17) +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/cid/Cid.java b/modules/vendor-nabu/src/main/java/io/ipfs/cid/Cid.java new file mode 100644 index 0000000..bc75225 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/cid/Cid.java @@ -0,0 +1,205 @@ +package io.ipfs.cid; + +import io.ipfs.multibase.Multibase; +import io.ipfs.multihash.Multihash; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Arrays; +import java.util.Map; +import java.util.TreeMap; + +public class Cid extends Multihash { + + public static final class CidEncodingException extends RuntimeException { + + public CidEncodingException(String message) { + super(message); + } + + public CidEncodingException(String message, Throwable cause) { + super(message, cause); + } + } + + public enum Codec { + // https://github.com/multiformats/multicodec/blob/master/table.csv + Cbor(0x51, "cbor"), + Raw(0x55, "raw"), + DagProtobuf(0x70, "dag-pb"), + DagCbor(0x71, "dag-cbor"), + Libp2pKey(0x72, "libp2p-key"), + EthereumBlock(0x90, "eth-block"), + EthereumTx(0x91, "eth-block-list"), + BitcoinBlock(0xb0, "bitcoin-block"), + BitcoinTx(0xb1, "bitcoin-tx"), + ZcashBlock(0xc0, "zcash-block"), + ZcashTx(0xc1, "zcash-tx"); + + public final long type; + public final String name; + + Codec(long type, String name) { + this.type = type; + this.name = name; + } + + private static Map<Long, Codec> lookup = new TreeMap<>(); + private static Map<String, Codec> nameLookup = new TreeMap<>(); + + static { + for (Codec c : Codec.values()) { + lookup.put(c.type, c); + nameLookup.put(c.name, c); + } + } + + public static Codec lookup(long c) { + Codec codec = lookup.get(c); + if (codec == null) throw new IllegalStateException("Unknown Codec type: " + c); + return codec; + } + + public static Codec lookupIPLDName(String name) { + Codec codec = nameLookup.get(name); + if (codec == null) throw new IllegalStateException("Unknown Codec type: " + name); + return codec; + } + } + + public final long version; + public final Codec codec; + + public Cid(long version, Codec codec, Multihash.Type type, byte[] hash) { + super(type, hash); + this.version = version; + this.codec = codec; + } + + public static Cid build(long version, Codec codec, Multihash h) { + return new Cid(version, codec, h.getType(), h.getHash()); + } + + private byte[] toBytesV0() { + return super.toBytes(); + } + + private byte[] toBytesV1() { + try { + ByteArrayOutputStream res = new ByteArrayOutputStream(); + putUvarint(res, version); + putUvarint(res, codec.type); + super.serialize(res); + return res.toByteArray(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public byte[] toBytes() { + if (version == 0) return toBytesV0(); + else if (version == 1) return toBytesV1(); + throw new IllegalStateException("Unknown CID version: " + version); + } + + @Override + public String toString() { + if (version == 0) { + return super.toString(); + } else if (version == 1) { + return Multibase.encode(Multibase.Base.Base32, toBytesV1()); + } + throw new IllegalStateException("Unknown CID version: " + version); + } + + @Override + public Multihash bareMultihash() { + return new Multihash(getType(), getHash()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Multihash)) return false; + if (!super.equals(o)) return false; + + if (o instanceof Cid) { + Cid cid = (Cid) o; + + if (version != cid.version) return false; + return codec == cid.codec; + } + // o must be a Multihash + return version == 0 && super.equals(o); + } + + @Override + public int hashCode() { + int result = super.hashCode(); + if (version == 0) return result; + result = 31 * result + (int) (version ^ (version >>> 32)); + result = 31 * result + (codec != null ? codec.hashCode() : 0); + return result; + } + + public static Cid buildV0(Multihash h) { + return Cid.build(0, Codec.DagProtobuf, h); + } + + public static Cid buildCidV1(Codec c, Multihash.Type type, byte[] hash) { + return new Cid(1, c, type, hash); + } + + public static Cid decode(String v) { + if (v.length() < 2) throw new IllegalStateException("CID too short: " + v); + + // support legacy format + if (v.length() == 46 && v.startsWith("Qm")) return buildV0(Multihash.fromBase58(v)); + + byte[] data = Multibase.decode(v); + return cast(data); + } + + public static Cid cast(byte[] data) { + if (data.length == 34 && data[0] == 18 && data[1] == 32) + return buildV0( + new Multihash(Type.lookup(data[0] & 0xff), Arrays.copyOfRange(data, 2, data.length))); + + InputStream in = new ByteArrayInputStream(data); + try { + long version = readVarint(in); + if (version != 0 && version != 1) + throw new CidEncodingException("Invalid CID version number: " + version); + + long codec = readVarint(in); + Multihash hash = Multihash.deserialize(in); + + return new Cid(version, Codec.lookup(codec), hash.getType(), hash.getHash()); + } catch (CidEncodingException cee) { + throw cee; + } catch (Exception e) { + throw new CidEncodingException("Invalid CID bytes: " + bytesToHex(data), e); + } + } + + private static String[] HEX_DIGITS = + new String[] {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}; + private static String[] HEX = new String[256]; + + static { + for (int i = 0; i < 256; i++) HEX[i] = HEX_DIGITS[(i >> 4) & 0xF] + HEX_DIGITS[i & 0xF]; + } + + private static String byteToHex(byte b) { + return HEX[b & 0xFF]; + } + + private static String bytesToHex(byte[] data) { + StringBuilder s = new StringBuilder(data.length * 2); + for (byte b : data) s.append(byteToHex(b)); + return s.toString(); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base16.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base16.java new file mode 100644 index 0000000..d26d12f --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base16.java @@ -0,0 +1,35 @@ +package io.ipfs.multibase; + +public class Base16 { + public static byte[] decode(String hex) { + if (hex.length() % 2 == 1) + throw new IllegalArgumentException( + "Must have an even number of hex digits to convert to bytes!"); + byte[] res = new byte[hex.length() / 2]; + for (int i = 0; i < res.length; i++) + res[i] = (byte) Integer.parseInt(hex.substring(2 * i, 2 * i + 2), 16); + return res; + } + + public static String encode(byte[] data) { + return bytesToHex(data); + } + + private static String[] HEX_DIGITS = + new String[] {"0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}; + private static String[] HEX = new String[256]; + + static { + for (int i = 0; i < 256; i++) HEX[i] = HEX_DIGITS[(i >> 4) & 0xF] + HEX_DIGITS[i & 0xF]; + } + + public static String byteToHex(byte b) { + return HEX[b & 0xFF]; + } + + public static String bytesToHex(byte[] data) { + StringBuilder s = new StringBuilder(); + for (byte b : data) s.append(byteToHex(b)); + return s.toString(); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base256Emoji.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base256Emoji.java new file mode 100644 index 0000000..e6368f3 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base256Emoji.java @@ -0,0 +1,105 @@ +package io.ipfs.multibase; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/* + * Copyright 2025 Michael Vorburger.ch + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * <a + * href="https://github.com/multiformats/multibase/blob/master/rfcs/Base256Emoji.md">Base256Emoji</a> + * is an encoding mapping each 0-255 byte value to (or from) a specific single Unicode Emoji + * character. + * + * @author <a href="https://www.vorburger.ch/">Michael Vorburger.ch</a> + */ +public class Base256Emoji { + + // from https://github.com/multiformats/multibase/blob/master/rfcs/Base256Emoji.md + private static final String[] EMOJIS = { + "๐", "๐ช", "โ", "๐ฐ", "๐", "๐", "๐", "๐", "๐", "๐", + "๐", "๐", "๐", "๐", "๐", "๐", "๐", "โ", "๐ป", "๐ฅ", + "๐พ", "๐ฟ", "๐", "โค", "๐", "๐คฃ", "๐", "๐", "๐", "๐ญ", + "๐", "๐", "๐
", "๐", "๐", "๐ฅ", "๐ฅฐ", "๐", "๐", "๐", + "๐ข", "๐ค", "๐", "๐", "๐ช", "๐", "โบ", "๐", "๐ค", "๐", + "๐", "๐", "๐", "๐น", "๐คฆ", "๐", "๐", "โ", "โจ", "๐คท", + "๐ฑ", "๐", "๐ธ", "๐", "๐", "๐", "๐", "๐", "๐", "๐", + "๐", "๐คฉ", "๐", "๐", "๐ค", "๐", "๐ฏ", "๐", "๐", "๐ถ", + "๐", "๐คญ", "โฃ", "๐", "๐", "๐", "๐ช", "๐", "๐ฅ", "๐", + "๐", "๐ฉ", "๐ก", "๐คช", "๐", "๐ฅณ", "๐ฅ", "๐คค", "๐", "๐", + "๐ณ", "โ", "๐", "๐", "๐ด", "๐", "๐ฌ", "๐", "๐", "๐ท", + "๐ป", "๐", "โญ", "โ
", "๐ฅบ", "๐", "๐", "๐ค", "๐ฆ", "โ", + "๐ฃ", "๐", "๐", "โน", "๐", "๐", "๐ ", "โ", "๐", "๐บ", + "๐", "๐ป", "๐", "๐", "๐", "๐", "๐น", "๐ฃ", "๐ซ", "๐", + "๐", "๐ต", "๐ค", "๐", "๐ด", "๐ค", "๐ผ", "๐ซ", "โฝ", "๐ค", + "โ", "๐", "๐คซ", "๐", "๐ฎ", "๐", "๐ป", "๐", "๐ถ", "๐", + "๐ฒ", "๐ฟ", "๐งก", "๐", "โก", "๐", "๐", "โ", "โ", "๐", + "๐ฐ", "๐คจ", "๐ถ", "๐ค", "๐ถ", "๐ฐ", "๐", "๐ข", "๐ค", "๐", + "๐จ", "๐จ", "๐คฌ", "โ", "๐", "๐บ", "๐ค", "๐", "๐", "๐ฑ", + "๐", "๐ถ", "๐ฅด", "โถ", "โก", "โ", "๐", "๐ธ", "โฌ", "๐จ", + "๐", "๐ฆ", "๐ท", "๐บ", "โ ", "๐
", "๐", "๐ต", "๐", "๐คฒ", + "๐ค ", "๐คง", "๐", "๐ต", "๐
", "๐ง", "๐พ", "๐", "๐", "๐ค", + "๐", "๐คฏ", "๐ท", "โ", "๐ง", "๐ฏ", "๐", "๐", "๐ค", "๐", + "๐", "โ", "๐ด", "๐ฃ", "๐ธ", "๐", "๐", "๐ฅ", "๐คข", "๐
", + "๐ก", "๐ฉ", "๐", "๐ธ", "๐ป", "๐ค", "๐คฎ", "๐ผ", "๐ฅต", "๐ฉ", + "๐", "๐", "๐ผ", "๐", "๐ฃ", "๐ฅ" + }; + + // TODO Propose adding a Guava dependency to use ImmutableMap instead of this + + private static final Map<String, Integer> EMOJI_TO_INDEX; + private static final int MAP_EXPECTED_SIZE = EMOJIS.length; + private static final float MAP_LOAD_FACTOR = 1.0f; + + static { + if (EMOJIS.length != 256) { + throw new IllegalStateException("EMOJIS.length must be 256, but is " + EMOJIS.length); + } + + Map<String, Integer> mutableMap = new HashMap<>(MAP_EXPECTED_SIZE, MAP_LOAD_FACTOR); + for (int i = 0; i < EMOJIS.length; i++) { + mutableMap.put(EMOJIS[i], i); + } + EMOJI_TO_INDEX = Collections.unmodifiableMap(mutableMap); + } + + public static String encode(byte[] in) { + StringBuilder sb = new StringBuilder(in.length); + for (byte b : in) { + sb.append(EMOJIS[b & 0xFF]); + } + return sb.toString(); + } + + public static byte[] decode(String in) { + int length = in.codePointCount(0, in.length()); + byte[] bytes = new byte[length]; + + for (int i = 0; i < in.codePointCount(0, in.length()); i++) { + int cp = in.codePointAt(in.offsetByCodePoints(0, i)); + String emoji = new String(Character.toChars(cp)); + Integer index = EMOJI_TO_INDEX.get(emoji); + if (index == null) { + throw new IllegalArgumentException("Unknown Base256Emoji character: " + emoji); + } + bytes[i] = (byte) (index & 0xFF); + } + + return bytes; + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base36.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base36.java new file mode 100644 index 0000000..cfd8ada --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base36.java @@ -0,0 +1,50 @@ +package io.ipfs.multibase; + +import java.math.BigInteger; + +public class Base36 { + + public static byte[] decode(String in) { + if (in.isEmpty()) { + return new byte[0]; + } + BigInteger value = new BigInteger(in, 36); + byte[] withoutLeadingZeroes = value.signum() == 0 ? new byte[0] : value.toByteArray(); + // BigInteger.toByteArray() prepends a 0x00 sign byte when the top magnitude byte has its + // high bit set; strip it so the only leading zeroes are the ones recorded in the string. + int start = withoutLeadingZeroes.length > 0 && withoutLeadingZeroes[0] == 0 ? 1 : 0; + int magnitudeLength = withoutLeadingZeroes.length - start; + int zeroPrefixLength = zeroPrefixLength(in); + byte[] res = new byte[zeroPrefixLength + magnitudeLength]; + System.arraycopy(withoutLeadingZeroes, start, res, zeroPrefixLength, magnitudeLength); + return res; + } + + public static String encode(byte[] in) { + BigInteger value = new BigInteger(1, in); + String withoutLeadingZeroes = value.signum() == 0 ? "" : value.toString(36); + int zeroPrefixLength = zeroPrefixLength(in); + StringBuilder b = new StringBuilder(); + for (int i = 0; i < zeroPrefixLength; i++) b.append("0"); + b.append(withoutLeadingZeroes); + return b.toString(); + } + + private static int zeroPrefixLength(byte[] bytes) { + for (int i = 0; i < bytes.length; i++) { + if (bytes[i] != 0) { + return i; + } + } + return bytes.length; + } + + private static int zeroPrefixLength(String in) { + for (int i = 0; i < in.length(); i++) { + if (in.charAt(i) != '0') { + return i; + } + } + return in.length(); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base58.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base58.java new file mode 100644 index 0000000..7d8219e --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Base58.java @@ -0,0 +1,169 @@ +package io.ipfs.multibase; + +/* + * Copyright 2011 Google Inc. + * Copyright 2018 Andreas Schildbach + * + * From https://github.com/bitcoinj/bitcoinj/blob/master/core/src/main/java/org/bitcoinj/core/Base58.java + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.math.BigInteger; +import java.util.Arrays; + +/** + * Base58 is a way to encode Bitcoin addresses (or arbitrary data) as alphanumeric strings. + * + * <p>Note that this is not the same base58 as used by Flickr, which you may find referenced around + * the Internet. + * + * <p>Satoshi explains: why base-58 instead of standard base-64 encoding? + * + * <ul> + * <li>Don't want 0OIl characters that look the same in some fonts and could be used to create + * visually identical looking account numbers. + * <li>A string with non-alphanumeric characters is not as easily accepted as an account number. + * <li>E-mail usually won't line-break if there's no punctuation to break at. + * <li>Doubleclicking selects the whole number as one word if it's all alphanumeric. + * </ul> + * + * <p>However, note that the encoding/decoding runs in O(n²) time, so it is not useful for + * large data. + * + * <p>The basic idea of the encoding is to treat the data bytes as a large number represented using + * base-256 digits, convert the number to be represented using base-58 digits, preserve the exact + * number of leading zeros (which are otherwise lost during the mathematical operations on the + * numbers), and finally represent the resulting base-58 digits as alphanumeric ASCII characters. + */ +public class Base58 { + public static final char[] ALPHABET = + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz".toCharArray(); + private static final char ENCODED_ZERO = ALPHABET[0]; + private static final int[] INDEXES = new int[128]; + + static { + Arrays.fill(INDEXES, -1); + for (int i = 0; i < ALPHABET.length; i++) { + INDEXES[ALPHABET[i]] = i; + } + } + + /** + * Encodes the given bytes as a base58 string (no checksum is appended). + * + * @param input the bytes to encode + * @return the base58-encoded string + */ + public static String encode(byte[] input) { + if (input.length == 0) { + return ""; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input.length && input[zeros] == 0) { + ++zeros; + } + // Convert base-256 digits to base-58 digits (plus conversion to ASCII characters) + input = Arrays.copyOf(input, input.length); // since we modify it in-place + char[] encoded = new char[input.length * 2]; // upper bound + int outputStart = encoded.length; + for (int inputStart = zeros; inputStart < input.length; ) { + encoded[--outputStart] = ALPHABET[divmod(input, inputStart, 256, 58)]; + if (input[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Preserve exactly as many leading encoded zeros in output as there were leading zeros in + // input. + while (outputStart < encoded.length && encoded[outputStart] == ENCODED_ZERO) { + ++outputStart; + } + while (--zeros >= 0) { + encoded[--outputStart] = ENCODED_ZERO; + } + // Return encoded string (including encoded leading zeros). + return new String(encoded, outputStart, encoded.length - outputStart); + } + + /** + * Decodes the given base58 string into the original data bytes. + * + * @param input the base58-encoded string to decode + * @return the decoded data bytes + */ + public static byte[] decode(String input) { + if (input.length() == 0) { + return new byte[0]; + } + // Convert the base58-encoded ASCII chars to a base58 byte sequence (base58 digits). + byte[] input58 = new byte[input.length()]; + for (int i = 0; i < input.length(); ++i) { + char c = input.charAt(i); + int digit = c < 128 ? INDEXES[c] : -1; + if (digit < 0) { + throw new IllegalArgumentException( + String.format("Invalid character in Base58: 0x%04x", (int) c)); + } + input58[i] = (byte) digit; + } + // Count leading zeros. + int zeros = 0; + while (zeros < input58.length && input58[zeros] == 0) { + ++zeros; + } + // Convert base-58 digits to base-256 digits. + byte[] decoded = new byte[input.length()]; + int outputStart = decoded.length; + for (int inputStart = zeros; inputStart < input58.length; ) { + decoded[--outputStart] = divmod(input58, inputStart, 58, 256); + if (input58[inputStart] == 0) { + ++inputStart; // optimization - skip leading zeros + } + } + // Ignore extra leading zeroes that were added during the calculation. + while (outputStart < decoded.length && decoded[outputStart] == 0) { + ++outputStart; + } + // Return decoded data (including original number of leading zeros). + return Arrays.copyOfRange(decoded, outputStart - zeros, decoded.length); + } + + public static BigInteger decodeToBigInteger(String input) { + return new BigInteger(1, decode(input)); + } + + /** + * Divides a number, represented as an array of bytes each containing a single digit in the + * specified base, by the given divisor. The given number is modified in-place to contain the + * quotient, and the return value is the remainder. + * + * @param number the number to divide + * @param firstDigit the index within the array of the first non-zero digit (this is used for + * optimization by skipping the leading zeros) + * @param base the base in which the number's digits are represented (up to 256) + * @param divisor the number to divide by (up to 256) + * @return the remainder of the division operation + */ + private static byte divmod(byte[] number, int firstDigit, int base, int divisor) { + // this is just long division which accounts for the base of the input digits + int remainder = 0; + for (int i = firstDigit; i < number.length; i++) { + int digit = (int) number[i] & 0xFF; + int temp = remainder * base + digit; + number[i] = (byte) (temp / divisor); + remainder = temp % divisor; + } + return (byte) remainder; + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/BinaryDecoder.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/BinaryDecoder.java new file mode 100644 index 0000000..14be1d3 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/BinaryDecoder.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Defines common decoding methods for byte array decoders. + * + * @version $Id$ + */ +public interface BinaryDecoder extends Decoder { + + /** + * Decodes a byte array and returns the results as a byte array. + * + * @param source A byte array which has been encoded with the appropriate encoder + * @return a byte array that contains decoded content + * @throws DecoderException A decoder exception is thrown if a Decoder encounters a failure + * condition during the decode process. + */ + byte[] decode(byte[] source) throws DecoderException; +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/BinaryEncoder.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/BinaryEncoder.java new file mode 100644 index 0000000..fe33ebf --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/BinaryEncoder.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Defines common encoding methods for byte array encoders. + * + * @version $Id$ + */ +public interface BinaryEncoder extends Encoder { + + /** + * Encodes a byte array and return the encoded data as a byte array. + * + * @param source Data to be encoded + * @return A byte array containing the encoded data + * @throws EncoderException thrown if the Encoder encounters a failure condition during the + * encoding process. + */ + byte[] encode(byte[] source) throws EncoderException; +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/CharEncoding.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/CharEncoding.java new file mode 100644 index 0000000..1eb8168 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/CharEncoding.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Character encoding names required of every implementation of the Java platform. + * + * <p>From the Java documentation <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a>: + * + * <p><cite>Every implementation of the Java platform is required to support the following character + * encodings. Consult the release documentation for your implementation to see if any other + * encodings are supported. Consult the release documentation for your implementation to see if any + * other encodings are supported.</cite> + * + * <ul> + * <li><code>US-ASCII</code><br> + * Seven-bit ASCII, a.k.a. ISO646-US, a.k.a. the Basic Latin block of the Unicode character + * set. + * <li><code>ISO-8859-1</code><br> + * ISO Latin Alphabet No. 1, a.k.a. ISO-LATIN-1. + * <li><code>UTF-8</code><br> + * Eight-bit Unicode Transformation Format. + * <li><code>UTF-16BE</code><br> + * Sixteen-bit Unicode Transformation Format, big-endian byte order. + * <li><code>UTF-16LE</code><br> + * Sixteen-bit Unicode Transformation Format, little-endian byte order. + * <li><code>UTF-16</code><br> + * Sixteen-bit Unicode Transformation Format, byte order specified by a mandatory initial + * byte-order mark (either order accepted on input, big-endian used on output.) + * </ul> + * + * This perhaps would best belong in the [lang] project. Even if a similar interface is defined in + * [lang], it is not foreseen that [codec] would be made to depend on [lang]. + * + * <p>This class is immutable and thread-safe. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + * @since 1.4 + * @version $Id$ + */ +public class CharEncoding { + /** + * CharEncodingISO Latin Alphabet No. 1, a.k.a. ISO-LATIN-1. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final String ISO_8859_1 = "ISO-8859-1"; + + /** + * Seven-bit ASCII, also known as ISO646-US, also known as the Basic Latin block of the Unicode + * character set. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final String US_ASCII = "US-ASCII"; + + /** + * Sixteen-bit Unicode Transformation Format, The byte order specified by a mandatory initial + * byte-order mark (either order accepted on input, big-endian used on output) + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final String UTF_16 = "UTF-16"; + + /** + * Sixteen-bit Unicode Transformation Format, big-endian byte order. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final String UTF_16BE = "UTF-16BE"; + + /** + * Sixteen-bit Unicode Transformation Format, little-endian byte order. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final String UTF_16LE = "UTF-16LE"; + + /** + * Eight-bit Unicode Transformation Format. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final String UTF_8 = "UTF-8"; +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Charsets.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Charsets.java new file mode 100644 index 0000000..3f0afb4 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Charsets.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.ipfs.multibase; + +import java.nio.charset.Charset; + +/** + * Charsets required of every implementation of the Java platform. + * + * <p>From the Java documentation <a + * href="http://docs.oracle.com/javase/6/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a>: + * + * <p><cite>Every implementation of the Java platform is required to support the following character + * encodings. Consult the release documentation for your implementation to see if any other + * encodings are supported. Consult the release documentation for your implementation to see if any + * other encodings are supported. </cite> + * + * <ul> + * <li><code>US-ASCII</code><br> + * Seven-bit ASCII, a.k.a. ISO646-US, a.k.a. the Basic Latin block of the Unicode character + * set. + * <li><code>ISO-8859-1</code><br> + * ISO Latin Alphabet No. 1, a.k.a. ISO-LATIN-1. + * <li><code>UTF-8</code><br> + * Eight-bit Unicode Transformation Format. + * <li><code>UTF-16BE</code><br> + * Sixteen-bit Unicode Transformation Format, big-endian byte order. + * <li><code>UTF-16LE</code><br> + * Sixteen-bit Unicode Transformation Format, little-endian byte order. + * <li><code>UTF-16</code><br> + * Sixteen-bit Unicode Transformation Format, byte order specified by a mandatory initial + * byte-order mark (either order accepted on input, big-endian used on output.) + * </ul> + * + * This perhaps would best belong in the Commons Lang project. Even if a similar class is defined in + * Commons Lang, it is not foreseen that Commons Codec would be made to depend on Commons Lang. + * + * <p>This class is immutable and thread-safe. + * + * @see <a href="http://docs.oracle.com/javase/6/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + * @since 1.7 + * @version $Id: CharEncoding.java 1173287 2011-09-20 18:16:19Z ggregory $ + */ +public class Charsets { + + // + // This class should only contain Charset instances for required encodings. This guarantees that + // it will load + // correctly and without delay on all Java platforms. + // + + /** + * Seven-bit ASCII, also known as ISO646-US, also known as the Basic Latin block of the Unicode + * character set. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * <p>On Java 7 or later, use {@link java.nio.charset.StandardCharsets#ISO_8859_1} instead. + * + * @see <a href="http://docs.oracle.com/javase/6/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final Charset US_ASCII = Charset.forName(CharEncoding.US_ASCII); + + /** + * Eight-bit Unicode Transformation Format. + * + * <p>Every implementation of the Java platform is required to support this character encoding. + * + * <p>On Java 7 or later, use {@link java.nio.charset.StandardCharsets#ISO_8859_1} instead. + * + * @see <a href="http://docs.oracle.com/javase/6/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static final Charset UTF_8 = Charset.forName(CharEncoding.UTF_8); +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Decoder.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Decoder.java new file mode 100644 index 0000000..6ae0d78 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Decoder.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Provides the highest level of abstraction for Decoders. + * + * <p>This is the sister interface of {@link Encoder}. All Decoders implement this common generic + * interface. Allows a user to pass a generic Object to any Decoder implementation in the codec + * package. + * + * <p>One of the two interfaces at the center of the codec package. + * + * @version $Id$ + */ +public interface Decoder { + + /** + * Decodes an "encoded" Object and returns a "decoded" Object. Note that the implementation of + * this interface will try to cast the Object parameter to the specific type expected by a + * particular Decoder implementation. If a {@link ClassCastException} occurs this decode method + * will throw a DecoderException. + * + * @param source the object to decode + * @return a 'decoded" object + * @throws DecoderException a decoder exception can be thrown for any number of reasons. Some good + * candidates are that the parameter passed to this method is null, a param cannot be cast to + * the appropriate type for a specific encoder. + */ + Object decode(Object source) throws DecoderException; +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/DecoderException.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/DecoderException.java new file mode 100644 index 0000000..ddfc94c --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/DecoderException.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Thrown when there is a failure condition during the decoding process. This exception is thrown + * when a {@link Decoder} encounters a decoding specific exception such as invalid data, or + * characters outside of the expected range. + * + * @version $Id$ + */ +public class DecoderException extends Exception { + + /** + * Declares the Serial Version Uid. + * + * @see <a href="http://c2.com/cgi/wiki?AlwaysDeclareSerialVersionUid">Always Declare Serial + * Version Uid</a> + */ + private static final long serialVersionUID = 1L; + + /** + * Constructs a new exception with <code>null</code> as its detail message. The cause is not + * initialized, and may subsequently be initialized by a call to {@link #initCause}. + * + * @since 1.4 + */ + public DecoderException() { + super(); + } + + /** + * Constructs a new exception with the specified detail message. The cause is not initialized, and + * may subsequently be initialized by a call to {@link #initCause}. + * + * @param message The detail message which is saved for later retrieval by the {@link + * #getMessage()} method. + */ + public DecoderException(final String message) { + super(message); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * <p>Note that the detail message associated with <code>cause</code> is not automatically + * incorporated into this exception's detail message. + * + * @param message The detail message which is saved for later retrieval by the {@link + * #getMessage()} method. + * @param cause The cause which is saved for later retrieval by the {@link #getCause()} method. A + * <code>null</code> value is permitted, and indicates that the cause is nonexistent or + * unknown. + * @since 1.4 + */ + public DecoderException(final String message, final Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified cause and a detail message of <code> + * (cause==null ? + * null : cause.toString())</code> (which typically contains the class and detail message of + * <code>cause</code>). This constructor is useful for exceptions that are little more than + * wrappers for other throwables. + * + * @param cause The cause which is saved for later retrieval by the {@link #getCause()} method. A + * <code>null</code> value is permitted, and indicates that the cause is nonexistent or + * unknown. + * @since 1.4 + */ + public DecoderException(final Throwable cause) { + super(cause); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Encoder.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Encoder.java new file mode 100644 index 0000000..facd21a --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Encoder.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Provides the highest level of abstraction for Encoders. + * + * <p>This is the sister interface of {@link Decoder}. Every implementation of Encoder provides this + * common generic interface which allows a user to pass a generic Object to any Encoder + * implementation in the codec package. + * + * @version $Id$ + */ +public interface Encoder { + + /** + * Encodes an "Object" and returns the encoded content as an Object. The Objects here may just be + * <code>byte[]</code> or <code>String</code>s depending on the implementation used. + * + * @param source An object to encode + * @return An "encoded" Object + * @throws EncoderException An encoder exception is thrown if the encoder experiences a failure + * condition during the encoding process. + */ + Object encode(Object source) throws EncoderException; +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/EncoderException.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/EncoderException.java new file mode 100644 index 0000000..47d38ee --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/EncoderException.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase; + +/** + * Thrown when there is a failure condition during the encoding process. This exception is thrown + * when an {@link Encoder} encounters a encoding specific exception such as invalid data, inability + * to calculate a checksum, characters outside of the expected range. + * + * @version $Id$ + */ +public class EncoderException extends Exception { + + /** + * Declares the Serial Version Uid. + * + * @see <a href="http://c2.com/cgi/wiki?AlwaysDeclareSerialVersionUid">Always Declare Serial + * Version Uid</a> + */ + private static final long serialVersionUID = 1L; + + /** + * Constructs a new exception with <code>null</code> as its detail message. The cause is not + * initialized, and may subsequently be initialized by a call to {@link #initCause}. + * + * @since 1.4 + */ + public EncoderException() { + super(); + } + + /** + * Constructs a new exception with the specified detail message. The cause is not initialized, and + * may subsequently be initialized by a call to {@link #initCause}. + * + * @param message a useful message relating to the encoder specific error. + */ + public EncoderException(final String message) { + super(message); + } + + /** + * Constructs a new exception with the specified detail message and cause. + * + * <p>Note that the detail message associated with <code>cause</code> is not automatically + * incorporated into this exception's detail message. + * + * @param message The detail message which is saved for later retrieval by the {@link + * #getMessage()} method. + * @param cause The cause which is saved for later retrieval by the {@link #getCause()} method. A + * <code>null</code> value is permitted, and indicates that the cause is nonexistent or + * unknown. + * @since 1.4 + */ + public EncoderException(final String message, final Throwable cause) { + super(message, cause); + } + + /** + * Constructs a new exception with the specified cause and a detail message of <code> + * (cause==null ? + * null : cause.toString())</code> (which typically contains the class and detail message of + * <code>cause</code>). This constructor is useful for exceptions that are little more than + * wrappers for other throwables. + * + * @param cause The cause which is saved for later retrieval by the {@link #getCause()} method. A + * <code>null</code> value is permitted, and indicates that the cause is nonexistent or + * unknown. + * @since 1.4 + */ + public EncoderException(final Throwable cause) { + super(cause); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Multibase.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Multibase.java new file mode 100644 index 0000000..0d2a7b7 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/Multibase.java @@ -0,0 +1,178 @@ +package io.ipfs.multibase; + +import io.ipfs.multibase.binary.Base32; +import io.ipfs.multibase.binary.Base64; +import java.util.Map; +import java.util.Optional; +import java.util.TreeMap; + +public class Multibase { + + public enum Base { + Base1("1"), + Base2("0"), + Base8("7"), + Base10("9"), + Base16("f"), + Base16Upper("F"), + Base32("b"), + Base32Upper("B"), + Base32Pad("c"), + Base32PadUpper("C"), + Base32Hex("v"), + Base32HexUpper("V"), + Base32HexPad("t"), + Base32HexPadUpper("T"), + Base36("k"), + Base36Upper("K"), + Base58BTC("z"), + Base58Flickr("Z"), + Base64("m"), + Base64Url("u"), + Base64Pad("M"), + Base64UrlPad("U"), + Base256Emoji("๐"); + + public String prefix; + + Base(String prefix) { + this.prefix = prefix; + } + + private static Map<String, Base> lookup = new TreeMap<>(); + + static { + for (Base b : Base.values()) lookup.put(b.prefix, b); + } + + private static Optional<Base> lookupOptional(String data) { + if (data == null || data.isEmpty()) return Optional.empty(); + String p = Character.toString(data.codePointAt(0)); + Base base = lookup.get(p); + if (base != null) return Optional.of(base); + if (data.startsWith(Base256Emoji.prefix)) return Optional.of(Base256Emoji); + return Optional.empty(); + } + + public static Base lookup(String data) { + return lookupOptional(data) + .orElseThrow(() -> new IllegalArgumentException("Unknown Multibase type: " + data)); + } + } + + public static String encode(Base b, byte[] data) { + switch (b) { + case Base58BTC: + return b.prefix + Base58.encode(data); + case Base16: + return b.prefix + Base16.encode(data); + case Base16Upper: + return b.prefix + Base16.encode(data).toUpperCase(); + case Base32: + return b.prefix + new String(new Base32().encode(data)).toLowerCase().replaceAll("=", ""); + case Base32Pad: + return b.prefix + new String(new Base32().encode(data)).toLowerCase(); + case Base32PadUpper: + return b.prefix + new String(new Base32().encode(data)); + case Base32Upper: + return b.prefix + new String(new Base32().encode(data)).replaceAll("=", ""); + case Base32Hex: + return b.prefix + + new String(new Base32(true).encode(data)).toLowerCase().replaceAll("=", ""); + case Base32HexPad: + return b.prefix + new String(new Base32(true).encode(data)).toLowerCase(); + case Base32HexPadUpper: + return b.prefix + new String(new Base32(true).encode(data)); + case Base32HexUpper: + return b.prefix + new String(new Base32(true).encode(data)).replaceAll("=", ""); + case Base36: + return b.prefix + Base36.encode(data); + case Base36Upper: + return b.prefix + Base36.encode(data).toUpperCase(); + case Base64: + return b.prefix + Base64.encodeBase64String(data).replaceAll("=", ""); + case Base64Url: + return b.prefix + Base64.encodeBase64URLSafeString(data).replaceAll("=", ""); + case Base64Pad: + return b.prefix + Base64.encodeBase64String(data); + case Base64UrlPad: + return b.prefix + + Base64.encodeBase64String(data).replaceAll("\\+", "-").replaceAll("/", "_"); + case Base256Emoji: + return b.prefix + Base256Emoji.encode(data); + default: + throw new UnsupportedOperationException("Unsupported base encoding: " + b.name()); + } + } + + public static Base encoding(String data) { + return Base.lookup(data); + } + + public static byte[] decode(String data) { + if (data.isEmpty()) { + throw new IllegalArgumentException("Cannot decode an empty string"); + } + Base b = encoding(data); + String rest = safeSubstringFromIndexOne(data); + switch (b) { + case Base58BTC: + return Base58.decode(rest); + case Base16: + return Base16.decode(rest); + case Base16Upper: + return Base16.decode(rest.toLowerCase()); + case Base32: + case Base32Pad: + return new Base32().decode(rest); + case Base32PadUpper: + case Base32Upper: + return new Base32().decode(rest.toLowerCase()); + case Base32Hex: + case Base32HexPad: + return new Base32(true).decode(rest); + case Base32HexPadUpper: + case Base32HexUpper: + return new Base32(true).decode(rest.toLowerCase()); + case Base36: + return Base36.decode(rest); + case Base36Upper: + return Base36.decode(rest.toLowerCase()); + case Base64: + case Base64Url: + case Base64Pad: + case Base64UrlPad: + return Base64.decodeBase64(rest); + case Base256Emoji: + return Base256Emoji.decode(rest); + default: + throw new UnsupportedOperationException("Unsupported base encoding: " + b.name()); + } + } + + private static String safeSubstringFromIndexOne(String data) { + // Check if there's at least 2 code points in the string + if (data.codePointCount(0, data.length()) <= 1) { + return ""; + } + + // If so, do an Emoji-safe data.substring(1) equivalent: + int charIndex = data.offsetByCodePoints(0, 1); + return data.substring(charIndex); + } + + /** + * Check if the given data has a valid multibase prefix. + * + * <p>Please note that "having a valid prefix" is NOT the same as "being an entirely valid + * multibase string"; even if <tt>true</tt>, it's still entirely possible for {@link + * #decode(String)} to throw an <tt>IllegalArgumentException</tt>, if prefix is valid, but the + * following data is not. + * + * @param data Multibase string to check. + * @return true if the data has a valid multibase prefix, false otherwise; but see above. + */ + public static boolean hasValidPrefix(String data) { + return Base.lookupOptional(data).isPresent(); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/Base32.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/Base32.java new file mode 100644 index 0000000..12cde40 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/Base32.java @@ -0,0 +1,648 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase.binary; + +/** + * Provides Base32 encoding and decoding as defined by <a + * href="http://www.ietf.org/rfc/rfc4648.txt">RFC 4648</a>. + * + * <p>From https://commons.apache.org/proper/commons-codec/ + * + * <p>The class can be parameterized in the following manner with various constructors: + * + * <ul> + * <li>Whether to use the "base32hex" variant instead of the default "base32" + * <li>Line length: Default 76. Line length that aren't multiples of 8 will still essentially end + * up being multiples of 8 in the encoded data. + * <li>Line separator: Default is CRLF ("\r\n") + * </ul> + * + * <p>This class operates directly on byte streams, and not character streams. + * + * <p>This class is thread-safe. + * + * @see <a href="http://www.ietf.org/rfc/rfc4648.txt">RFC 4648</a> + * @since 1.5 + * @version $Id$ + */ +public class Base32 extends BaseNCodec { + + /** + * BASE32 characters are 5 bits in length. They are formed by taking a block of five octets to + * form a 40-bit string, which is converted into eight BASE32 characters. + */ + private static final int BITS_PER_ENCODED_BYTE = 5; + + private static final int BYTES_PER_ENCODED_BLOCK = 8; + private static final int BYTES_PER_UNENCODED_BLOCK = 5; + + /** + * Chunk separator per RFC 2045 section 2.1. + * + * @see <a href="http://www.ietf.org/rfc/rfc2045.txt">RFC 2045 section 2.1</a> + */ + private static final byte[] CHUNK_SEPARATOR = {'\r', '\n'}; + + /** + * This array is a lookup table that translates Unicode characters drawn from the "Base32 + * Alphabet" (as specified in Table 3 of RFC 4648) into their 5-bit positive integer equivalents. + * Characters that are not in the Base32 alphabet but fall within the bounds of the array are + * translated to -1. + */ + private static final byte[] DECODE_TABLE = { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 00-0f + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 10-1f + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 20-2f + -1, -1, 26, 27, 28, 29, 30, 31, -1, -1, -1, -1, -1, -1, -1, -1, // 30-3f 2-7 + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 40-4f A-O + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, // 50-5a P-Z + -1, -1, -1, -1, -1, // 5b - 5f + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 60 - 6f a-o + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, // 70 - 7a p-z/**/ + }; + + /** + * This array is a lookup table that translates 5-bit positive integer index values into their + * "Base32 Alphabet" equivalents as specified in Table 3 of RFC 4648. + */ + private static final byte[] ENCODE_TABLE = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + '2', '3', '4', '5', '6', '7', + }; + + /** + * This array is a lookup table that translates Unicode characters drawn from the "Base32 Hex + * Alphabet" (as specified in Table 4 of RFC 4648) into their 5-bit positive integer equivalents. + * Characters that are not in the Base32 Hex alphabet but fall within the bounds of the array are + * translated to -1. + */ + private static final byte[] HEX_DECODE_TABLE = { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, // 00-0f + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, // 10-1f + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, // 20-2f + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + -1, + -1, + -1, + -1, + -1, + -1, // 30-3f 2-7 + -1, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, // 40-4f A-O + 25, + 26, + 27, + 28, + 29, + 30, + 31, // 50-56 P-V + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, + -1, // 57-5f Z-_ + -1, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 17, + 18, + 19, + 20, + 21, + 22, + 23, + 24, // 60-6f `-o + 25, + 26, + 27, + 28, + 29, + 30, + 31 // 70-76 p-v + }; + + /** + * This array is a lookup table that translates 5-bit positive integer index values into their + * "Base32 Hex Alphabet" equivalents as specified in Table 4 of RFC 4648. + */ + private static final byte[] HEX_ENCODE_TABLE = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', + 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', + }; + + /** Mask used to extract 5 bits, used when encoding Base32 bytes */ + private static final int MASK_5BITS = 0x1f; + + // The static final fields above are used for the original static byte[] methods on Base32. + // The private member fields below are used with the new streaming approach, which requires + // some state be preserved between calls of encode() and decode(). + + /** + * Place holder for the bytes we're dealing with for our based logic. Bitwise operations store and + * extract the encoding or decoding from this variable. + */ + + /** + * Convenience variable to help us determine when our buffer is going to run out of room and needs + * resizing. <code>decodeSize = {@link #BYTES_PER_ENCODED_BLOCK} - 1 + lineSeparator.length; + * </code> + */ + private final int decodeSize; + + /** Decode table to use. */ + private final byte[] decodeTable; + + /** + * Convenience variable to help us determine when our buffer is going to run out of room and needs + * resizing. <code>encodeSize = {@link #BYTES_PER_ENCODED_BLOCK} + lineSeparator.length;</code> + */ + private final int encodeSize; + + /** Encode table to use. */ + private final byte[] encodeTable; + + /** Line separator for encoding. Not used when decoding. Only used if lineLength > 0. */ + private final byte[] lineSeparator; + + /** + * Creates a Base32 codec used for decoding and encoding. + * + * <p>When encoding the line length is 0 (no chunking). + */ + public Base32() { + this(false); + } + + /** + * Creates a Base32 codec used for decoding and encoding. + * + * <p>When encoding the line length is 0 (no chunking). + * + * @param pad byte used as padding byte. + */ + public Base32(final byte pad) { + this(false, pad); + } + + /** + * Creates a Base32 codec used for decoding and encoding. + * + * <p>When encoding the line length is 0 (no chunking). + * + * @param useHex if {@code true} then use Base32 Hex alphabet + */ + public Base32(final boolean useHex) { + this(0, null, useHex, PAD_DEFAULT); + } + + /** + * Creates a Base32 codec used for decoding and encoding. + * + * <p>When encoding the line length is 0 (no chunking). + * + * @param useHex if {@code true} then use Base32 Hex alphabet + * @param pad byte used as padding byte. + */ + public Base32(final boolean useHex, final byte pad) { + this(0, null, useHex, pad); + } + + /** + * Creates a Base32 codec used for decoding and encoding. + * + * <p>When encoding the line length is given in the constructor, the line separator is CRLF. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 8). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + */ + public Base32(final int lineLength) { + this(lineLength, CHUNK_SEPARATOR); + } + + /** + * Creates a Base32 codec used for decoding and encoding. + * + * <p>When encoding the line length and line separator are given in the constructor. + * + * <p>Line lengths that aren't multiples of 8 will still essentially end up being multiples of 8 + * in the encoded data. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 8). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + * @param lineSeparator Each line of encoded data will end with this sequence of bytes. + * @throws IllegalArgumentException The provided lineSeparator included some Base32 characters. + * That's not going to work! + */ + public Base32(final int lineLength, final byte[] lineSeparator) { + this(lineLength, lineSeparator, false, PAD_DEFAULT); + } + + /** + * Creates a Base32 / Base32 Hex codec used for decoding and encoding. + * + * <p>When encoding the line length and line separator are given in the constructor. + * + * <p>Line lengths that aren't multiples of 8 will still essentially end up being multiples of 8 + * in the encoded data. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 8). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + * @param lineSeparator Each line of encoded data will end with this sequence of bytes. + * @param useHex if {@code true}, then use Base32 Hex alphabet, otherwise use Base32 alphabet + * @throws IllegalArgumentException The provided lineSeparator included some Base32 characters. + * That's not going to work! Or the lineLength > 0 and lineSeparator is null. + */ + public Base32(final int lineLength, final byte[] lineSeparator, final boolean useHex) { + this(lineLength, lineSeparator, useHex, PAD_DEFAULT); + } + + /** + * Creates a Base32 / Base32 Hex codec used for decoding and encoding. + * + * <p>When encoding the line length and line separator are given in the constructor. + * + * <p>Line lengths that aren't multiples of 8 will still essentially end up being multiples of 8 + * in the encoded data. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 8). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + * @param lineSeparator Each line of encoded data will end with this sequence of bytes. + * @param useHex if {@code true}, then use Base32 Hex alphabet, otherwise use Base32 alphabet + * @param pad byte used as padding byte. + * @throws IllegalArgumentException The provided lineSeparator included some Base32 characters. + * That's not going to work! Or the lineLength > 0 and lineSeparator is null. + */ + public Base32( + final int lineLength, final byte[] lineSeparator, final boolean useHex, final byte pad) { + super( + BYTES_PER_UNENCODED_BLOCK, + BYTES_PER_ENCODED_BLOCK, + lineLength, + lineSeparator == null ? 0 : lineSeparator.length, + pad); + if (useHex) { + this.encodeTable = HEX_ENCODE_TABLE; + this.decodeTable = HEX_DECODE_TABLE; + } else { + this.encodeTable = ENCODE_TABLE; + this.decodeTable = DECODE_TABLE; + } + if (lineLength > 0) { + if (lineSeparator == null) { + throw new IllegalArgumentException( + "lineLength " + lineLength + " > 0, but lineSeparator is null"); + } + // Must be done after initializing the tables + if (containsAlphabetOrPad(lineSeparator)) { + final String sep = StringUtils.newStringUtf8(lineSeparator); + throw new IllegalArgumentException( + "lineSeparator must not contain Base32 characters: [" + sep + "]"); + } + this.encodeSize = BYTES_PER_ENCODED_BLOCK + lineSeparator.length; + this.lineSeparator = new byte[lineSeparator.length]; + System.arraycopy(lineSeparator, 0, this.lineSeparator, 0, lineSeparator.length); + } else { + this.encodeSize = BYTES_PER_ENCODED_BLOCK; + this.lineSeparator = null; + } + this.decodeSize = this.encodeSize - 1; + + if (isInAlphabet(pad) || isWhiteSpace(pad)) { + throw new IllegalArgumentException("pad must not be in alphabet or whitespace"); + } + } + + /** + * Decodes all of the provided data, starting at inPos, for inAvail bytes. Should be called at + * least twice: once with the data to decode, and once with inAvail set to "-1" to alert decoder + * that EOF has been reached. The "-1" call is not necessary when decoding, but it doesn't hurt, + * either. + * + * <p>Ignores all non-Base32 characters. This is how chunked (e.g. 76 character) data is handled, + * since CR and LF are silently ignored, but has implications for other bytes, too. This method + * subscribes to the garbage-in, garbage-out philosophy: it will not check the provided data for + * validity. + * + * @param in byte[] array of ascii data to Base32 decode. + * @param inPos Position to start reading data from. + * @param inAvail Amount of bytes available from input for encoding. + * @param context the context to be used + * <p>Output is written to {@link Context#buffer} as 8-bit octets, using {@link Context#pos} + * as the buffer position + */ + @Override + void decode(final byte[] in, int inPos, final int inAvail, final Context context) { + // package protected for access from I/O streams + + if (context.eof) { + return; + } + if (inAvail < 0) { + context.eof = true; + } + for (int i = 0; i < inAvail; i++) { + final byte b = in[inPos++]; + if (b == pad) { + // We're done. + context.eof = true; + break; + } + final byte[] buffer = ensureBufferSize(decodeSize, context); + if (b >= 0 && b < this.decodeTable.length) { + final int result = this.decodeTable[b]; + if (result >= 0) { + context.modulus = (context.modulus + 1) % BYTES_PER_ENCODED_BLOCK; + // collect decoded bytes + context.lbitWorkArea = (context.lbitWorkArea << BITS_PER_ENCODED_BYTE) + result; + if (context.modulus == 0) { // we can output the 5 bytes + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 32) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 24) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 16) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) (context.lbitWorkArea & MASK_8BITS); + } + } + } + } + + // Two forms of EOF as far as Base32 decoder is concerned: actual + // EOF (-1) and first time '=' character is encountered in stream. + // This approach makes the '=' padding characters completely optional. + if (context.eof && context.modulus >= 2) { // if modulus < 2, nothing to do + final byte[] buffer = ensureBufferSize(decodeSize, context); + + // we ignore partial bytes, i.e. only multiples of 8 count + switch (context.modulus) { + case 2: // 10 bits, drop 2 and output one byte + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 2) & MASK_8BITS); + break; + case 3: // 15 bits, drop 7 and output 1 byte + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 7) & MASK_8BITS); + break; + case 4: // 20 bits = 2*8 + 4 + context.lbitWorkArea = context.lbitWorkArea >> 4; // drop 4 bits + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea) & MASK_8BITS); + break; + case 5: // 25bits = 3*8 + 1 + context.lbitWorkArea = context.lbitWorkArea >> 1; + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 16) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea) & MASK_8BITS); + break; + case 6: // 30bits = 3*8 + 6 + context.lbitWorkArea = context.lbitWorkArea >> 6; + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 16) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea) & MASK_8BITS); + break; + case 7: // 35 = 4*8 +3 + context.lbitWorkArea = context.lbitWorkArea >> 3; + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 24) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 16) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.lbitWorkArea) & MASK_8BITS); + break; + default: + // modulus can be 0-7, and we excluded 0,1 already + throw new IllegalStateException("Impossible modulus " + context.modulus); + } + } + } + + /** + * Encodes all of the provided data, starting at inPos, for inAvail bytes. Must be called at least + * twice: once with the data to encode, and once with inAvail set to "-1" to alert encoder that + * EOF has been reached, so flush last remaining bytes (if not multiple of 5). + * + * @param in byte[] array of binary data to Base32 encode. + * @param inPos Position to start reading data from. + * @param inAvail Amount of bytes available from input for encoding. + * @param context the context to be used + */ + @Override + void encode(final byte[] in, int inPos, final int inAvail, final Context context) { + // package protected for access from I/O streams + + if (context.eof) { + return; + } + // inAvail < 0 is how we're informed of EOF in the underlying data we're + // encoding. + if (inAvail < 0) { + context.eof = true; + if (0 == context.modulus && lineLength == 0) { + return; // no leftovers to process and not using chunking + } + final byte[] buffer = ensureBufferSize(encodeSize, context); + final int savedPos = context.pos; + switch (context.modulus) { // % 5 + case 0: + break; + case 1: // Only 1 octet; take top 5 bits then remainder + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 3) & MASK_5BITS]; // 8-1*5 = 3 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea << 2) & MASK_5BITS]; // 5-3=2 + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + break; + case 2: // 2 octets = 16 bits to use + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 11) & MASK_5BITS]; // 16-1*5 = 11 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 6) & MASK_5BITS]; // 16-2*5 = 6 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 1) & MASK_5BITS]; // 16-3*5 = 1 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea << 4) & MASK_5BITS]; // 5-1 = 4 + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + break; + case 3: // 3 octets = 24 bits to use + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 19) & MASK_5BITS]; // 24-1*5 = 19 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 14) & MASK_5BITS]; // 24-2*5 = 14 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 9) & MASK_5BITS]; // 24-3*5 = 9 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 4) & MASK_5BITS]; // 24-4*5 = 4 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea << 1) & MASK_5BITS]; // 5-4 = 1 + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + break; + case 4: // 4 octets = 32 bits to use + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 27) & MASK_5BITS]; // 32-1*5 = 27 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 22) & MASK_5BITS]; // 32-2*5 = 22 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 17) & MASK_5BITS]; // 32-3*5 = 17 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 12) & MASK_5BITS]; // 32-4*5 = 12 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 7) & MASK_5BITS]; // 32-5*5 = 7 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea >> 2) & MASK_5BITS]; // 32-6*5 = 2 + buffer[context.pos++] = + encodeTable[(int) (context.lbitWorkArea << 3) & MASK_5BITS]; // 5-2 = 3 + buffer[context.pos++] = pad; + break; + default: + throw new IllegalStateException("Impossible modulus " + context.modulus); + } + context.currentLinePos += context.pos - savedPos; // keep track of current line position + // if currentPos == 0 we are at the start of a line, so don't add CRLF + if (lineLength > 0 && context.currentLinePos > 0) { // add chunk separator if required + System.arraycopy(lineSeparator, 0, buffer, context.pos, lineSeparator.length); + context.pos += lineSeparator.length; + } + } else { + for (int i = 0; i < inAvail; i++) { + final byte[] buffer = ensureBufferSize(encodeSize, context); + context.modulus = (context.modulus + 1) % BYTES_PER_UNENCODED_BLOCK; + int b = in[inPos++]; + if (b < 0) { + b += 256; + } + context.lbitWorkArea = (context.lbitWorkArea << 8) + b; // BITS_PER_BYTE + if (0 == context.modulus) { // we have enough bytes to create our output + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 35) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 30) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 25) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 20) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 15) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 10) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) (context.lbitWorkArea >> 5) & MASK_5BITS]; + buffer[context.pos++] = encodeTable[(int) context.lbitWorkArea & MASK_5BITS]; + context.currentLinePos += BYTES_PER_ENCODED_BLOCK; + if (lineLength > 0 && lineLength <= context.currentLinePos) { + System.arraycopy(lineSeparator, 0, buffer, context.pos, lineSeparator.length); + context.pos += lineSeparator.length; + context.currentLinePos = 0; + } + } + } + } + } + + /** + * Returns whether or not the {@code octet} is in the Base32 alphabet. + * + * @param octet The value to test + * @return {@code true} if the value is defined in the the Base32 alphabet {@code false} + * otherwise. + */ + @Override + public boolean isInAlphabet(final byte octet) { + return octet >= 0 && octet < decodeTable.length && decodeTable[octet] != -1; + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/Base64.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/Base64.java new file mode 100644 index 0000000..be3cff4 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/Base64.java @@ -0,0 +1,745 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase.binary; + +import java.math.BigInteger; + +/** + * Provides Base64 encoding and decoding as defined by <a + * href="http://www.ietf.org/rfc/rfc2045.txt">RFC 2045</a>. + * + * <p>From https://commons.apache.org/proper/commons-codec/ + * + * <p>This class implements section <cite>6.8. Base64 Content-Transfer-Encoding</cite> from RFC 2045 + * <cite>Multipurpose Internet Mail Extensions (MIME) Part One: Format of Internet Message + * Bodies</cite> by Freed and Borenstein. + * + * <p>The class can be parameterized in the following manner with various constructors: + * + * <ul> + * <li>URL-safe mode: Default off. + * <li>Line length: Default 76. Line length that aren't multiples of 4 will still essentially end + * up being multiples of 4 in the encoded data. + * <li>Line separator: Default is CRLF ("\r\n") + * </ul> + * + * <p>The URL-safe parameter is only applied to encode operations. Decoding seamlessly handles both + * modes. + * + * <p>Since this class operates directly on byte streams, and not character streams, it is + * hard-coded to only encode/decode character encodings which are compatible with the lower 127 + * ASCII chart (ISO-8859-1, Windows-1252, UTF-8, etc). + * + * <p>This class is thread-safe. + * + * @see <a href="http://www.ietf.org/rfc/rfc2045.txt">RFC 2045</a> + * @since 1.0 + * @version $Id$ + */ +public class Base64 extends BaseNCodec { + + /** + * BASE32 characters are 6 bits in length. They are formed by taking a block of 3 octets to form a + * 24-bit string, which is converted into 4 BASE64 characters. + */ + private static final int BITS_PER_ENCODED_BYTE = 6; + + private static final int BYTES_PER_UNENCODED_BLOCK = 3; + private static final int BYTES_PER_ENCODED_BLOCK = 4; + + /** + * Chunk separator per RFC 2045 section 2.1. + * + * <p>N.B. The next major release may break compatibility and make this field private. + * + * @see <a href="http://www.ietf.org/rfc/rfc2045.txt">RFC 2045 section 2.1</a> + */ + static final byte[] CHUNK_SEPARATOR = {'\r', '\n'}; + + /** + * This array is a lookup table that translates 6-bit positive integer index values into their + * "Base64 Alphabet" equivalents as specified in Table 1 of RFC 2045. + * + * <p>Thanks to "commons" project in ws.apache.org for this code. + * http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ + */ + private static final byte[] STANDARD_ENCODE_TABLE = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' + }; + + /** + * This is a copy of the STANDARD_ENCODE_TABLE above, but with + and / changed to - and _ to make + * the encoded Base64 results more URL-SAFE. This table is only used when the Base64's mode is set + * to URL-SAFE. + */ + private static final byte[] URL_SAFE_ENCODE_TABLE = { + 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', + 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', + 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', + 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '-', '_' + }; + + /** + * This array is a lookup table that translates Unicode characters drawn from the "Base64 + * Alphabet" (as specified in Table 1 of RFC 2045) into their 6-bit positive integer equivalents. + * Characters that are not in the Base64 alphabet but fall within the bounds of the array are + * translated to -1. + * + * <p>Note: '+' and '-' both decode to 62. '/' and '_' both decode to 63. This means decoder + * seamlessly handles both URL_SAFE and STANDARD base64. (The encoder, on the other hand, needs to + * know ahead of time what to emit). + * + * <p>Thanks to "commons" project in ws.apache.org for this code. + * http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ + */ + private static final byte[] DECODE_TABLE = { + // 0 1 2 3 4 5 6 7 8 9 A B C D E F + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 00-0f + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // 10-1f + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, 62, -1, 63, // 20-2f + - / + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, // 30-3f 0-9 + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, // 40-4f A-O + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, 63, // 50-5f P-Z _ + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, // 60-6f a-o + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51 // 70-7a p-z + }; + + /** Base64 uses 6-bit fields. */ + /** Mask used to extract 6 bits, used when encoding */ + private static final int MASK_6BITS = 0x3f; + + // The static final fields above are used for the original static byte[] methods on Base64. + // The private member fields below are used with the new streaming approach, which requires + // some state be preserved between calls of encode() and decode(). + + /** + * Encode table to use: either STANDARD or URL_SAFE. Note: the DECODE_TABLE above remains static + * because it is able to decode both STANDARD and URL_SAFE streams, but the encodeTable must be a + * member variable so we can switch between the two modes. + */ + private final byte[] encodeTable; + + // Only one decode table currently; keep for consistency with Base32 code + private final byte[] decodeTable = DECODE_TABLE; + + /** Line separator for encoding. Not used when decoding. Only used if lineLength > 0. */ + private final byte[] lineSeparator; + + /** + * Convenience variable to help us determine when our buffer is going to run out of room and needs + * resizing. <code>decodeSize = 3 + lineSeparator.length;</code> + */ + private final int decodeSize; + + /** + * Convenience variable to help us determine when our buffer is going to run out of room and needs + * resizing. <code>encodeSize = 4 + lineSeparator.length;</code> + */ + private final int encodeSize; + + /** + * Creates a Base64 codec used for decoding (all modes) and encoding in URL-unsafe mode. + * + * <p>When encoding the line length is 0 (no chunking), and the encoding table is + * STANDARD_ENCODE_TABLE. + * + * <p>When decoding all variants are supported. + */ + public Base64() { + this(0); + } + + /** + * Creates a Base64 codec used for decoding (all modes) and encoding in the given URL-safe mode. + * + * <p>When encoding the line length is 76, the line separator is CRLF, and the encoding table is + * STANDARD_ENCODE_TABLE. + * + * <p>When decoding all variants are supported. + * + * @param urlSafe if <code>true</code>, URL-safe encoding is used. In most cases this should be + * set to <code>false</code>. + * @since 1.4 + */ + public Base64(final boolean urlSafe) { + this(MIME_CHUNK_SIZE, CHUNK_SEPARATOR, urlSafe); + } + + /** + * Creates a Base64 codec used for decoding (all modes) and encoding in URL-unsafe mode. + * + * <p>When encoding the line length is given in the constructor, the line separator is CRLF, and + * the encoding table is STANDARD_ENCODE_TABLE. + * + * <p>Line lengths that aren't multiples of 4 will still essentially end up being multiples of 4 + * in the encoded data. + * + * <p>When decoding all variants are supported. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 4). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + * @since 1.4 + */ + public Base64(final int lineLength) { + this(lineLength, CHUNK_SEPARATOR); + } + + /** + * Creates a Base64 codec used for decoding (all modes) and encoding in URL-unsafe mode. + * + * <p>When encoding the line length and line separator are given in the constructor, and the + * encoding table is STANDARD_ENCODE_TABLE. + * + * <p>Line lengths that aren't multiples of 4 will still essentially end up being multiples of 4 + * in the encoded data. + * + * <p>When decoding all variants are supported. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 4). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + * @param lineSeparator Each line of encoded data will end with this sequence of bytes. + * @throws IllegalArgumentException Thrown when the provided lineSeparator included some base64 + * characters. + * @since 1.4 + */ + public Base64(final int lineLength, final byte[] lineSeparator) { + this(lineLength, lineSeparator, false); + } + + /** + * Creates a Base64 codec used for decoding (all modes) and encoding in URL-unsafe mode. + * + * <p>When encoding the line length and line separator are given in the constructor, and the + * encoding table is STANDARD_ENCODE_TABLE. + * + * <p>Line lengths that aren't multiples of 4 will still essentially end up being multiples of 4 + * in the encoded data. + * + * <p>When decoding all variants are supported. + * + * @param lineLength Each line of encoded data will be at most of the given length (rounded down + * to nearest multiple of 4). If lineLength <= 0, then the output will not be divided into + * lines (chunks). Ignored when decoding. + * @param lineSeparator Each line of encoded data will end with this sequence of bytes. + * @param urlSafe Instead of emitting '+' and '/' we emit '-' and '_' respectively. urlSafe is + * only applied to encode operations. Decoding seamlessly handles both modes. <b>Note: no + * padding is added when using the URL-safe alphabet.</b> + * @throws IllegalArgumentException The provided lineSeparator included some base64 characters. + * That's not going to work! + * @since 1.4 + */ + public Base64(final int lineLength, final byte[] lineSeparator, final boolean urlSafe) { + super( + BYTES_PER_UNENCODED_BLOCK, + BYTES_PER_ENCODED_BLOCK, + lineLength, + lineSeparator == null ? 0 : lineSeparator.length); + // TODO could be simplified if there is no requirement to reject invalid line sep when length + // <=0 + // @see test case Base64Test.testConstructors() + if (lineSeparator != null) { + if (containsAlphabetOrPad(lineSeparator)) { + final String sep = StringUtils.newStringUtf8(lineSeparator); + throw new IllegalArgumentException( + "lineSeparator must not contain base64 characters: [" + sep + "]"); + } + if (lineLength > 0) { // null line-sep forces no chunking rather than throwing IAE + this.encodeSize = BYTES_PER_ENCODED_BLOCK + lineSeparator.length; + this.lineSeparator = new byte[lineSeparator.length]; + System.arraycopy(lineSeparator, 0, this.lineSeparator, 0, lineSeparator.length); + } else { + this.encodeSize = BYTES_PER_ENCODED_BLOCK; + this.lineSeparator = null; + } + } else { + this.encodeSize = BYTES_PER_ENCODED_BLOCK; + this.lineSeparator = null; + } + this.decodeSize = this.encodeSize - 1; + this.encodeTable = urlSafe ? URL_SAFE_ENCODE_TABLE : STANDARD_ENCODE_TABLE; + } + + /** + * Returns our current encode mode. True if we're URL-SAFE, false otherwise. + * + * @return true if we're in URL-SAFE mode, false otherwise. + * @since 1.4 + */ + public boolean isUrlSafe() { + return this.encodeTable == URL_SAFE_ENCODE_TABLE; + } + + /** + * Encodes all of the provided data, starting at inPos, for inAvail bytes. Must be called at least + * twice: once with the data to encode, and once with inAvail set to "-1" to alert encoder that + * EOF has been reached, to flush last remaining bytes (if not multiple of 3). + * + * <p><b>Note: no padding is added when encoding using the URL-safe alphabet.</b> + * + * <p>Thanks to "commons" project in ws.apache.org for the bitwise operations, and general + * approach. http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ + * + * @param in byte[] array of binary data to base64 encode. + * @param inPos Position to start reading data from. + * @param inAvail Amount of bytes available from input for encoding. + * @param context the context to be used + */ + @Override + void encode(final byte[] in, int inPos, final int inAvail, final Context context) { + if (context.eof) { + return; + } + // inAvail < 0 is how we're informed of EOF in the underlying data we're + // encoding. + if (inAvail < 0) { + context.eof = true; + if (0 == context.modulus && lineLength == 0) { + return; // no leftovers to process and not using chunking + } + final byte[] buffer = ensureBufferSize(encodeSize, context); + final int savedPos = context.pos; + switch (context.modulus) { // 0-2 + case 0: // nothing to do here + break; + case 1: // 8 bits = 6 + 2 + // top 6 bits: + buffer[context.pos++] = encodeTable[(context.ibitWorkArea >> 2) & MASK_6BITS]; + // remaining 2: + buffer[context.pos++] = encodeTable[(context.ibitWorkArea << 4) & MASK_6BITS]; + // URL-SAFE skips the padding to further reduce size. + if (encodeTable == STANDARD_ENCODE_TABLE) { + buffer[context.pos++] = pad; + buffer[context.pos++] = pad; + } + break; + + case 2: // 16 bits = 6 + 6 + 4 + buffer[context.pos++] = encodeTable[(context.ibitWorkArea >> 10) & MASK_6BITS]; + buffer[context.pos++] = encodeTable[(context.ibitWorkArea >> 4) & MASK_6BITS]; + buffer[context.pos++] = encodeTable[(context.ibitWorkArea << 2) & MASK_6BITS]; + // URL-SAFE skips the padding to further reduce size. + if (encodeTable == STANDARD_ENCODE_TABLE) { + buffer[context.pos++] = pad; + } + break; + default: + throw new IllegalStateException("Impossible modulus " + context.modulus); + } + context.currentLinePos += context.pos - savedPos; // keep track of current line position + // if currentPos == 0 we are at the start of a line, so don't add CRLF + if (lineLength > 0 && context.currentLinePos > 0) { + System.arraycopy(lineSeparator, 0, buffer, context.pos, lineSeparator.length); + context.pos += lineSeparator.length; + } + } else { + for (int i = 0; i < inAvail; i++) { + final byte[] buffer = ensureBufferSize(encodeSize, context); + context.modulus = (context.modulus + 1) % BYTES_PER_UNENCODED_BLOCK; + int b = in[inPos++]; + if (b < 0) { + b += 256; + } + context.ibitWorkArea = (context.ibitWorkArea << 8) + b; // BITS_PER_BYTE + if (0 == context.modulus) { // 3 bytes = 24 bits = 4 * 6 bits to extract + buffer[context.pos++] = encodeTable[(context.ibitWorkArea >> 18) & MASK_6BITS]; + buffer[context.pos++] = encodeTable[(context.ibitWorkArea >> 12) & MASK_6BITS]; + buffer[context.pos++] = encodeTable[(context.ibitWorkArea >> 6) & MASK_6BITS]; + buffer[context.pos++] = encodeTable[context.ibitWorkArea & MASK_6BITS]; + context.currentLinePos += BYTES_PER_ENCODED_BLOCK; + if (lineLength > 0 && lineLength <= context.currentLinePos) { + System.arraycopy(lineSeparator, 0, buffer, context.pos, lineSeparator.length); + context.pos += lineSeparator.length; + context.currentLinePos = 0; + } + } + } + } + } + + /** + * Decodes all of the provided data, starting at inPos, for inAvail bytes. Should be called at + * least twice: once with the data to decode, and once with inAvail set to "-1" to alert decoder + * that EOF has been reached. The "-1" call is not necessary when decoding, but it doesn't hurt, + * either. + * + * <p>Ignores all non-base64 characters. This is how chunked (e.g. 76 character) data is handled, + * since CR and LF are silently ignored, but has implications for other bytes, too. This method + * subscribes to the garbage-in, garbage-out philosophy: it will not check the provided data for + * validity. + * + * <p>Thanks to "commons" project in ws.apache.org for the bitwise operations, and general + * approach. http://svn.apache.org/repos/asf/webservices/commons/trunk/modules/util/ + * + * @param in byte[] array of ascii data to base64 decode. + * @param inPos Position to start reading data from. + * @param inAvail Amount of bytes available from input for encoding. + * @param context the context to be used + */ + @Override + void decode(final byte[] in, int inPos, final int inAvail, final Context context) { + if (context.eof) { + return; + } + if (inAvail < 0) { + context.eof = true; + } + for (int i = 0; i < inAvail; i++) { + final byte[] buffer = ensureBufferSize(decodeSize, context); + final byte b = in[inPos++]; + if (b == pad) { + // We're done. + context.eof = true; + break; + } + if (b >= 0 && b < DECODE_TABLE.length) { + final int result = DECODE_TABLE[b]; + if (result >= 0) { + context.modulus = (context.modulus + 1) % BYTES_PER_ENCODED_BLOCK; + context.ibitWorkArea = (context.ibitWorkArea << BITS_PER_ENCODED_BYTE) + result; + if (context.modulus == 0) { + buffer[context.pos++] = (byte) ((context.ibitWorkArea >> 16) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.ibitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) (context.ibitWorkArea & MASK_8BITS); + } + } + } + } + + // Two forms of EOF as far as base64 decoder is concerned: actual + // EOF (-1) and first time '=' character is encountered in stream. + // This approach makes the '=' padding characters completely optional. + if (context.eof && context.modulus != 0) { + final byte[] buffer = ensureBufferSize(decodeSize, context); + + // We have some spare bits remaining + // Output all whole multiples of 8 bits and ignore the rest + switch (context.modulus) { + // case 0 : // impossible, as excluded above + case 1: // 6 bits - ignore entirely + // TODO not currently tested; perhaps it is impossible? + break; + case 2: // 12 bits = 8 + 4 + context.ibitWorkArea = context.ibitWorkArea >> 4; // dump the extra 4 bits + buffer[context.pos++] = (byte) ((context.ibitWorkArea) & MASK_8BITS); + break; + case 3: // 18 bits = 8 + 8 + 2 + context.ibitWorkArea = context.ibitWorkArea >> 2; // dump 2 bits + buffer[context.pos++] = (byte) ((context.ibitWorkArea >> 8) & MASK_8BITS); + buffer[context.pos++] = (byte) ((context.ibitWorkArea) & MASK_8BITS); + break; + default: + throw new IllegalStateException("Impossible modulus " + context.modulus); + } + } + } + + /** + * Tests a given byte array to see if it contains only valid characters within the Base64 + * alphabet. Currently the method treats whitespace as valid. + * + * @param arrayOctet byte array to test + * @return <code>true</code> if all bytes are valid characters in the Base64 alphabet or if the + * byte array is empty; <code>false</code>, otherwise + * @deprecated 1.5 Use {@link #isBase64(byte[])}, will be removed in 2.0. + */ + @Deprecated + public static boolean isArrayByteBase64(final byte[] arrayOctet) { + return isBase64(arrayOctet); + } + + /** + * Returns whether or not the <code>octet</code> is in the base 64 alphabet. + * + * @param octet The value to test + * @return <code>true</code> if the value is defined in the the base 64 alphabet, <code>false + * </code> otherwise. + * @since 1.4 + */ + public static boolean isBase64(final byte octet) { + return octet == PAD_DEFAULT + || (octet >= 0 && octet < DECODE_TABLE.length && DECODE_TABLE[octet] != -1); + } + + /** + * Tests a given String to see if it contains only valid characters within the Base64 alphabet. + * Currently the method treats whitespace as valid. + * + * @param base64 String to test + * @return <code>true</code> if all characters in the String are valid characters in the Base64 + * alphabet or if the String is empty; <code>false</code>, otherwise + * @since 1.5 + */ + public static boolean isBase64(final String base64) { + return isBase64(StringUtils.getBytesUtf8(base64)); + } + + /** + * Tests a given byte array to see if it contains only valid characters within the Base64 + * alphabet. Currently the method treats whitespace as valid. + * + * @param arrayOctet byte array to test + * @return <code>true</code> if all bytes are valid characters in the Base64 alphabet or if the + * byte array is empty; <code>false</code>, otherwise + * @since 1.5 + */ + public static boolean isBase64(final byte[] arrayOctet) { + for (int i = 0; i < arrayOctet.length; i++) { + if (!isBase64(arrayOctet[i]) && !isWhiteSpace(arrayOctet[i])) { + return false; + } + } + return true; + } + + /** + * Encodes binary data using the base64 algorithm but does not chunk the output. + * + * @param binaryData binary data to encode + * @return byte[] containing Base64 characters in their UTF-8 representation. + */ + public static byte[] encodeBase64(final byte[] binaryData) { + return encodeBase64(binaryData, false); + } + + /** + * Encodes binary data using the base64 algorithm but does not chunk the output. + * + * <p>NOTE: We changed the behaviour of this method from multi-line chunking (commons-codec-1.4) + * to single-line non-chunking (commons-codec-1.5). + * + * @param binaryData binary data to encode + * @return String containing Base64 characters. + * @since 1.4 (NOTE: 1.4 chunked the output, whereas 1.5 does not). + */ + public static String encodeBase64String(final byte[] binaryData) { + return StringUtils.newStringUsAscii(encodeBase64(binaryData, false)); + } + + /** + * Encodes binary data using a URL-safe variation of the base64 algorithm but does not chunk the + * output. The url-safe variation emits - and _ instead of + and / characters. <b>Note: no padding + * is added.</b> + * + * @param binaryData binary data to encode + * @return byte[] containing Base64 characters in their UTF-8 representation. + * @since 1.4 + */ + public static byte[] encodeBase64URLSafe(final byte[] binaryData) { + return encodeBase64(binaryData, false, true); + } + + /** + * Encodes binary data using a URL-safe variation of the base64 algorithm but does not chunk the + * output. The url-safe variation emits - and _ instead of + and / characters. <b>Note: no padding + * is added.</b> + * + * @param binaryData binary data to encode + * @return String containing Base64 characters + * @since 1.4 + */ + public static String encodeBase64URLSafeString(final byte[] binaryData) { + return StringUtils.newStringUsAscii(encodeBase64(binaryData, false, true)); + } + + /** + * Encodes binary data using the base64 algorithm and chunks the encoded output into 76 character + * blocks + * + * @param binaryData binary data to encode + * @return Base64 characters chunked in 76 character blocks + */ + public static byte[] encodeBase64Chunked(final byte[] binaryData) { + return encodeBase64(binaryData, true); + } + + /** + * Encodes binary data using the base64 algorithm, optionally chunking the output into 76 + * character blocks. + * + * @param binaryData Array containing binary data to encode. + * @param isChunked if <code>true</code> this encoder will chunk the base64 output into 76 + * character blocks + * @return Base64-encoded data. + * @throws IllegalArgumentException Thrown when the input array needs an output array bigger than + * {@link Integer#MAX_VALUE} + */ + public static byte[] encodeBase64(final byte[] binaryData, final boolean isChunked) { + return encodeBase64(binaryData, isChunked, false); + } + + /** + * Encodes binary data using the base64 algorithm, optionally chunking the output into 76 + * character blocks. + * + * @param binaryData Array containing binary data to encode. + * @param isChunked if <code>true</code> this encoder will chunk the base64 output into 76 + * character blocks + * @param urlSafe if <code>true</code> this encoder will emit - and _ instead of the usual + and / + * characters. <b>Note: no padding is added when encoding using the URL-safe alphabet.</b> + * @return Base64-encoded data. + * @throws IllegalArgumentException Thrown when the input array needs an output array bigger than + * {@link Integer#MAX_VALUE} + * @since 1.4 + */ + public static byte[] encodeBase64( + final byte[] binaryData, final boolean isChunked, final boolean urlSafe) { + return encodeBase64(binaryData, isChunked, urlSafe, Integer.MAX_VALUE); + } + + /** + * Encodes binary data using the base64 algorithm, optionally chunking the output into 76 + * character blocks. + * + * @param binaryData Array containing binary data to encode. + * @param isChunked if <code>true</code> this encoder will chunk the base64 output into 76 + * character blocks + * @param urlSafe if <code>true</code> this encoder will emit - and _ instead of the usual + and / + * characters. <b>Note: no padding is added when encoding using the URL-safe alphabet.</b> + * @param maxResultSize The maximum result size to accept. + * @return Base64-encoded data. + * @throws IllegalArgumentException Thrown when the input array needs an output array bigger than + * maxResultSize + * @since 1.4 + */ + public static byte[] encodeBase64( + final byte[] binaryData, + final boolean isChunked, + final boolean urlSafe, + final int maxResultSize) { + if (binaryData == null || binaryData.length == 0) { + return binaryData; + } + + // Create this so can use the super-class method + // Also ensures that the same roundings are performed by the ctor and the code + final Base64 b64 = isChunked ? new Base64(urlSafe) : new Base64(0, CHUNK_SEPARATOR, urlSafe); + final long len = b64.getEncodedLength(binaryData); + if (len > maxResultSize) { + throw new IllegalArgumentException( + "Input array too big, the output array would be bigger (" + + len + + ") than the specified maximum size of " + + maxResultSize); + } + + return b64.encode(binaryData); + } + + /** + * Decodes a Base64 String into octets. + * + * <p><b>Note:</b> this method seamlessly handles data encoded in URL-safe or normal mode. + * + * @param base64String String containing Base64 data + * @return Array containing decoded data. + * @since 1.4 + */ + public static byte[] decodeBase64(final String base64String) { + return new Base64().decode(base64String); + } + + /** + * Decodes Base64 data into octets. + * + * <p><b>Note:</b> this method seamlessly handles data encoded in URL-safe or normal mode. + * + * @param base64Data Byte array containing Base64 data + * @return Array containing decoded data. + */ + public static byte[] decodeBase64(final byte[] base64Data) { + return new Base64().decode(base64Data); + } + + // Implementation of the Encoder Interface + + // Implementation of integer encoding used for crypto + /** + * Decodes a byte64-encoded integer according to crypto standards such as W3C's XML-Signature. + * + * @param pArray a byte array containing base64 character data + * @return A BigInteger + * @since 1.4 + */ + public static BigInteger decodeInteger(final byte[] pArray) { + return new BigInteger(1, decodeBase64(pArray)); + } + + /** + * Encodes to a byte64-encoded integer according to crypto standards such as W3C's XML-Signature. + * + * @param bigInt a BigInteger + * @return A byte array containing base64 character data + * @throws NullPointerException if null is passed in + * @since 1.4 + */ + public static byte[] encodeInteger(final BigInteger bigInt) { + if (bigInt == null) { + throw new NullPointerException("encodeInteger called with null parameter"); + } + return encodeBase64(toIntegerBytes(bigInt), false); + } + + /** + * Returns a byte-array representation of a <code>BigInteger</code> without sign bit. + * + * @param bigInt <code>BigInteger</code> to be converted + * @return a byte array representation of the BigInteger parameter + */ + static byte[] toIntegerBytes(final BigInteger bigInt) { + int bitlen = bigInt.bitLength(); + // round bitlen + bitlen = ((bitlen + 7) >> 3) << 3; + final byte[] bigBytes = bigInt.toByteArray(); + + if (((bigInt.bitLength() % 8) != 0) && (((bigInt.bitLength() / 8) + 1) == (bitlen / 8))) { + return bigBytes; + } + // set up params for copying everything but sign bit + int startSrc = 0; + int len = bigBytes.length; + + // if bigInt is exactly byte-aligned, just skip signbit in copy + if ((bigInt.bitLength() % 8) == 0) { + startSrc = 1; + len--; + } + final int startDst = bitlen / 8 - len; // to pad w/ nulls as per spec + final byte[] resizedBytes = new byte[bitlen / 8]; + System.arraycopy(bigBytes, startSrc, resizedBytes, startDst, len); + return resizedBytes; + } + + /** + * Returns whether or not the <code>octet</code> is in the Base64 alphabet. + * + * @param octet The value to test + * @return <code>true</code> if the value is defined in the the Base64 alphabet <code>false</code> + * otherwise. + */ + @Override + protected boolean isInAlphabet(final byte octet) { + return octet >= 0 && octet < decodeTable.length && decodeTable[octet] != -1; + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/BaseNCodec.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/BaseNCodec.java new file mode 100644 index 0000000..704fd25 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/BaseNCodec.java @@ -0,0 +1,512 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase.binary; + +import io.ipfs.multibase.BinaryDecoder; +import io.ipfs.multibase.BinaryEncoder; +import io.ipfs.multibase.DecoderException; +import io.ipfs.multibase.EncoderException; + +/** + * Abstract superclass for Base-N encoders and decoders. + * + * <p>From https://commons.apache.org/proper/commons-codec/ + * + * <p>This class is thread-safe. + * + * @version $Id$ + */ +public abstract class BaseNCodec implements BinaryEncoder, BinaryDecoder { + + /** + * Holds thread context so classes can be thread-safe. + * + * <p>This class is not itself thread-safe; each thread must allocate its own copy. + * + * @since 1.7 + */ + static class Context { + + /** + * Place holder for the bytes we're dealing with for our based logic. Bitwise operations store + * and extract the encoding or decoding from this variable. + */ + int ibitWorkArea; + + /** + * Place holder for the bytes we're dealing with for our based logic. Bitwise operations store + * and extract the encoding or decoding from this variable. + */ + long lbitWorkArea; + + /** Buffer for streaming. */ + byte[] buffer; + + /** Position where next character should be written in the buffer. */ + int pos; + + /** Position where next character should be read from the buffer. */ + int readPos; + + /** + * Boolean flag to indicate the EOF has been reached. Once EOF has been reached, this object + * becomes useless, and must be thrown away. + */ + boolean eof; + + /** + * Variable tracks how many characters have been written to the current line. Only used when + * encoding. We use it to make sure each encoded line never goes beyond lineLength (if + * lineLength > 0). + */ + int currentLinePos; + + /** + * Writes to the buffer only occur after every 3/5 reads when encoding, and every 4/8 reads when + * decoding. This variable helps track that. + */ + int modulus; + + Context() {} + } + + /** + * EOF + * + * @since 1.7 + */ + static final int EOF = -1; + + /** + * MIME chunk size per RFC 2045 section 6.8. + * + * <p>The {@value} character limit does not count the trailing CRLF, but counts all other + * characters, including any equal signs. + * + * @see <a href="http://www.ietf.org/rfc/rfc2045.txt">RFC 2045 section 6.8</a> + */ + public static final int MIME_CHUNK_SIZE = 76; + + /** + * PEM chunk size per RFC 1421 section 4.3.2.4. + * + * <p>The {@value} character limit does not count the trailing CRLF, but counts all other + * characters, including any equal signs. + * + * @see <a href="http://tools.ietf.org/html/rfc1421">RFC 1421 section 4.3.2.4</a> + */ + public static final int PEM_CHUNK_SIZE = 64; + + private static final int DEFAULT_BUFFER_RESIZE_FACTOR = 2; + + /** + * Defines the default buffer size - currently {@value} - must be large enough for at least one + * encoded block+separator + */ + private static final int DEFAULT_BUFFER_SIZE = 8192; + + /** Mask used to extract 8 bits, used in decoding bytes */ + protected static final int MASK_8BITS = 0xff; + + /** Byte used to pad output. */ + protected static final byte PAD_DEFAULT = '='; // Allow static access to default + + /** + * @deprecated Use {@link #pad}. Will be removed in 2.0. + */ + @Deprecated + protected final byte PAD = PAD_DEFAULT; // instance variable just in case it needs to vary later + + protected final byte pad; // instance variable just in case it needs to vary later + + /** Number of bytes in each full block of unencoded data, e.g. 4 for Base64 and 5 for Base32 */ + private final int unencodedBlockSize; + + /** Number of bytes in each full block of encoded data, e.g. 3 for Base64 and 8 for Base32 */ + private final int encodedBlockSize; + + /** + * Chunksize for encoding. Not used when decoding. A value of zero or less implies no chunking of + * the encoded data. Rounded down to nearest multiple of encodedBlockSize. + */ + protected final int lineLength; + + /** Size of chunk separator. Not used unless {@link #lineLength} > 0. */ + private final int chunkSeparatorLength; + + /** + * Note <code>lineLength</code> is rounded down to the nearest multiple of {@link + * #encodedBlockSize} If <code>chunkSeparatorLength</code> is zero, then chunking is disabled. + * + * @param unencodedBlockSize the size of an unencoded block (e.g. Base64 = 3) + * @param encodedBlockSize the size of an encoded block (e.g. Base64 = 4) + * @param lineLength if > 0, use chunking with a length <code>lineLength</code> + * @param chunkSeparatorLength the chunk separator length, if relevant + */ + protected BaseNCodec( + final int unencodedBlockSize, + final int encodedBlockSize, + final int lineLength, + final int chunkSeparatorLength) { + this(unencodedBlockSize, encodedBlockSize, lineLength, chunkSeparatorLength, PAD_DEFAULT); + } + + /** + * Note <code>lineLength</code> is rounded down to the nearest multiple of {@link + * #encodedBlockSize} If <code>chunkSeparatorLength</code> is zero, then chunking is disabled. + * + * @param unencodedBlockSize the size of an unencoded block (e.g. Base64 = 3) + * @param encodedBlockSize the size of an encoded block (e.g. Base64 = 4) + * @param lineLength if > 0, use chunking with a length <code>lineLength</code> + * @param chunkSeparatorLength the chunk separator length, if relevant + * @param pad byte used as padding byte. + */ + protected BaseNCodec( + final int unencodedBlockSize, + final int encodedBlockSize, + final int lineLength, + final int chunkSeparatorLength, + final byte pad) { + this.unencodedBlockSize = unencodedBlockSize; + this.encodedBlockSize = encodedBlockSize; + final boolean useChunking = lineLength > 0 && chunkSeparatorLength > 0; + this.lineLength = useChunking ? (lineLength / encodedBlockSize) * encodedBlockSize : 0; + this.chunkSeparatorLength = chunkSeparatorLength; + + this.pad = pad; + } + + /** + * Returns true if this object has buffered data for reading. + * + * @param context the context to be used + * @return true if there is data still available for reading. + */ + boolean hasData(final Context context) { // package protected for access from I/O streams + return context.buffer != null; + } + + /** + * Returns the amount of buffered data available for reading. + * + * @param context the context to be used + * @return The amount of buffered data available for reading. + */ + int available(final Context context) { // package protected for access from I/O streams + return context.buffer != null ? context.pos - context.readPos : 0; + } + + /** + * Get the default buffer size. Can be overridden. + * + * @return {@link #DEFAULT_BUFFER_SIZE} + */ + protected int getDefaultBufferSize() { + return DEFAULT_BUFFER_SIZE; + } + + /** + * Increases our buffer by the {@link #DEFAULT_BUFFER_RESIZE_FACTOR}. + * + * @param context the context to be used + */ + private byte[] resizeBuffer(final Context context) { + if (context.buffer == null) { + context.buffer = new byte[getDefaultBufferSize()]; + context.pos = 0; + context.readPos = 0; + } else { + final byte[] b = new byte[context.buffer.length * DEFAULT_BUFFER_RESIZE_FACTOR]; + System.arraycopy(context.buffer, 0, b, 0, context.buffer.length); + context.buffer = b; + } + return context.buffer; + } + + /** + * Ensure that the buffer has room for <code>size</code> bytes + * + * @param size minimum spare space required + * @param context the context to be used + * @return the buffer + */ + protected byte[] ensureBufferSize(final int size, final Context context) { + if ((context.buffer == null) || (context.buffer.length < context.pos + size)) { + return resizeBuffer(context); + } + return context.buffer; + } + + /** + * Extracts buffered data into the provided byte[] array, starting at position bPos, up to a + * maximum of bAvail bytes. Returns how many bytes were actually extracted. + * + * <p>Package protected for access from I/O streams. + * + * @param b byte[] array to extract the buffered data into. + * @param bPos position in byte[] array to start extraction at. + * @param bAvail amount of bytes we're allowed to extract. We may extract fewer (if fewer are + * available). + * @param context the context to be used + * @return The number of bytes successfully extracted into the provided byte[] array. + */ + int readResults(final byte[] b, final int bPos, final int bAvail, final Context context) { + if (context.buffer != null) { + final int len = Math.min(available(context), bAvail); + System.arraycopy(context.buffer, context.readPos, b, bPos, len); + context.readPos += len; + if (context.readPos >= context.pos) { + context.buffer = null; // so hasData() will return false, and this method can return -1 + } + return len; + } + return context.eof ? EOF : 0; + } + + /** + * Checks if a byte value is whitespace or not. Whitespace is taken to mean: space, tab, CR, LF + * + * @param byteToCheck the byte to check + * @return true if byte is whitespace, false otherwise + */ + protected static boolean isWhiteSpace(final byte byteToCheck) { + switch (byteToCheck) { + case ' ': + case '\n': + case '\r': + case '\t': + return true; + default: + return false; + } + } + + /** + * Encodes an Object using the Base-N algorithm. This method is provided in order to satisfy the + * requirements of the Encoder interface, and will throw an EncoderException if the supplied + * object is not of type byte[]. + * + * @param obj Object to encode + * @return An object (of type byte[]) containing the Base-N encoded data which corresponds to the + * byte[] supplied. + * @throws EncoderException if the parameter supplied is not of type byte[] + */ + @Override + public Object encode(final Object obj) throws EncoderException { + if (!(obj instanceof byte[])) { + throw new EncoderException("Parameter supplied to Base-N encode is not a byte[]"); + } + return encode((byte[]) obj); + } + + /** + * Encodes a byte[] containing binary data, into a String containing characters in the Base-N + * alphabet. Uses UTF8 encoding. + * + * @param pArray a byte array containing binary data + * @return A String containing only Base-N character data + */ + public String encodeToString(final byte[] pArray) { + return StringUtils.newStringUtf8(encode(pArray)); + } + + /** + * Encodes a byte[] containing binary data, into a String containing characters in the appropriate + * alphabet. Uses UTF8 encoding. + * + * @param pArray a byte array containing binary data + * @return String containing only character data in the appropriate alphabet. + * @since 1.5 This is a duplicate of {@link #encodeToString(byte[])}; it was merged during + * refactoring. + */ + public String encodeAsString(final byte[] pArray) { + return StringUtils.newStringUtf8(encode(pArray)); + } + + /** + * Decodes an Object using the Base-N algorithm. This method is provided in order to satisfy the + * requirements of the Decoder interface, and will throw a DecoderException if the supplied object + * is not of type byte[] or String. + * + * @param obj Object to decode + * @return An object (of type byte[]) containing the binary data which corresponds to the byte[] + * or String supplied. + * @throws DecoderException if the parameter supplied is not of type byte[] + */ + @Override + public Object decode(final Object obj) throws DecoderException { + if (obj instanceof byte[]) { + return decode((byte[]) obj); + } else if (obj instanceof String) { + return decode((String) obj); + } else { + throw new DecoderException("Parameter supplied to Base-N decode is not a byte[] or a String"); + } + } + + /** + * Decodes a String containing characters in the Base-N alphabet. + * + * @param pArray A String containing Base-N character data + * @return a byte array containing binary data + */ + public byte[] decode(final String pArray) { + return decode(StringUtils.getBytesUtf8(pArray)); + } + + /** + * Decodes a byte[] containing characters in the Base-N alphabet. + * + * @param pArray A byte array containing Base-N character data + * @return a byte array containing binary data + */ + @Override + public byte[] decode(final byte[] pArray) { + if (pArray == null || pArray.length == 0) { + return pArray; + } + final Context context = new Context(); + decode(pArray, 0, pArray.length, context); + decode(pArray, 0, EOF, context); // Notify decoder of EOF. + final byte[] result = new byte[context.pos]; + readResults(result, 0, result.length, context); + return result; + } + + /** + * Encodes a byte[] containing binary data, into a byte[] containing characters in the alphabet. + * + * @param pArray a byte array containing binary data + * @return A byte array containing only the base N alphabetic character data + */ + @Override + public byte[] encode(final byte[] pArray) { + if (pArray == null || pArray.length == 0) { + return pArray; + } + return encode(pArray, 0, pArray.length); + } + + /** + * Encodes a byte[] containing binary data, into a byte[] containing characters in the alphabet. + * + * @param pArray a byte array containing binary data + * @param offset initial offset of the subarray. + * @param length length of the subarray. + * @return A byte array containing only the base N alphabetic character data + * @since 1.11 + */ + public byte[] encode(final byte[] pArray, final int offset, final int length) { + if (pArray == null || pArray.length == 0) { + return pArray; + } + final Context context = new Context(); + encode(pArray, offset, length, context); + encode(pArray, offset, EOF, context); // Notify encoder of EOF. + final byte[] buf = new byte[context.pos - context.readPos]; + readResults(buf, 0, buf.length, context); + return buf; + } + + // package protected for access from I/O streams + abstract void encode(byte[] pArray, int i, int length, Context context); + + // package protected for access from I/O streams + abstract void decode(byte[] pArray, int i, int length, Context context); + + /** + * Returns whether or not the <code>octet</code> is in the current alphabet. Does not allow + * whitespace or pad. + * + * @param value The value to test + * @return <code>true</code> if the value is defined in the current alphabet, <code>false</code> + * otherwise. + */ + protected abstract boolean isInAlphabet(byte value); + + /** + * Tests a given byte array to see if it contains only valid characters within the alphabet. The + * method optionally treats whitespace and pad as valid. + * + * @param arrayOctet byte array to test + * @param allowWSPad if <code>true</code>, then whitespace and PAD are also allowed + * @return <code>true</code> if all bytes are valid characters in the alphabet or if the byte + * array is empty; <code>false</code>, otherwise + */ + public boolean isInAlphabet(final byte[] arrayOctet, final boolean allowWSPad) { + for (final byte octet : arrayOctet) { + if (!isInAlphabet(octet) && (!allowWSPad || (octet != pad) && !isWhiteSpace(octet))) { + return false; + } + } + return true; + } + + /** + * Tests a given String to see if it contains only valid characters within the alphabet. The + * method treats whitespace and PAD as valid. + * + * @param basen String to test + * @return <code>true</code> if all characters in the String are valid characters in the alphabet + * or if the String is empty; <code>false</code>, otherwise + * @see #isInAlphabet(byte[], boolean) + */ + public boolean isInAlphabet(final String basen) { + return isInAlphabet(StringUtils.getBytesUtf8(basen), true); + } + + /** + * Tests a given byte array to see if it contains any characters within the alphabet or PAD. + * + * <p>Intended for use in checking line-ending arrays + * + * @param arrayOctet byte array to test + * @return <code>true</code> if any byte is a valid character in the alphabet or PAD; <code>false + * </code> otherwise + */ + protected boolean containsAlphabetOrPad(final byte[] arrayOctet) { + if (arrayOctet == null) { + return false; + } + for (final byte element : arrayOctet) { + if (pad == element || isInAlphabet(element)) { + return true; + } + } + return false; + } + + /** + * Calculates the amount of space needed to encode the supplied array. + * + * @param pArray byte[] array which will later be encoded + * @return amount of space needed to encoded the supplied array. Returns a long since a max-len + * array will require > Integer.MAX_VALUE + */ + public long getEncodedLength(final byte[] pArray) { + // Calculate non-chunked size - rounded up to allow for padding + // cast to long is needed to avoid possibility of overflow + long len = + ((pArray.length + unencodedBlockSize - 1) / unencodedBlockSize) * (long) encodedBlockSize; + if (lineLength > 0) { // We're using chunking + // Round up to nearest multiple + len += ((len + lineLength - 1) / lineLength) * chunkSeparatorLength; + } + return len; + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/StringUtils.java b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/StringUtils.java new file mode 100644 index 0000000..9d8bd67 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multibase/binary/StringUtils.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.ipfs.multibase.binary; + +import io.ipfs.multibase.CharEncoding; +import io.ipfs.multibase.Charsets; +import java.nio.charset.Charset; + +/** + * Converts String to and from bytes using the encodings required by the Java specification. These + * encodings are specified in <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a>. + * + * <p>This class is immutable and thread-safe. + * + * @see CharEncoding + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + * @version $Id$ + * @since 1.4 + */ +public class StringUtils { + + /** + * Calls {@link String#getBytes(Charset)} + * + * @param string The string to encode (if null, return null). + * @param charset The {@link Charset} to encode the <code>String</code> + * @return the encoded bytes + */ + private static byte[] getBytes(final String string, final Charset charset) { + if (string == null) { + return null; + } + return string.getBytes(charset); + } + + /** + * Encodes the given string into a sequence of bytes using the UTF-8 charset, storing the result + * into a new byte array. + * + * @param string the String to encode, may be <code>null</code> + * @return encoded bytes, or <code>null</code> if the input string was <code>null</code> + * @throws NullPointerException Thrown if {@link Charsets#UTF_8} is not initialized, which should + * never happen since it is required by the Java platform specification. + * @since As of 1.7, throws {@link NullPointerException} instead of UnsupportedEncodingException + * @see <a + * href="http://download.oracle.com/javase/7/docs/api/java/nio/charset/Charset.html">Standard + * charsets</a> + */ + public static byte[] getBytesUtf8(final String string) { + return getBytes(string, Charset.forName("UTF-8")); + } + + /** + * Constructs a new <code>String</code> by decoding the specified array of bytes using the given + * charset. + * + * @param bytes The bytes to be decoded into characters + * @param charset The {@link Charset} to encode the <code>String</code>; not {@code null} + * @return A new <code>String</code> decoded from the specified array of bytes using the given + * charset, or <code>null</code> if the input byte array was <code>null</code>. + * @throws NullPointerException Thrown if charset is {@code null} + */ + private static String newString(final byte[] bytes, final Charset charset) { + return bytes == null ? null : new String(bytes, charset); + } + + /** + * Constructs a new <code>String</code> by decoding the specified array of bytes using the + * US-ASCII charset. + * + * @param bytes The bytes to be decoded into characters + * @return A new <code>String</code> decoded from the specified array of bytes using the US-ASCII + * charset, or <code>null</code> if the input byte array was <code>null</code>. + * @throws NullPointerException Thrown if {@link Charsets#US_ASCII} is not initialized, which + * should never happen since it is required by the Java platform specification. + * @since As of 1.7, throws {@link NullPointerException} instead of UnsupportedEncodingException + */ + public static String newStringUsAscii(final byte[] bytes) { + return newString(bytes, Charset.forName("US-ASCII")); + } + + /** + * Constructs a new <code>String</code> by decoding the specified array of bytes using the UTF-8 + * charset. + * + * @param bytes The bytes to be decoded into characters + * @return A new <code>String</code> decoded from the specified array of bytes using the UTF-8 + * charset, or <code>null</code> if the input byte array was <code>null</code>. + * @throws NullPointerException Thrown if {@link Charsets#UTF_8} is not initialized, which should + * never happen since it is required by the Java platform specification. + * @since As of 1.7, throws {@link NullPointerException} instead of UnsupportedEncodingException + */ + public static String newStringUtf8(final byte[] bytes) { + return newString(bytes, Charset.forName("UTF-8")); + } +} diff --git a/modules/vendor-nabu/src/main/java/io/ipfs/multihash/Multihash.java b/modules/vendor-nabu/src/main/java/io/ipfs/multihash/Multihash.java new file mode 100644 index 0000000..7894a74 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/io/ipfs/multihash/Multihash.java @@ -0,0 +1,313 @@ +package io.ipfs.multihash; + +import io.ipfs.multibase.Base16; +import io.ipfs.multibase.Base58; +import io.ipfs.multibase.Multibase; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +public class Multihash { + public static final int MAX_IDENTITY_HASH_LENGTH = 1024 * 1024; + + // From https://github.com/multiformats/multicodec/blob/master/table.csv + public enum Type { + id(0, -1), + md5(0xd5, 16), + sha1(0x11, 20), + sha2_256(0x12, 32), + sha2_512(0x13, 64), + dbl_sha2_256(0x56, 32), + sha3_224(0x17, 24), + sha3_256(0x16, 32), + sha3_512(0x14, 64), + shake_128(0x18, 32), + shake_256(0x19, 64), + keccak_224(0x1a, 24), + keccak_256(0x1b, 32), + keccak_384(0x1c, 48), + keccak_512(0x1d, 64), + blake3(0x1e, 32), + murmur3(0x22, 4), + + // blake2b (64 codes) + blake2b_8(0xb201, 1), + blake2b_16(0xb202, 2), + blake2b_24(0xb203, 3), + blake2b_32(0xb204, 4), + blake2b_40(0xb205, 5), + blake2b_48(0xb206, 6), + blake2b_56(0xb207, 7), + blake2b_64(0xb208, 8), + blake2b_72(0xb209, 9), + blake2b_80(0xb20a, 10), + blake2b_88(0xb20b, 11), + blake2b_96(0xb20c, 12), + blake2b_104(0xb20d, 13), + blake2b_112(0xb20e, 14), + blake2b_120(0xb20f, 15), + blake2b_128(0xb210, 16), + blake2b_136(0xb211, 17), + blake2b_144(0xb212, 18), + blake2b_152(0xb213, 19), + blake2b_160(0xb214, 20), + blake2b_168(0xb215, 21), + blake2b_176(0xb216, 22), + blake2b_184(0xb217, 23), + blake2b_192(0xb218, 24), + blake2b_200(0xb219, 25), + blake2b_208(0xb21a, 26), + blake2b_216(0xb21b, 27), + blake2b_224(0xb21c, 28), + blake2b_232(0xb21d, 29), + blake2b_240(0xb21e, 30), + blake2b_248(0xb21f, 31), + blake2b_256(0xb220, 32), + blake2b_264(0xb221, 33), + blake2b_272(0xb222, 34), + blake2b_280(0xb223, 35), + blake2b_288(0xb224, 36), + blake2b_296(0xb225, 37), + blake2b_304(0xb226, 38), + blake2b_312(0xb227, 39), + blake2b_320(0xb228, 40), + blake2b_328(0xb229, 41), + blake2b_336(0xb22a, 42), + blake2b_344(0xb22b, 43), + blake2b_352(0xb22c, 44), + blake2b_360(0xb22d, 45), + blake2b_368(0xb22e, 46), + blake2b_376(0xb22f, 47), + blake2b_384(0xb230, 48), + blake2b_392(0xb231, 49), + blake2b_400(0xb232, 50), + blake2b_408(0xb233, 51), + blake2b_416(0xb234, 52), + blake2b_424(0xb235, 53), + blake2b_432(0xb236, 54), + blake2b_440(0xb237, 55), + blake2b_448(0xb238, 56), + blake2b_456(0xb239, 57), + blake2b_464(0xb23a, 58), + blake2b_472(0xb23b, 59), + blake2b_480(0xb23c, 60), + blake2b_488(0xb23d, 61), + blake2b_496(0xb23e, 62), + blake2b_504(0xb23f, 63), + blake2b_512(0xb240, 64), + + // blake2s (32 codes) + blake2s_8(0xb241, 1), + blake2s_16(0xb242, 2), + blake2s_24(0xb243, 3), + blake2s_32(0xb244, 4), + blake2s_40(0xb245, 5), + blake2s_48(0xb246, 6), + blake2s_56(0xb247, 7), + blake2s_64(0xb248, 8), + blake2s_72(0xb249, 9), + blake2s_80(0xb24a, 10), + blake2s_88(0xb24b, 11), + blake2s_96(0xb24c, 12), + blake2s_104(0xb24d, 13), + blake2s_112(0xb24e, 14), + blake2s_120(0xb24f, 15), + blake2s_128(0xb250, 16), + blake2s_136(0xb251, 17), + blake2s_144(0xb252, 18), + blake2s_152(0xb253, 19), + blake2s_160(0xb254, 20), + blake2s_168(0xb255, 21), + blake2s_176(0xb256, 22), + blake2s_184(0xb257, 23), + blake2s_192(0xb258, 24), + blake2s_200(0xb259, 25), + blake2s_208(0xb25a, 26), + blake2s_216(0xb25b, 27), + blake2s_224(0xb25c, 28), + blake2s_232(0xb25d, 29), + blake2s_240(0xb25e, 30), + blake2s_248(0xb25f, 31), + blake2s_256(0xb260, 32), + + // Murmur + murmur3_x64_64(0x22, 8), + murmur3_x64_128(0x1022, 16); // DRAFT status + public final int index, length; + + Type(final int index, final int length) { + this.index = index; + this.length = length; + } + + private static Map<Integer, Type> lookup = new HashMap<>(); + + static { + for (Type t : Type.values()) lookup.put(t.index, t); + } + + public static Type lookup(int t) { + Type type = lookup.get(t); + if (type == null) + throw new IllegalStateException(String.format("Unknown Multihash type: 0x%x", t)); + return type; + } + } + + private final Type type; + private final byte[] hash; + + public Multihash(Type type, byte[] hash) { + if (hash.length > 127 && type != Type.id) + throw new IllegalStateException("Unsupported hash size: " + hash.length); + if (hash.length > MAX_IDENTITY_HASH_LENGTH) + throw new IllegalStateException("Unsupported hash size: " + hash.length); + if (hash.length != type.length && type != Type.id) + throw new IllegalStateException( + "Incorrect hash length: " + hash.length + " != " + type.length); + this.type = type; + this.hash = hash; + } + + public Multihash(Multihash toClone) { + this(toClone.type, toClone.hash); // N.B. despite being a byte[], hash is immutable + } + + public byte[] toBytes() { + try { + ByteArrayOutputStream res = new ByteArrayOutputStream(); + putUvarint(res, type.index); + putUvarint(res, hash.length); + res.write(hash); + return res.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Type getType() { + return type; + } + + public byte[] getHash() { + return Arrays.copyOf(hash, hash.length); + } + + public void serialize(OutputStream out) { + try { + putUvarint(out, type.index); + putUvarint(out, hash.length); + out.write(hash); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public Multihash bareMultihash() { + return this; + } + + public static Multihash deserialize(InputStream din) throws IOException { + int type = (int) readVarint(din); + int len = (int) readVarint(din); + Type t = Type.lookup(type); + byte[] hash = new byte[len]; + int total = 0; + while (total < len) { + int read = din.read(hash); + if (read < 0) throw new EOFException(); + else total += read; + } + return new Multihash(t, hash); + } + + public static Multihash deserialize(byte[] raw) { + try { + return deserialize(new ByteArrayInputStream(raw)); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * @param encoded A multibase encoded serialization of a Multihash + * @return + * @throws IOException + */ + public static Multihash decode(String encoded) { + if (encoded.length() == 46 && encoded.startsWith("Qm")) + return deserialize(Base58.decode(encoded)); + return deserialize(Multibase.decode(encoded)); + } + + @Override + public String toString() { + return toBase58(); + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof Multihash)) return false; + return type == ((Multihash) o).type && Arrays.equals(hash, ((Multihash) o).hash); + } + + @Override + public int hashCode() { + return Arrays.hashCode(hash) ^ type.hashCode(); + } + + public String toHex() { + return Base16.encode(toBytes()); + } + + public String toBase58() { + return Base58.encode(toBytes()); + } + + public static Multihash fromHex(String hex) { + if (hex.length() % 2 != 0) throw new IllegalStateException("Odd number of hex digits!"); + + try (ByteArrayOutputStream bout = new ByteArrayOutputStream()) { + for (int i = 0; i < hex.length() - 1; i += 2) + bout.write(Integer.valueOf(hex.substring(i, i + 2), 16)); + return Multihash.deserialize(bout.toByteArray()); + } catch (IOException e) { + throw new IllegalStateException("Unable to handle Multihash conversion to Hex properly"); + } + } + + public static Multihash fromBase58(String base58) { + return Multihash.deserialize(Base58.decode(base58)); + } + + public static long readVarint(InputStream in) throws IOException { + long x = 0; + int s = 0; + for (int i = 0; i < 10; i++) { + int b = in.read(); + if (b < 0x80) { + if (i == 9 && b > 1) { + throw new IllegalStateException("Overflow reading varint!"); + } + return x | (((long) b) << s); + } + x |= ((long) b & 0x7f) << s; + s += 7; + } + throw new IllegalStateException("Varint too long!"); + } + + public static void putUvarint(OutputStream out, long x) throws IOException { + while (x >= 0x80) { + out.write((byte) (x | 0x80)); + x >>= 7; + } + out.write((byte) x); + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/Blockstore.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/Blockstore.java new file mode 100644 index 0000000..f845b3a --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/Blockstore.java @@ -0,0 +1,45 @@ +package org.peergos.blockstore; + +import io.ipfs.cid.*; +import io.ipfs.multibase.binary.Base32; +import io.ipfs.multihash.Multihash; +import org.peergos.blockstore.metadatadb.BlockMetadata; +import org.peergos.blockstore.metadatadb.BlockMetadataStore; + +import java.util.*; +import java.util.concurrent.*; +import java.util.function.Consumer; + +public interface Blockstore { + + default String hashToKey(Multihash hash) { + String padded = new Base32().encodeAsString(hash.toBytes()); + int padStart = padded.indexOf("="); + return padStart > 0 ? padded.substring(0, padStart) : padded; + } + + default Cid keyToHash(String key) { + byte[] decoded = new Base32().decode(key); + return Cid.cast(decoded); + } + + CompletableFuture<Boolean> has(Cid c); + + CompletableFuture<Boolean> hasAny(Multihash h); + + CompletableFuture<Optional<byte[]>> get(Cid c); + + CompletableFuture<Cid> put(byte[] block, Cid.Codec codec); + + CompletableFuture<Boolean> rm(Cid c); + + CompletableFuture<Long> count(boolean useBlockstore); + + CompletableFuture<List<Cid>> refs(boolean useBlockstore); + + CompletableFuture<Boolean> applyToAll(Consumer<Cid> action, boolean useBlockstore); + + CompletableFuture<Boolean> bloomAdd(Cid cid); + + CompletableFuture<BlockMetadata> getBlockMetadata(Cid h); +}
\ No newline at end of file diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/BloomFilter.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/BloomFilter.java new file mode 100644 index 0000000..06fe6bc --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/BloomFilter.java @@ -0,0 +1,441 @@ +/** + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + */ + +package org.peergos.blockstore; + +import java.io.Serializable; +import java.nio.charset.Charset; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.BitSet; +import java.util.Collection; + +/** + * Implementation of a Bloom-filter, as described here: + * http://en.wikipedia.org/wiki/Bloom_filter + * + * For updates and bugfixes, see http://github.com/magnuss/java-bloomfilter + * + * Inspired by the SimpleBloomFilter-class written by Ian Clarke. This + * implementation provides a more evenly distributed Hash-function by + * using a proper digest instead of the Java RNG. Many of the changes + * were proposed in comments in his blog: + * http://blog.locut.us/2008/01/12/a-decent-stand-alone-java-bloom-filter-implementation/ + * + * @param <E> Object type that is to be inserted into the Bloom filter, e.g. String or Integer. + * @author Magnus Skjegstad <magnus@skjegstad.com> + */ +public class BloomFilter<E> implements Serializable { + private BitSet bitset; + private int bitSetSize; + private double bitsPerElement; + private int expectedNumberOfFilterElements; // expected (maximum) number of elements to be added + private int numberOfAddedElements; // number of elements actually added to the Bloom filter + private int k; // number of hash functions + + static final Charset charset = Charset.forName("UTF-8"); // encoding used for storing hash values as strings + + static final String hashName = "MD5"; // MD5 gives good enough accuracy in most circumstances. Change to SHA1 if it's needed + static final MessageDigest digestFunction; + static { // The digest method is reused between instances + MessageDigest tmp; + try { + tmp = java.security.MessageDigest.getInstance(hashName); + } catch (NoSuchAlgorithmException e) { + tmp = null; + } + digestFunction = tmp; + } + + /** + * Constructs an empty Bloom filter. The total length of the Bloom filter will be + * c*n. + * + * @param c is the number of bits used per element. + * @param n is the expected number of elements the filter will contain. + * @param k is the number of hash functions used. + */ + public BloomFilter(double c, int n, int k) { + this.expectedNumberOfFilterElements = n; + this.k = k; + this.bitsPerElement = c; + this.bitSetSize = (int)Math.ceil(c * n); + numberOfAddedElements = 0; + this.bitset = new BitSet(bitSetSize); + } + + /** + * Constructs an empty Bloom filter. The optimal number of hash functions (k) is estimated from the total size of the Bloom + * and the number of expected elements. + * + * @param bitSetSize defines how many bits should be used in total for the filter. + * @param expectedNumberOElements defines the maximum number of elements the filter is expected to contain. + */ + public BloomFilter(int bitSetSize, int expectedNumberOElements) { + this(bitSetSize / (double)expectedNumberOElements, + expectedNumberOElements, + (int) Math.round((bitSetSize / (double)expectedNumberOElements) * Math.log(2.0))); + } + + /** + * Constructs an empty Bloom filter with a given false positive probability. The number of bits per + * element and the number of hash functions is estimated + * to match the false positive probability. + * + * @param falsePositiveProbability is the desired false positive probability. + * @param expectedNumberOfElements is the expected number of elements in the Bloom filter. + */ + public BloomFilter(double falsePositiveProbability, int expectedNumberOfElements) { + this(Math.ceil(-(Math.log(falsePositiveProbability) / Math.log(2))) / Math.log(2), // c = k / ln(2) + expectedNumberOfElements, + (int)Math.ceil(-(Math.log(falsePositiveProbability) / Math.log(2)))); // k = ceil(-log_2(false prob.)) + } + + /** + * Construct a new Bloom filter based on existing Bloom filter data. + * + * @param bitSetSize defines how many bits should be used for the filter. + * @param expectedNumberOfFilterElements defines the maximum number of elements the filter is expected to contain. + * @param actualNumberOfFilterElements specifies how many elements have been inserted into the <code>filterData</code> BitSet. + * @param filterData a BitSet representing an existing Bloom filter. + */ + public BloomFilter(int bitSetSize, int expectedNumberOfFilterElements, int actualNumberOfFilterElements, BitSet filterData) { + this(bitSetSize, expectedNumberOfFilterElements); + this.bitset = filterData; + this.numberOfAddedElements = actualNumberOfFilterElements; + } + + /** + * Generates a digest based on the contents of a String. + * + * @param val specifies the input data. + * @param charset specifies the encoding of the input data. + * @return digest as long. + */ + public static int createHash(String val, Charset charset) { + return createHash(val.getBytes(charset)); + } + + /** + * Generates a digest based on the contents of a String. + * + * @param val specifies the input data. The encoding is expected to be UTF-8. + * @return digest as long. + */ + public static int createHash(String val) { + return createHash(val, charset); + } + + /** + * Generates a digest based on the contents of an array of bytes. + * + * @param data specifies input data. + * @return digest as long. + */ + public static int createHash(byte[] data) { + return createHashes(data, 1)[0]; + } + + /** + * Generates digests based on the contents of an array of bytes and splits the result into 4-byte int's and store them in an array. The + * digest function is called until the required number of int's are produced. For each call to digest a salt + * is prepended to the data. The salt is increased by 1 for each call. + * + * @param data specifies input data. + * @param hashes number of hashes/int's to produce. + * @return array of int-sized hashes + */ + public static int[] createHashes(byte[] data, int hashes) { + int[] result = new int[hashes]; + + int k = 0; + byte salt = 0; + while (k < hashes) { + byte[] digest; + synchronized (digestFunction) { + digestFunction.update(salt); + salt++; + digest = digestFunction.digest(data); + } + + for (int i = 0; i < digest.length/4 && k < hashes; i++) { + int h = 0; + for (int j = (i*4); j < (i*4)+4; j++) { + h <<= 8; + h |= ((int) digest[j]) & 0xFF; + } + result[k] = h; + k++; + } + } + return result; + } + + /** + * Compares the contents of two instances to see if they are equal. + * + * @param obj is the object to compare to. + * @return True if the contents of the objects are equal. + */ + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + final BloomFilter<E> other = (BloomFilter<E>) obj; + if (this.expectedNumberOfFilterElements != other.expectedNumberOfFilterElements) { + return false; + } + if (this.k != other.k) { + return false; + } + if (this.bitSetSize != other.bitSetSize) { + return false; + } + if (this.bitset != other.bitset && (this.bitset == null || !this.bitset.equals(other.bitset))) { + return false; + } + return true; + } + + /** + * Calculates a hash code for this class. + * @return hash code representing the contents of an instance of this class. + */ + @Override + public int hashCode() { + int hash = 7; + hash = 61 * hash + (this.bitset != null ? this.bitset.hashCode() : 0); + hash = 61 * hash + this.expectedNumberOfFilterElements; + hash = 61 * hash + this.bitSetSize; + hash = 61 * hash + this.k; + return hash; + } + + + /** + * Calculates the expected probability of false positives based on + * the number of expected filter elements and the size of the Bloom filter. + * <br /><br /> + * The value returned by this method is the <i>expected</i> rate of false + * positives, assuming the number of inserted elements equals the number of + * expected elements. If the number of elements in the Bloom filter is less + * than the expected value, the true probability of false positives will be lower. + * + * @return expected probability of false positives. + */ + public double expectedFalsePositiveProbability() { + return getFalsePositiveProbability(expectedNumberOfFilterElements); + } + + /** + * Calculate the probability of a false positive given the specified + * number of inserted elements. + * + * @param numberOfElements number of inserted elements. + * @return probability of a false positive. + */ + public double getFalsePositiveProbability(double numberOfElements) { + // (1 - e^(-k * n / m)) ^ k + return Math.pow((1 - Math.exp(-k * (double) numberOfElements + / (double) bitSetSize)), k); + + } + + /** + * Get the current probability of a false positive. The probability is calculated from + * the size of the Bloom filter and the current number of elements added to it. + * + * @return probability of false positives. + */ + public double getFalsePositiveProbability() { + return getFalsePositiveProbability(numberOfAddedElements); + } + + + /** + * Returns the value chosen for K.<br /> + * <br /> + * K is the optimal number of hash functions based on the size + * of the Bloom filter and the expected number of inserted elements. + * + * @return optimal k. + */ + public int getK() { + return k; + } + + /** + * Sets all bits to false in the Bloom filter. + */ + public void clear() { + bitset.clear(); + numberOfAddedElements = 0; + } + + /** + * Adds an object to the Bloom filter. The output from the object's + * toString() method is used as input to the hash functions. + * + * @param element is an element to register in the Bloom filter. + */ + public void add(E element) { + add(element.toString().getBytes(charset)); + } + + /** + * Adds an array of bytes to the Bloom filter. + * + * @param bytes array of bytes to add to the Bloom filter. + */ + public void add(byte[] bytes) { + int[] hashes = createHashes(bytes, k); + for (int hash : hashes) + bitset.set(Math.abs(hash % bitSetSize), true); + numberOfAddedElements ++; + } + + /** + * Adds all elements from a Collection to the Bloom filter. + * @param c Collection of elements. + */ + public void addAll(Collection<? extends E> c) { + for (E element : c) + add(element); + } + + /** + * Returns true if the element could have been inserted into the Bloom filter. + * Use getFalsePositiveProbability() to calculate the probability of this + * being correct. + * + * @param element element to check. + * @return true if the element could have been inserted into the Bloom filter. + */ + public boolean contains(E element) { + return contains(element.toString().getBytes(charset)); + } + + /** + * Returns true if the array of bytes could have been inserted into the Bloom filter. + * Use getFalsePositiveProbability() to calculate the probability of this + * being correct. + * + * @param bytes array of bytes to check. + * @return true if the array could have been inserted into the Bloom filter. + */ + public boolean contains(byte[] bytes) { + int[] hashes = createHashes(bytes, k); + for (int hash : hashes) { + if (!bitset.get(Math.abs(hash % bitSetSize))) { + return false; + } + } + return true; + } + + /** + * Returns true if all the elements of a Collection could have been inserted + * into the Bloom filter. Use getFalsePositiveProbability() to calculate the + * probability of this being correct. + * @param c elements to check. + * @return true if all the elements in c could have been inserted into the Bloom filter. + */ + public boolean containsAll(Collection<? extends E> c) { + for (E element : c) + if (!contains(element)) + return false; + return true; + } + + /** + * Read a single bit from the Bloom filter. + * @param bit the bit to read. + * @return true if the bit is set, false if it is not. + */ + public boolean getBit(int bit) { + return bitset.get(bit); + } + + /** + * Set a single bit in the Bloom filter. + * @param bit is the bit to set. + * @param value If true, the bit is set. If false, the bit is cleared. + */ + public void setBit(int bit, boolean value) { + bitset.set(bit, value); + } + + /** + * Return the bit set used to store the Bloom filter. + * @return bit set representing the Bloom filter. + */ + public BitSet getBitSet() { + return bitset; + } + + /** + * Returns the number of bits in the Bloom filter. Use count() to retrieve + * the number of inserted elements. + * + * @return the size of the bitset used by the Bloom filter. + */ + public int size() { + return this.bitSetSize; + } + + /** + * Returns the number of elements added to the Bloom filter after it + * was constructed or after clear() was called. + * + * @return number of elements added to the Bloom filter. + */ + public int count() { + return this.numberOfAddedElements; + } + + /** + * Returns the expected number of elements to be inserted into the filter. + * This value is the same value as the one passed to the constructor. + * + * @return expected number of elements. + */ + public int getExpectedNumberOfElements() { + return expectedNumberOfFilterElements; + } + + /** + * Get expected number of bits per element when the Bloom filter is full. This value is set by the constructor + * when the Bloom filter is created. See also getBitsPerElement(). + * + * @return expected number of bits per element. + */ + public double getExpectedBitsPerElement() { + return this.bitsPerElement; + } + + /** + * Get actual number of bits per element based on the number of elements that have currently been inserted and the length + * of the Bloom filter. See also getExpectedBitsPerElement(). + * + * @return number of bits per element. + */ + public double getBitsPerElement() { + return this.bitSetSize / (double)numberOfAddedElements; + } +}
\ No newline at end of file diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/CidBloomFilter.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/CidBloomFilter.java new file mode 100644 index 0000000..9d4e7c6 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/CidBloomFilter.java @@ -0,0 +1,36 @@ +package org.peergos.blockstore; + +import io.ipfs.cid.*; + +import java.util.*; + +public class CidBloomFilter implements Filter { + + private final BloomFilter<Cid> bloom; + + public CidBloomFilter(BloomFilter<Cid> bloom) { + this.bloom = bloom; + } + + @Override + public boolean has(Cid c) { + return bloom.contains(c); + } + + @Override + public Cid add(Cid c) { + bloom.add(c); + return c; + } + + public static CidBloomFilter build(Blockstore bs, double falsePositiveRate) { + List<Cid> refs = bs.refs(false).join(); + BloomFilter<Cid> bloom = new BloomFilter<>(falsePositiveRate, refs.size()); + refs.forEach(bloom::add); + return new CidBloomFilter(bloom); + } + + public static CidBloomFilter build(Blockstore bs) { + return build(bs, 0.01); + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/FileBlockstore.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/FileBlockstore.java new file mode 100644 index 0000000..7bcd40c --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/FileBlockstore.java @@ -0,0 +1,189 @@ +package org.peergos.blockstore; + +import io.ipfs.cid.Cid; +import io.ipfs.multihash.Multihash; +import org.peergos.Hash; +import org.peergos.blockstore.metadatadb.BlockMetadata; +import org.peergos.util.*; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class FileBlockstore implements Blockstore { + + private static final Logger LOG = Logging.LOG(); + + private final Path blocksRoot; + private final String BLOCKS = "blocks"; + private final String BLOCK_FILE_SUFFIX = ".data"; + + public FileBlockstore(Path root) { + if (root == null || !root.toFile().isDirectory()) { + throw new IllegalStateException("Path must be a directory! " + root); + } + Path blocksPath = root.resolve(BLOCKS); + File blocksDirectory = blocksPath.toFile(); + if (!blocksDirectory.exists()) { + if (!blocksDirectory.mkdirs()) { + throw new IllegalStateException("Unable to make blocks directory"); + } + } else if (blocksDirectory.isFile()) { + throw new IllegalStateException("Unable to create blocks directory"); + } + this.blocksRoot = blocksPath; + LOG.info("Using FileBlockStore at location: " + blocksPath); + } + + public Path getFilePath(Cid cid) { + String key = hashToKey(cid); + String folder = key.substring(key.length() -3, key.length()-1); + String filename = key + BLOCK_FILE_SUFFIX; + + Path path = Paths.get(folder); + path = path.resolve(filename); + return path; + } + + @Override + public CompletableFuture<Boolean> has(Cid cid) { + Path path = getFilePath(cid); + File file = blocksRoot.resolve(path).toFile(); + return CompletableFuture.completedFuture(file.exists()); + } + + @Override + public CompletableFuture<Boolean> hasAny(Multihash h) { + return Futures.of(Stream.of(Cid.Codec.DagCbor, Cid.Codec.Raw, Cid.Codec.DagProtobuf) + .anyMatch(c -> has(new Cid(1, c, h.getType(), h.getHash())).join())); + } + + @Override + public CompletableFuture<Optional<byte[]>> get(Cid cid) { + try { + Path path = getFilePath(cid); + File file = blocksRoot.resolve(path).toFile(); + if (!file.exists()) { + return CompletableFuture.completedFuture(Optional.empty()); + } + try (DataInputStream din = new DataInputStream(new BufferedInputStream(new FileInputStream(file)))) { + byte[] buffer = new byte[1024]; + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + for (int len; (len = din.read(buffer)) != -1; ) { + bout.write(buffer, 0, len); + } + return CompletableFuture.completedFuture(Optional.of(bout.toByteArray())); + } + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + public CompletableFuture<Cid> put(byte[] block, Cid.Codec codec) { + Cid cid = new Cid(1, codec, Multihash.Type.sha2_256, Hash.sha256(block)); + try { + Path filePath = getFilePath(cid); + Path target = blocksRoot.resolve(filePath); + Path parent = target.getParent(); + File parentDir = parent.toFile(); + + if (!parentDir.exists()) + Files.createDirectories(parent); + + for (Path someParent = parent; !someParent.equals(blocksRoot); someParent = someParent.getParent()) { + File someParentFile = someParent.toFile(); + if (!someParentFile.canWrite()) { + final boolean b = someParentFile.setWritable(true, false); + if (!b) + throw new IllegalStateException("Could not make " + someParent.toString() + ", ancestor of " + parentDir.toString() + " writable"); + } + } + Files.write(target, block, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.CREATE, StandardOpenOption.WRITE); + return CompletableFuture.completedFuture(cid); + } catch (IOException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + @Override + public CompletableFuture<Boolean> rm(Cid cid) { + Path path = getFilePath(cid); + File file = blocksRoot.resolve(path).toFile(); + if (file.exists()) { + return CompletableFuture.completedFuture(file.delete()); + } else { + return CompletableFuture.completedFuture(false); + } + } + + @Override + public CompletableFuture<Boolean> bloomAdd(Cid cid) { + //not implemented + return CompletableFuture.completedFuture(false); + } + + @Override + public CompletableFuture<List<Cid>> refs(boolean useBlockstore) { + List<Path> result = new ArrayList<>(); + try (Stream<Path> walk = Files.walk(blocksRoot)) { + result = walk.filter(f -> Files.isRegularFile(f) && + f.toFile().length() > 0 && + f.getFileName().toString().endsWith(BLOCK_FILE_SUFFIX)) + .collect(Collectors.toList()); + } catch (IOException ioe) { + LOG.log(Level.WARNING, "Unable to retrieve local refs: " + ioe); + } + List<Cid> cidList = result.stream().map(p -> { + String filename = p.getFileName().toString(); + return keyToHash(filename.substring(0, filename.length() - BLOCK_FILE_SUFFIX.length())); + }).collect(Collectors.toList()); + return CompletableFuture.completedFuture(cidList); + } + + @Override + public CompletableFuture<Long> count(boolean useBlockstore) { + try { + return Futures.of(Files.walk(blocksRoot) + .filter(f -> Files.isRegularFile(f) && + f.toFile().length() > 0 && + f.getFileName().toString().endsWith(BLOCK_FILE_SUFFIX)) + .count()); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public CompletableFuture<Boolean> applyToAll(Consumer<Cid> action, boolean useBlockstore) { + try { + Files.walk(blocksRoot) + .filter(f -> Files.isRegularFile(f) && + f.toFile().length() > 0 && + f.getFileName().toString().endsWith(BLOCK_FILE_SUFFIX)) + .map(p -> { + String filename = p.getFileName().toString(); + return keyToHash(filename.substring(0, filename.length() - BLOCK_FILE_SUFFIX.length())); + }).forEach(action); + return Futures.of(true); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public CompletableFuture<BlockMetadata> getBlockMetadata(Cid h) { + throw new IllegalStateException("Unsupported operation!"); + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/Filter.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/Filter.java new file mode 100644 index 0000000..60c489d --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/Filter.java @@ -0,0 +1,15 @@ +package org.peergos.blockstore; + +import io.ipfs.cid.*; + +public interface Filter { + + boolean has(Cid c); + + /** + * + * @param c + * @return the argument c + */ + Cid add(Cid c); +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/RamBlockstore.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/RamBlockstore.java new file mode 100644 index 0000000..5b67a63 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/RamBlockstore.java @@ -0,0 +1,79 @@ +package org.peergos.blockstore; + +import io.ipfs.cid.*; +import io.ipfs.multihash.*; +import org.peergos.*; +import org.peergos.blockstore.metadatadb.BlockMetadata; +import org.peergos.cbor.*; +import org.peergos.util.*; + +import java.util.*; +import java.util.concurrent.*; +import java.util.function.Consumer; +import java.util.stream.*; + +public class RamBlockstore implements Blockstore { + + private final ConcurrentHashMap<Cid, byte[]> blocks = new ConcurrentHashMap<>(); + + @Override + public CompletableFuture<Boolean> has(Cid c) { + return CompletableFuture.completedFuture(blocks.containsKey(c)); + } + + @Override + public CompletableFuture<Boolean> hasAny(Multihash h) { + return Futures.of(Stream.of(Cid.Codec.DagCbor, Cid.Codec.Raw, Cid.Codec.DagProtobuf) + .anyMatch(c -> has(new Cid(1, c, h.getType(), h.getHash())).join())); + } + + @Override + public CompletableFuture<Optional<byte[]>> get(Cid c) { + return CompletableFuture.completedFuture(Optional.ofNullable(blocks.get(c))); + } + + @Override + public CompletableFuture<Cid> put(byte[] block, Cid.Codec codec) { + Cid cid = new Cid(1, codec, Multihash.Type.sha2_256, Hash.sha256(block)); + blocks.put(cid, block); + return CompletableFuture.completedFuture(cid); + } + + @Override + public CompletableFuture<Boolean> rm(Cid c) { + if (blocks.containsKey(c)) { + blocks.remove(c); + return CompletableFuture.completedFuture(true); + } else { + return CompletableFuture.completedFuture(false); + } + } + + @Override + public CompletableFuture<Boolean> bloomAdd(Cid cid) { + //not implemented + return CompletableFuture.completedFuture(false); + } + + @Override + public CompletableFuture<List<Cid>> refs(boolean useBlockstore) { + return CompletableFuture.completedFuture(new ArrayList(blocks.keySet())); + } + + @Override + public CompletableFuture<Long> count(boolean useBlockstore) { + return Futures.of((long)blocks.size()); + } + + @Override + public CompletableFuture<Boolean> applyToAll(Consumer<Cid> action, boolean useBlockstore) { + blocks.keySet().stream().forEach(action); + return Futures.of(true); + } + + @Override + public CompletableFuture<BlockMetadata> getBlockMetadata(Cid h) { + byte[] block = get(h).join().get(); + return Futures.of(new BlockMetadata(block.length, CborObject.getLinks(h, block))); + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/blockstore/TypeLimitedBlockstore.java b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/TypeLimitedBlockstore.java new file mode 100644 index 0000000..9e3ed29 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/blockstore/TypeLimitedBlockstore.java @@ -0,0 +1,95 @@ +package org.peergos.blockstore; + +import io.ipfs.cid.Cid; +import io.ipfs.multihash.*; +import org.peergos.blockstore.metadatadb.BlockMetadata; +import org.peergos.util.*; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; +import java.util.stream.*; + +public class TypeLimitedBlockstore implements Blockstore { + + private final Blockstore blocks; + private final Set<Cid.Codec> allowedCodecs; + + public TypeLimitedBlockstore(Blockstore blocks, Set<Cid.Codec> allowedCodecs) { + this.blocks = blocks; + this.allowedCodecs = allowedCodecs; + } + + public CompletableFuture<Boolean> bloomAdd(Cid cid) { + if (allowedCodecs.contains(cid.codec)) { + return blocks.bloomAdd(cid); + } + throw new IllegalArgumentException("Unsupported codec: " + cid.codec); + } + + @Override + public CompletableFuture<Boolean> hasAny(Multihash h) { + for (Cid.Codec codec : allowedCodecs) { + if (has(new Cid(1, codec, h.getType(), h.getHash())).join()) + return Futures.of(true); + } + return Futures.of(false); + } + + @Override + public CompletableFuture<Boolean> has(Cid cid) { + if (allowedCodecs.contains(cid.codec)) { + return blocks.has(cid); + } + return CompletableFuture.completedFuture(false); + } + + @Override + public CompletableFuture<Optional<byte[]>> get(Cid cid) { + if (allowedCodecs.contains(cid.codec)) { + return blocks.get(cid); + } + return CompletableFuture.completedFuture(Optional.empty()); + } + + @Override + public CompletableFuture<Cid> put(byte[] block, Cid.Codec codec) { + if (allowedCodecs.contains(codec)) { + return blocks.put(block, codec); + } + throw new IllegalArgumentException("Unsupported codec: " + codec); + } + + @Override + public CompletableFuture<Boolean> rm(Cid cid) { + if (allowedCodecs.contains(cid.codec)) { + return blocks.rm(cid); + } + throw new IllegalArgumentException("Unsupported codec: " + cid.codec); + } + + @Override + public CompletableFuture<List<Cid>> refs(boolean useBlockstore) { + return blocks.refs(useBlockstore).thenApply(res -> res.stream() + .filter(c -> allowedCodecs.contains(c.codec)) + .collect(Collectors.toList())); + } + + @Override + public CompletableFuture<Long> count(boolean useBlockstore) { + return blocks.count(useBlockstore); + } + + @Override + public CompletableFuture<Boolean> applyToAll(Consumer<Cid> action, boolean useBlockstore) { + return blocks.applyToAll(action, useBlockstore); + } + + @Override + public CompletableFuture<BlockMetadata> getBlockMetadata(Cid h) { + return blocks.getBlockMetadata(h); + } + +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborConstants.java b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborConstants.java new file mode 100644 index 0000000..001c14c --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborConstants.java @@ -0,0 +1,90 @@ +package org.peergos.cbor; + +/* + * JACOB - CBOR implementation in Java. + * + * (C) Copyright - 2013 - J.W. Janssen <j.w.janssen@lxtreme.nl> + * + * Licensed under Apache License v2.0. + */ + +/** + * Constant values used by the CBOR format. + */ +public interface CborConstants { + /** Major type 0: unsigned integers. */ + int TYPE_UNSIGNED_INTEGER = 0x00; + /** Major type 1: negative integers. */ + int TYPE_NEGATIVE_INTEGER = 0x01; + /** Major type 2: byte string. */ + int TYPE_BYTE_STRING = 0x02; + /** Major type 3: text/UTF8 string. */ + int TYPE_TEXT_STRING = 0x03; + /** Major type 4: array of items. */ + int TYPE_ARRAY = 0x04; + /** Major type 5: map of pairs. */ + int TYPE_MAP = 0x05; + /** Major type 6: semantic tags. */ + int TYPE_TAG = 0x06; + /** Major type 7: floating point, simple data types. */ + int TYPE_FLOAT_SIMPLE = 0x07; + + /** Denotes a one-byte value (uint8). */ + int ONE_BYTE = 0x18; + /** Denotes a two-byte value (uint16). */ + int TWO_BYTES = 0x19; + /** Denotes a four-byte value (uint32). */ + int FOUR_BYTES = 0x1a; + /** Denotes a eight-byte value (uint64). */ + int EIGHT_BYTES = 0x1b; + + /** The CBOR-encoded boolean <code>false</code> value (encoded as "simple value": {@link #MT_SIMPLE}). */ + int FALSE = 0x14; + /** The CBOR-encoded boolean <code>true</code> value (encoded as "simple value": {@link #MT_SIMPLE}). */ + int TRUE = 0x15; + /** The CBOR-encoded <code>null</code> value (encoded as "simple value": {@link #MT_SIMPLE}). */ + int NULL = 0x16; + /** The CBOR-encoded "undefined" value (encoded as "simple value": {@link #MT_SIMPLE}). */ + int UNDEFINED = 0x17; + /** Denotes a half-precision float (two-byte IEEE 754, see {@link #MT_FLOAT}). */ + int HALF_PRECISION_FLOAT = 0x19; + /** Denotes a single-precision float (four-byte IEEE 754, see {@link #MT_FLOAT}). */ + int SINGLE_PRECISION_FLOAT = 0x1a; + /** Denotes a double-precision float (eight-byte IEEE 754, see {@link #MT_FLOAT}). */ + int DOUBLE_PRECISION_FLOAT = 0x1b; + /** The CBOR-encoded "break" stop code for unlimited arrays/maps. */ + int BREAK = 0x1f; + + /** Semantic tag value describing date/time values in the standard format (UTF8 string, RFC3339). */ + int TAG_STANDARD_DATE_TIME = 0; + /** Semantic tag value describing date/time values as Epoch timestamp (numeric, RFC3339). */ + int TAG_EPOCH_DATE_TIME = 1; + /** Semantic tag value describing a positive big integer value (byte string). */ + int TAG_POSITIVE_BIGINT = 2; + /** Semantic tag value describing a negative big integer value (byte string). */ + int TAG_NEGATIVE_BIGINT = 3; + /** Semantic tag value describing a decimal fraction value (two-element array, base 10). */ + int TAG_DECIMAL_FRACTION = 4; + /** Semantic tag value describing a big decimal value (two-element array, base 2). */ + int TAG_BIGDECIMAL = 5; + /** Semantic tag value describing an expected conversion to base64url encoding. */ + int TAG_EXPECTED_BASE64_URL_ENCODED = 21; + /** Semantic tag value describing an expected conversion to base64 encoding. */ + int TAG_EXPECTED_BASE64_ENCODED = 22; + /** Semantic tag value describing an expected conversion to base16 encoding. */ + int TAG_EXPECTED_BASE16_ENCODED = 23; + /** Semantic tag value describing an encoded CBOR data item (byte string). */ + int TAG_CBOR_ENCODED = 24; + /** Semantic tag value describing an URL (UTF8 string). */ + int TAG_URI = 32; + /** Semantic tag value describing a base64url encoded string (UTF8 string). */ + int TAG_BASE64_URL_ENCODED = 33; + /** Semantic tag value describing a base64 encoded string (UTF8 string). */ + int TAG_BASE64_ENCODED = 34; + /** Semantic tag value describing a regular expression string (UTF8 string, PCRE). */ + int TAG_REGEXP = 35; + /** Semantic tag value describing a MIME message (UTF8 string, RFC2045). */ + int TAG_MIME_MESSAGE = 36; + /** Semantic tag value describing CBOR content. */ + int TAG_CBOR_MARKER = 55799; +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborDecoder.java b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborDecoder.java new file mode 100644 index 0000000..63f4a3e --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborDecoder.java @@ -0,0 +1,491 @@ +package org.peergos.cbor; + +/* + * JACOB - CBOR implementation in Java. + * + * (C) Copyright - 2013 - J.W. Janssen <j.w.janssen@lxtreme.nl> + * Apache Public License v2.0 + */ + +import java.io.*; +import java.util.*; + +import static org.peergos.cbor.CborConstants.*; +import static org.peergos.cbor.CborType.*; + +/** + * Provides a decoder capable of handling CBOR encoded data from a {@link InputStream}. + */ +public class CborDecoder { + protected final PushbackInputStream m_is; + + /** + * Creates a new {@link CborDecoder} instance. + * + * @param is the actual input stream to read the CBOR-encoded data from, cannot be <code>null</code>. + */ + public CborDecoder(InputStream is) { + if (is == null) { + throw new IllegalArgumentException("InputStream cannot be null!"); + } + m_is = (is instanceof PushbackInputStream) ? (PushbackInputStream) is : new PushbackInputStream(is); + } + + private static void fail(String msg, Object... args) throws IOException { + throw new IOException(msg + Arrays.toString(args)); + } + + private static String lengthToString(int len) { + return (len < 0) ? "no payload" : (len == ONE_BYTE) ? "one byte" : (len == TWO_BYTES) ? "two bytes" + : (len == FOUR_BYTES) ? "four bytes" : (len == EIGHT_BYTES) ? "eight bytes" : "(unknown)"; + } + + /** + * Peeks in the input stream for the upcoming type. + * + * @return the upcoming type in the stream, or <code>null</code> in case of an end-of-stream. + * @throws IOException in case of I/O problems reading the CBOR-type from the underlying input stream. + */ + public CborType peekType() throws IOException { + int p = m_is.read(); + if (p < 0) { + // EOF, nothing to peek at... + return null; + } + m_is.unread(p); + return valueOf(p); + } + + /** + * Prolog to reading an array value in CBOR format. + * + * @return the number of elements in the array to read, or <tt>-1</tt> in case of infinite-length arrays. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public long readArrayLength() throws IOException { + return readMajorTypeWithSize(TYPE_ARRAY); + } + + /** + * Reads a boolean value in CBOR format. + * + * @return the read boolean. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public boolean readBoolean() throws IOException { + int b = readMajorType(TYPE_FLOAT_SIMPLE); + if (b != FALSE && b != TRUE) { + fail("Unexpected boolean value: %d!", b); + } + return b == TRUE; + } + + /** + * Reads a "break"/stop value in CBOR format. + * + * @return always <code>null</code>. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public Object readBreak() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, BREAK); + + return null; + } + + /** + * Reads a byte string value in CBOR format. + * + * @return the read byte string, never <code>null</code>. In case the encoded string has a length of <tt>0</tt>, an empty string is returned. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public byte[] readByteString(int maxLen) throws IOException { + long len = readMajorTypeWithSize(TYPE_BYTE_STRING); + if (len < 0) + fail("Infinite-length byte strings not supported!"); + if (len > Integer.MAX_VALUE) + fail("String length too long!"); + if (len > maxLen) + fail("Invalid cbor: byte string longer than original bytes!"); + return readFully(new byte[(int) len]); + } + + /** + * Prolog to reading a byte string value in CBOR format. + * + * @return the number of bytes in the string to read, or <tt>-1</tt> in case of infinite-length strings. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public long readByteStringLength() throws IOException { + return readMajorTypeWithSize(TYPE_BYTE_STRING); + } + + /** + * Reads a double-precision float value in CBOR format. + * + * @return the read double value, values from {@link Float#MIN_VALUE} to {@link Float#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public double readDouble() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, DOUBLE_PRECISION_FLOAT); + + return Double.longBitsToDouble(readUInt64()); + } + + /** + * Reads a single-precision float value in CBOR format. + * + * @return the read float value, values from {@link Float#MIN_VALUE} to {@link Float#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public float readFloat() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, SINGLE_PRECISION_FLOAT); + + return Float.intBitsToFloat((int) readUInt32()); + } + + /** + * Reads a half-precision float value in CBOR format. + * + * @return the read half-precision float value, values from {@link Float#MIN_VALUE} to {@link Float#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public double readHalfPrecisionFloat() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, HALF_PRECISION_FLOAT); + + int half = readUInt16(); + int exp = (half >> 10) & 0x1f; + int mant = half & 0x3ff; + + double val; + if (exp == 0) { + val = mant * Math.pow(2, -24); + } else if (exp != 31) { + val = (mant + 1024) * Math.pow(2, exp - 25); + } else if (mant != 0) { + val = Double.NaN; + } else { + val = Double.POSITIVE_INFINITY; + } + + return ((half & 0x8000) == 0) ? val : -val; + } + + /** + * Reads a signed or unsigned integer value in CBOR format. + * + * @return the read integer value, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public long readInt() throws IOException { + int ib = m_is.read(); + + // in case of negative integers, extends the sign to all bits; otherwise zero... + long ui = expectIntegerType(ib); + // in case of negative integers does a ones complement + return ui ^ readUInt(ib & 0x1f, false /* breakAllowed */); + } + + /** + * Reads a signed or unsigned 16-bit integer value in CBOR format. + * + * @read the small integer value, values from <tt>[-65536..65535]</tt> are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying output stream. + */ + public int readInt16() throws IOException { + int ib = m_is.read(); + + // in case of negative integers, extends the sign to all bits; otherwise zero... + long ui = expectIntegerType(ib); + // in case of negative integers does a ones complement + return (int) (ui ^ readUIntExact(TWO_BYTES, ib & 0x1f)); + } + + /** + * Reads a signed or unsigned 32-bit integer value in CBOR format. + * + * @read the small integer value, values in the range <tt>[-4294967296..4294967295]</tt> are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying output stream. + */ + public long readInt32() throws IOException { + int ib = m_is.read(); + + // in case of negative integers, extends the sign to all bits; otherwise zero... + long ui = expectIntegerType(ib); + // in case of negative integers does a ones complement + return ui ^ readUIntExact(FOUR_BYTES, ib & 0x1f); + } + + /** + * Reads a signed or unsigned 64-bit integer value in CBOR format. + * + * @read the small integer value, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying output stream. + */ + public long readInt64() throws IOException { + int ib = m_is.read(); + + // in case of negative integers, extends the sign to all bits; otherwise zero... + long ui = expectIntegerType(ib); + // in case of negative integers does a ones complement + return ui ^ readUIntExact(EIGHT_BYTES, ib & 0x1f); + } + + /** + * Reads a signed or unsigned 8-bit integer value in CBOR format. + * + * @read the small integer value, values in the range <tt>[-256..255]</tt> are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying output stream. + */ + public int readInt8() throws IOException { + int ib = m_is.read(); + + // in case of negative integers, extends the sign to all bits; otherwise zero... + long ui = expectIntegerType(ib); + // in case of negative integers does a ones complement + return (int) (ui ^ readUIntExact(ONE_BYTE, ib & 0x1f)); + } + + /** + * Prolog to reading a map of key-value pairs in CBOR format. + * + * @return the number of entries in the map, >= 0. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public long readMapLength() throws IOException { + return readMajorTypeWithSize(TYPE_MAP); + } + + /** + * Reads a <code>null</code>-value in CBOR format. + * + * @return always <code>null</code>. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public Object readNull() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, NULL); + return null; + } + + /** + * Reads a single byte value in CBOR format. + * + * @return the read byte value. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public byte readSimpleValue() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, ONE_BYTE); + return (byte) readUInt8(); + } + + /** + * Reads a signed or unsigned small (<= 23) integer value in CBOR format. + * + * @read the small integer value, values in the range <tt>[-24..23]</tt> are supported. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying output stream. + */ + public int readSmallInt() throws IOException { + int ib = m_is.read(); + + // in case of negative integers, extends the sign to all bits; otherwise zero... + long ui = expectIntegerType(ib); + // in case of negative integers does a ones complement + return (int) (ui ^ readUIntExact(-1, ib & 0x1f)); + } + + /** + * Reads a semantic tag value in CBOR format. + * + * @return the read tag value. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public long readTag() throws IOException { + return readUInt(readMajorType(TYPE_TAG), false /* breakAllowed */); + } + + /** + * Reads an UTF-8 encoded string value in CBOR format. + * + * @return the read UTF-8 encoded string, never <code>null</code>. In case the encoded string has a length of <tt>0</tt>, an empty string is returned. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public String readTextString(int maxLen) throws IOException { + long len = readMajorTypeWithSize(TYPE_TEXT_STRING); + if (len < 0) + fail("Infinite-length text strings not supported!"); + if (len > Integer.MAX_VALUE) + fail("String length too long!"); + if (len > maxLen) + fail("Invalid cbor: text string longer than original bytes!"); + return new String(readFully(new byte[(int) len]), "UTF-8"); + } + + /** + * Prolog to reading an UTF-8 encoded string value in CBOR format. + * + * @return the length of the string to read, or <tt>-1</tt> in case of infinite-length strings. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public long readTextStringLength() throws IOException { + return readMajorTypeWithSize(TYPE_TEXT_STRING); + } + + /** + * Reads an undefined value in CBOR format. + * + * @return always <code>null</code>. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + public Object readUndefined() throws IOException { + readMajorTypeExact(TYPE_FLOAT_SIMPLE, UNDEFINED); + return null; + } + + protected long expectIntegerType(int ib) throws IOException { + int majorType = ((ib & 0xFF) >>> 5); + if ((majorType != TYPE_UNSIGNED_INTEGER) && (majorType != TYPE_NEGATIVE_INTEGER)) { + fail("Unexpected type: %s, expected type %s or %s!", getName(majorType), getName(TYPE_UNSIGNED_INTEGER), + getName(TYPE_NEGATIVE_INTEGER)); + } + return -majorType; + } + + /** + * Reads the next major type from the underlying input stream, and verifies whether it matches the given expectation. + * + * @param majorType the expected major type, cannot be <code>null</code> (unchecked). + * @return the read subtype, or payload, of the read major type. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + protected int readMajorType(int majorType) throws IOException { + int ib = m_is.read(); + if (majorType != ((ib >>> 5) & 0x07)) { + fail("Unexpected type: %s, expected: %s!", getName(ib), getName(majorType)); + } + return ib & 0x1F; + } + + /** + * Reads the next major type from the underlying input stream, and verifies whether it matches the given expectations. + * + * @param majorType the expected major type, cannot be <code>null</code> (unchecked); + * @param subtype the expected subtype. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + protected void readMajorTypeExact(int majorType, int subtype) throws IOException { + int st = readMajorType(majorType); + if ((st ^ subtype) != 0) { + fail("Unexpected subtype: %d, expected: %d!", st, subtype); + } + } + + /** + * Reads the next major type from the underlying input stream, verifies whether it matches the given expectation, and decodes the payload into a size. + * + * @param majorType the expected major type, cannot be <code>null</code> (unchecked). + * @return the number of succeeding bytes, >= 0, or <tt>-1</tt> if an infinite-length type is read. + * @throws IOException in case of I/O problems reading the CBOR-encoded value from the underlying input stream. + */ + protected long readMajorTypeWithSize(int majorType) throws IOException { + return readUInt(readMajorType(majorType), true /* breakAllowed */); + } + + /** + * Reads an unsigned integer with a given length-indicator. + * + * @param length the length indicator to use; + * @return the read unsigned integer, as long value. + * @throws IOException in case of I/O problems reading the unsigned integer from the underlying input stream. + */ + protected long readUInt(int length, boolean breakAllowed) throws IOException { + long result = -1; + if (length < ONE_BYTE) { + result = length; + } else if (length == ONE_BYTE) { + result = readUInt8(); + } else if (length == TWO_BYTES) { + result = readUInt16(); + } else if (length == FOUR_BYTES) { + result = readUInt32(); + } else if (length == EIGHT_BYTES) { + result = readUInt64(); + } else if (breakAllowed && length == BREAK) { + return -1; + } + if (result < 0) { + fail("Not well-formed CBOR integer found, invalid length: %d!", result); + } + return result; + } + + /** + * Reads an unsigned 16-bit integer value + * + * @return value the read value, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected int readUInt16() throws IOException { + byte[] buf = readFully(new byte[2]); + return (buf[0] & 0xFF) << 8 | (buf[1] & 0xFF); + } + + /** + * Reads an unsigned 32-bit integer value + * + * @return value the read value, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected long readUInt32() throws IOException { + byte[] buf = readFully(new byte[4]); + return ((buf[0] & 0xFF) << 24 | (buf[1] & 0xFF) << 16 | (buf[2] & 0xFF) << 8 | (buf[3] & 0xFF)) & 0xffffffffL; + } + + /** + * Reads an unsigned 64-bit integer value + * + * @return value the read value, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected long readUInt64() throws IOException { + byte[] buf = readFully(new byte[8]); + return (buf[0] & 0xFFL) << 56 | (buf[1] & 0xFFL) << 48 | (buf[2] & 0xFFL) << 40 | (buf[3] & 0xFFL) << 32 | // + (buf[4] & 0xFFL) << 24 | (buf[5] & 0xFFL) << 16 | (buf[6] & 0xFFL) << 8 | (buf[7] & 0xFFL); + } + + /** + * Reads an unsigned 8-bit integer value + * + * @return value the read value, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected int readUInt8() throws IOException { + return m_is.read() & 0xff; + } + + /** + * Reads an unsigned integer with a given length-indicator. + * + * @param length the length indicator to use; + * @return the read unsigned integer, as long value. + * @throws IOException in case of I/O problems reading the unsigned integer from the underlying input stream. + */ + protected long readUIntExact(int expectedLength, int length) throws IOException { + if (((expectedLength == -1) && (length >= ONE_BYTE)) || ((expectedLength >= 0) && (length != expectedLength))) { + fail("Unexpected payload/length! Expected %s, but got %s.", lengthToString(expectedLength), + lengthToString(length)); + } + return readUInt(length, false /* breakAllowed */); + } + + private byte[] readFully(byte[] buf) throws IOException { + int len = buf.length; + int n = 0, off = 0; + while (n < len) { + int count = m_is.read(buf, off + n, len - n); + if (count < 0) { + throw new EOFException(); + } + n += count; + } + return buf; + } +}
\ No newline at end of file diff --git a/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborEncoder.java b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborEncoder.java new file mode 100644 index 0000000..3235db8 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborEncoder.java @@ -0,0 +1,490 @@ +package org.peergos.cbor; + +/* + * JACOB - CBOR implementation in Java. + * + * (C) Copyright - 2013 - J.W. Janssen <j.w.janssen@lxtreme.nl> + * + * Licensed under Apache License v2.0. + */ + +import java.io.*; + +import static org.peergos.cbor.CborConstants.*; + +/** + * Provides an encoder capable of encoding data into CBOR format to a given {@link OutputStream}. + */ +public class CborEncoder { + private static final int NEG_INT_MASK = TYPE_NEGATIVE_INTEGER << 5; + + private final OutputStream m_os; + + /** + * Creates a new {@link CborEncoder} instance. + * + * @param os the actual output stream to write the CBOR-encoded data to, cannot be <code>null</code>. + */ + public CborEncoder(OutputStream os) { + if (os == null) { + throw new IllegalArgumentException("OutputStream cannot be null!"); + } + m_os = os; + } + + /** + * Interprets a given float-value as a half-precision float value and + * converts it to its raw integer form, as defined in IEEE 754. + * <p> + * Taken from: <a href="http://stackoverflow.com/a/6162687/229140">this Stack Overflow answer</a>. + * </p> + * + * @param fval the value to convert. + * @return the raw integer representation of the given float value. + */ + static int halfPrecisionToRawIntBits(float fval) { + int fbits = Float.floatToIntBits(fval); + int sign = (fbits >>> 16) & 0x8000; + int val = (fbits & 0x7fffffff) + 0x1000; + + // might be or become NaN/Inf + if (val >= 0x47800000) { + if ((fbits & 0x7fffffff) >= 0x47800000) { // is or must become NaN/Inf + if (val < 0x7f800000) { + // was value but too large, make it +/-Inf + return sign | 0x7c00; + } + return sign | 0x7c00 | (fbits & 0x007fffff) >>> 13; // keep NaN (and Inf) bits + } + return sign | 0x7bff; // unrounded not quite Inf + } + if (val >= 0x38800000) { + // remains normalized value + return sign | val - 0x38000000 >>> 13; // exp - 127 + 15 + } + if (val < 0x33000000) { + // too small for subnormal + return sign; // becomes +/-0 + } + + val = (fbits & 0x7fffffff) >>> 23; + // add subnormal bit, round depending on cut off and div by 2^(1-(exp-127+15)) and >> 13 | exp=0 + return sign | ((fbits & 0x7fffff | 0x800000) + (0x800000 >>> val - 102) >>> 126 - val); + } + + /** + * Writes the start of an indefinite-length array. + * <p> + * After calling this method, one is expected to write the given number of array elements, which can be of any type. No length checks are performed.<br/> + * After all array elements are written, one should write a single break value to end the array, see {@link #writeBreak()}. + * </p> + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeArrayStart() throws IOException { + writeSimpleType(TYPE_ARRAY, BREAK); + } + + /** + * Writes the start of a definite-length array. + * <p> + * After calling this method, one is expected to write the given number of array elements, which can be of any type. No length checks are performed. + * </p> + * + * @param length the number of array elements to write, should >= 0. + * @throws IllegalArgumentException in case the given length was negative; + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeArrayStart(int length) throws IOException { + if (length < 0) { + throw new IllegalArgumentException("Invalid array-length!"); + } + writeType(TYPE_ARRAY, length); + } + + /** + * Writes a boolean value in canonical CBOR format. + * + * @param value the boolean to write. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeBoolean(boolean value) throws IOException { + writeSimpleType(TYPE_FLOAT_SIMPLE, value ? TRUE : FALSE); + } + + /** + * Writes a "break" stop-value in canonical CBOR format. + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeBreak() throws IOException { + writeSimpleType(TYPE_FLOAT_SIMPLE, BREAK); + } + + /** + * Writes a byte string in canonical CBOR-format. + * + * @param bytes the byte string to write, can be <code>null</code> in which case a byte-string of length <tt>0</tt> is written. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeByteString(byte[] bytes) throws IOException { + writeString(TYPE_BYTE_STRING, bytes); + } + + /** + * Writes the start of an indefinite-length byte string. + * <p> + * After calling this method, one is expected to write the given number of string parts. No length checks are performed.<br/> + * After all string parts are written, one should write a single break value to end the string, see {@link #writeBreak()}. + * </p> + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeByteStringStart() throws IOException { + writeSimpleType(TYPE_BYTE_STRING, BREAK); + } + + /** + * Writes a double-precision float value in canonical CBOR format. + * + * @param value the value to write, values from {@link Double#MIN_VALUE} to {@link Double#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeDouble(double value) throws IOException { + throw new IllegalStateException("Unimplemented!"); +// writeUInt64(TYPE_FLOAT_SIMPLE << 5, Double.doubleToRawLongBits(value)); + } + + /** + * Writes a single-precision float value in canonical CBOR format. + * + * @param value the value to write, values from {@link Float#MIN_VALUE} to {@link Float#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeFloat(float value) throws IOException { + throw new IllegalStateException("Unimplemented!"); +// writeUInt32(TYPE_FLOAT_SIMPLE << 5, Float.floatToRawIntBits(value)); + } + + /** + * Writes a half-precision float value in canonical CBOR format. + * + * @param value the value to write, values from {@link Float#MIN_VALUE} to {@link Float#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeHalfPrecisionFloat(float value) throws IOException { + writeUInt16(TYPE_FLOAT_SIMPLE << 5, halfPrecisionToRawIntBits(value)); + } + + /** + * Writes a signed or unsigned integer value in canonical CBOR format, that is, tries to encode it in a little bytes as possible.. + * + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeInt(long value) throws IOException { + // extends the sign over all bits... + long sign = value >> 63; + // in case value is negative, this bit should be set... + int mt = (int) (sign & NEG_INT_MASK); + // complement negative value... + value = (sign ^ value); + + writeUInt(mt, value); + } + + /** + * Writes a signed or unsigned 16-bit integer value in CBOR format. + * + * @param value the value to write, values from <tt>[-65536..65535]</tt> are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeInt16(int value) throws IOException { + // extends the sign over all bits... + int sign = value >> 31; + // in case value is negative, this bit should be set... + int mt = (int) (sign & NEG_INT_MASK); + // complement negative value... + writeUInt16(mt, (sign ^ value) & 0xffff); + } + + /** + * Writes a signed or unsigned 32-bit integer value in CBOR format. + * + * @param value the value to write, values in the range <tt>[-4294967296..4294967295]</tt> are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeInt32(long value) throws IOException { + // extends the sign over all bits... + long sign = value >> 63; + // in case value is negative, this bit should be set... + int mt = (int) (sign & NEG_INT_MASK); + // complement negative value... + writeUInt32(mt, (int) ((sign ^ value) & 0xffffffffL)); + } + + /** + * Writes a signed or unsigned 64-bit integer value in CBOR format. + * + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeInt64(long value) throws IOException { + // extends the sign over all bits... + long sign = value >> 63; + // in case value is negative, this bit should be set... + int mt = (int) (sign & NEG_INT_MASK); + // complement negative value... + writeUInt64(mt, sign ^ value); + } + + /** + * Writes a signed or unsigned 8-bit integer value in CBOR format. + * + * @param value the value to write, values in the range <tt>[-256..255]</tt> are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeInt8(int value) throws IOException { + // extends the sign over all bits... + int sign = value >> 31; + // in case value is negative, this bit should be set... + int mt = (int) (sign & NEG_INT_MASK); + // complement negative value... + writeUInt8(mt, (sign ^ value) & 0xff); + } + + /** + * Writes the start of an indefinite-length map. + * <p> + * After calling this method, one is expected to write any number of map entries, as separate key and value. Keys and values can both be of any type. No length checks are performed.<br/> + * After all map entries are written, one should write a single break value to end the map, see {@link #writeBreak()}. + * </p> + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeMapStart() throws IOException { + writeSimpleType(TYPE_MAP, BREAK); + } + + /** + * Writes the start of a finite-length map. + * <p> + * After calling this method, one is expected to write any number of map entries, as separate key and value. Keys and values can both be of any type. No length checks are performed. + * </p> + * + * @param length the number of map entries to write, should >= 0. + * @throws IllegalArgumentException in case the given length was negative; + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeMapStart(int length) throws IOException { + if (length < 0) { + throw new IllegalArgumentException("Invalid length of map!"); + } + writeType(TYPE_MAP, length); + } + + /** + * Writes a <code>null</code> value in canonical CBOR format. + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeNull() throws IOException { + writeSimpleType(TYPE_FLOAT_SIMPLE, NULL); + } + + /** + * Writes a simple value, i.e., an "atom" or "constant" value in canonical CBOR format. + * + * @param value the (unsigned byte) value to write, values from <tt>32</tt> to <tt>255</tt> are supported (though not enforced). + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeSimpleValue(byte simpleValue) throws IOException { + // convert to unsigned value... + int value = (simpleValue & 0xff); + writeType(TYPE_FLOAT_SIMPLE, value); + } + + /** + * Writes a signed or unsigned small (<= 23) integer value in CBOR format. + * + * @param value the value to write, values in the range <tt>[-24..23]</tt> are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeSmallInt(int value) throws IOException { + // extends the sign over all bits... + int sign = value >> 31; + // in case value is negative, this bit should be set... + int mt = (int) (sign & NEG_INT_MASK); + // complement negative value... + value = Math.min(0x17, (sign ^ value)); + + m_os.write((int) (mt | value)); + } + + /** + * Writes a semantic tag in canonical CBOR format. + * + * @param tag the tag to write, should >= 0. + * @throws IllegalArgumentException in case the given tag was negative; + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeTag(long tag) throws IOException { + if (tag < 0) { + throw new IllegalArgumentException("Invalid tag specification, cannot be negative!"); + } + writeType(TYPE_TAG, tag); + } + + /** + * Writes an UTF-8 string in canonical CBOR-format. + * <p> + * Note that this method is <em>platform</em> specific, as the given string value will be encoded in a byte array + * using the <em>platform</em> encoding! This means that the encoding must be standardized and known. + * </p> + * + * @param value the UTF-8 string to write, can be <code>null</code> in which case an UTF-8 string of length <tt>0</tt> is written. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeTextString(String value) throws IOException { + writeString(TYPE_TEXT_STRING, value == null ? null : value.getBytes("UTF-8")); + } + + /** + * Writes the start of an indefinite-length UTF-8 string. + * <p> + * After calling this method, one is expected to write the given number of string parts. No length checks are performed.<br/> + * After all string parts are written, one should write a single break value to end the string, see {@link #writeBreak()}. + * </p> + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeTextStringStart() throws IOException { + writeSimpleType(TYPE_TEXT_STRING, BREAK); + } + + /** + * Writes an "undefined" value in canonical CBOR format. + * + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + public void writeUndefined() throws IOException { + writeSimpleType(TYPE_FLOAT_SIMPLE, UNDEFINED); + } + + /** + * Encodes and writes the major type and value as a simple type. + * + * @param majorType the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from [0..31] are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeSimpleType(int majorType, int value) throws IOException { + m_os.write((majorType << 5) | (value & 0x1f)); + } + + /** + * Writes a byte string in canonical CBOR-format. + * + * @param majorType the major type of the string, should be either 0x40 or 0x60; + * @param bytes the byte string to write, can be <code>null</code> in which case a byte-string of length <tt>0</tt> is written. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeString(int majorType, byte[] bytes) throws IOException { + int len = (bytes == null) ? 0 : bytes.length; + writeType(majorType, len); + if (len > 0){ + m_os.write(bytes); + } + } + + /** + * Encodes and writes the major type indicator with a given payload (length). + * + * @param majorType the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeType(int majorType, long value) throws IOException { + writeUInt((majorType << 5), value); + } + + /** + * Encodes and writes an unsigned integer value, that is, tries to encode it in a little bytes as possible. + * + * @param mt the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeUInt(int mt, long value) throws IOException { + if (value < 0x18L) { + m_os.write((int) (mt | value)); + } else if (value < 0x100L) { + writeUInt8(mt, (int) value); + } else if (value < 0x10000L) { + writeUInt16(mt, (int) value); + } else if (value < 0x100000000L) { + writeUInt32(mt, (int) value); + } else { + writeUInt64(mt, value); + } + } + + /** + * Encodes and writes an unsigned 16-bit integer value + * + * @param mt the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeUInt16(int mt, int value) throws IOException { + m_os.write(mt | TWO_BYTES); + m_os.write(value >> 8); + m_os.write(value & 0xFF); + } + + /** + * Encodes and writes an unsigned 32-bit integer value + * + * @param mt the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeUInt32(int mt, int value) throws IOException { + m_os.write(mt | FOUR_BYTES); + m_os.write(value >> 24); + m_os.write(value >> 16); + m_os.write(value >> 8); + m_os.write(value & 0xFF); + } + + /** + * Encodes and writes an unsigned 64-bit integer value + * + * @param mt the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeUInt64(int mt, long value) throws IOException { + m_os.write(mt | EIGHT_BYTES); + m_os.write((int) (value >> 56)); + m_os.write((int) (value >> 48)); + m_os.write((int) (value >> 40)); + m_os.write((int) (value >> 32)); + m_os.write((int) (value >> 24)); + m_os.write((int) (value >> 16)); + m_os.write((int) (value >> 8)); + m_os.write((int) (value & 0xFF)); + } + + /** + * Encodes and writes an unsigned 8-bit integer value + * + * @param mt the major type of the value to write, denotes what semantics the written value has; + * @param value the value to write, values from {@link Long#MIN_VALUE} to {@link Long#MAX_VALUE} are supported. + * @throws IOException in case of I/O problems writing the CBOR-encoded value to the underlying output stream. + */ + protected void writeUInt8(int mt, int value) throws IOException { + m_os.write(mt | ONE_BYTE); + m_os.write(value & 0xFF); + } +}
\ No newline at end of file diff --git a/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborObject.java b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborObject.java new file mode 100644 index 0000000..145842b --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborObject.java @@ -0,0 +1,692 @@ +package org.peergos.cbor; + +import io.ipfs.cid.*; +import io.ipfs.multihash.*; + +import java.io.*; +import java.util.*; +import java.util.function.*; +import java.util.stream.*; + +import static org.peergos.cbor.CborConstants.*; + +public interface CborObject extends Cborable { + + void serialize(CborEncoder encoder); + + List<Multihash> links(); + + static List<Cid> getLinks(Cid h, byte[] data) { + return h.codec == Cid.Codec.Raw ? + Collections.emptyList() : + CborObject.fromByteArray(data) + .links() + .stream() + .map(m -> (Cid) m) + .collect(Collectors.toList()); + } + + default byte[] toByteArray() { + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + CborEncoder encoder = new CborEncoder(bout); + serialize(encoder); + return bout.toByteArray(); + } + + @Override + default CborObject toCbor() { + return this; + } + + int LINK_TAG = 42; + + static CborObject fromByteArray(byte[] cbor) { + return deserialize(new CborDecoder(new ByteArrayInputStream(cbor)), cbor.length); + } + + static CborObject read(InputStream in, int maxBytes) { + return deserialize(new CborDecoder(in), maxBytes); + } + + static CborObject deserialize(CborDecoder decoder, int maxGroupSize) { + try { + CborType type = decoder.peekType(); + switch (type.getMajorType()) { + case TYPE_TEXT_STRING: + return new CborString(decoder.readTextString(maxGroupSize)); + case CborConstants.TYPE_BYTE_STRING: + return new CborByteArray(decoder.readByteString(maxGroupSize)); + case CborConstants.TYPE_UNSIGNED_INTEGER: + return new CborLong(decoder.readInt()); + case CborConstants.TYPE_NEGATIVE_INTEGER: + return new CborLong(decoder.readInt()); + case CborConstants.TYPE_FLOAT_SIMPLE: + if (type.getAdditionalInfo() == CborConstants.NULL) { + decoder.readNull(); + return new CborNull(); + } + if (type.getAdditionalInfo() == CborConstants.TRUE) { + decoder.readBoolean(); + return new CborBoolean(true); + } + if (type.getAdditionalInfo() == CborConstants.FALSE) { + decoder.readBoolean(); + return new CborBoolean(false); + } + throw new IllegalStateException("Unimplemented simple type! " + type.getAdditionalInfo()); + case CborConstants.TYPE_MAP: { + long nValues = decoder.readMapLength(); + if (nValues > maxGroupSize) + throw new IllegalStateException("Invalid cbor: more map elements than original bytes!"); + SortedMap<CborString, CborObject> result = new TreeMap<>(); + for (long i=0; i < nValues; i++) { + CborString key = (CborString) deserialize(decoder, maxGroupSize); + CborObject value = deserialize(decoder, maxGroupSize); + result.put(key, value); + } + return new CborMap(result); + } + case CborConstants.TYPE_ARRAY: + long nItems = decoder.readArrayLength(); + if (nItems > maxGroupSize) + throw new IllegalStateException("Invalid cbor: more array elements than original bytes!"); + List<CborObject> res = new ArrayList<>((int) nItems); + for (long i=0; i < nItems; i++) + res.add(deserialize(decoder, maxGroupSize)); + return new CborList(res); + case CborConstants.TYPE_TAG: + long tag = decoder.readTag(); + if (tag == LINK_TAG) { + CborObject value = deserialize(decoder, maxGroupSize); + if (value instanceof CborString) + return new CborMerkleLink(Cid.decode(((CborString) value).value)); + if (value instanceof CborByteArray) { + byte[] bytes = ((CborByteArray) value).value; + if (bytes[0] == 0) // multibase for binary + return new CborMerkleLink(Cid.cast(Arrays.copyOfRange(bytes, 1, bytes.length))); + throw new IllegalStateException("Unknown Multibase decoding Merkle link: " + bytes[0]); + } + throw new IllegalStateException("Invalid type for merkle link: " + value); + } + throw new IllegalStateException("Unknown TAG in CBOR: " + type.getAdditionalInfo()); + default: + throw new IllegalStateException("Unimplemented cbor type: " + type); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + final class CborMap implements CborObject { + // Only String keys should be used in IPLD dag-cbor maps + private final SortedMap<CborString, CborObject> values; + + private CborMap(SortedMap<CborString, CborObject> values) { + this.values = values; + } + + public static CborMap build(Map<String, Cborable> values) { + SortedMap<CborString, CborObject> transformed = values.entrySet() + .stream() + .collect(Collectors.toMap( + e -> new CborString(e.getKey()), + e -> e.getValue().toCbor(), + (a, b) -> a, TreeMap::new)); + return new CborMap(transformed); + } + + public void put(String key, CborObject val) { + values.put(new CborString(key), val); + } + + public boolean containsKey(String key) { + return values.containsKey(new CborString(key)); + } + + public Set<String> keySet() { + return values.keySet().stream() + .map(c -> c.value) + .collect(Collectors.toSet()); + } + + public Cborable get(String key) { + return values.get(new CborString(key)); + } + + public <T> T getObject(String key, Function<Cborable, T> fromCbor) { + return fromCbor.apply(get(key)); + } + + public String getString(String key) { + return ((CborString) get(key)).value; + } + + public String getString(String key, String defaultValue) { + CborString cborKey = new CborString(key); + Cborable val = values.get(cborKey); + return val != null ? ((CborString) val).value : defaultValue; + } + + public long getLong(String key) { + return ((CborLong) get(key)).value; + } + + public Multihash getMerkleLink(String key) { + return ((CborMerkleLink) get(key)).target; + } + + public boolean getBoolean(String key) { + CborBoolean val = (CborBoolean) get(key); + return val != null && val.value; + } + + public boolean getBoolean(String key, boolean def) { + Cborable val = get(key); + if (val == null) + return def; + return ((CborBoolean) val).value; + } + + public Optional<byte[]> getOptionalByteArray(String key) { + return Optional.ofNullable((CborByteArray) get(key)).map(c -> c.value); + } + + public byte[] getByteArray(String key) { + return ((CborByteArray) get(key)).value; + } + + public Optional<Cborable> getOptional(String key) { + return Optional.ofNullable(get(key)); + } + + public <T> Optional<T> getOptional(String key, Function<Cborable, T> fromCbor) { + return Optional.ofNullable(get(key)).map(fromCbor); + } + + public Optional<Long> getOptionalLong(String key) { + return Optional.ofNullable((CborLong) get(key)).map(c -> c.value); + } + + public Optional<String> getOptionalString(String key) { + return Optional.ofNullable((CborString) get(key)).map(c -> c.value); + } + + public <T> List<T> getList(String key, Function<Cborable, T> fromCbor) { + CborList cborList = (CborList) get(key); + if (cborList == null) + return Collections.emptyList(); + return cborList.value + .stream() + .map(fromCbor) + .collect(Collectors.toList()); + } + + public void applyToAll(BiConsumer<String, Cborable> func) { + values.entrySet().forEach(e -> func.accept(e.getKey().value, e.getValue())); + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeMapStart(values.size()); + for (Map.Entry<CborString, CborObject> entry : values.entrySet()) { + entry.getKey().serialize(encoder); + entry.getValue().toCbor().serialize(encoder); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return values.values().stream() + .flatMap(cbor -> cbor.toCbor().links().stream()) + .collect(Collectors.toList()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborMap cborMap = (CborMap) o; + + return values != null ? values.equals(cborMap.values) : cborMap.values == null; + + } + + @Override + public int hashCode() { + return values != null ? values.hashCode() : 0; + } + + public CborList getList(String key) { + return (CborList) get(key); + } + + public <T> T get(String key, Function<? super Cborable, T> fromCbor) { + return fromCbor.apply(get(key)); + } + + public <K,V> Map<K,V> toMap(Function<? super Cborable, K> toKey, Function<? super Cborable, V> toValue) { + return values.entrySet().stream() + .collect(Collectors.toMap( + e -> toKey.apply(e.getKey()), + e -> toValue.apply(e.getValue()) + )); + } + + public <K,V> Map<K,V> getMap(String key, Function<? super Cborable, K> toKey, Function<? super Cborable, V> toValue) { + CborMap val = (CborMap) get(key); + if (val == null) + return Collections.emptyMap(); + return val.toMap(toKey, toValue); + } + + public <K,V> Map<K,V> getListMap(String key, Function<? super Cborable, K> toKey, Function<? super Cborable, V> toValue) { + CborList val = (CborList) get(key); + if (val == null) + return Collections.emptyMap(); + return val.getMap(toKey, toValue); + } + } + + static int compare(Multihash a, Multihash b) { + byte[] aHash = a.getHash(); + byte[] bHash = b.getHash(); + int compare = Integer.compare(aHash.length, bHash.length); + if (compare != 0) + return compare; + for (int i = 0; i < aHash.length; i++) { + compare = Byte.compare(aHash[i], bHash[i]); + if (compare != 0) + return compare; + } + return Integer.compare(a.getType().index, b.getType().index); + } + + final class CborMerkleLink implements CborObject, Comparable<CborMerkleLink> { + public final Multihash target; + + public CborMerkleLink(Multihash target) { + this.target = target; + } + + @Override + public int compareTo(CborMerkleLink that) { + return compare(this.target, that.target); + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeTag(LINK_TAG); + byte[] cid = target.toBytes(); + byte[] withMultibaseHeader = new byte[cid.length + 1]; + System.arraycopy(cid, 0, withMultibaseHeader, 1, cid.length); + encoder.writeByteString(withMultibaseHeader); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return Collections.singletonList(target); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborMerkleLink that = (CborMerkleLink) o; + + return target != null ? target.equals(that.target) : that.target == null; + + } + + @Override + public int hashCode() { + return target != null ? target.hashCode() : 0; + } + + @Override + public String toString() { + return target.toString(); + } + } + + final class CborList implements CborObject, Cborable { + public final List<? extends Cborable> value; + + public CborList(List<? extends Cborable> value) { + this.value = value; + } + + public CborList(Map<? extends Cborable, ? extends Cborable> map) { + this.value = map.entrySet().stream() + .flatMap(e -> Stream.of(e.getKey(), e.getValue())) + .collect(Collectors.toList()); + } + + public static <T> CborList build(List<T> in, Function<T, Cborable> toCbor) { + return new CborList(in.stream().map(toCbor).collect(Collectors.toList())); + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeArrayStart(value.size()); + for (Cborable object : value) { + object.toCbor().serialize(encoder); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return value.stream() + .flatMap(cbor -> cbor.toCbor().links().stream()) + .collect(Collectors.toList()); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborList cborList = (CborList) o; + + return value != null ? value.equals(cborList.value) : cborList.value == null; + } + + @Override + public int hashCode() { + return value != null ? value.hashCode() : 0; + } + + public <T> List<T> map(Function<? super Cborable, T> fromCbor) { + return value.stream() + .map(fromCbor) + .collect(Collectors.toList()); + } + + public long getLong(int index) { + return ((CborLong)value.get(index)).value; + } + + public <T> T get(int index, Function<? super Cborable, T> fromCbor) { + return fromCbor.apply(value.get(index)); + } + + public <K,V> Map<K, V> getMap(Function<? super Cborable, K> toKey, Function<? super Cborable, V> toValue) { + if (value.size() % 2 != 0) + throw new IllegalStateException(); + + Map<K, V> map = new HashMap<>(); + for (int i = 0; i < value.size(); i += 2) { + K key = toKey.apply(value.get(i)); + V _value = toValue.apply(value.get(i + 1)); + map.put(key, _value); + } + return map; + } + } + + final class CborBoolean implements CborObject { + public final boolean value; + + public CborBoolean(boolean value) { + this.value = value; + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeBoolean(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return Collections.emptyList(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborBoolean that = (CborBoolean) o; + + return value == that.value; + + } + + @Override + public int hashCode() { + return (value ? 1 : 0); + } + + @Override + public String toString() { + return "CborBoolean{" + + value + + '}'; + } + } + + final class CborByteArray implements CborObject, Comparable<CborByteArray> { + public final byte[] value; + + public CborByteArray(byte[] value) { + this.value = value; + } + + @Override + public int compareTo(CborByteArray other) { + return compare(value, other.value); + } + + /** This only matter so that we can have byte[]'s as keys in a sorted map deterministically + * + * @param a + * @param b + * @return + */ + public static int compare(byte[] a, byte[] b) + { + if (a.length != b.length) + return a.length - b.length; + for (int i=0; i < a.length; i++) + if (a[i] != b[i]) + return a[i] & 0xff - b[i] & 0xff; + return 0; + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeByteString(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return Collections.emptyList(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborByteArray that = (CborByteArray) o; + + return Arrays.equals(value, that.value); + + } + + @Override + public int hashCode() { + return Arrays.hashCode(value); + } + } + + final class CborString implements CborObject, Comparable<CborString> { + + public final String value; + + public CborString(String value) { + this.value = value; + } + + @Override + public int compareTo(CborString cborString) { + int lenDiff = value.length() - cborString.value.length(); + if (lenDiff != 0) + return lenDiff; + return value.compareTo(cborString.value); + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeTextString(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return Collections.emptyList(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborString that = (CborString) o; + + return value.equals(that.value); + + } + + @Override + public int hashCode() { + return value.hashCode(); + } + + @Override + public String toString() { + return "CborString{\"" + + value + + "\"}"; + } + + public static String getString(Cborable cbor) { + return ((CborString)cbor).value; + } + } + + final class CborLong implements CborObject, Comparable<CborLong> { + public final long value; + + public CborLong(long value) { + this.value = value; + } + + @Override + public int compareTo(CborLong other) { + return Long.compare(value, other.value); + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeInt(value); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return Collections.emptyList(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + CborLong cborLong = (CborLong) o; + + return value == cborLong.value; + + } + + @Override + public int hashCode() { + return (int) (value ^ (value >>> 32)); + } + + @Override + public String toString() { + return "CborLong{" + + value + + '}'; + } + } + + final class CborNull implements CborObject, Comparable<CborNull> { + public CborNull() {} + + @Override + public int compareTo(CborNull cborNull) { + return 0; + } + + @Override + public void serialize(CborEncoder encoder) { + try { + encoder.writeNull(); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public List<Multihash> links() { + return Collections.emptyList(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + return true; + } + + @Override + public int hashCode() { + return 0; + } + + @Override + public String toString() { + return "CborNull{}"; + } + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborType.java b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborType.java new file mode 100644 index 0000000..56bede0 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/cbor/CborType.java @@ -0,0 +1,143 @@ +package org.peergos.cbor; + +/* + * JACOB - CBOR implementation in Java. + * + * (C) Copyright - 2013 - J.W. Janssen <j.w.janssen@lxtreme.nl> + * + * Licensed under Apache License v2.0. + */ + +import static org.peergos.cbor.CborConstants.*; + +/** + * Represents the various major types in CBOR, along with their . + * <p> + * The major type is encoded in the upper three bits of each initial byte. The lower 5 bytes represent any additional information. + * </p> + */ +public class CborType { + private final int m_major; + private final int m_additional; + + private CborType(int major, int additional) { + m_major = major; + m_additional = additional; + } + + /** + * Returns a descriptive string for the given major type. + * + * @param mt the major type to return as string, values from [0..7] are supported. + * @return the name of the given major type, as String, never <code>null</code>. + * @throws IllegalArgumentException in case the given major type is not supported. + */ + public static String getName(int mt) { + switch (mt) { + case TYPE_ARRAY: + return "array"; + case TYPE_BYTE_STRING: + return "byte string"; + case TYPE_FLOAT_SIMPLE: + return "float/simple value"; + case TYPE_MAP: + return "map"; + case TYPE_NEGATIVE_INTEGER: + return "negative integer"; + case TYPE_TAG: + return "tag"; + case TYPE_TEXT_STRING: + return "text string"; + case TYPE_UNSIGNED_INTEGER: + return "unsigned integer"; + default: + throw new IllegalArgumentException("Invalid major type: " + mt); + } + } + + /** + * Decodes a given byte value to a {@link CborType} value. + * + * @param i the input byte (8-bit) to decode into a {@link CborType} instance. + * @return a {@link CborType} instance, never <code>null</code>. + */ + public static CborType valueOf(int i) { + return new CborType((i & 0xff) >>> 5, i & 0x1f); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + + CborType other = (CborType) obj; + return (m_major == other.m_major) && (m_additional == other.m_additional); + } + + /** + * @return the additional information of this type, as integer value from [0..31]. + */ + public int getAdditionalInfo() { + return m_additional; + } + + /** + * @return the major type, as integer value from [0..7]. + */ + public int getMajorType() { + return m_major; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + m_additional; + result = prime * result + m_major; + return result; + } + + /** + * @return <code>true</code> if this type allows for an infinite-length payload, + * <code>false</code> if only definite-length payloads are allowed. + */ + public boolean isBreakAllowed() { + return m_major == TYPE_ARRAY || m_major == TYPE_BYTE_STRING || m_major == TYPE_MAP + || m_major == TYPE_TEXT_STRING; + } + + /** + * Determines whether the major type of a given {@link CborType} equals the major type of this {@link CborType}. + * + * @param other the {@link CborType} to compare against, cannot be <code>null</code>. + * @return <code>true</code> if the given {@link CborType} is of the same major type as this {@link CborType}, <code>false</code> otherwise. + * @throws IllegalArgumentException in case the given argument was <code>null</code>. + */ + public boolean isEqualType(CborType other) { + if (other == null) { + throw new IllegalArgumentException("Parameter cannot be null!"); + } + return m_major == other.m_major; + } + + /** + * Determines whether the major type of a given byte value (representing an encoded {@link CborType}) equals the major type of this {@link CborType}. + * + * @param encoded the encoded CBOR type to compare. + * @return <code>true</code> if the given byte value represents the same major type as this {@link CborType}, <code>false</code> otherwise. + */ + public boolean isEqualType(int encoded) { + return m_major == ((encoded & 0xff) >>> 5); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(getName(m_major)).append('(').append(m_additional).append(')'); + return sb.toString(); + } +}
\ No newline at end of file diff --git a/modules/vendor-nabu/src/main/java/org/peergos/cbor/Cborable.java b/modules/vendor-nabu/src/main/java/org/peergos/cbor/Cborable.java new file mode 100644 index 0000000..8227da4 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/cbor/Cborable.java @@ -0,0 +1,16 @@ +package org.peergos.cbor; + +import java.util.function.*; + +public interface Cborable { + + CborObject toCbor(); + + default byte[] serialize() { + return toCbor().toByteArray(); + } + + static <T> Function<byte[], T> parser(Function<Cborable, T> parser) { + return arr -> parser.apply(CborObject.fromByteArray(arr)); + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/util/ArrayOps.java b/modules/vendor-nabu/src/main/java/org/peergos/util/ArrayOps.java new file mode 100644 index 0000000..c4725a3 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/util/ArrayOps.java @@ -0,0 +1,42 @@ +package org.peergos.util; + +public class ArrayOps { + + private static String[] HEX_DIGITS = new String[]{ + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "a", "b", "c", "d", "e", "f"}; + private static String[] HEX = new String[256]; + static { + for (int i=0; i < 256; i++) + HEX[i] = HEX_DIGITS[(i >> 4) & 0xF] + HEX_DIGITS[i & 0xF]; + } + + public static String byteToHex(byte b) { + return HEX[b & 0xFF]; + } + + public static String bytesToHex(byte[] data) + { + StringBuilder s = new StringBuilder(); + for (byte b : data) + s.append(byteToHex(b)); + return s.toString(); + } + + public static byte[] hexToBytes(String hex) { + byte[] res = new byte[hex.length()/2]; + for (int i=0; i < res.length; i++) + res[i] = (byte) Integer.parseInt(hex.substring(2*i, 2*i+2), 16); + return res; + } + + public static boolean equalArrays(byte[] a, int aStart, int aEnd, byte[] b, int bStart, int bEnd) { + int len = aEnd - aStart; + if (len != bEnd - bStart) + return false; + for (int i=0; i < len; i++) { + if (a[aStart + i] != b[bStart + i]) + return false; + } + return true; + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/util/Exceptions.java b/modules/vendor-nabu/src/main/java/org/peergos/util/Exceptions.java new file mode 100644 index 0000000..c1d62d7 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/util/Exceptions.java @@ -0,0 +1,14 @@ +package org.peergos.util; + +import java.util.concurrent.ExecutionException; + +public class Exceptions { + public static Throwable getRootCause(Throwable t) { + Throwable cause = t.getCause(); + if (t instanceof ExecutionException) + return getRootCause(cause); + if (t instanceof RuntimeException && cause != null && cause != t) + return getRootCause(cause); + return t; + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/util/LRUCache.java b/modules/vendor-nabu/src/main/java/org/peergos/util/LRUCache.java new file mode 100644 index 0000000..36ed6c9 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/util/LRUCache.java @@ -0,0 +1,16 @@ +package org.peergos.util; + +import java.util.*; + +public class LRUCache<K, V> extends LinkedHashMap<K, V> { + private final int cacheSize; + + public LRUCache(int cacheSize) { + super(16, 0.75f, true); + this.cacheSize = cacheSize; + } + + protected boolean removeEldestEntry(Map.Entry<K, V> eldest) { + return size() >= cacheSize; + } +} diff --git a/modules/vendor-nabu/src/main/java/org/peergos/util/Pair.java b/modules/vendor-nabu/src/main/java/org/peergos/util/Pair.java new file mode 100755 index 0000000..0edbc31 --- /dev/null +++ b/modules/vendor-nabu/src/main/java/org/peergos/util/Pair.java @@ -0,0 +1,47 @@ +package org.peergos.util; + +import java.util.function.Function; + +public class Pair<L,R> { + public final L left; + public final R right; + + public Pair(L left, R right) { + this.left = left; + this.right = right; + } + + public <B,D> Pair<B, D> apply(Function<L, B> applyLeft, Function<R, D> applyRight) { + return new Pair<>( + applyLeft.apply(left), + applyRight.apply(right)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + + Pair<?, ?> pair = (Pair<?, ?>) o; + + if (left != null ? !left.equals(pair.left) : pair.left != null) return false; + return right != null ? right.equals(pair.right) : pair.right == null; + + } + + @Override + public int hashCode() { + int result = left != null ? left.hashCode() : 0; + result = 31 * result + (right != null ? right.hashCode() : 0); + return result; + } + + @Override + public String toString() { + return "(" + left.toString() + ", " + right.toString() + ")"; + } + + public static <E, F> Pair<E, F> of(E left, F right) { + return new Pair<>(left, right); + } +} |
