use core::cmp::{self, Ordering};
use core::fmt::{self, Display, Formatter};
use hashes::{sha256d, siphash24, Hash};
use internals::write_err;
use io::{Read, Write};
use crate::blockdata::block::{Block, BlockHash};
use crate::blockdata::script::Script;
use crate::blockdata::transaction::OutPoint;
use crate::consensus::encode::VarInt;
use crate::consensus::{Decodable, Encodable};
use crate::internal_macros::impl_hashencode;
use crate::prelude::*;
const P: u8 = 19;
const M: u64 = 784931;
hashes::hash_newtype! {
pub struct FilterHash(sha256d::Hash);
pub struct FilterHeader(sha256d::Hash);
}
impl_hashencode!(FilterHash);
impl_hashencode!(FilterHeader);
#[derive(Debug)]
#[non_exhaustive]
pub enum Error {
UtxoMissing(OutPoint),
Io(io::Error),
}
internals::impl_from_infallible!(Error);
impl Display for Error {
fn fmt(&self, f: &mut Formatter) -> Result<(), fmt::Error> {
use Error::*;
match *self {
UtxoMissing(ref coin) => write!(f, "unresolved UTXO {}", coin),
Io(ref e) => write_err!(f, "IO error"; e),
}
}
}
#[cfg(feature = "std")]
impl std::error::Error for Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
use Error::*;
match *self {
UtxoMissing(_) => None,
Io(ref e) => Some(e),
}
}
}
impl From<io::Error> for Error {
fn from(io: io::Error) -> Self { Error::Io(io) }
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct BlockFilter {
pub content: Vec<u8>,
}
impl FilterHash {
pub fn filter_header(&self, previous_filter_header: &FilterHeader) -> FilterHeader {
let mut header_data = [0u8; 64];
header_data[0..32].copy_from_slice(&self[..]);
header_data[32..64].copy_from_slice(&previous_filter_header[..]);
FilterHeader::hash(&header_data)
}
}
impl BlockFilter {
pub fn new(content: &[u8]) -> BlockFilter { BlockFilter { content: content.to_vec() } }
pub fn new_script_filter<M, S>(block: &Block, script_for_coin: M) -> Result<BlockFilter, Error>
where
M: Fn(&OutPoint) -> Result<S, Error>,
S: Borrow<Script>,
{
let mut out = Vec::new();
let mut writer = BlockFilterWriter::new(&mut out, block);
writer.add_output_scripts();
writer.add_input_scripts(script_for_coin)?;
writer.finish()?;
Ok(BlockFilter { content: out })
}
pub fn filter_header(&self, previous_filter_header: &FilterHeader) -> FilterHeader {
let filter_hash = FilterHash::hash(self.content.as_slice());
filter_hash.filter_header(previous_filter_header)
}
pub fn match_any<I>(&self, block_hash: &BlockHash, query: I) -> Result<bool, Error>
where
I: Iterator,
I::Item: Borrow<[u8]>,
{
let filter_reader = BlockFilterReader::new(block_hash);
filter_reader.match_any(&mut self.content.as_slice(), query)
}
pub fn match_all<I>(&self, block_hash: &BlockHash, query: I) -> Result<bool, Error>
where
I: Iterator,
I::Item: Borrow<[u8]>,
{
let filter_reader = BlockFilterReader::new(block_hash);
filter_reader.match_all(&mut self.content.as_slice(), query)
}
}
pub struct BlockFilterWriter<'a, W> {
block: &'a Block,
writer: GcsFilterWriter<'a, W>,
}
impl<'a, W: Write> BlockFilterWriter<'a, W> {
pub fn new(writer: &'a mut W, block: &'a Block) -> BlockFilterWriter<'a, W> {
let block_hash_as_int = block.block_hash().to_byte_array();
let k0 = u64::from_le_bytes(block_hash_as_int[0..8].try_into().expect("8 byte slice"));
let k1 = u64::from_le_bytes(block_hash_as_int[8..16].try_into().expect("8 byte slice"));
let writer = GcsFilterWriter::new(writer, k0, k1, M, P);
BlockFilterWriter { block, writer }
}
pub fn add_output_scripts(&mut self) {
for transaction in &self.block.txdata {
for output in &transaction.output {
if !output.script_pubkey.is_op_return() {
self.add_element(output.script_pubkey.as_bytes());
}
}
}
}
pub fn add_input_scripts<M, S>(&mut self, script_for_coin: M) -> Result<(), Error>
where
M: Fn(&OutPoint) -> Result<S, Error>,
S: Borrow<Script>,
{
for script in self
.block
.txdata
.iter()
.skip(1) .flat_map(|t| t.input.iter().map(|i| &i.previous_output))
.map(script_for_coin)
{
match script {
Ok(script) => self.add_element(script.borrow().as_bytes()),
Err(e) => return Err(e),
}
}
Ok(())
}
pub fn add_element(&mut self, data: &[u8]) { self.writer.add_element(data); }
pub fn finish(&mut self) -> Result<usize, io::Error> { self.writer.finish() }
}
pub struct BlockFilterReader {
reader: GcsFilterReader,
}
impl BlockFilterReader {
pub fn new(block_hash: &BlockHash) -> BlockFilterReader {
let block_hash_as_int = block_hash.to_byte_array();
let k0 = u64::from_le_bytes(block_hash_as_int[0..8].try_into().expect("8 byte slice"));
let k1 = u64::from_le_bytes(block_hash_as_int[8..16].try_into().expect("8 byte slice"));
BlockFilterReader { reader: GcsFilterReader::new(k0, k1, M, P) }
}
pub fn match_any<I, R>(&self, reader: &mut R, query: I) -> Result<bool, Error>
where
I: Iterator,
I::Item: Borrow<[u8]>,
R: Read + ?Sized,
{
self.reader.match_any(reader, query)
}
pub fn match_all<I, R>(&self, reader: &mut R, query: I) -> Result<bool, Error>
where
I: Iterator,
I::Item: Borrow<[u8]>,
R: Read + ?Sized,
{
self.reader.match_all(reader, query)
}
}
pub struct GcsFilterReader {
filter: GcsFilter,
m: u64,
}
impl GcsFilterReader {
pub fn new(k0: u64, k1: u64, m: u64, p: u8) -> GcsFilterReader {
GcsFilterReader { filter: GcsFilter::new(k0, k1, p), m }
}
pub fn match_any<I, R>(&self, reader: &mut R, query: I) -> Result<bool, Error>
where
I: Iterator,
I::Item: Borrow<[u8]>,
R: Read + ?Sized,
{
let n_elements: VarInt = Decodable::consensus_decode(reader).unwrap_or(VarInt(0));
let nm = n_elements.0 * self.m;
let mut mapped =
query.map(|e| map_to_range(self.filter.hash(e.borrow()), nm)).collect::<Vec<_>>();
mapped.sort_unstable();
if mapped.is_empty() {
return Ok(true);
}
if n_elements.0 == 0 {
return Ok(false);
}
let mut reader = BitStreamReader::new(reader);
let mut data = self.filter.golomb_rice_decode(&mut reader)?;
let mut remaining = n_elements.0 - 1;
for p in mapped {
loop {
match data.cmp(&p) {
Ordering::Equal => return Ok(true),
Ordering::Less =>
if remaining > 0 {
data += self.filter.golomb_rice_decode(&mut reader)?;
remaining -= 1;
} else {
return Ok(false);
},
Ordering::Greater => break,
}
}
}
Ok(false)
}
pub fn match_all<I, R>(&self, reader: &mut R, query: I) -> Result<bool, Error>
where
I: Iterator,
I::Item: Borrow<[u8]>,
R: Read + ?Sized,
{
let n_elements: VarInt = Decodable::consensus_decode(reader).unwrap_or(VarInt(0));
let nm = n_elements.0 * self.m;
let mut mapped =
query.map(|e| map_to_range(self.filter.hash(e.borrow()), nm)).collect::<Vec<_>>();
mapped.sort_unstable();
mapped.dedup();
if mapped.is_empty() {
return Ok(true);
}
if n_elements.0 == 0 {
return Ok(false);
}
let mut reader = BitStreamReader::new(reader);
let mut data = self.filter.golomb_rice_decode(&mut reader)?;
let mut remaining = n_elements.0 - 1;
for p in mapped {
loop {
match data.cmp(&p) {
Ordering::Equal => break,
Ordering::Less =>
if remaining > 0 {
data += self.filter.golomb_rice_decode(&mut reader)?;
remaining -= 1;
} else {
return Ok(false);
},
Ordering::Greater => return Ok(false),
}
}
}
Ok(true)
}
}
fn map_to_range(hash: u64, nm: u64) -> u64 { ((hash as u128 * nm as u128) >> 64) as u64 }
pub struct GcsFilterWriter<'a, W> {
filter: GcsFilter,
writer: &'a mut W,
elements: BTreeSet<Vec<u8>>,
m: u64,
}
impl<'a, W: Write> GcsFilterWriter<'a, W> {
pub fn new(writer: &'a mut W, k0: u64, k1: u64, m: u64, p: u8) -> GcsFilterWriter<'a, W> {
GcsFilterWriter { filter: GcsFilter::new(k0, k1, p), writer, elements: BTreeSet::new(), m }
}
pub fn add_element(&mut self, element: &[u8]) {
if !element.is_empty() {
self.elements.insert(element.to_vec());
}
}
pub fn finish(&mut self) -> Result<usize, io::Error> {
let nm = self.elements.len() as u64 * self.m;
let mut mapped: Vec<_> = self
.elements
.iter()
.map(|e| map_to_range(self.filter.hash(e.as_slice()), nm))
.collect();
mapped.sort_unstable();
let mut wrote = VarInt::from(mapped.len()).consensus_encode(self.writer)?;
let mut writer = BitStreamWriter::new(self.writer);
let mut last = 0;
for data in mapped {
wrote += self.filter.golomb_rice_encode(&mut writer, data - last)?;
last = data;
}
wrote += writer.flush()?;
Ok(wrote)
}
}
struct GcsFilter {
k0: u64, k1: u64, p: u8,
}
impl GcsFilter {
fn new(k0: u64, k1: u64, p: u8) -> GcsFilter { GcsFilter { k0, k1, p } }
fn golomb_rice_encode<W>(
&self,
writer: &mut BitStreamWriter<'_, W>,
n: u64,
) -> Result<usize, io::Error>
where
W: Write,
{
let mut wrote = 0;
let mut q = n >> self.p;
while q > 0 {
let nbits = cmp::min(q, 64);
wrote += writer.write(!0u64, nbits as u8)?;
q -= nbits;
}
wrote += writer.write(0, 1)?;
wrote += writer.write(n, self.p)?;
Ok(wrote)
}
fn golomb_rice_decode<R>(&self, reader: &mut BitStreamReader<R>) -> Result<u64, io::Error>
where
R: Read + ?Sized,
{
let mut q = 0u64;
while reader.read(1)? == 1 {
q += 1;
}
let r = reader.read(self.p)?;
Ok((q << self.p) + r)
}
fn hash(&self, element: &[u8]) -> u64 {
siphash24::Hash::hash_to_u64_with_keys(self.k0, self.k1, element)
}
}
pub struct BitStreamReader<'a, R: ?Sized> {
buffer: [u8; 1],
offset: u8,
reader: &'a mut R,
}
impl<'a, R: Read + ?Sized> BitStreamReader<'a, R> {
pub fn new(reader: &'a mut R) -> BitStreamReader<'a, R> {
BitStreamReader { buffer: [0u8], reader, offset: 8 }
}
pub fn read(&mut self, mut nbits: u8) -> Result<u64, io::Error> {
if nbits > 64 {
return Err(io::Error::new(
io::ErrorKind::Other,
"can not read more than 64 bits at once",
));
}
let mut data = 0u64;
while nbits > 0 {
if self.offset == 8 {
self.reader.read_exact(&mut self.buffer)?;
self.offset = 0;
}
let bits = cmp::min(8 - self.offset, nbits);
data <<= bits;
data |= ((self.buffer[0] << self.offset) >> (8 - bits)) as u64;
self.offset += bits;
nbits -= bits;
}
Ok(data)
}
}
pub struct BitStreamWriter<'a, W> {
buffer: [u8; 1],
offset: u8,
writer: &'a mut W,
}
impl<'a, W: Write> BitStreamWriter<'a, W> {
pub fn new(writer: &'a mut W) -> BitStreamWriter<'a, W> {
BitStreamWriter { buffer: [0u8], writer, offset: 0 }
}
pub fn write(&mut self, data: u64, mut nbits: u8) -> Result<usize, io::Error> {
if nbits > 64 {
return Err(io::Error::new(
io::ErrorKind::Other,
"can not write more than 64 bits at once",
));
}
let mut wrote = 0;
while nbits > 0 {
let bits = cmp::min(8 - self.offset, nbits);
self.buffer[0] |= ((data << (64 - nbits)) >> (64 - 8 + self.offset)) as u8;
self.offset += bits;
nbits -= bits;
if self.offset == 8 {
wrote += self.flush()?;
}
}
Ok(wrote)
}
pub fn flush(&mut self) -> Result<usize, io::Error> {
if self.offset > 0 {
self.writer.write_all(&self.buffer)?;
self.buffer[0] = 0u8;
self.offset = 0;
Ok(1)
} else {
Ok(0)
}
}
}
#[cfg(test)]
mod test {
use std::collections::HashMap;
use hex::test_hex_unwrap as hex;
use serde_json::Value;
use super::*;
use crate::consensus::encode::deserialize;
use crate::ScriptBuf;
#[test]
fn test_blockfilters() {
let data = include_str!("../tests/data/blockfilters.json");
let testdata = serde_json::from_str::<Value>(data).unwrap().as_array().unwrap().clone();
for t in testdata.iter().skip(1) {
let block_hash = t.get(1).unwrap().as_str().unwrap().parse::<BlockHash>().unwrap();
let block: Block = deserialize(&hex!(t.get(2).unwrap().as_str().unwrap())).unwrap();
assert_eq!(block.block_hash(), block_hash);
let scripts = t.get(3).unwrap().as_array().unwrap();
let previous_filter_header =
t.get(4).unwrap().as_str().unwrap().parse::<FilterHeader>().unwrap();
let filter_content = hex!(t.get(5).unwrap().as_str().unwrap());
let filter_header =
t.get(6).unwrap().as_str().unwrap().parse::<FilterHeader>().unwrap();
let mut txmap = HashMap::new();
let mut si = scripts.iter();
for tx in block.txdata.iter().skip(1) {
for input in tx.input.iter() {
txmap.insert(
input.previous_output,
ScriptBuf::from(hex!(si.next().unwrap().as_str().unwrap())),
);
}
}
let filter = BlockFilter::new_script_filter(&block, |o| {
if let Some(s) = txmap.get(o) {
Ok(s.clone())
} else {
Err(Error::UtxoMissing(*o))
}
})
.unwrap();
let test_filter = BlockFilter::new(filter_content.as_slice());
assert_eq!(test_filter.content, filter.content);
let block_hash = &block.block_hash();
assert!(filter
.match_all(
block_hash,
&mut txmap.iter().filter_map(|(_, s)| if !s.is_empty() {
Some(s.as_bytes())
} else {
None
})
)
.unwrap());
for script in txmap.values() {
let query = [script];
if !script.is_empty() {
assert!(filter
.match_any(block_hash, &mut query.iter().map(|s| s.as_bytes()))
.unwrap());
}
}
assert_eq!(filter_header, filter.filter_header(&previous_filter_header));
}
}
#[test]
fn test_filter() {
let mut patterns = BTreeSet::new();
patterns.insert(hex!("000000"));
patterns.insert(hex!("111111"));
patterns.insert(hex!("222222"));
patterns.insert(hex!("333333"));
patterns.insert(hex!("444444"));
patterns.insert(hex!("555555"));
patterns.insert(hex!("666666"));
patterns.insert(hex!("777777"));
patterns.insert(hex!("888888"));
patterns.insert(hex!("999999"));
patterns.insert(hex!("aaaaaa"));
patterns.insert(hex!("bbbbbb"));
patterns.insert(hex!("cccccc"));
patterns.insert(hex!("dddddd"));
patterns.insert(hex!("eeeeee"));
patterns.insert(hex!("ffffff"));
let mut out = Vec::new();
{
let mut writer = GcsFilterWriter::new(&mut out, 0, 0, M, P);
for p in &patterns {
writer.add_element(p.as_slice());
}
writer.finish().unwrap();
}
let bytes = out;
{
let query = [hex!("abcdef"), hex!("eeeeee")];
let reader = GcsFilterReader::new(0, 0, M, P);
assert!(reader
.match_any(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
}
{
let query = [hex!("abcdef"), hex!("123456")];
let reader = GcsFilterReader::new(0, 0, M, P);
assert!(!reader
.match_any(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
}
{
let reader = GcsFilterReader::new(0, 0, M, P);
let mut query = Vec::new();
for p in &patterns {
query.push(p.clone());
}
assert!(reader
.match_all(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
}
{
let reader = GcsFilterReader::new(0, 0, M, P);
let mut query = Vec::new();
for p in &patterns {
query.push(p.clone());
}
query.push(hex!("abcdef"));
assert!(!reader
.match_all(&mut bytes.as_slice(), &mut query.iter().map(|v| v.as_slice()))
.unwrap());
}
}
#[test]
fn test_bit_stream() {
let mut out = Vec::new();
{
let mut writer = BitStreamWriter::new(&mut out);
writer.write(0, 1).unwrap(); writer.write(2, 2).unwrap(); writer.write(6, 3).unwrap(); writer.write(11, 4).unwrap(); writer.write(1, 5).unwrap(); writer.write(32, 6).unwrap(); writer.write(7, 7).unwrap(); writer.flush().unwrap();
}
let bytes = out;
assert_eq!(
"01011010110000110000000001110000",
format!("{:08b}{:08b}{:08b}{:08b}", bytes[0], bytes[1], bytes[2], bytes[3])
);
{
let mut input = bytes.as_slice();
let mut reader = BitStreamReader::new(&mut input);
assert_eq!(reader.read(1).unwrap(), 0);
assert_eq!(reader.read(2).unwrap(), 2);
assert_eq!(reader.read(3).unwrap(), 6);
assert_eq!(reader.read(4).unwrap(), 11);
assert_eq!(reader.read(5).unwrap(), 1);
assert_eq!(reader.read(6).unwrap(), 32);
assert_eq!(reader.read(7).unwrap(), 7);
assert!(reader.read(5).is_err());
}
}
}