pini/duniter-gva
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

perf(indexer): change compression algo: lz4 -> deflate

Browse source
This commit is contained in:
librelois 2021-05-17 23:43:05 +02:00
parent 6d7a35dd8b
commit cef485f67a
3 changed files with 26 additions and 11 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
Cargo.lock generated
View file

@ -10,6 +10,12 @@ dependencies = [
"regex", "regex",
] ]
[[package]]
name = "adler"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]] [[package]]
name = "aho-corasick" name = "aho-corasick"
version = "0.7.15" version = "0.7.15"
@ -1022,8 +1028,8 @@ dependencies = [
"dubp", "dubp",
"duniter-core", "duniter-core",
"duniter-gva-db", "duniter-gva-db",
"lz4_flex",
"maplit", "maplit",
"miniz_oxide",
"once_cell", "once_cell",
"parking_lot", "parking_lot",
"resiter", "resiter",
@ -1686,12 +1692,6 @@ dependencies = [
"cfg-if 1.0.0", "cfg-if 1.0.0",
] ]
[[package]]
name = "lz4_flex"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05304f8e67dfc93d1b4b990137fd1a7a4c6ad44b60a9c486c8c4486f9d2027ae"
[[package]] [[package]]
name = "maplit" name = "maplit"
version = "1.0.2" version = "1.0.2"
@ -1735,6 +1735,16 @@ dependencies = [
"unicase", "unicase",
] ]
[[package]]
name = "miniz_oxide"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b"
dependencies = [
"adler",
"autocfg",
]
[[package]] [[package]]
name = "mio" name = "mio"
version = "0.7.11" version = "0.7.11"

2
indexer/Cargo.toml
View file

@ -17,7 +17,7 @@ bincode = "1.3"
duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" } duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" }
duniter-gva-db = { path = "../db" } duniter-gva-db = { path = "../db" }
dubp = { version = "0.54.1", features = ["duniter"] } dubp = { version = "0.54.1", features = ["duniter"] }
lz4_flex = { version = "0.7", default-features = false } miniz_oxide = "0.4.4"
once_cell = "1.7" once_cell = "1.7"
resiter = "0.4.0" resiter = "0.4.0"

11
indexer/src/blocks_chunks.rs
View file

@ -35,7 +35,9 @@ pub(super) fn apply_block_blocks_chunk<B: Backend>(
.serialize(&current_chunk) .serialize(&current_chunk)
.map_err(|e| KvError::DeserError(e.into()))?; .map_err(|e| KvError::DeserError(e.into()))?;
let chunk_hash = Hash::compute_blake3(current_chunk_bin.as_ref()); let chunk_hash = Hash::compute_blake3(current_chunk_bin.as_ref());
let compressed_chunk = lz4_flex::compress_prepend_size(current_chunk_bin.as_ref());
let compressed_chunk = miniz_oxide::deflate::compress_to_vec(current_chunk_bin.as_ref(), 6);
let chunk_index = U32BE(block_number / CHUNK_SIZE); let chunk_index = U32BE(block_number / CHUNK_SIZE);
gva_db gva_db
.blocks_chunk_hash .blocks_chunk_hash
@ -59,8 +61,11 @@ pub(super) fn revert_block_blocks_chunk<B: Backend>(
if let Some(compressed_chunk) = gva_db.compressed_blocks_chunk.get(&chunk_index)? { if let Some(compressed_chunk) = gva_db.compressed_blocks_chunk.get(&chunk_index)? {
gva_db.blocks_chunk_hash.remove(chunk_index); gva_db.blocks_chunk_hash.remove(chunk_index);
gva_db.compressed_blocks_chunk.remove(chunk_index); gva_db.compressed_blocks_chunk.remove(chunk_index);
let current_chunk_bin = lz4_flex::decompress_size_prepended(compressed_chunk.as_ref())
.map_err(|e| KvError::Custom(format!("{:?}", e).into()))?; let current_chunk_bin =
miniz_oxide::inflate::decompress_to_vec(compressed_chunk.as_ref())
.map_err(|e| KvError::Custom(format!("{:?}", e).into()))?;
let current_chunk: Vec<GvaBlockDbV1> = bincode_db() let current_chunk: Vec<GvaBlockDbV1> = bincode_db()
.deserialize(current_chunk_bin.as_ref()) .deserialize(current_chunk_bin.as_ref())
.map_err(|e| KvError::DeserError(e.into()))?; .map_err(|e| KvError::DeserError(e.into()))?;