diff --git a/Cargo.lock b/Cargo.lock index 6b5d99b..5500a07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -986,6 +986,7 @@ dependencies = [ "duniter-bca-types", "duniter-core", "duniter-gva-db", + "flate2", "maplit", "mockall", "resiter", @@ -1028,6 +1029,7 @@ dependencies = [ "dubp", "duniter-core", "duniter-gva-db", + "duniter-gva-dbs-reader", "flate2", "log", "maplit", diff --git a/dbs-reader/Cargo.toml b/dbs-reader/Cargo.toml index 21332a6..791231e 100644 --- a/dbs-reader/Cargo.toml +++ b/dbs-reader/Cargo.toml @@ -21,6 +21,7 @@ duniter-bca-types = { path = "../bca/types" } duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" } duniter-gva-db = { path = "../db" } dubp = { version = "0.54.1", features = ["duniter"] } +flate2 = { version = "1.0", features = ["zlib-ng-compat"], default-features = false } mockall = { version = "0.9.1", optional = true } resiter = "0.4.0" diff --git a/dbs-reader/src/blocks_chunks.rs b/dbs-reader/src/blocks_chunks.rs new file mode 100644 index 0000000..606b7cb --- /dev/null +++ b/dbs-reader/src/blocks_chunks.rs @@ -0,0 +1,51 @@ +// Copyright (C) 2021 Pascal Engélibert +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +use crate::*; +use flate2::read::ZlibDecoder; + +pub const CHUNK_FILE_PREFIX: &str = "_"; +pub const CHUNK_FILE_EXT: &str = ".bin.gz"; + +/// Read and decompress chunk file +pub fn read_compressed_chunk( + chunk_index: u32, + chunks_folder_path: &Path, + remove: bool, +) -> std::io::Result>> { + let file_path = chunks_folder_path.join(format!( + "{}{}{}", + CHUNK_FILE_PREFIX, chunk_index, CHUNK_FILE_EXT + )); + if !file_path.exists() { + return Ok(None); + } + if std::fs::metadata(file_path.as_path())?.len() > 0 { + let file = std::fs::File::open(file_path.as_path())?; + let mut z = ZlibDecoder::new(file); + let mut decompressed_bytes = Vec::new(); + z.read_to_end(&mut decompressed_bytes)?; + + if remove { + std::fs::remove_file(file_path)?; + } + Ok(Some(decompressed_bytes)) + } else { + if remove { + std::fs::remove_file(file_path)?; + } + Ok(None) + } +} diff --git a/dbs-reader/src/lib.rs b/dbs-reader/src/lib.rs index 838f724..ba6d79e 100644 --- a/dbs-reader/src/lib.rs +++ b/dbs-reader/src/lib.rs @@ -23,6 +23,7 @@ )] pub mod block; +pub mod blocks_chunks; pub mod current_frame; pub mod find_inputs; pub mod idty; @@ -61,7 +62,9 @@ use resiter::flatten::Flatten; use resiter::map::Map; use std::{ collections::{BTreeSet, VecDeque}, + io::Read, num::NonZeroUsize, + path::Path, str::FromStr, }; diff --git a/indexer/Cargo.toml b/indexer/Cargo.toml index 908c872..6b15dbc 100644 --- a/indexer/Cargo.toml +++ b/indexer/Cargo.toml @@ -16,6 +16,7 @@ anyhow = "1.0.34" bincode = "1.3" duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" } duniter-gva-db = { path = "../db" } +duniter-gva-dbs-reader = { path = "../dbs-reader" } dubp = { version = "0.54.1", features = ["duniter"] } flate2 = { version = "1.0", features = ["zlib-ng-compat"], default-features = false } log = "0.4" diff --git a/indexer/src/blocks_chunks.rs b/indexer/src/blocks_chunks.rs index 3afdf70..e4d7d17 100644 --- a/indexer/src/blocks_chunks.rs +++ b/indexer/src/blocks_chunks.rs @@ -14,13 +14,11 @@ // along with this program. If not, see . use crate::*; -use flate2::read::ZlibDecoder; +use duniter_gva_dbs_reader::blocks_chunks::{CHUNK_FILE_EXT, CHUNK_FILE_PREFIX}; use flate2::write::ZlibEncoder; use flate2::Compression; const CHUNK_SIZE: u32 = 4_096; -const CHUNK_FILE_PREFIX: &str = "_"; -const CHUNK_FILE_EXT: &str = ".bin.gz"; pub fn apply_block_blocks_chunk( block: &DubpBlockV10, @@ -72,7 +70,11 @@ pub fn revert_block_blocks_chunk( // Uncompress last compressed chunk and replace it in current chunk let chunk_index = U32BE(block_number / CHUNK_SIZE); if let Some(current_chunk_bin) = - read_and_remove_compressed_chunk(chunk_index.0, chunks_folder_path.as_path())? + duniter_gva_dbs_reader::blocks_chunks::read_compressed_chunk( + chunk_index.0, + chunks_folder_path.as_path(), + true, + )? { db.blocks_chunk_hash.remove(chunk_index); @@ -97,30 +99,6 @@ pub fn revert_block_blocks_chunk( }) } -/// Read and decompress bytes from file -fn read_and_remove_compressed_chunk( - chunk_index: u32, - chunks_folder_path: &Path, -) -> std::io::Result>> { - let file_path = chunks_folder_path.join(format!( - "{}{}{}", - CHUNK_FILE_PREFIX, chunk_index, CHUNK_FILE_EXT - )); - if !file_path.exists() { - return Ok(None); - } - if std::fs::metadata(file_path.as_path())?.len() > 0 { - let file = std::fs::File::open(file_path)?; - let mut z = ZlibDecoder::new(file); - let mut decompressed_bytes = Vec::new(); - z.read_to_end(&mut decompressed_bytes)?; - - Ok(Some(decompressed_bytes)) - } else { - Ok(None) - } -} - /// Write and compress chunk in file fn write_and_compress_chunk_in_file( chunk: &[u8],