ref(indexer):blocks_chunks: move chunk read in crate dbs-reader

Cargo.lock

@@ -986,6 +986,7 @@ dependencies = [
  "duniter-bca-types",
  "duniter-core",
  "duniter-gva-db",
+ "flate2",
  "maplit",
  "mockall",
  "resiter",
@@ -1028,6 +1029,7 @@ dependencies = [
  "dubp",
  "duniter-core",
  "duniter-gva-db",
+ "duniter-gva-dbs-reader",
  "flate2",
  "log",
  "maplit",

dbs-reader/Cargo.toml

@@ -21,6 +21,7 @@ duniter-bca-types = { path = "../bca/types" }
 duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" }
 duniter-gva-db = { path = "../db" }
 dubp = { version = "0.54.1", features = ["duniter"] }
+flate2 = { version = "1.0", features = ["zlib-ng-compat"], default-features = false }
 mockall = { version = "0.9.1", optional = true }
 resiter = "0.4.0"
 

dbs-reader/src/blocks_chunks.rs

@@ -0,0 +1,51 @@
+//  Copyright (C) 2021 Pascal Engélibert
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+use crate::*;
+use flate2::read::ZlibDecoder;
+
+pub const CHUNK_FILE_PREFIX: &str = "_";
+pub const CHUNK_FILE_EXT: &str = ".bin.gz";
+
+/// Read and decompress chunk file
+pub fn read_compressed_chunk(
+    chunk_index: u32,
+    chunks_folder_path: &Path,
+    remove: bool,
+) -> std::io::Result<Option<Vec<u8>>> {
+    let file_path = chunks_folder_path.join(format!(
+        "{}{}{}",
+        CHUNK_FILE_PREFIX, chunk_index, CHUNK_FILE_EXT
+    ));
+    if !file_path.exists() {
+        return Ok(None);
+    }
+    if std::fs::metadata(file_path.as_path())?.len() > 0 {
+        let file = std::fs::File::open(file_path.as_path())?;
+        let mut z = ZlibDecoder::new(file);
+        let mut decompressed_bytes = Vec::new();
+        z.read_to_end(&mut decompressed_bytes)?;
+
+        if remove {
+            std::fs::remove_file(file_path)?;
+        }
+        Ok(Some(decompressed_bytes))
+    } else {
+        if remove {
+            std::fs::remove_file(file_path)?;
+        }
+        Ok(None)
+    }
+}

dbs-reader/src/lib.rs

@@ -23,6 +23,7 @@
 )]
 
 pub mod block;
+pub mod blocks_chunks;
 pub mod current_frame;
 pub mod find_inputs;
 pub mod idty;
@@ -61,7 +62,9 @@ use resiter::flatten::Flatten;
 use resiter::map::Map;
 use std::{
     collections::{BTreeSet, VecDeque},
+    io::Read,
     num::NonZeroUsize,
+    path::Path,
     str::FromStr,
 };
 

indexer/Cargo.toml

@@ -16,6 +16,7 @@ anyhow = "1.0.34"
 bincode = "1.3"
 duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" }
 duniter-gva-db = { path = "../db" }
+duniter-gva-dbs-reader = { path = "../dbs-reader" }
 dubp = { version = "0.54.1", features = ["duniter"] }
 flate2 = { version = "1.0", features = ["zlib-ng-compat"], default-features = false }
 log = "0.4"

indexer/src/blocks_chunks.rs

@@ -14,13 +14,11 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 use crate::*;
-use flate2::read::ZlibDecoder;
+use duniter_gva_dbs_reader::blocks_chunks::{CHUNK_FILE_EXT, CHUNK_FILE_PREFIX};
 use flate2::write::ZlibEncoder;
 use flate2::Compression;
 
 const CHUNK_SIZE: u32 = 4_096;
-const CHUNK_FILE_PREFIX: &str = "_";
-const CHUNK_FILE_EXT: &str = ".bin.gz";
 
 pub fn apply_block_blocks_chunk<B: Backend>(
     block: &DubpBlockV10,
@@ -72,7 +70,11 @@ pub fn revert_block_blocks_chunk<B: Backend>(
             // Uncompress last compressed chunk and replace it in current chunk
             let chunk_index = U32BE(block_number / CHUNK_SIZE);
             if let Some(current_chunk_bin) =
-                read_and_remove_compressed_chunk(chunk_index.0, chunks_folder_path.as_path())?
+                duniter_gva_dbs_reader::blocks_chunks::read_compressed_chunk(
+                    chunk_index.0,
+                    chunks_folder_path.as_path(),
+                    true,
+                )?
             {
                 db.blocks_chunk_hash.remove(chunk_index);
 
@@ -97,30 +99,6 @@ pub fn revert_block_blocks_chunk<B: Backend>(
     })
 }
 
-/// Read and decompress bytes from file
-fn read_and_remove_compressed_chunk(
-    chunk_index: u32,
-    chunks_folder_path: &Path,
-) -> std::io::Result<Option<Vec<u8>>> {
-    let file_path = chunks_folder_path.join(format!(
-        "{}{}{}",
-        CHUNK_FILE_PREFIX, chunk_index, CHUNK_FILE_EXT
-    ));
-    if !file_path.exists() {
-        return Ok(None);
-    }
-    if std::fs::metadata(file_path.as_path())?.len() > 0 {
-        let file = std::fs::File::open(file_path)?;
-        let mut z = ZlibDecoder::new(file);
-        let mut decompressed_bytes = Vec::new();
-        z.read_to_end(&mut decompressed_bytes)?;
-
-        Ok(Some(decompressed_bytes))
-    } else {
-        Ok(None)
-    }
-}
-
 /// Write and compress chunk in file
 fn write_and_compress_chunk_in_file(
     chunk: &[u8],