duniter-gva/indexer/src/blocks_chunks.rs

//  Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <https://www.gnu.org/licenses/>.

use crate::*;
use flate2::read::ZlibDecoder;
use flate2::write::ZlibEncoder;
use flate2::Compression;

const CHUNK_SIZE: u32 = 4_096;

pub fn apply_block_blocks_chunk<B: Backend>(
    block: &DubpBlockV10,
    gva_db: &GvaV1Db<B>,
    profile_path: &Path,
) -> KvResult<()> {
    let block_number = block.number().0;
    let chunks_folder_path = profile_path.join("data/gva_v1_blocks_chunks");
    gva_db.write(|mut db| {
        db.current_blocks_chunk.upsert(
            U32BE(block_number),
            GvaBlockDbV1(DubpBlock::V10(block.clone())),
        );

        if (block_number + 1) % CHUNK_SIZE == 0 {
            let current_chunk: Vec<GvaBlockDbV1> = db
                .current_blocks_chunk
                .iter(.., |it| it.values().collect::<Result<Vec<_>, _>>())?;
            let current_chunk_bin = bincode_db()
                .serialize(&current_chunk)
                .map_err(|e| KvError::DeserError(e.into()))?;
            let chunk_hash = Hash::compute_blake3(current_chunk_bin.as_ref());
            let chunk_index = U32BE(block_number / CHUNK_SIZE);
            db.blocks_chunk_hash.upsert(chunk_index, HashDb(chunk_hash));

            write_and_compress_chunk_in_file(
                current_chunk_bin.as_ref(),
                chunk_index.0,
                chunks_folder_path.as_path(),
            )
            .map_err(|e| KvError::Custom(e.into()))?;
        }
        Ok(())
    })
}

/// Read and decompress bytes from file
fn read_and_remove_compressed_chunk(
    chunk_index: u32,
    chunks_folder_path: &Path,
) -> std::io::Result<Option<Vec<u8>>> {
    let file_path = chunks_folder_path.join(format!("_{}", chunk_index));
    if !file_path.exists() {
        return Ok(None);
    }
    if std::fs::metadata(file_path.as_path())?.len() > 0 {
        let file = std::fs::File::open(file_path)?;
        let mut z = ZlibDecoder::new(file);
        let mut decompressed_bytes = Vec::new();
        z.read_to_end(&mut decompressed_bytes)?;

        Ok(Some(decompressed_bytes))
    } else {
        Ok(None)
    }
}

/// Write and compress chunk in file
fn write_and_compress_chunk_in_file(
    chunk: &[u8],
    chunk_index: u32,
    chunks_folder_path: &Path,
) -> Result<(), std::io::Error> {
    log::info!("blocks_chunk_{}: {} bytes", chunk_index, chunk.len());
    let file = std::fs::File::create(chunks_folder_path.join(format!("_{}", chunk_index)))?;
    let mut e = ZlibEncoder::new(file, Compression::new(3));
    e.write_all(chunk)?;
    e.finish()?;

    Ok(())
}

pub fn revert_block_blocks_chunk<B: Backend>(
    block: &DubpBlockV10,
    gva_db: &GvaV1Db<B>,
    profile_path: &Path,
) -> KvResult<()> {
    let block_number = block.number().0;
    let chunks_folder_path = profile_path.join("data/gva_v1_blocks_chunks");
    gva_db.write(|mut db| {
        if (block_number + 1) % CHUNK_SIZE == 0 {
            // Uncompress last compressed chunk and replace it in current chunk
            let chunk_index = U32BE(block_number / CHUNK_SIZE);
            if let Some(current_chunk_bin) =
                read_and_remove_compressed_chunk(chunk_index.0, chunks_folder_path.as_path())?
            {
                db.blocks_chunk_hash.remove(chunk_index);

                let current_chunk: Vec<GvaBlockDbV1> = bincode_db()
                    .deserialize(current_chunk_bin.as_ref())
                    .map_err(|e| KvError::DeserError(e.into()))?;
                let current_chunk_begin = block_number - CHUNK_SIZE + 1;
                for (i, block) in current_chunk.into_iter().enumerate() {
                    db.current_blocks_chunk
                        .upsert(U32BE(current_chunk_begin + i as u32), block);
                }
            } else {
                return Err(KvError::DbCorrupted(
                    "Not found last compressed chunk".to_owned(),
                ));
            }
        } else {
            db.current_blocks_chunk.remove(U32BE(block_number));
        }

        Ok(())
    })
}