feat(bda): impl requests CompressedBlockChunk, CurrentBlockstamp & Sync

This commit is contained in:
librelois 2021-06-13 19:40:17 +02:00
parent 92f5dbea20
commit c8d178fd89
10 changed files with 216 additions and 64 deletions

56
Cargo.lock generated
View file

@ -694,9 +694,9 @@ checksum = "4bb454f0228b18c7f4c3b0ebbee346ed9c52e7443b0999cd543ff3571205701d"
[[package]] [[package]]
name = "dubp" name = "dubp"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "66f0006f2695beb0ebeaf9018bf7fc16f92ad2a60bc5522ca9813d257dc501f4" checksum = "429ad631310968b6f0ef8b0621ff685ff03a1f69304174c7d8e7dbed47ec68aa"
dependencies = [ dependencies = [
"dubp-block", "dubp-block",
"dubp-common", "dubp-common",
@ -710,9 +710,9 @@ dependencies = [
[[package]] [[package]]
name = "dubp-block" name = "dubp-block"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4524d9bfb9dff0857d705663138dfc3de0761a322ed7c9a429f2777411f3395c" checksum = "8481be98e093e9f599ef685f8b2880ac273145ce4b12bae45a6fe28b158acb38"
dependencies = [ dependencies = [
"dubp-documents", "dubp-documents",
"dubp-documents-parser", "dubp-documents-parser",
@ -725,9 +725,9 @@ dependencies = [
[[package]] [[package]]
name = "dubp-common" name = "dubp-common"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "270b8470070985a1291e0bde3ec76b61b79a148be135a6d2e372e699b5c50d1c" checksum = "ab01b4870cd960917b638dfd951996dcb19850fdc9868a46d81c7a48c1882f2a"
dependencies = [ dependencies = [
"dup-crypto", "dup-crypto",
"serde", "serde",
@ -738,9 +738,9 @@ dependencies = [
[[package]] [[package]]
name = "dubp-documents" name = "dubp-documents"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd04417e58905daf5e0a7c392e6deb19c6503f6d1aaff92fd12895d701681427" checksum = "38c2667a2c02b657ecce47a2f8d421ddd8f193a77277518aa74585bd42303246"
dependencies = [ dependencies = [
"beef", "beef",
"dubp-wallet", "dubp-wallet",
@ -752,9 +752,9 @@ dependencies = [
[[package]] [[package]]
name = "dubp-documents-parser" name = "dubp-documents-parser"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f4e2aa1ce0bd09b26f710ba725e55901ec0ec47fbe26b43b26071f8cf439962" checksum = "9597df5d36e1e269802341908d8970d58ccde6162b402748f8b8b1bd50c1bec6"
dependencies = [ dependencies = [
"dubp-documents", "dubp-documents",
"json-pest-parser", "json-pest-parser",
@ -766,9 +766,9 @@ dependencies = [
[[package]] [[package]]
name = "dubp-wallet" name = "dubp-wallet"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8ee60a2d7fe343782fa4ab3e281e23cdf4313d8ffb029d64bb600dd53a347bb" checksum = "d044529bbf5928e9519b9040516365e6e20f2d65f8c8e565930f5561b8ff0126"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"dubp-common", "dubp-common",
@ -781,7 +781,7 @@ dependencies = [
[[package]] [[package]]
name = "dubp-wot" name = "dubp-wot"
version = "0.11.0" version = "0.11.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"log", "log",
"once_cell", "once_cell",
@ -793,7 +793,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-bc-reader" name = "duniter-bc-reader"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"dubp", "dubp",
@ -824,9 +824,9 @@ dependencies = [
[[package]] [[package]]
name = "duniter-bda-types" name = "duniter-bda-types"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "023475a8fefc5dc75f150d3fb0afea984df5be359e1eb061f37a18a1b74be9e7" checksum = "a04e49bc24390430f986d07c43a5cafd431ded3954d31d554efd6c34b77c2788"
dependencies = [ dependencies = [
"arrayvec 0.7.0", "arrayvec 0.7.0",
"bincode", "bincode",
@ -840,7 +840,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-conf" name = "duniter-conf"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"dubp", "dubp",
@ -854,7 +854,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-core" name = "duniter-core"
version = "1.8.1" version = "1.8.1"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"dubp", "dubp",
"dubp-wot", "dubp-wot",
@ -869,7 +869,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-dbs" name = "duniter-dbs"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"arrayvec 0.7.0", "arrayvec 0.7.0",
"bincode", "bincode",
@ -892,7 +892,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-dbs-write-ops" name = "duniter-dbs-write-ops"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"chrono", "chrono",
"dubp", "dubp",
@ -907,7 +907,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-global" name = "duniter-global"
version = "1.8.1" version = "1.8.1"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"async-rwlock", "async-rwlock",
"dubp", "dubp",
@ -1041,7 +1041,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-mempools" name = "duniter-mempools"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"dubp", "dubp",
"duniter-bc-reader", "duniter-bc-reader",
@ -1054,7 +1054,7 @@ dependencies = [
[[package]] [[package]]
name = "duniter-module" name = "duniter-module"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"async-mutex", "async-mutex",
@ -1073,9 +1073,9 @@ dependencies = [
[[package]] [[package]]
name = "duniter-peer" name = "duniter-peer"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1604f510b17e284453cb06f46a9304be9ed65728163ac35f3b22cd5f058952d3" checksum = "952ebba568abb1a8a1a7f83ff960a88b4c1de9cd29c74c6885ecc83ca56cccaf"
dependencies = [ dependencies = [
"beef", "beef",
"dubp-common", "dubp-common",
@ -1088,9 +1088,9 @@ dependencies = [
[[package]] [[package]]
name = "dup-crypto" name = "dup-crypto"
version = "0.57.0" version = "0.58.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83e883538079a059b2126a94a13e1d1c62a3a93cd14174eea9d4dd3b4f40cb73" checksum = "f74bc2e8282dd156cd62e54686bc77a22dbaa711facac93b27d6f2c72ff5d07a"
dependencies = [ dependencies = [
"base64", "base64",
"blake3", "blake3",
@ -1641,7 +1641,7 @@ dependencies = [
[[package]] [[package]]
name = "kv_typed" name = "kv_typed"
version = "0.1.0" version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#06eca4f3410523109b16582510c57ac5e03fa9c3" source = "git+https://git.duniter.org/nodes/rust/duniter-core#eebd168525aa1d1424ad12ad71da768471207d83"
dependencies = [ dependencies = [
"byteorder", "byteorder",
"cfg-if 0.1.10", "cfg-if 0.1.10",

View file

@ -41,6 +41,7 @@ tokio = { version = "1.2", features = ["macros", "rt-multi-thread", "time"] }
unwrap = "1.2.1" unwrap = "1.2.1"
[features] [features]
default = ["explorer"]
explorer = ["duniter-gva-db/explorer", "duniter-gva-indexer/explorer"] explorer = ["duniter-gva-db/explorer", "duniter-gva-indexer/explorer"]
[workspace] [workspace]

View file

@ -20,6 +20,7 @@ mod members_count;
mod peers; mod peers;
mod prepare_simple_payment; mod prepare_simple_payment;
mod send_txs; mod send_txs;
mod sync;
mod utxos; mod utxos;
use duniter_core::crypto::keys::KeyPair; use duniter_core::crypto::keys::KeyPair;
@ -47,30 +48,26 @@ pub(super) async fn execute_req_type(
BcaReqTypeV0::BalancesOfPubkeys(pubkeys) => { BcaReqTypeV0::BalancesOfPubkeys(pubkeys) => {
balances::exec_req_balances_of_pubkeys(bda_executor, pubkeys).await balances::exec_req_balances_of_pubkeys(bda_executor, pubkeys).await
} }
BcaReqTypeV0::BalancesOfScripts(scripts) => {
balances::exec_req_balances_of_scripts(bda_executor, scripts).await
}
BcaReqTypeV0::CompressedBlockChunk { chunk_id } => {
sync::exec_req_compressed_block_chunk(bda_executor, chunk_id).await
}
BcaReqTypeV0::CurrentBlockstamp => {
if let Some(current_meta) = bda_executor.cm_accessor.get_current_meta(|cm| *cm).await {
Ok(BcaRespTypeV0::CurrentBlockstamp(
current_meta.current_block_meta.blockstamp(),
))
} else {
Err("no blockchain".into())
}
}
BcaReqTypeV0::CurrentUd => current_ud::exec_req_current_ud(bda_executor).await,
BcaReqTypeV0::FirstUtxosOfPubkeys { BcaReqTypeV0::FirstUtxosOfPubkeys {
amount_target_opt, amount_target_opt,
pubkeys, pubkeys,
} => utxos::exec_req_first_utxos_of_pubkeys(bda_executor, amount_target_opt, pubkeys).await, } => utxos::exec_req_first_utxos_of_pubkeys(bda_executor, amount_target_opt, pubkeys).await,
BcaReqTypeV0::LastBlockstampOutOfForkWindow => {
last_blockstamp_out_of_fork_window::exec_req_last_blockstamp_out_of_fork_window(
bda_executor,
)
.await
}
BcaReqTypeV0::MembersCount => members_count::exec_req_members_count(bda_executor).await,
BcaReqTypeV0::PrepareSimplePayment(params) => {
prepare_simple_payment::exec_req_prepare_simple_payment(bda_executor, params).await
}
BcaReqTypeV0::ProofServerPubkey { challenge } => Ok(BcaRespTypeV0::ProofServerPubkey {
challenge,
server_pubkey: bda_executor.self_keypair.public_key(),
sig: bda_executor
.self_keypair
.generate_signator()
.sign(&challenge),
}),
BcaReqTypeV0::Ping => Ok(BcaRespTypeV0::Pong),
BcaReqTypeV0::SendTxs(txs) => send_txs::send_txs(bda_executor, txs).await,
BcaReqTypeV0::Identities(pubkeys) => { BcaReqTypeV0::Identities(pubkeys) => {
let dbs_reader = bda_executor.dbs_reader(); let dbs_reader = bda_executor.dbs_reader();
Ok(BcaRespTypeV0::Identities( Ok(BcaRespTypeV0::Identities(
@ -92,10 +89,27 @@ pub(super) async fn execute_req_type(
.await??, .await??,
)) ))
} }
BcaReqTypeV0::CurrentUd => current_ud::exec_req_current_ud(bda_executor).await, BcaReqTypeV0::LastBlockstampOutOfForkWindow => {
BcaReqTypeV0::BalancesOfScripts(scripts) => { last_blockstamp_out_of_fork_window::exec_req_last_blockstamp_out_of_fork_window(
balances::exec_req_balances_of_scripts(bda_executor, scripts).await bda_executor,
)
.await
} }
BcaReqTypeV0::MembersCount => members_count::exec_req_members_count(bda_executor).await,
BcaReqTypeV0::PrepareSimplePayment(params) => {
prepare_simple_payment::exec_req_prepare_simple_payment(bda_executor, params).await
}
BcaReqTypeV0::ProofServerPubkey { challenge } => Ok(BcaRespTypeV0::ProofServerPubkey {
challenge,
server_pubkey: bda_executor.self_keypair.public_key(),
sig: bda_executor
.self_keypair
.generate_signator()
.sign(&challenge),
}),
BcaReqTypeV0::Ping => Ok(BcaRespTypeV0::Pong),
BcaReqTypeV0::PeersV10 { n } => peers::exec_req_peers_v1(bda_executor, n).await, BcaReqTypeV0::PeersV10 { n } => peers::exec_req_peers_v1(bda_executor, n).await,
BcaReqTypeV0::SendTxs(txs) => send_txs::send_txs(bda_executor, txs).await,
BcaReqTypeV0::Sync { from, to } => sync::exec_req_sync(bda_executor, from, to).await,
} }
} }

View file

@ -0,0 +1,96 @@
// Copyright (C) 2020 Éloïs SANCHEZ.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as
// published by the Free Software Foundation, either version 3 of the
// License, or (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::*;
use duniter_core::bda_types::peer::Peer;
use duniter_gva_db::BLOCKS_CHUNK_SIZE;
pub(super) async fn exec_req_compressed_block_chunk(
bda_executor: &BdaExecutor,
chunk_id: usize,
) -> Result<BcaRespTypeV0, ExecReqTypeError> {
if let Some(ref profile_path) = bda_executor.profile_path_opt {
let chunks_folder_path = profile_path.join("data/gva_v1_blocks_chunks");
if let Some(chunk_data) = duniter_gva_dbs_reader::blocks_chunks::read_compressed_chunk(
chunk_id as u32,
&chunks_folder_path,
false,
)? {
Ok(BcaRespTypeV0::CompressedBlockChunk {
chunk_id,
compressed_data: chunk_data,
})
} else {
Err("not found".into())
}
} else {
Err("Cannot get compressed block chunk in memony mode".into())
}
}
pub(super) async fn exec_req_sync(
bda_executor: &BdaExecutor,
from: u32,
to_opt: Option<u32>,
) -> Result<BcaRespTypeV0, ExecReqTypeError> {
let (target_blockstamp, last_chunk_id_opt) = if let Some(to) = to_opt {
if to < from {
return Err("to < from".into());
}
let dbs_reader = bda_executor.dbs_reader();
if let Some(block_meta) = bda_executor
.dbs_pool
.execute(move |shared_dbs| dbs_reader.block(&shared_dbs.bc_db_ro, U32BE(to)))
.await??
{
(block_meta.blockstamp(), Some(to / BLOCKS_CHUNK_SIZE))
} else {
return Err("Not enough blocks, try another server".into());
}
} else if let Some(current_meta) = bda_executor.cm_accessor.get_current_meta(|cm| *cm).await {
(current_meta.current_block_meta.blockstamp(), None)
} else {
return Err("no blockchain".into());
};
let first_chunk_id = (from + 1) / BLOCKS_CHUNK_SIZE;
let dbs_reader = bda_executor.dbs_reader();
let (peers, blocks_chunks_hashs) = bda_executor
.dbs_pool
.execute(move |shared_dbs| {
Ok::<_, KvError>((
dbs_reader
.get_some_peers(&shared_dbs.dunp_db, 0)?
.into_iter()
.map(|peer_db| Peer {
peer: peer_db.peer,
is_member: peer_db.member,
is_up: peer_db.status,
})
.collect(),
dbs_reader.blocks_chunks_hashs(first_chunk_id, last_chunk_id_opt)?,
))
})
.await??;
Ok(BcaRespTypeV0::Sync {
blocks_chunks_hashs,
peers,
target_blockstamp,
})
}

View file

@ -24,9 +24,7 @@
mod exec_req_type; mod exec_req_type;
const MAX_BATCH_SIZE: usize = 10; pub use duniter_core::dbs::kv_typed::prelude::*;
const RESP_MIN_SIZE: usize = 64;
type RespBytes = SmallVec<[u8; RESP_MIN_SIZE]>;
use crate::exec_req_type::ExecReqTypeError; use crate::exec_req_type::ExecReqTypeError;
#[cfg(test)] #[cfg(test)]
@ -40,17 +38,20 @@ use duniter_core::bda_types::{
BcaResp, BcaRespTypeV0, BcaRespV0, BcaResp, BcaRespTypeV0, BcaRespV0,
}; };
use duniter_core::crypto::keys::{ed25519::Ed25519KeyPair, Signator}; use duniter_core::crypto::keys::{ed25519::Ed25519KeyPair, Signator};
pub use duniter_core::dbs::kv_typed::prelude::*;
use duniter_core::dbs::{FileBackend, SharedDbs}; use duniter_core::dbs::{FileBackend, SharedDbs};
#[cfg(not(test))] #[cfg(not(test))]
use duniter_core::global::AsyncAccessor; use duniter_core::global::AsyncAccessor;
use duniter_gva_dbs_reader::DbsReader; use duniter_gva_dbs_reader::DbsReader;
use futures::{prelude::stream::FuturesUnordered, StreamExt, TryStream, TryStreamExt}; use futures::{prelude::stream::FuturesUnordered, StreamExt, TryStream, TryStreamExt};
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use smallvec::SmallVec; use smallvec::SmallVec;
use std::path::PathBuf;
use tokio::task::JoinError; use tokio::task::JoinError;
const MAX_BATCH_SIZE: usize = 10;
const RESP_MIN_SIZE: usize = 64;
type RespBytes = SmallVec<[u8; RESP_MIN_SIZE]>;
#[cfg(test)] #[cfg(test)]
use crate::tests::DbsReaderImpl; use crate::tests::DbsReaderImpl;
#[cfg(not(test))] #[cfg(not(test))]
@ -58,11 +59,13 @@ use duniter_gva_dbs_reader::DbsReaderImpl;
static BDA_EXECUTOR: OnceCell<BdaExecutor> = OnceCell::new(); static BDA_EXECUTOR: OnceCell<BdaExecutor> = OnceCell::new();
#[allow(clippy::too_many_arguments)]
pub fn set_bda_executor( pub fn set_bda_executor(
currency: String, currency: String,
cm_accessor: AsyncAccessor, cm_accessor: AsyncAccessor,
dbs_pool: fast_threadpool::ThreadPoolAsyncHandler<SharedDbs<FileBackend>>, dbs_pool: fast_threadpool::ThreadPoolAsyncHandler<SharedDbs<FileBackend>>,
dbs_reader: DbsReaderImpl, dbs_reader: DbsReaderImpl,
profile_path_opt: Option<PathBuf>,
self_keypair: Ed25519KeyPair, self_keypair: Ed25519KeyPair,
software_version: &'static str, software_version: &'static str,
txs_mempool: duniter_core::mempools::TxsMempool, txs_mempool: duniter_core::mempools::TxsMempool,
@ -73,6 +76,7 @@ pub fn set_bda_executor(
cm_accessor, cm_accessor,
dbs_pool, dbs_pool,
dbs_reader, dbs_reader,
profile_path_opt,
self_keypair, self_keypair,
software_version, software_version,
txs_mempool, txs_mempool,
@ -100,6 +104,7 @@ struct BdaExecutor {
currency: String, currency: String,
dbs_pool: fast_threadpool::ThreadPoolAsyncHandler<SharedDbs<FileBackend>>, dbs_pool: fast_threadpool::ThreadPoolAsyncHandler<SharedDbs<FileBackend>>,
dbs_reader: DbsReaderImpl, dbs_reader: DbsReaderImpl,
profile_path_opt: Option<PathBuf>,
self_keypair: Ed25519KeyPair, self_keypair: Ed25519KeyPair,
software_version: &'static str, software_version: &'static str,
txs_mempool: duniter_core::mempools::TxsMempool, txs_mempool: duniter_core::mempools::TxsMempool,
@ -264,6 +269,7 @@ mod tests {
currency: "g1".to_owned(), currency: "g1".to_owned(),
dbs_pool: threadpool.into_async_handler(), dbs_pool: threadpool.into_async_handler(),
dbs_reader: duniter_core::dbs::kv_typed::prelude::Arc::new(mock_dbs_reader), dbs_reader: duniter_core::dbs::kv_typed::prelude::Arc::new(mock_dbs_reader),
profile_path_opt: None,
self_keypair: Ed25519KeyPair::from_seed( self_keypair: Ed25519KeyPair::from_seed(
Seed32::random().expect("fail to gen random seed"), Seed32::random().expect("fail to gen random seed"),
), ),

View file

@ -49,6 +49,8 @@ pub(crate) use duniter_core::wot::WotId;
pub(crate) use serde::{Deserialize, Serialize}; pub(crate) use serde::{Deserialize, Serialize};
pub(crate) use std::collections::BTreeSet; pub(crate) use std::collections::BTreeSet;
pub const BLOCKS_CHUNK_SIZE: u32 = 4_096;
db_schema!( db_schema!(
GvaV1, GvaV1,
[ [

View file

@ -19,6 +19,26 @@ use flate2::read::ZlibDecoder;
pub const CHUNK_FILE_PREFIX: &str = "_"; pub const CHUNK_FILE_PREFIX: &str = "_";
pub const CHUNK_FILE_EXT: &str = ".bin.gz"; pub const CHUNK_FILE_EXT: &str = ".bin.gz";
impl DbsReaderImpl {
pub(super) fn blocks_chunks_hashs_(
&self,
from: u32,
to_opt: Option<u32>,
) -> KvResult<Vec<Hash>> {
if let Some(to) = to_opt {
self.0
.blocks_chunk_hash()
.iter(U32BE(from)..=U32BE(to), |it| {
it.values().map_ok(|hash_db| hash_db.0).collect()
})
} else {
self.0.blocks_chunk_hash().iter(U32BE(from).., |it| {
it.values().map_ok(|hash_db| hash_db.0).collect()
})
}
}
}
/// Read and decompress chunk file /// Read and decompress chunk file
pub fn read_compressed_chunk( pub fn read_compressed_chunk(
chunk_index: u32, chunk_index: u32,

View file

@ -86,6 +86,7 @@ pub trait DbsReader {
bc_db: &BcV2DbRo<FileBackend>, bc_db: &BcV2DbRo<FileBackend>,
page_info: PageInfo<block::BlockCursor>, page_info: PageInfo<block::BlockCursor>,
) -> KvResult<PagedData<Vec<(block::BlockCursor, BlockMetaV2)>>>; ) -> KvResult<PagedData<Vec<(block::BlockCursor, BlockMetaV2)>>>;
fn blocks_chunks_hashs(&self, from: u32, to_opt: Option<u32>) -> KvResult<Vec<Hash>>;
fn endpoints<Db: 'static + NetworkV1DbReadable>( fn endpoints<Db: 'static + NetworkV1DbReadable>(
&self, &self,
network_db: &Db, network_db: &Db,
@ -229,6 +230,10 @@ impl DbsReader for DbsReaderImpl {
self.blocks_(bc_db, page_info) self.blocks_(bc_db, page_info)
} }
fn blocks_chunks_hashs(&self, from: u32, to_opt: Option<u32>) -> KvResult<Vec<Hash>> {
self.blocks_chunks_hashs_(from, to_opt)
}
fn endpoints<Db: 'static + NetworkV1DbReadable>( fn endpoints<Db: 'static + NetworkV1DbReadable>(
&self, &self,
network_db: &Db, network_db: &Db,

View file

@ -18,8 +18,6 @@ use duniter_gva_dbs_reader::blocks_chunks::{CHUNK_FILE_EXT, CHUNK_FILE_PREFIX};
use flate2::write::ZlibEncoder; use flate2::write::ZlibEncoder;
use flate2::Compression; use flate2::Compression;
const CHUNK_SIZE: u32 = 4_096;
pub fn apply_block_blocks_chunk<B: Backend>( pub fn apply_block_blocks_chunk<B: Backend>(
block: &DubpBlockV10, block: &DubpBlockV10,
gva_db: &GvaV1Db<B>, gva_db: &GvaV1Db<B>,
@ -33,7 +31,7 @@ pub fn apply_block_blocks_chunk<B: Backend>(
GvaBlockDbV1(DubpBlock::V10(block.clone())), GvaBlockDbV1(DubpBlock::V10(block.clone())),
)?; )?;
if (block_number + 1) % CHUNK_SIZE == 0 { if (block_number + 1) % BLOCKS_CHUNK_SIZE == 0 {
let current_chunk: Vec<GvaBlockDbV1> = gva_db let current_chunk: Vec<GvaBlockDbV1> = gva_db
.current_blocks_chunk() .current_blocks_chunk()
.iter(.., |it| it.values().collect::<Result<Vec<_>, _>>())?; .iter(.., |it| it.values().collect::<Result<Vec<_>, _>>())?;
@ -41,7 +39,7 @@ pub fn apply_block_blocks_chunk<B: Backend>(
.serialize(&current_chunk) .serialize(&current_chunk)
.map_err(|e| KvError::DeserError(e.into()))?; .map_err(|e| KvError::DeserError(e.into()))?;
let chunk_hash = Hash::compute_blake3(current_chunk_bin.as_ref()); let chunk_hash = Hash::compute_blake3(current_chunk_bin.as_ref());
let chunk_index = U32BE(block_number / CHUNK_SIZE); let chunk_index = U32BE(block_number / BLOCKS_CHUNK_SIZE);
gva_db gva_db
.blocks_chunk_hash_write() .blocks_chunk_hash_write()
.upsert(chunk_index, HashDb(chunk_hash))?; .upsert(chunk_index, HashDb(chunk_hash))?;
@ -66,9 +64,9 @@ pub fn revert_block_blocks_chunk<B: Backend>(
let block_number = block.number().0; let block_number = block.number().0;
let chunks_folder_path = profile_path.join("data/gva_v1_blocks_chunks"); let chunks_folder_path = profile_path.join("data/gva_v1_blocks_chunks");
gva_db.write(|mut db| { gva_db.write(|mut db| {
if (block_number + 1) % CHUNK_SIZE == 0 { if (block_number + 1) % BLOCKS_CHUNK_SIZE == 0 {
// Uncompress last compressed chunk and replace it in current chunk // Uncompress last compressed chunk and replace it in current chunk
let chunk_index = U32BE(block_number / CHUNK_SIZE); let chunk_index = U32BE(block_number / BLOCKS_CHUNK_SIZE);
if let Some(current_chunk_bin) = if let Some(current_chunk_bin) =
duniter_gva_dbs_reader::blocks_chunks::read_compressed_chunk( duniter_gva_dbs_reader::blocks_chunks::read_compressed_chunk(
chunk_index.0, chunk_index.0,
@ -81,7 +79,7 @@ pub fn revert_block_blocks_chunk<B: Backend>(
let current_chunk: Vec<GvaBlockDbV1> = bincode_db() let current_chunk: Vec<GvaBlockDbV1> = bincode_db()
.deserialize(current_chunk_bin.as_ref()) .deserialize(current_chunk_bin.as_ref())
.map_err(|e| KvError::DeserError(e.into()))?; .map_err(|e| KvError::DeserError(e.into()))?;
let current_chunk_begin = block_number - CHUNK_SIZE + 1; let current_chunk_begin = block_number - BLOCKS_CHUNK_SIZE + 1;
for (i, block) in current_chunk.into_iter().enumerate() { for (i, block) in current_chunk.into_iter().enumerate() {
db.current_blocks_chunk db.current_blocks_chunk
.upsert(U32BE(current_chunk_begin + i as u32), block); .upsert(U32BE(current_chunk_begin + i as u32), block);

View file

@ -46,7 +46,10 @@ use duniter_gva_db::*;
use duniter_gva_gql::{GvaSchema, QueryContext}; use duniter_gva_gql::{GvaSchema, QueryContext};
use duniter_gva_indexer::{get_gva_db_ro, get_gva_db_rw}; use duniter_gva_indexer::{get_gva_db_ro, get_gva_db_rw};
use futures::{StreamExt, TryStreamExt}; use futures::{StreamExt, TryStreamExt};
use std::{convert::Infallible, path::Path}; use std::{
convert::Infallible,
path::{Path, PathBuf},
};
#[cfg(test)] #[cfg(test)]
use tests::get_public_ips; use tests::get_public_ips;
use warp::{http::Response as HttpResponse, Filter as _, Rejection}; use warp::{http::Response as HttpResponse, Filter as _, Rejection};
@ -62,6 +65,7 @@ pub struct GvaModule {
gva_db_ro: &'static GvaV1DbRo<FileBackend>, gva_db_ro: &'static GvaV1DbRo<FileBackend>,
mempools: Mempools, mempools: Mempools,
mode: DuniterMode, mode: DuniterMode,
profile_path_opt: Option<PathBuf>,
self_keypair: Ed25519KeyPair, self_keypair: Ed25519KeyPair,
software_version: &'static str, software_version: &'static str,
} }
@ -125,6 +129,7 @@ impl duniter_core::module::DuniterModule for GvaModule {
gva_db_ro: get_gva_db_ro(profile_path_opt), gva_db_ro: get_gva_db_ro(profile_path_opt),
mempools, mempools,
mode, mode,
profile_path_opt: profile_path_opt.map(ToOwned::to_owned),
self_keypair, self_keypair,
software_version, software_version,
}, },
@ -142,6 +147,7 @@ impl duniter_core::module::DuniterModule for GvaModule {
gva_db_ro, gva_db_ro,
mempools, mempools,
mode, mode,
profile_path_opt,
self_keypair, self_keypair,
software_version, software_version,
} = self; } = self;
@ -154,6 +160,7 @@ impl duniter_core::module::DuniterModule for GvaModule {
dbs_pool, dbs_pool,
gva_db_ro, gva_db_ro,
mempools, mempools,
profile_path_opt,
self_keypair, self_keypair,
software_version, software_version,
) )
@ -235,12 +242,14 @@ impl duniter_core::module::DuniterModule for GvaModule {
} }
impl GvaModule { impl GvaModule {
#[allow(clippy::too_many_arguments)]
async fn start_inner( async fn start_inner(
conf: GvaConf, conf: GvaConf,
currency: String, currency: String,
dbs_pool: fast_threadpool::ThreadPoolAsyncHandler<SharedDbs<FileBackend>>, dbs_pool: fast_threadpool::ThreadPoolAsyncHandler<SharedDbs<FileBackend>>,
gva_db_ro: &'static GvaV1DbRo<FileBackend>, gva_db_ro: &'static GvaV1DbRo<FileBackend>,
mempools: Mempools, mempools: Mempools,
profile_path_opt: Option<PathBuf>,
self_keypair: Ed25519KeyPair, self_keypair: Ed25519KeyPair,
software_version: &'static str, software_version: &'static str,
) { ) {
@ -253,6 +262,7 @@ impl GvaModule {
AsyncAccessor::new(), AsyncAccessor::new(),
dbs_pool.clone(), dbs_pool.clone(),
duniter_gva_dbs_reader::create_dbs_reader(gva_db_ro), duniter_gva_dbs_reader::create_dbs_reader(gva_db_ro),
profile_path_opt,
self_keypair, self_keypair,
software_version, software_version,
mempools.txs, mempools.txs,