perf(db): serialize integers as varint

This commit is contained in:
librelois 2021-05-14 20:18:19 +02:00
parent 9450429421
commit f5d8f70bce
22 changed files with 82 additions and 58 deletions

41
Cargo.lock generated
View file

@ -54,6 +54,12 @@ name = "arrayvec"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "arrayvec"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a2f58b0bb10c380af2b26e57212856b8c9a59e0925b4c20f4a174a49734eaf7"
dependencies = [
"serde",
]
@ -261,7 +267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9ff35b701f3914bdb8fad3368d822c766ef2858b2583198e41639b936f09d3f"
dependencies = [
"arrayref",
"arrayvec",
"arrayvec 0.5.2",
"cc",
"cfg-if 0.1.10",
"constant_time_eq",
@ -766,7 +772,7 @@ dependencies = [
[[package]]
name = "duniter-bc-reader"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"anyhow",
"dubp",
@ -779,7 +785,7 @@ name = "duniter-bca"
version = "0.1.0"
dependencies = [
"anyhow",
"arrayvec",
"arrayvec 0.7.0",
"async-bincode",
"async_io_stream",
"bincode",
@ -801,7 +807,7 @@ dependencies = [
name = "duniter-bca-types"
version = "0.1.0"
dependencies = [
"arrayvec",
"arrayvec 0.7.0",
"bincode",
"dubp",
"serde",
@ -812,7 +818,7 @@ dependencies = [
[[package]]
name = "duniter-conf"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"anyhow",
"dubp",
@ -826,7 +832,7 @@ dependencies = [
[[package]]
name = "duniter-core"
version = "1.8.1"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"duniter-bc-reader",
"duniter-conf",
@ -839,9 +845,9 @@ dependencies = [
[[package]]
name = "duniter-dbs"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"arrayvec",
"arrayvec 0.7.0",
"bincode",
"byteorder",
"chrono",
@ -862,7 +868,7 @@ dependencies = [
[[package]]
name = "duniter-dbs-write-ops"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"chrono",
"dubp",
@ -877,7 +883,7 @@ dependencies = [
[[package]]
name = "duniter-global"
version = "1.8.1"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"async-rwlock",
"dubp",
@ -893,7 +899,7 @@ name = "duniter-gva"
version = "0.1.0"
dependencies = [
"anyhow",
"arrayvec",
"arrayvec 0.7.0",
"async-graphql",
"async-mutex",
"async-trait",
@ -955,7 +961,7 @@ name = "duniter-gva-dbs-reader"
version = "0.1.0"
dependencies = [
"anyhow",
"arrayvec",
"arrayvec 0.7.0",
"dubp",
"duniter-bca-types",
"duniter-core",
@ -972,10 +978,11 @@ name = "duniter-gva-gql"
version = "0.1.0"
dependencies = [
"anyhow",
"arrayvec",
"arrayvec 0.7.0",
"async-graphql",
"async-trait",
"dubp",
"duniter-bca-types",
"duniter-core",
"duniter-gva-db",
"duniter-gva-dbs-reader",
@ -1011,7 +1018,7 @@ dependencies = [
[[package]]
name = "duniter-mempools"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"dubp",
"duniter-bc-reader",
@ -1024,7 +1031,7 @@ dependencies = [
[[package]]
name = "duniter-module"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"anyhow",
"async-mutex",
@ -1047,7 +1054,7 @@ version = "0.53.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c530e25cc0a03ae36229401ca322002931b55fc180e6001bdab20a9087f297c"
dependencies = [
"arrayvec",
"arrayvec 0.5.2",
"base64",
"blake3",
"bs58",
@ -1595,7 +1602,7 @@ dependencies = [
[[package]]
name = "kv_typed"
version = "0.1.0"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#520d3893ec9bc3f4ca523af5de27933a0b0bd52d"
source = "git+https://git.duniter.org/nodes/rust/duniter-core#116bdbe459f3eaabd14fe0611527379689385d6f"
dependencies = [
"byteorder",
"cfg-if 0.1.10",

View file

@ -8,7 +8,7 @@ repository = "https://git.duniter.org/nodes/rust/modules/duniter-gva"
[dependencies]
anyhow = "1.0.33"
arrayvec = "0.5.1"
arrayvec = { version = "0.7", features = ["serde"] }
async-graphql = { version = "2.8", features = ["log"] }
async-mutex = "1.4.0"
async-trait = "0.1.41"

View file

@ -7,7 +7,7 @@ edition = "2018"
[dependencies]
anyhow = "1.0.33"
arrayvec = { version = "0.5.1", features = ["serde"] }
arrayvec = { version = "0.7", features = ["serde"] }
async-bincode = "0.6.1"
async_io_stream = { version = "0.3.1", features = [ "tokio_io"] }
bincode = "1.3"

View file

@ -86,7 +86,7 @@ pub(super) async fn execute_req_type(
})
})
})
.collect::<KvResult<ArrayVec<_>>>()
.collect::<KvResult<ArrayVec<_, 16>>>()
})
.await??,
))

View file

@ -18,7 +18,7 @@ use dubp::{crypto::keys::ed25519::PublicKey, wallet::prelude::WalletScriptV10};
pub(super) async fn exec_req_balances_of_pubkeys(
bca_executor: &BcaExecutor,
pubkeys: ArrayVec<[PublicKey; 16]>,
pubkeys: ArrayVec<PublicKey, 16>,
) -> Result<BcaRespTypeV0, ExecReqTypeError> {
let dbs_reader = bca_executor.dbs_reader();
Ok(BcaRespTypeV0::Balances(
@ -32,7 +32,7 @@ pub(super) async fn exec_req_balances_of_pubkeys(
.get_account_balance(&WalletScriptV10::single_sig(pubkey))
.map(|balance_opt| balance_opt.map(|balance| balance.0))
})
.collect::<Result<ArrayVec<_>, _>>()
.collect::<Result<ArrayVec<_, 16>, _>>()
})
.await??,
))
@ -40,7 +40,7 @@ pub(super) async fn exec_req_balances_of_pubkeys(
pub(super) async fn exec_req_balances_of_scripts(
bca_executor: &BcaExecutor,
scripts: ArrayVec<[WalletScriptV10; 16]>,
scripts: ArrayVec<WalletScriptV10, 16>,
) -> Result<BcaRespTypeV0, ExecReqTypeError> {
let dbs_reader = bca_executor.dbs_reader();
Ok(BcaRespTypeV0::Balances(
@ -54,7 +54,7 @@ pub(super) async fn exec_req_balances_of_scripts(
.get_account_balance(&script)
.map(|balance_opt| balance_opt.map(|balance| balance.0))
})
.collect::<Result<ArrayVec<_>, _>>()
.collect::<Result<ArrayVec<_, 16>, _>>()
})
.await??,
))

View file

@ -19,7 +19,7 @@ use dubp::{crypto::keys::ed25519::PublicKey, wallet::prelude::WalletScriptV10};
pub(super) async fn exec_req_first_utxos_of_pubkeys(
bca_executor: &BcaExecutor,
amount_target_opt: Option<Amount>,
pubkeys: ArrayVec<[PublicKey; 16]>,
pubkeys: ArrayVec<PublicKey, 16>,
) -> Result<BcaRespTypeV0, ExecReqTypeError> {
if let Some(current_ud) = bca_executor
.cm_accessor
@ -27,7 +27,7 @@ pub(super) async fn exec_req_first_utxos_of_pubkeys(
.await
{
let dbs_reader = bca_executor.dbs_reader();
let scripts: ArrayVec<[WalletScriptV10; 16]> = pubkeys
let scripts: ArrayVec<WalletScriptV10, 16> = pubkeys
.into_iter()
.map(WalletScriptV10::single_sig)
.collect();

View file

@ -6,7 +6,7 @@ license = "AGPL-3.0"
edition = "2018"
[dependencies]
arrayvec = { version = "0.5.1", features = ["serde"] }
arrayvec = { version = "0.7", features = ["serde"] }
bincode = "1.3"
dubp = { version = "0.53.1" }
serde = { version = "1.0.105", features = ["derive"] }

View file

@ -63,10 +63,10 @@ pub struct BcaReqV0 {
#[allow(clippy::large_enum_variant)]
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
pub enum BcaReqTypeV0 {
BalancesOfPubkeys(ArrayVec<[PublicKey; 16]>),
BalancesOfPubkeys(ArrayVec<PublicKey, 16>),
FirstUtxosOfPubkeys {
amount_target_opt: Option<Amount>,
pubkeys: ArrayVec<[PublicKey; 16]>,
pubkeys: ArrayVec<PublicKey, 16>,
},
LastBlockstampOutOfForkWindow,
MembersCount,
@ -76,9 +76,9 @@ pub enum BcaReqTypeV0 {
},
Ping,
SendTxs(Txs),
Identities(ArrayVec<[PublicKey; 16]>),
Identities(ArrayVec<PublicKey, 16>),
CurrentUd,
BalancesOfScripts(ArrayVec<[WalletScriptV10; 16]>),
BalancesOfScripts(ArrayVec<WalletScriptV10, 16>),
}
// Request types helpers
@ -103,8 +103,8 @@ pub struct BcaRespV0 {
#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)]
pub enum BcaRespTypeV0 {
Error(String),
Balances(ArrayVec<[Option<SourceAmount>; 16]>),
FirstUtxosOfPubkeys(Vec<ArrayVec<[Utxo; MAX_FIRST_UTXOS]>>),
Balances(ArrayVec<Option<SourceAmount>, 16>),
FirstUtxosOfPubkeys(Vec<ArrayVec<Utxo, MAX_FIRST_UTXOS>>),
ProofServerPubkey {
challenge: [u8; 16],
server_pubkey: PublicKey,
@ -115,7 +115,7 @@ pub enum BcaRespTypeV0 {
PrepareSimplePayment(PrepareSimplePaymentResp),
Pong,
RejectedTxs(Vec<rejected_tx::RejectedTx>),
Identities(ArrayVec<[Option<Identity>; 16]>),
Identities(ArrayVec<Option<Identity>, 16>),
CurrentUd(SourceAmount),
}

View file

@ -33,6 +33,7 @@ pub use values::gva_tx::GvaTxDbV1;
pub use values::wallet_script_array::WalletScriptArrayV2;
pub use values::HashDb;
pub(crate) use bincode::Options as _;
pub(crate) use dubp::common::prelude::*;
pub(crate) use dubp::crypto::hashs::Hash;
pub(crate) use dubp::wallet::prelude::*;
@ -41,7 +42,8 @@ pub(crate) use duniter_core::dbs::kv_typed::db_schema;
pub(crate) use duniter_core::dbs::kv_typed::prelude::*;
pub(crate) use duniter_core::dbs::smallvec::SmallVec;
pub(crate) use duniter_core::dbs::{
CorruptedBytes, HashKeyV2, PubKeyKeyV2, SourceAmountValV2, ToDumpString, WalletConditionsV2,
bincode_db, CorruptedBytes, HashKeyV2, PubKeyKeyV2, SourceAmountValV2, ToDumpString,
WalletConditionsV2,
};
pub(crate) use serde::{Deserialize, Serialize};
pub(crate) use std::collections::BTreeSet;

View file

@ -28,7 +28,9 @@ impl Default for GvaBlockDbV1 {
impl AsBytes for GvaBlockDbV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
f(&bincode::serialize(&self).unwrap_or_else(|_| unreachable!()))
f(&bincode_db()
.serialize(&self)
.unwrap_or_else(|_| unreachable!()))
}
}
@ -36,7 +38,7 @@ impl kv_typed::prelude::FromBytes for GvaBlockDbV1 {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
bincode::deserialize(bytes)
bincode_db().deserialize(bytes)
}
}

View file

@ -25,7 +25,9 @@ pub struct GvaIdtyDbV1 {
impl AsBytes for GvaIdtyDbV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
f(&bincode::serialize(&self).unwrap_or_else(|_| unreachable!()))
f(&bincode_db()
.serialize(&self)
.unwrap_or_else(|_| unreachable!()))
}
}
@ -33,7 +35,7 @@ impl kv_typed::prelude::FromBytes for GvaIdtyDbV1 {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
bincode::deserialize(bytes)
bincode_db().deserialize(bytes)
}
}

View file

@ -26,7 +26,9 @@ pub struct GvaTxDbV1 {
impl AsBytes for GvaTxDbV1 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
let bytes = bincode::serialize(self).unwrap_or_else(|_| unreachable!());
let bytes = bincode_db()
.serialize(self)
.unwrap_or_else(|_| unreachable!());
f(bytes.as_ref())
}
}
@ -35,7 +37,9 @@ impl kv_typed::prelude::FromBytes for GvaTxDbV1 {
type Err = CorruptedBytes;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
bincode::deserialize(&bytes).map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
bincode_db()
.deserialize(&bytes)
.map_err(|e| CorruptedBytes(format!("{}: '{:?}'", e, bytes)))
}
}

View file

@ -19,7 +19,9 @@ pub struct WalletScriptArrayV2(pub std::collections::HashSet<WalletScriptV10>);
impl AsBytes for WalletScriptArrayV2 {
fn as_bytes<T, F: FnMut(&[u8]) -> T>(&self, mut f: F) -> T {
f(&bincode::serialize(&self.0).unwrap_or_else(|_| unreachable!()))
f(&bincode_db()
.serialize(&self.0)
.unwrap_or_else(|_| unreachable!()))
}
}
@ -27,7 +29,7 @@ impl kv_typed::prelude::FromBytes for WalletScriptArrayV2 {
type Err = bincode::Error;
fn from_bytes(bytes: &[u8]) -> std::result::Result<Self, Self::Err> {
Ok(Self(bincode::deserialize(bytes)?))
Ok(Self(bincode_db().deserialize(bytes)?))
}
}

View file

@ -16,7 +16,7 @@ mock = ["mockall"]
[dependencies]
anyhow = "1.0.34"
arrayvec = "0.5.1"
arrayvec = { version = "0.7", features = ["serde"] }
duniter-bca-types = { path = "../bca/types" }
duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" }
duniter-gva-db = { path = "../db" }

View file

@ -116,7 +116,7 @@ pub trait DbsReader {
amount_target_opt: Option<SourceAmount>,
first: usize,
scripts: &[WalletScriptV10],
) -> anyhow::Result<Vec<arrayvec::ArrayVec<[Utxo; MAX_FIRST_UTXOS]>>>;
) -> anyhow::Result<Vec<arrayvec::ArrayVec<Utxo, MAX_FIRST_UTXOS>>>;
fn get_account_balance(
&self,
account_script: &WalletScriptV10,
@ -240,7 +240,7 @@ impl DbsReader for DbsReaderImpl {
amount_target_opt: Option<SourceAmount>,
first: usize,
scripts: &[WalletScriptV10],
) -> anyhow::Result<Vec<ArrayVec<[Utxo; MAX_FIRST_UTXOS]>>> {
) -> anyhow::Result<Vec<ArrayVec<Utxo, MAX_FIRST_UTXOS>>> {
self.first_scripts_utxos_(amount_target_opt, first, scripts)
}

View file

@ -174,7 +174,7 @@ impl DbsReaderImpl {
amount_target_opt: Option<SourceAmount>,
first: usize,
scripts: &[WalletScriptV10],
) -> anyhow::Result<Vec<ArrayVec<[Utxo; MAX_FIRST_UTXOS]>>> {
) -> anyhow::Result<Vec<ArrayVec<Utxo, MAX_FIRST_UTXOS>>> {
let iter = scripts.iter().map(|script| {
let (k_min, k_max) =
GvaUtxoIdDbV1::script_interval(Hash::compute(script.to_string().as_bytes()));
@ -191,7 +191,7 @@ impl DbsReaderImpl {
if let Some(amount_target) = amount_target_opt {
let mut sum = SourceAmount::ZERO;
Ok(iter
.take_while(|utxos_res: &KvResult<ArrayVec<[Utxo; MAX_FIRST_UTXOS]>>| {
.take_while(|utxos_res: &KvResult<ArrayVec<Utxo, MAX_FIRST_UTXOS>>| {
if let Ok(utxos) = utxos_res {
sum = sum + utxos.iter().map(|utxo| utxo.amount).sum();
sum <= amount_target
@ -344,7 +344,7 @@ mod tests {
]
.iter()
.copied()
.collect::<ArrayVec<_>>(),
.collect::<ArrayVec<_, MAX_FIRST_UTXOS>>(),
[
Utxo {
amount: SourceAmount::with_base0(400),
@ -359,7 +359,7 @@ mod tests {
]
.iter()
.copied()
.collect::<ArrayVec<_>>()
.collect::<ArrayVec<_, MAX_FIRST_UTXOS>>()
]
);

View file

@ -7,10 +7,11 @@ edition = "2018"
[dependencies]
anyhow = "1.0.33"
arrayvec = "0.5.1"
arrayvec = { version = "0.7", features = ["serde"] }
async-graphql = { version = "2.8", features = ["log"] }
async-trait = "0.1.41"
dubp = { version = "0.53.1", features = ["duniter"] }
duniter-bca-types = { path = "../bca/types" }
duniter-core = { git = "https://git.duniter.org/nodes/rust/duniter-core" }
duniter-gva-db = { path = "../db" }
duniter-gva-dbs-reader = { path = "../dbs-reader" }

View file

@ -15,7 +15,7 @@
use crate::*;
pub(crate) struct UtxosGva(pub arrayvec::ArrayVec<[UtxoGva; 40]>);
pub(crate) struct UtxosGva(pub arrayvec::ArrayVec<UtxoGva, 40>);
impl async_graphql::Type for UtxosGva {
fn type_name() -> Cow<'static, str> {
Cow::Owned(format!("[{}]", UtxoGva::qualified_type_name()))

View file

@ -61,6 +61,7 @@ use dubp::documents::prelude::*;
use dubp::documents::transaction::{TransactionDocumentTrait, TransactionDocumentV10};
use dubp::documents_parser::prelude::*;
use dubp::wallet::prelude::*;
use duniter_bca_types::MAX_FIRST_UTXOS;
use duniter_core::dbs::databases::txs_mp_v2::TxsMpV2DbReadable;
use duniter_core::dbs::prelude::*;
use duniter_core::dbs::{kv_typed::prelude::*, FileBackend};

View file

@ -40,7 +40,7 @@ impl FirstUtxosQuery {
let data = ctx.data::<GvaSchemaData>()?;
let db_reader = data.dbs_reader();
let utxos_matrice: Vec<arrayvec::ArrayVec<_>> = data
let utxos_matrice: Vec<arrayvec::ArrayVec<_, MAX_FIRST_UTXOS>> = data
.dbs_pool
.execute(move |_| db_reader.first_scripts_utxos(None, first as usize, &scripts))
.await??;

View file

@ -29,8 +29,9 @@ pub(super) fn apply_block_blocks_chunk<B: Backend>(
let current_chunk: Vec<GvaBlockDbV1> = gva_db
.current_blocks_chunk
.iter(.., |it| it.values().collect::<Result<Vec<_>, _>>())?;
let current_chunk_bin =
bincode::serialize(&current_chunk).map_err(|e| KvError::DeserError(e.into()))?;
let current_chunk_bin = bincode_db()
.serialize(&current_chunk)
.map_err(|e| KvError::DeserError(e.into()))?;
let chunk_hash = Hash::compute_blake3(current_chunk_bin.as_ref());
let compressed_chunk = lz4_flex::compress_prepend_size(current_chunk_bin.as_ref());
let chunk_index = U32BE(block_number / 1_000);
@ -58,7 +59,8 @@ pub(super) fn revert_block_blocks_chunk<B: Backend>(
gva_db.compressed_blocks_chunk.remove(chunk_index);
let current_chunk_bin = lz4_flex::decompress_size_prepended(compressed_chunk.as_ref())
.map_err(|e| KvError::Custom(format!("{:?}", e).into()))?;
let current_chunk: Vec<GvaBlockDbV1> = bincode::deserialize(current_chunk_bin.as_ref())
let current_chunk: Vec<GvaBlockDbV1> = bincode_db()
.deserialize(current_chunk_bin.as_ref())
.map_err(|e| KvError::DeserError(e.into()))?;
let current_chunk_begin = block_number - 999;
for (i, block) in current_chunk.into_iter().enumerate() {

View file

@ -27,6 +27,7 @@ mod identities;
mod tx;
mod utxos;
use bincode::Options as _;
use dubp::block::prelude::*;
use dubp::common::crypto::hashs::Hash;
use dubp::common::prelude::*;
@ -35,8 +36,8 @@ use dubp::documents::{
};
use dubp::wallet::prelude::*;
use duniter_core::dbs::{
kv_typed::prelude::*, prelude::*, FileBackend, HashKeyV2, PubKeyKeyV2, SourceAmountValV2,
WalletConditionsV2,
bincode_db, kv_typed::prelude::*, prelude::*, FileBackend, HashKeyV2, PubKeyKeyV2,
SourceAmountValV2, WalletConditionsV2,
};
use duniter_gva_db::*;
use resiter::filter::Filter;