From 366c3ee5fe3aa682ed9264f93c81f2cd743d4d5b Mon Sep 17 00:00:00 2001 From: Artur Yurii Korchynskyi <42449190+akorchyn@users.noreply.github.com> Date: Wed, 25 Dec 2024 03:44:07 +0200 Subject: [PATCH] bugfix: handle new rate-limits for querying pools (#433) Closes: #431 * Added fast-near for testnet * Rate limited querying pools to 140 / 30s Ideally, if Fastnear is stable enough, the user shouldn't be able to hit this limit as we will query information within the pools the user has staked before. But in case the fastener fails, we would need to query all active pools, which number more than 150. | Network | FastNear | Non fast near | | :---: | :---: | :---: | | Testnet | < 140 (mostly 0-5) | 563 pools | Mainnet | < 140 ( mostly 0-5 ) | State of contract poolv1.near is too large to be viewed | @race-of-sloths --- .../account/view_account_summary/mod.rs | 89 +++++++++++++------ src/config/migrations.rs | 30 ++++++- src/config/mod.rs | 7 +- 3 files changed, 97 insertions(+), 29 deletions(-) diff --git a/src/commands/account/view_account_summary/mod.rs b/src/commands/account/view_account_summary/mod.rs index 0f5fd8a65..bd1f8fc37 100644 --- a/src/commands/account/view_account_summary/mod.rs +++ b/src/commands/account/view_account_summary/mod.rs @@ -108,8 +108,8 @@ fn get_account_inquiry( crate::common::fetch_historically_delegated_staking_pools(fastnear_url, account_id) .ok() }); - let validators = if let Some(validators) = historically_delegated_validators { - Ok(validators) + let pools_to_query = if let Some(user_staked_pools) = historically_delegated_validators { + Ok(user_staked_pools) } else if let Some(staking_pools_factory_account_id) = &network_config.staking_pools_factory_account_id { @@ -124,32 +124,69 @@ fn get_account_inquiry( let runtime = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()?; - let concurrency = 10; + // Staring from Feb 2025, the rate limit is 150 requests per 30 seconds for mainnet. + // We will limit the number of requests per batch to 140 to be conservative. + let batch_size = 140; + let batch_cooldown = tokio::time::Duration::from_secs(30); + let concurrency = 10; // Process 10 requests concurrently within each batch + let delegated_stake: color_eyre::Result< std::collections::BTreeMap, - > = match validators { - Ok(validators) => Ok(runtime.block_on( - futures::stream::iter(validators) - .map(|validator_account_id| async { - let balance = get_delegated_staked_balance( - &json_rpc_client, - block_reference, - &validator_account_id, - account_id, - ) - .await?; - Ok::<_, color_eyre::eyre::Report>((validator_account_id, balance)) - }) - .buffer_unordered(concurrency) - .filter(|balance_result| { - futures::future::ready(if let Ok((_, balance)) = balance_result { - !balance.is_zero() - } else { - true - }) - }) - .try_collect(), - )?), + > = match pools_to_query { + Ok(validators) => { + let mut all_results = Ok(std::collections::BTreeMap::new()); + let validators: Vec<_> = validators.into_iter().collect(); + + for (batch_index, validator_batch) in validators + .chunks(batch_size) + .map(|x| x.to_vec()) + .enumerate() + { + if batch_index > 0 { + // Wait 30 seconds before starting next batch + tracing::info!( + "Waiting for 30 seconds before fetching next batch of stake information" + ); + runtime.block_on(async { tokio::time::sleep(batch_cooldown).await }); + } + + let batch_results = runtime.block_on( + futures::stream::iter(validator_batch) + .map(|validator_account_id| async { + let balance = get_delegated_staked_balance( + &json_rpc_client, + block_reference, + &validator_account_id, + account_id, + ) + .await?; + Ok::<_, color_eyre::eyre::Report>((validator_account_id, balance)) + }) + .buffer_unordered(concurrency) + .filter(|balance_result| { + futures::future::ready(if let Ok((_, balance)) = balance_result { + !balance.is_zero() + } else { + true + }) + }) + .try_collect::>(), + ); + + match batch_results { + Ok(batch_results) => { + let _ = all_results.as_mut().map(|all_results| { + all_results.extend(batch_results); + }); + } + Err(err) => { + all_results = Err(err); + break; + } + }; + } + all_results + } Err(err) => Err(err), }; diff --git a/src/config/migrations.rs b/src/config/migrations.rs index 4995da5f4..914c2e27b 100644 --- a/src/config/migrations.rs +++ b/src/config/migrations.rs @@ -1,4 +1,4 @@ -use crate::config::Config as ConfigV2; +use crate::config::Config as ConfigV3; use crate::config::NetworkConfig as NetworkConfigV2; #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] @@ -7,6 +7,12 @@ pub struct ConfigV1 { pub network_connection: linked_hash_map::LinkedHashMap, } +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ConfigV2 { + pub credentials_home_dir: std::path::PathBuf, + pub network_connection: linked_hash_map::LinkedHashMap, +} + #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct NetworkConfigV1 { pub network_name: String, @@ -35,6 +41,25 @@ impl From for ConfigV2 { } } +impl From for ConfigV3 { + fn from(config: ConfigV2) -> Self { + ConfigV3 { + credentials_home_dir: config.credentials_home_dir, + network_connection: config + .network_connection + .into_iter() + .map(|(network_name, mut network_config)| { + if network_name == "testnet" && network_config.faucet_url.is_none() { + network_config.fastnear_url = + Some("https://test.api.fastnear.com/".parse().unwrap()); + } + (network_name, network_config) + }) + .collect(), + } + } +} + impl From for NetworkConfigV2 { fn from(network_config: NetworkConfigV1) -> Self { match network_config.network_name.as_str() { @@ -94,4 +119,7 @@ pub enum ConfigVersion { V1(ConfigV1), #[serde(rename = "2")] V2(ConfigV2), + // Adds fastnear_url to the testnet config if it's not present + #[serde(rename = "3")] + V3(ConfigV3), } diff --git a/src/config/mod.rs b/src/config/mod.rs index 319542df5..b652305ad 100644 --- a/src/config/mod.rs +++ b/src/config/mod.rs @@ -74,7 +74,7 @@ impl Config { } pub fn into_latest_version(self) -> migrations::ConfigVersion { - migrations::ConfigVersion::V2(self) + migrations::ConfigVersion::V3(self) } pub fn get_config_toml() -> color_eyre::eyre::Result { @@ -193,7 +193,10 @@ impl From for Config { migrations::ConfigVersion::V2(config_v1.into()) } migrations::ConfigVersion::V2(config_v2) => { - break config_v2; + migrations::ConfigVersion::V3(config_v2.into()) + } + migrations::ConfigVersion::V3(config_v3) => { + break config_v3; } }; }