1use std::{borrow::Borrow, collections::HashSet, iter::once, str::FromStr, sync::Arc};
2
3use alloy::primitives::{
4 Address, B256, U256,
5 utils::{ParseUnits, parse_units},
6};
7use anyhow::{Context, bail, ensure};
8use ark_serialize::{
9 CanonicalDeserialize, CanonicalSerialize, Compress, Read, SerializationError, Valid, Validate,
10};
11use espresso_utils::{
12 impl_serde_from_string_or_integer, impl_to_fixed_bytes, ser::FromStringOrInteger,
13};
14use hotshot_contract_adapter::reward::RewardProofSiblings;
15use hotshot_types::{
16 data::{EpochNumber, ViewNumber},
17 epoch_membership::EpochMembershipCoordinator,
18 signature_key::BLSPubKey,
19 traits::election::{Membership, MembershipSnapshot},
20 utils::epoch_from_block_number,
21};
22use jf_merkle_tree_compat::{
23 ForgetableMerkleTreeScheme, ForgetableUniversalMerkleTreeScheme, LookupResult,
24 MerkleTreeScheme, ToTraversalPath, UniversalMerkleTreeScheme, prelude::MerkleNode,
25};
26use num_traits::CheckedSub;
27use tokio::task::JoinHandle;
28use vbs::version::Version;
29use versions::{DRB_AND_HEADER_UPGRADE_VERSION, EPOCH_REWARD_VERSION, EPOCH_VERSION};
30
31use super::{
32 Leaf2, NodeState, ValidatedState,
33 v0_3::{AuthenticatedValidator, COMMISSION_BASIS_POINTS, RewardAmount},
34 v0_4::{
35 RewardAccountProofV2, RewardAccountQueryDataV2, RewardAccountV2, RewardMerkleCommitmentV2,
36 RewardMerkleProofV2, RewardMerkleTreeV2, forgotten_accounts_include,
37 },
38};
39use crate::{
40 EpochSnapshot, FeeAccount, SeqTypes,
41 eth_signature_key::EthKeyPair,
42 v0_3::{
43 RewardAccountProofV1, RewardAccountV1, RewardMerkleCommitmentV1, RewardMerkleProofV1,
44 RewardMerkleTreeV1,
45 },
46 v0_4::{Delta, REWARD_MERKLE_TREE_V2_ARITY, REWARD_MERKLE_TREE_V2_HEIGHT},
47 v0_5::LeaderCounts,
48};
49
50impl_serde_from_string_or_integer!(RewardAmount);
51impl_to_fixed_bytes!(RewardAmount, U256);
52
53impl From<u64> for RewardAmount {
54 fn from(amt: u64) -> Self {
55 Self(U256::from(amt))
56 }
57}
58
59impl CheckedSub for RewardAmount {
60 fn checked_sub(&self, v: &Self) -> Option<Self> {
61 self.0.checked_sub(v.0).map(RewardAmount)
62 }
63}
64
65impl FromStr for RewardAmount {
66 type Err = <U256 as FromStr>::Err;
67
68 fn from_str(s: &str) -> Result<Self, Self::Err> {
69 Ok(Self(s.parse()?))
70 }
71}
72
73impl FromStringOrInteger for RewardAmount {
74 type Binary = U256;
75 type Integer = u64;
76
77 fn from_binary(b: Self::Binary) -> anyhow::Result<Self> {
78 Ok(Self(b))
79 }
80
81 fn from_integer(i: Self::Integer) -> anyhow::Result<Self> {
82 Ok(i.into())
83 }
84
85 fn from_string(s: String) -> anyhow::Result<Self> {
86 if let Some(s) = s.strip_prefix("0x") {
89 return Ok(Self(s.parse()?));
90 }
91
92 let (base, unit) = s
94 .split_once(char::is_whitespace)
95 .unwrap_or((s.as_str(), "wei"));
96 match parse_units(base, unit)? {
97 ParseUnits::U256(n) => Ok(Self(n)),
98 ParseUnits::I256(_) => bail!("amount cannot be negative"),
99 }
100 }
101
102 fn to_binary(&self) -> anyhow::Result<Self::Binary> {
103 Ok(self.0)
104 }
105
106 fn to_string(&self) -> anyhow::Result<String> {
107 Ok(format!("{self}"))
108 }
109}
110
111impl RewardAmount {
112 pub fn as_u64(&self) -> Option<u64> {
113 if self.0 <= U256::from(u64::MAX) {
114 Some(self.0.to::<u64>())
115 } else {
116 None
117 }
118 }
119}
120
121impl From<[u8; 20]> for RewardAccountV1 {
122 fn from(bytes: [u8; 20]) -> Self {
123 Self(Address::from(bytes))
124 }
125}
126
127impl AsRef<[u8]> for RewardAccountV1 {
128 fn as_ref(&self) -> &[u8] {
129 self.0.as_slice()
130 }
131}
132
133impl<const ARITY: usize> ToTraversalPath<ARITY> for RewardAccountV1 {
134 fn to_traversal_path(&self, height: usize) -> Vec<usize> {
135 self.0
136 .as_slice()
137 .iter()
138 .take(height)
139 .map(|i| *i as usize)
140 .collect()
141 }
142}
143
144impl RewardAccountV2 {
145 pub fn address(&self) -> Address {
147 self.0
148 }
149 pub fn as_bytes(&self) -> &[u8] {
151 self.0.as_slice()
152 }
153 pub fn to_fixed_bytes(self) -> [u8; 20] {
155 self.0.into_array()
156 }
157 pub fn test_key_pair() -> EthKeyPair {
158 EthKeyPair::from_mnemonic(
159 "test test test test test test test test test test test junk",
160 0u32,
161 )
162 .unwrap()
163 }
164}
165
166impl RewardAccountV1 {
167 pub fn address(&self) -> Address {
169 self.0
170 }
171 pub fn as_bytes(&self) -> &[u8] {
173 self.0.as_slice()
174 }
175 pub fn to_fixed_bytes(self) -> [u8; 20] {
177 self.0.into_array()
178 }
179 pub fn test_key_pair() -> EthKeyPair {
180 EthKeyPair::from_mnemonic(
181 "test test test test test test test test test test test junk",
182 0u32,
183 )
184 .unwrap()
185 }
186}
187
188impl FromStr for RewardAccountV2 {
189 type Err = anyhow::Error;
190
191 fn from_str(s: &str) -> Result<Self, Self::Err> {
192 Ok(Self(s.parse()?))
193 }
194}
195
196impl FromStr for RewardAccountV1 {
197 type Err = anyhow::Error;
198
199 fn from_str(s: &str) -> Result<Self, Self::Err> {
200 Ok(Self(s.parse()?))
201 }
202}
203
204impl Valid for RewardAmount {
205 fn check(&self) -> Result<(), SerializationError> {
206 Ok(())
207 }
208}
209
210impl Valid for RewardAccountV2 {
211 fn check(&self) -> Result<(), SerializationError> {
212 Ok(())
213 }
214}
215
216impl Valid for RewardAccountV1 {
217 fn check(&self) -> Result<(), SerializationError> {
218 Ok(())
219 }
220}
221
222impl CanonicalSerialize for RewardAmount {
223 fn serialize_with_mode<W: std::io::prelude::Write>(
224 &self,
225 mut writer: W,
226 _compress: Compress,
227 ) -> Result<(), SerializationError> {
228 Ok(writer.write_all(&self.to_fixed_bytes())?)
229 }
230
231 fn serialized_size(&self, _compress: Compress) -> usize {
232 core::mem::size_of::<U256>()
233 }
234}
235impl CanonicalDeserialize for RewardAmount {
236 fn deserialize_with_mode<R: Read>(
237 mut reader: R,
238 _compress: Compress,
239 _validate: Validate,
240 ) -> Result<Self, SerializationError> {
241 let mut bytes = [0u8; core::mem::size_of::<U256>()];
242 reader.read_exact(&mut bytes)?;
243 let value = U256::from_le_slice(&bytes);
244 Ok(Self(value))
245 }
246}
247
248impl CanonicalSerialize for RewardAccountV2 {
249 fn serialize_with_mode<W: std::io::prelude::Write>(
250 &self,
251 mut writer: W,
252 _compress: Compress,
253 ) -> Result<(), SerializationError> {
254 Ok(writer.write_all(self.0.as_slice())?)
255 }
256
257 fn serialized_size(&self, _compress: Compress) -> usize {
258 core::mem::size_of::<Address>()
259 }
260}
261impl CanonicalDeserialize for RewardAccountV2 {
262 fn deserialize_with_mode<R: Read>(
263 mut reader: R,
264 _compress: Compress,
265 _validate: Validate,
266 ) -> Result<Self, SerializationError> {
267 let mut bytes = [0u8; core::mem::size_of::<Address>()];
268 reader.read_exact(&mut bytes)?;
269 let value = Address::from_slice(&bytes);
270 Ok(Self(value))
271 }
272}
273
274impl CanonicalSerialize for RewardAccountV1 {
275 fn serialize_with_mode<W: std::io::prelude::Write>(
276 &self,
277 mut writer: W,
278 _compress: Compress,
279 ) -> Result<(), SerializationError> {
280 Ok(writer.write_all(self.0.as_slice())?)
281 }
282
283 fn serialized_size(&self, _compress: Compress) -> usize {
284 core::mem::size_of::<Address>()
285 }
286}
287impl CanonicalDeserialize for RewardAccountV1 {
288 fn deserialize_with_mode<R: Read>(
289 mut reader: R,
290 _compress: Compress,
291 _validate: Validate,
292 ) -> Result<Self, SerializationError> {
293 let mut bytes = [0u8; core::mem::size_of::<Address>()];
294 reader.read_exact(&mut bytes)?;
295 let value = Address::from_slice(&bytes);
296 Ok(Self(value))
297 }
298}
299
300impl From<[u8; 20]> for RewardAccountV2 {
301 fn from(bytes: [u8; 20]) -> Self {
302 Self(Address::from(bytes))
303 }
304}
305
306impl AsRef<[u8]> for RewardAccountV2 {
307 fn as_ref(&self) -> &[u8] {
308 self.0.as_slice()
309 }
310}
311
312impl<const ARITY: usize> ToTraversalPath<ARITY> for RewardAccountV2 {
313 fn to_traversal_path(&self, height: usize) -> Vec<usize> {
314 let mut result = vec![0; height];
315
316 let mut value = U256::from_be_slice(self.0.as_slice());
318
319 for item in result.iter_mut().take(height) {
321 let digit = (value % U256::from(ARITY)).to::<usize>();
322 *item = digit;
323 value /= U256::from(ARITY);
324 }
325
326 result
327 }
328}
329
330impl RewardAccountProofV2 {
331 pub fn presence(
332 pos: FeeAccount,
333 proof: <RewardMerkleTreeV2 as MerkleTreeScheme>::MembershipProof,
334 ) -> Self {
335 Self {
336 account: pos.into(),
337 proof: RewardMerkleProofV2::Presence(proof),
338 }
339 }
340
341 pub fn absence(
342 pos: RewardAccountV2,
343 proof: <RewardMerkleTreeV2 as UniversalMerkleTreeScheme>::NonMembershipProof,
344 ) -> Self {
345 Self {
346 account: pos.into(),
347 proof: RewardMerkleProofV2::Absence(proof),
348 }
349 }
350
351 pub fn prove(tree: &RewardMerkleTreeV2, account: Address) -> Option<(Self, U256)> {
352 match tree.universal_lookup(RewardAccountV2(account)) {
353 LookupResult::Ok(balance, proof) => Some((
354 Self {
355 account,
356 proof: RewardMerkleProofV2::Presence(proof),
357 },
358 balance.0,
359 )),
360 LookupResult::NotFound(proof) => Some((
361 Self {
362 account,
363 proof: RewardMerkleProofV2::Absence(proof),
364 },
365 U256::ZERO,
366 )),
367 LookupResult::NotInMemory => None,
368 }
369 }
370
371 pub fn verify(&self, comm: &RewardMerkleCommitmentV2) -> anyhow::Result<U256> {
372 match &self.proof {
373 RewardMerkleProofV2::Presence(proof) => {
374 ensure!(
375 RewardMerkleTreeV2::verify(comm, RewardAccountV2(self.account), proof)?.is_ok(),
376 "invalid proof"
377 );
378 Ok(proof
379 .elem()
380 .context("presence proof is missing account balance")?
381 .0)
382 },
383 RewardMerkleProofV2::Absence(proof) => {
384 let tree = RewardMerkleTreeV2::from_commitment(comm);
385 ensure!(
386 RewardMerkleTreeV2::non_membership_verify(
387 tree.commitment(),
388 RewardAccountV2(self.account),
389 proof
390 )?,
391 "invalid proof"
392 );
393 Ok(U256::ZERO)
394 },
395 }
396 }
397
398 pub fn remember(&self, tree: &mut RewardMerkleTreeV2) -> anyhow::Result<()> {
399 match &self.proof {
400 RewardMerkleProofV2::Presence(proof) => {
401 tree.remember(
402 RewardAccountV2(self.account),
403 proof
404 .elem()
405 .context("presence proof is missing account balance")?,
406 proof,
407 )?;
408 Ok(())
409 },
410 RewardMerkleProofV2::Absence(proof) => {
411 tree.non_membership_remember(RewardAccountV2(self.account), proof)?;
412 Ok(())
413 },
414 }
415 }
416}
417
418impl TryInto<RewardProofSiblings> for RewardAccountProofV2 {
419 type Error = anyhow::Error;
420
421 fn try_into(self) -> anyhow::Result<RewardProofSiblings> {
426 let proof = if let RewardMerkleProofV2::Presence(proof) = &self.proof {
428 proof
429 } else {
430 bail!("only presence proofs supported")
431 };
432
433 let path = ToTraversalPath::<{ REWARD_MERKLE_TREE_V2_ARITY }>::to_traversal_path(
434 &RewardAccountV2(self.account),
435 REWARD_MERKLE_TREE_V2_HEIGHT,
436 );
437
438 if path.len() != REWARD_MERKLE_TREE_V2_HEIGHT {
439 bail!("Invalid proof: unexpected path length: {}", path.len());
440 };
441
442 let siblings: [B256; REWARD_MERKLE_TREE_V2_HEIGHT] = proof
443 .proof
444 .iter()
445 .enumerate()
446 .skip(1) .filter_map(|(level_idx, node)| match node {
448 MerkleNode::Branch { children, .. } => {
449 let path_direction = path.get(level_idx - 1).copied().expect("exists");
451 let sibling_idx = if path_direction == 0 { 1 } else { 0 };
452 if sibling_idx >= children.len() {
453 panic!(
454 "Invalid proof: index={sibling_idx} length={}",
455 children.len()
456 );
457 };
458
459 match children[sibling_idx].as_ref() {
460 MerkleNode::Empty => Some(B256::ZERO),
461 MerkleNode::Leaf { value, .. } => {
462 let bytes = value.as_ref();
463 Some(B256::from_slice(bytes))
464 },
465 MerkleNode::Branch { value, .. } => {
466 let bytes = value.as_ref();
467 Some(B256::from_slice(bytes))
468 },
469 MerkleNode::ForgettenSubtree { value } => {
470 let bytes = value.as_ref();
471 Some(B256::from_slice(bytes))
472 },
473 }
474 },
475 _ => None,
476 })
477 .collect::<Vec<B256>>()
478 .try_into()
479 .map_err(|err: Vec<_>| {
480 panic!(
481 "Invalid proof length: {:?}, this should never happen",
482 err.len()
483 )
484 })
485 .unwrap();
486
487 Ok(siblings.into())
488 }
489}
490
491impl RewardAccountProofV1 {
492 pub fn presence(
493 pos: FeeAccount,
494 proof: <RewardMerkleTreeV1 as MerkleTreeScheme>::MembershipProof,
495 ) -> Self {
496 Self {
497 account: pos.into(),
498 proof: RewardMerkleProofV1::Presence(proof),
499 }
500 }
501
502 pub fn absence(
503 pos: RewardAccountV1,
504 proof: <RewardMerkleTreeV1 as UniversalMerkleTreeScheme>::NonMembershipProof,
505 ) -> Self {
506 Self {
507 account: pos.into(),
508 proof: RewardMerkleProofV1::Absence(proof),
509 }
510 }
511
512 pub fn prove(tree: &RewardMerkleTreeV1, account: Address) -> Option<(Self, U256)> {
513 match tree.universal_lookup(RewardAccountV1(account)) {
514 LookupResult::Ok(balance, proof) => Some((
515 Self {
516 account,
517 proof: RewardMerkleProofV1::Presence(proof),
518 },
519 balance.0,
520 )),
521 LookupResult::NotFound(proof) => Some((
522 Self {
523 account,
524 proof: RewardMerkleProofV1::Absence(proof),
525 },
526 U256::ZERO,
527 )),
528 LookupResult::NotInMemory => None,
529 }
530 }
531
532 pub fn verify(&self, comm: &RewardMerkleCommitmentV1) -> anyhow::Result<U256> {
533 match &self.proof {
534 RewardMerkleProofV1::Presence(proof) => {
535 ensure!(
536 RewardMerkleTreeV1::verify(comm, RewardAccountV1(self.account), proof)?.is_ok(),
537 "invalid proof"
538 );
539 Ok(proof
540 .elem()
541 .context("presence proof is missing account balance")?
542 .0)
543 },
544 RewardMerkleProofV1::Absence(proof) => {
545 let tree = RewardMerkleTreeV1::from_commitment(comm);
546 ensure!(
547 RewardMerkleTreeV1::non_membership_verify(
548 tree.commitment(),
549 RewardAccountV1(self.account),
550 proof
551 )?,
552 "invalid proof"
553 );
554 Ok(U256::ZERO)
555 },
556 }
557 }
558
559 pub fn remember(&self, tree: &mut RewardMerkleTreeV1) -> anyhow::Result<()> {
560 match &self.proof {
561 RewardMerkleProofV1::Presence(proof) => {
562 tree.remember(
563 RewardAccountV1(self.account),
564 proof
565 .elem()
566 .context("presence proof is missing account balance")?,
567 proof,
568 )?;
569 Ok(())
570 },
571 RewardMerkleProofV1::Absence(proof) => {
572 tree.non_membership_remember(RewardAccountV1(self.account), proof)?;
573 Ok(())
574 },
575 }
576 }
577}
578
579impl From<(RewardAccountProofV2, U256)> for RewardAccountQueryDataV2 {
580 fn from((proof, balance): (RewardAccountProofV2, U256)) -> Self {
581 Self { balance, proof }
582 }
583}
584
585#[derive(Clone, Debug)]
586pub struct ComputedRewards {
587 leader_address: Address,
588 leader_commission: RewardAmount,
590 delegators: Vec<(Address, RewardAmount)>,
592}
593
594impl ComputedRewards {
595 pub fn new(
596 delegators: Vec<(Address, RewardAmount)>,
597 leader_address: Address,
598 leader_commission: RewardAmount,
599 ) -> Self {
600 Self {
601 delegators,
602 leader_address,
603 leader_commission,
604 }
605 }
606
607 pub fn leader_commission(&self) -> &RewardAmount {
608 &self.leader_commission
609 }
610
611 pub fn delegators(&self) -> &Vec<(Address, RewardAmount)> {
612 &self.delegators
613 }
614
615 pub fn all_rewards(self) -> Vec<(Address, RewardAmount)> {
617 self.delegators
618 .into_iter()
619 .chain(once((self.leader_address, self.leader_commission)))
620 .collect()
621 }
622}
623
624pub struct ValidatorLeaderCounts(Vec<(AuthenticatedValidator<BLSPubKey>, u16)>);
626
627impl ValidatorLeaderCounts {
628 pub fn new(snapshot: &EpochSnapshot, leader_counts: LeaderCounts) -> anyhow::Result<Self> {
635 let entries: Vec<_> = snapshot
636 .stake_table()
637 .zip(leader_counts.iter().copied())
638 .map(|(entry, count)| {
639 let validator = snapshot.validator_config(&entry.stake_table_entry.stake_key)?;
640 Ok((validator.clone(), count))
641 })
642 .collect::<anyhow::Result<_>>()?;
643
644 Ok(Self(entries))
645 }
646
647 pub fn active_leaders(
649 &self,
650 ) -> impl Iterator<Item = (&AuthenticatedValidator<BLSPubKey>, u16)> {
651 self.0
652 .iter()
653 .filter(|(_, count)| *count > 0)
654 .map(|(v, count)| (v, *count))
655 }
656
657 fn all_reward_accounts(&self) -> Vec<RewardAccountV2> {
659 self.active_leaders()
660 .flat_map(|(v, _)| {
661 std::iter::once(RewardAccountV2(v.account))
662 .chain(v.delegators.keys().map(|d| RewardAccountV2(*d)))
663 })
664 .collect()
665 }
666}
667
668pub struct RewardDistributor {
669 validator: AuthenticatedValidator<BLSPubKey>,
670 block_reward: RewardAmount,
671 total_distributed: RewardAmount,
672}
673
674impl RewardDistributor {
675 pub fn new(
676 validator: AuthenticatedValidator<BLSPubKey>,
677 block_reward: RewardAmount,
678 total_distributed: RewardAmount,
679 ) -> Self {
680 Self {
681 validator,
682 block_reward,
683 total_distributed,
684 }
685 }
686
687 pub fn validator(&self) -> AuthenticatedValidator<BLSPubKey> {
688 self.validator.clone()
689 }
690
691 pub fn block_reward(&self) -> RewardAmount {
692 self.block_reward
693 }
694
695 pub fn total_distributed(&self) -> RewardAmount {
696 self.total_distributed
697 }
698
699 pub fn update_rewards_delta(&self, delta: &mut Delta) -> anyhow::Result<()> {
700 delta
702 .rewards_delta
703 .insert(RewardAccountV2(self.validator().account));
704 delta.rewards_delta.extend(
705 self.validator()
706 .delegators
707 .keys()
708 .map(|d| RewardAccountV2(*d)),
709 );
710
711 Ok(())
712 }
713
714 pub fn update_reward_balance<P>(
715 tree: &mut P,
716 account: &P::Index,
717 amount: P::Element,
718 ) -> anyhow::Result<()>
719 where
720 P: UniversalMerkleTreeScheme<Element = RewardAmount>,
721 P::Index: Borrow<<P as MerkleTreeScheme>::Index> + std::fmt::Display,
722 {
723 let mut err = None;
724 tree.update_with(account.clone(), |balance| {
725 let balance = balance.copied();
726 match balance.unwrap_or_default().0.checked_add(amount.0) {
727 Some(updated) => Some(updated.into()),
728 None => {
729 err = Some(format!("overflowed reward balance for account {account}"));
730 balance
731 },
732 }
733 })?;
734
735 if let Some(error) = err {
736 tracing::warn!(error);
737 bail!(error)
738 }
739
740 Ok(())
741 }
742
743 pub fn apply_rewards(
744 &mut self,
745 version: Version,
746 state: &mut ValidatedState,
747 ) -> anyhow::Result<()> {
748 let computed_rewards = self.compute_rewards()?;
749
750 if version <= EPOCH_VERSION {
751 for (address, reward) in computed_rewards.all_rewards() {
752 Self::update_reward_balance(
753 &mut state.reward_merkle_tree_v1,
754 &RewardAccountV1(address),
755 reward,
756 )?;
757 tracing::debug!(%address, %reward, "applied v1 rewards");
758 }
759 } else {
760 for (address, reward) in computed_rewards.all_rewards() {
761 Self::update_reward_balance(
762 &mut state.reward_merkle_tree_v2,
763 &RewardAccountV2(address),
764 reward,
765 )?;
766 tracing::debug!(%address, %reward, "applied v2 rewards");
767 }
768 }
769
770 self.total_distributed += self.block_reward();
771
772 Ok(())
773 }
774
775 pub fn compute_rewards(&self) -> anyhow::Result<ComputedRewards> {
783 ensure!(
784 self.validator.commission <= COMMISSION_BASIS_POINTS,
785 "commission must not exceed {COMMISSION_BASIS_POINTS}"
786 );
787
788 let mut rewards = Vec::new();
789
790 let total_reward = self.block_reward.0;
791 let delegators_ratio_basis_points = U256::from(COMMISSION_BASIS_POINTS)
792 .checked_sub(U256::from(self.validator.commission))
793 .context("overflow")?;
794 let delegators_reward = delegators_ratio_basis_points
795 .checked_mul(total_reward)
796 .context("overflow")?;
797
798 let total_stake = self.validator.stake;
800 let mut delegators_total_reward_distributed = U256::from(0);
801 for (delegator_address, delegator_stake) in &self.validator.delegators {
802 let delegator_reward = RewardAmount::from(
803 (delegator_stake
804 .checked_mul(delegators_reward)
805 .context("overflow")?
806 .checked_div(total_stake)
807 .context("overflow")?)
808 .checked_div(U256::from(COMMISSION_BASIS_POINTS))
809 .context("overflow")?,
810 );
811
812 delegators_total_reward_distributed += delegator_reward.0;
813
814 rewards.push((*delegator_address, delegator_reward));
815 }
816
817 let leader_commission = total_reward
818 .checked_sub(delegators_total_reward_distributed)
819 .context("overflow")?;
820
821 Ok(ComputedRewards::new(
822 rewards,
823 self.validator.account,
824 leader_commission.into(),
825 ))
826 }
827}
828
829pub async fn distribute_block_reward(
836 instance_state: &NodeState,
837 validated_state: &mut ValidatedState,
838 parent_leaf: &Leaf2,
839 view_number: ViewNumber,
840 version: Version,
841) -> anyhow::Result<Option<RewardDistributor>> {
842 let height = parent_leaf.height() + 1;
843
844 let epoch_height = instance_state
845 .epoch_height
846 .context("epoch height not found")?;
847 let epoch = EpochNumber::new(epoch_from_block_number(height, epoch_height));
848 let coordinator = instance_state.coordinator.clone();
849 let first_epoch = coordinator.membership().first_epoch();
850
851 if epoch <= first_epoch + 1 {
854 return Ok(None);
855 }
856
857 let leader = get_leader_and_fetch_missing_rewards(
861 instance_state,
862 validated_state,
863 parent_leaf,
864 view_number,
865 )
866 .await?;
867
868 let parent_header = parent_leaf.block_header();
869
870 let mut previously_distributed = parent_header.total_reward_distributed().unwrap_or_default();
872
873 let block_reward = if version == DRB_AND_HEADER_UPGRADE_VERSION {
875 instance_state
876 .block_reward(EpochNumber::new(*epoch))
877 .await
878 .with_context(|| format!("block reward is None for epoch {epoch}"))?
879 } else {
880 instance_state.fixed_block_reward().await?
881 };
882
883 if version == DRB_AND_HEADER_UPGRADE_VERSION && parent_header.version() == EPOCH_VERSION {
888 ensure!(
889 instance_state.epoch_start_block != 0,
890 "epoch_start_block is zero"
891 );
892
893 let fixed_block_reward = instance_state.fixed_block_reward().await?;
894
895 let first_reward_block = (*first_epoch + 1) * epoch_height + 1;
901 if height > first_reward_block {
905 let blocks = height.checked_sub(first_reward_block).with_context(|| {
908 format!("height ({height}) - first_reward_block ({first_reward_block}) underflowed")
909 })?;
910 previously_distributed = U256::from(blocks)
911 .checked_mul(fixed_block_reward.0)
912 .with_context(|| {
913 format!(
914 "overflow during total_distributed calculation: blocks={blocks}, \
915 fixed_block_reward={}",
916 fixed_block_reward.0
917 )
918 })?
919 .into();
920 }
921 }
922
923 if block_reward.0.is_zero() {
924 tracing::info!("block reward is zero. height={height}. epoch={epoch}");
925 return Ok(None);
926 }
927
928 let mut reward_distributor =
929 RewardDistributor::new(leader, block_reward, previously_distributed);
930
931 reward_distributor.apply_rewards(version, validated_state)?;
932
933 Ok(Some(reward_distributor))
934}
935
936pub async fn get_leader_and_fetch_missing_rewards(
937 instance_state: &NodeState,
938 validated_state: &mut ValidatedState,
939 parent_leaf: &Leaf2,
940 view: ViewNumber,
941) -> anyhow::Result<AuthenticatedValidator<BLSPubKey>> {
942 let parent_height = parent_leaf.height();
943 let parent_view = parent_leaf.view_number();
944 let new_height = parent_height + 1;
945
946 let epoch_height = instance_state
947 .epoch_height
948 .context("epoch height not found")?;
949 if epoch_height == 0 {
950 bail!("epoch height is 0. can not catchup reward accounts");
951 }
952 let epoch = EpochNumber::new(epoch_from_block_number(new_height, epoch_height));
953
954 let coordinator = instance_state.coordinator.clone();
955
956 let membership = coordinator.membership_for_epoch(Some(epoch))?;
957
958 let snapshot = membership
959 .snapshot()
960 .context(format!("no committee for epoch {epoch:?}"))?;
961
962 let leader: BLSPubKey = snapshot
963 .leader(view)
964 .context(format!("leader for epoch {epoch:?} not found"))?;
965
966 tracing::debug!("Selected leader: {leader} for view {view} and epoch {epoch}");
967
968 let validator = snapshot
969 .validator_config(&leader)
970 .context("validator not found")?;
971
972 let parent_header = parent_leaf.block_header();
973
974 if parent_header.version() <= EPOCH_VERSION {
975 let mut reward_accounts = HashSet::new();
976 reward_accounts.insert(validator.account.into());
977 let delegators = validator
978 .delegators
979 .keys()
980 .cloned()
981 .map(|a| a.into())
982 .collect::<Vec<RewardAccountV2>>();
983
984 reward_accounts.extend(delegators.clone());
985
986 let accts: HashSet<_> = reward_accounts
987 .into_iter()
988 .map(RewardAccountV1::from)
989 .collect();
990 let missing_reward_accts = validated_state.forgotten_reward_accounts_v1(accts);
991
992 if !missing_reward_accts.is_empty() {
993 tracing::warn!(
994 parent_height,
995 ?parent_view,
996 ?missing_reward_accts,
997 "fetching missing v1 reward accounts from peers"
998 );
999
1000 let missing_account_proofs = instance_state
1001 .state_catchup
1002 .fetch_reward_accounts_v1(
1003 instance_state,
1004 parent_height,
1005 parent_view,
1006 validated_state.reward_merkle_tree_v1.commitment(),
1007 missing_reward_accts,
1008 )
1009 .await?;
1010
1011 for proof in missing_account_proofs.iter() {
1012 proof
1013 .remember(&mut validated_state.reward_merkle_tree_v1)
1014 .expect("proof previously verified");
1015 }
1016 }
1017 } else {
1018 let reward_accounts = Arc::new(
1019 std::iter::once(validator.account.into())
1020 .chain(validator.delegators.keys().cloned().map(Into::into))
1021 .collect::<Vec<_>>(),
1022 );
1023
1024 let reward_merkle_tree_root = validated_state.reward_merkle_tree_v2.commitment();
1025 if forgotten_accounts_include(&validated_state.reward_merkle_tree_v2, &reward_accounts) {
1026 tracing::warn!(
1027 parent_height,
1028 ?parent_view,
1029 %reward_merkle_tree_root,
1030 "fetching reward merkle tree from peers"
1031 );
1032
1033 validated_state.reward_merkle_tree_v2 = instance_state
1034 .state_catchup
1035 .fetch_reward_merkle_tree_v2(
1036 parent_height,
1037 parent_view,
1038 reward_merkle_tree_root,
1039 reward_accounts,
1040 )
1041 .await?
1042 .tree;
1043
1044 tracing::warn!(
1045 parent_height,
1046 ?parent_view,
1047 %reward_merkle_tree_root,
1048 "successfully fetched reward merkle tree from peers"
1049 );
1050 }
1051 }
1052
1053 Ok(validator.clone())
1054}
1055
1056#[derive(Debug, Clone)]
1058pub struct EpochRewardsResult {
1059 pub epoch: EpochNumber,
1061 pub reward_tree: RewardMerkleTreeV2,
1063 pub total_distributed: RewardAmount,
1065 pub changed_accounts: HashSet<RewardAccountV2>,
1067}
1068
1069#[derive(Debug, Default)]
1072pub struct EpochRewardsCalculator {
1073 pending: Option<PendingRewards>,
1075 applied: Option<EpochRewardsResult>,
1080}
1081
1082#[derive(Debug)]
1084struct PendingRewards {
1085 epoch: EpochNumber,
1086 leader_counts: Option<LeaderCounts>,
1089 handle: JoinHandle<anyhow::Result<EpochRewardsResult>>,
1090}
1091
1092impl EpochRewardsCalculator {
1093 pub fn new() -> Self {
1094 Self::default()
1095 }
1096
1097 pub fn is_calculating(&self, epoch: EpochNumber) -> bool {
1099 self.pending.as_ref().is_some_and(|p| p.epoch == epoch)
1100 || self.applied.as_ref().is_some_and(|r| r.epoch == epoch)
1101 }
1102
1103 pub async fn get_result(
1109 &mut self,
1110 epoch: EpochNumber,
1111 ) -> Option<anyhow::Result<EpochRewardsResult>> {
1112 if let Some(applied) = self.applied.as_ref().filter(|r| r.epoch == epoch) {
1113 return Some(Ok(applied.clone()));
1114 }
1115 let pending = self.pending.take()?;
1116 if pending.epoch != epoch {
1117 self.pending = Some(pending);
1119 return None;
1120 }
1121
1122 let result = match pending.handle.await {
1123 Ok(Ok(result)) => {
1124 tracing::info!(%epoch, total = %result.total_distributed.0, "epoch rewards calculation completed");
1125 self.applied = Some(result.clone());
1126 Ok(result)
1127 },
1128 Ok(Err(e)) => {
1129 tracing::error!(%epoch, error = %e, "epoch rewards calculation failed");
1130 Err(e)
1131 },
1132 Err(e) => {
1133 tracing::error!(%epoch, error = %e, "epoch rewards task panicked");
1134 Err(anyhow::Error::new(e).context("epoch rewards task panicked"))
1135 },
1136 };
1137 Some(result)
1138 }
1139
1140 pub fn discard(&mut self, epoch: EpochNumber) {
1148 if self.applied.take_if(|r| r.epoch == epoch).is_some() {
1149 tracing::warn!(%epoch, "discarding applied epoch rewards result");
1150 }
1151 if let Some(pending) = self.pending.take_if(|p| p.epoch == epoch) {
1152 tracing::warn!(%epoch, "discarding pending epoch rewards task");
1153 pending.handle.abort();
1154 }
1155 }
1156
1157 pub fn spawn_background_task(
1162 &mut self,
1163 epoch: EpochNumber,
1164 epoch_height: u64,
1165 reward_tree: RewardMerkleTreeV2,
1166 instance_state: NodeState,
1167 coordinator: EpochMembershipCoordinator<SeqTypes>,
1168 leader_counts: Option<LeaderCounts>,
1169 ) {
1170 if !self.needs_spawn(epoch, leader_counts.as_ref()) {
1171 tracing::debug!(%epoch, "calculation already in progress, skipping");
1172 return;
1173 }
1174
1175 if let Some(stale) = self.pending.take() {
1177 tracing::info!(stale_epoch = %stale.epoch, %epoch, "aborting stale epoch rewards task");
1178 stale.handle.abort();
1179 }
1180
1181 tracing::info!(
1182 %epoch,
1183 has_leader_counts = leader_counts.is_some(),
1184 "starting background epoch rewards task"
1185 );
1186
1187 let handle = tokio::spawn(async move {
1188 Self::fetch_and_calculate(
1189 epoch,
1190 epoch_height,
1191 reward_tree,
1192 instance_state,
1193 coordinator,
1194 leader_counts,
1195 )
1196 .await
1197 });
1198 self.pending = Some(PendingRewards {
1199 epoch,
1200 leader_counts,
1201 handle,
1202 });
1203 }
1204
1205 fn needs_spawn(&self, epoch: EpochNumber, leader_counts: Option<&LeaderCounts>) -> bool {
1211 match &self.pending {
1212 Some(p) if p.epoch == epoch => {
1213 leader_counts.is_some_and(|lc| p.leader_counts.as_ref() != Some(lc))
1214 },
1215 _ => !self.applied.as_ref().is_some_and(|r| r.epoch == epoch),
1216 }
1217 }
1218
1219 async fn fetch_and_calculate(
1220 epoch: EpochNumber,
1221 epoch_height: u64,
1222 mut reward_tree: RewardMerkleTreeV2,
1223 instance_state: NodeState,
1224 coordinator: EpochMembershipCoordinator<SeqTypes>,
1225 leader_counts: Option<LeaderCounts>,
1226 ) -> anyhow::Result<EpochRewardsResult> {
1227 let epoch_last_block_height = (*epoch) * epoch_height;
1228
1229 tracing::info!(
1230 %epoch,
1231 epoch_last_block_height,
1232 has_leader_counts = leader_counts.is_some(),
1233 "fetch_and_calculate: starting"
1234 );
1235
1236 if let Err(err) = coordinator.membership_for_epoch(Some(epoch)) {
1238 tracing::info!(%epoch, "stake table missing for epoch, triggering catchup: {err:#}");
1239 coordinator
1240 .wait_for_catchup(epoch)
1241 .await
1242 .context(format!("failed to catch up for epoch={epoch}"))?;
1243 }
1244
1245 let leader_counts = if let Some(lc) = leader_counts {
1247 lc
1248 } else {
1249 let leaf = instance_state
1252 .state_catchup
1253 .as_ref()
1254 .fetch_leaf(coordinator.clone(), epoch_last_block_height)
1255 .await
1256 .with_context(|| {
1257 format!(
1258 "failed to fetch leaf at height {epoch_last_block_height} for epoch \
1259 {epoch}"
1260 )
1261 })?;
1262 let header = leaf.block_header();
1263
1264 tracing::info!(
1265 %epoch,
1266 header_height = header.height(),
1267 header_version = %header.version(),
1268 header_reward_merkle_tree_root = %header.reward_merkle_tree_root(),
1269 "fetch_and_calculate: fetched leaf"
1270 );
1271
1272 if header.version() < EPOCH_REWARD_VERSION {
1273 tracing::info!(
1274 %epoch,
1275 header_version = %header.version(),
1276 "no rewards to distribute"
1277 );
1278 return Ok(EpochRewardsResult {
1279 epoch,
1280 reward_tree,
1281 total_distributed: RewardAmount::default(),
1282 changed_accounts: HashSet::new(),
1283 });
1284 }
1285
1286 let expected_root = header.reward_merkle_tree_root().right();
1292 let actual_root = reward_tree.commitment();
1293 if expected_root != Some(actual_root) {
1294 tracing::warn!(
1295 %epoch,
1296 ?expected_root,
1297 ?actual_root,
1298 "reward merkle tree root mismatch, using empty tree"
1299 );
1300 reward_tree = RewardMerkleTreeV2::new(REWARD_MERKLE_TREE_V2_HEIGHT);
1301 }
1302
1303 *header
1304 .leader_counts()
1305 .expect("V6+ header must have leader_counts")
1306 };
1307
1308 let snapshot = coordinator
1309 .membership()
1310 .snapshot(epoch)
1311 .with_context(|| format!("no committee for epoch={epoch}"))?;
1312 let validator_leader_counts = ValidatorLeaderCounts::new(&snapshot, leader_counts)?;
1313 let block_reward = snapshot
1314 .epoch_block_reward()
1315 .context("block reward not found for epoch")?;
1316
1317 tracing::info!(
1318 %epoch,
1319 %block_reward,
1320 "fetch_and_calculate: got block_reward"
1321 );
1322
1323 let accounts_to_update = validator_leader_counts.all_reward_accounts();
1325
1326 let missing_accounts: Vec<_> = accounts_to_update
1327 .iter()
1328 .filter(|account| reward_tree.lookup(**account).expect_not_in_memory().is_ok())
1329 .cloned()
1330 .collect();
1331
1332 if !missing_accounts.is_empty() {
1335 tracing::info!(
1336 %epoch,
1337 num_missing = missing_accounts.len(),
1338 "missing accounts detected, fetching reward merkle tree from peers"
1339 );
1340
1341 let reward_merkle_tree_root = reward_tree.commitment();
1342 reward_tree = instance_state
1343 .state_catchup
1344 .as_ref()
1345 .fetch_reward_merkle_tree_v2(
1346 epoch_last_block_height,
1347 ViewNumber::new(0),
1348 reward_merkle_tree_root,
1349 Arc::new(missing_accounts),
1350 )
1351 .await
1352 .with_context(|| {
1353 format!(
1354 "failed to fetch reward merkle tree at height {epoch_last_block_height} \
1355 for epoch {epoch}"
1356 )
1357 })?
1358 .tree;
1359
1360 tracing::info!(
1361 %epoch,
1362 reward_tree_commitment = %reward_tree.commitment(),
1363 "reward tree fetched successfully"
1364 );
1365 }
1366
1367 tracing::info!(
1368 %epoch,
1369 reward_tree_commitment = %reward_tree.commitment(),
1370 "starting final epoch calculation"
1371 );
1372
1373 Self::calculate_all_rewards(epoch, validator_leader_counts, reward_tree, block_reward).await
1374 }
1375
1376 async fn calculate_all_rewards(
1378 epoch: EpochNumber,
1379 validator_leader_counts: ValidatorLeaderCounts,
1380 mut reward_tree: RewardMerkleTreeV2,
1381 block_reward: RewardAmount,
1382 ) -> anyhow::Result<EpochRewardsResult> {
1383 let mut total_distributed = U256::ZERO;
1384 let mut changed_accounts = HashSet::new();
1385
1386 for (validator, count) in validator_leader_counts.active_leaders() {
1387 let validator_reward = block_reward
1389 .0
1390 .checked_mul(U256::from(count))
1391 .context("overflow in validator reward calculation")?;
1392
1393 if validator_reward.is_zero() {
1394 continue;
1395 }
1396
1397 changed_accounts.insert(RewardAccountV2(validator.account));
1398 changed_accounts.extend(validator.delegators.keys().map(|d| RewardAccountV2(*d)));
1399
1400 let distributor = RewardDistributor::new(
1401 validator.clone(),
1402 RewardAmount(validator_reward),
1403 Default::default(),
1404 );
1405
1406 let computed_rewards = distributor.compute_rewards()?;
1407
1408 for (address, reward) in computed_rewards.all_rewards() {
1409 RewardDistributor::update_reward_balance(
1410 &mut reward_tree,
1411 &RewardAccountV2(address),
1412 reward,
1413 )?;
1414 tracing::debug!(%epoch, %address, %reward, "applied epoch reward");
1415 }
1416
1417 total_distributed += validator_reward;
1418 }
1419
1420 tracing::info!(
1421 %epoch,
1422 total_distributed = %total_distributed,
1423 num_changed_accounts = changed_accounts.len(),
1424 "epoch rewards calculation complete"
1425 );
1426
1427 Ok(EpochRewardsResult {
1428 epoch,
1429 reward_tree,
1430 total_distributed: RewardAmount(total_distributed),
1431 changed_accounts,
1432 })
1433 }
1434}
1435
1436#[cfg(test)]
1437pub mod tests {
1438
1439 use super::*;
1440
1441 fn make_distributor(commission: u16) -> RewardDistributor {
1442 RewardDistributor::new(
1443 AuthenticatedValidator::mock_with_commission(commission),
1444 RewardAmount(U256::from(1902000000000000000_u128)),
1445 U256::ZERO.into(),
1446 )
1447 }
1448
1449 fn total_rewards(rewards: ComputedRewards) -> U256 {
1450 rewards
1451 .all_rewards()
1452 .iter()
1453 .fold(U256::ZERO, |acc, (_, r)| acc + r.0)
1454 }
1455
1456 #[tokio::test]
1460 async fn test_epoch_rewards_result_survives_reapplying_a_boundary() {
1461 let epoch = EpochNumber::new(3);
1462 let result = EpochRewardsResult {
1463 epoch,
1464 reward_tree: RewardMerkleTreeV2::new(REWARD_MERKLE_TREE_V2_HEIGHT),
1465 total_distributed: RewardAmount(U256::from(7)),
1466 changed_accounts: HashSet::new(),
1467 };
1468 let mut calculator = EpochRewardsCalculator::new();
1469 let ready = result.clone();
1470 calculator.pending = Some(PendingRewards {
1471 epoch,
1472 leader_counts: None,
1473 handle: tokio::spawn(async move { Ok(ready) }),
1474 });
1475
1476 let first = calculator.get_result(epoch).await.unwrap().unwrap();
1477 assert!(calculator.is_calculating(epoch));
1478 let second = calculator.get_result(epoch).await.unwrap().unwrap();
1479
1480 assert_eq!(first.epoch, epoch);
1481 assert_eq!(second.epoch, epoch);
1482 assert_eq!(second.total_distributed, result.total_distributed);
1483 }
1484
1485 #[tokio::test]
1489 async fn test_epoch_rewards_discard_reopens_recovery_after_root_mismatch() {
1490 let epoch = EpochNumber::new(3);
1491 let result = EpochRewardsResult {
1492 epoch,
1493 reward_tree: RewardMerkleTreeV2::new(REWARD_MERKLE_TREE_V2_HEIGHT),
1494 total_distributed: RewardAmount(U256::from(7)),
1495 changed_accounts: HashSet::new(),
1496 };
1497 let mut calculator = EpochRewardsCalculator::new();
1498 calculator.pending = Some(PendingRewards {
1499 epoch,
1500 leader_counts: Some([1; crate::v0_3::MAX_VALIDATORS]),
1501 handle: tokio::spawn(async move { Ok(result) }),
1502 });
1503 calculator.get_result(epoch).await.unwrap().unwrap();
1504 assert!(calculator.is_calculating(epoch));
1505 assert!(!calculator.needs_spawn(epoch, None));
1506
1507 calculator.discard(epoch + 1);
1509 assert!(calculator.is_calculating(epoch));
1510
1511 calculator.discard(epoch);
1512
1513 assert!(!calculator.is_calculating(epoch));
1514 assert!(calculator.needs_spawn(epoch, None));
1515 assert!(calculator.get_result(epoch).await.is_none());
1516 }
1517
1518 #[tokio::test]
1519 async fn test_epoch_rewards_discard_aborts_pending_task() {
1520 let epoch = EpochNumber::new(3);
1521 let mut calculator = EpochRewardsCalculator::new();
1522 calculator.pending = Some(PendingRewards {
1523 epoch,
1524 leader_counts: Some([1; crate::v0_3::MAX_VALIDATORS]),
1525 handle: tokio::spawn(std::future::pending()),
1526 });
1527
1528 calculator.discard(epoch);
1529
1530 assert!(!calculator.is_calculating(epoch));
1531 assert!(calculator.pending.is_none());
1532 }
1533
1534 #[tokio::test]
1538 async fn test_epoch_rewards_task_restarts_on_different_leader_counts() {
1539 let epoch = EpochNumber::new(3);
1540 let counts_v: LeaderCounts = [1; crate::v0_3::MAX_VALIDATORS];
1541 let mut counts_v1 = counts_v;
1542 counts_v1[0] += 1;
1543
1544 let mut calculator = EpochRewardsCalculator::new();
1545 calculator.pending = Some(PendingRewards {
1546 epoch,
1547 leader_counts: Some(counts_v),
1548 handle: tokio::spawn(std::future::pending()),
1549 });
1550
1551 assert!(!calculator.needs_spawn(epoch, Some(&counts_v)));
1552 assert!(!calculator.needs_spawn(epoch, None));
1553 assert!(calculator.needs_spawn(epoch, Some(&counts_v1)));
1554 assert!(calculator.needs_spawn(epoch + 1, None));
1555 }
1556
1557 #[test]
1560 fn test_reward_calculation_sanity_checks() {
1561 let distributor = make_distributor(500);
1566 let rewards = distributor.compute_rewards().unwrap();
1567 assert_eq!(total_rewards(rewards.clone()), distributor.block_reward.0);
1568
1569 let distributor = make_distributor(0);
1570 let rewards = distributor.compute_rewards().unwrap();
1571 assert_eq!(total_rewards(rewards.clone()), distributor.block_reward.0);
1572
1573 let distributor = make_distributor(10000);
1574 let rewards = distributor.compute_rewards().unwrap();
1575 assert_eq!(total_rewards(rewards.clone()), distributor.block_reward.0);
1576 let leader_commission = rewards.leader_commission();
1577 assert_eq!(*leader_commission, distributor.block_reward);
1578
1579 let distributor = make_distributor(10001);
1580 assert!(
1581 distributor
1582 .compute_rewards()
1583 .err()
1584 .unwrap()
1585 .to_string()
1586 .contains("must not exceed")
1587 );
1588 }
1589
1590 #[test]
1591 fn test_compute_rewards_validator_commission() {
1592 let distributor = make_distributor(0);
1593 let rewards = distributor.compute_rewards().unwrap();
1594 let leader_commission = rewards.leader_commission();
1595 let percentage =
1596 leader_commission.0 * U256::from(COMMISSION_BASIS_POINTS) / distributor.block_reward.0;
1597 assert_eq!(percentage, U256::ZERO);
1598
1599 let distributor = make_distributor(300);
1601 let rewards = distributor.compute_rewards().unwrap();
1602 let leader_commission = rewards.leader_commission();
1603 let percentage =
1604 leader_commission.0 * U256::from(COMMISSION_BASIS_POINTS) / distributor.block_reward.0;
1605 println!("percentage: {percentage:?}");
1606 assert_eq!(percentage, U256::from(300));
1607
1608 let distributor = make_distributor(10000);
1610 let rewards = distributor.compute_rewards().unwrap();
1611 let leader_commission = rewards.leader_commission();
1612 assert_eq!(*leader_commission, distributor.block_reward);
1613 }
1614}