hotshot_query_service/data_source/storage/
pruning.rs

1// Copyright (c) 2022 Espresso Systems (espressosys.com)
2// This file is part of the HotShot Query Service library.
3//
4// This program is free software: you can redistribute it and/or modify it under the terms of the GNU
5// General Public License as published by the Free Software Foundation, either version 3 of the
6// License, or (at your option) any later version.
7// This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
8// even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
9// General Public License for more details.
10// You should have received a copy of the GNU General Public License along with this program. If not,
11// see <https://www.gnu.org/licenses/>.
12
13use std::{fmt::Debug, time::Duration};
14
15use anyhow::bail;
16use async_trait::async_trait;
17
18#[derive(Clone, Debug)]
19pub struct PrunerCfg {
20    pruning_threshold: Option<u64>,
21    minimum_retention: Duration,
22    target_retention: Duration,
23    state_minimum_retention: Duration,
24    state_target_retention: Duration,
25    batch_size: u64,
26    max_usage: u16,
27    interval: Duration,
28    incremental_vacuum_pages: u64,
29    state_tables: Vec<String>,
30}
31
32#[async_trait]
33pub trait PruneStorage: PrunerConfig {
34    type Pruner<'a>: Default + Send
35    where
36        Self: 'a;
37
38    async fn prune<'a>(&'a self, _pruner: &mut Self::Pruner<'a>) -> anyhow::Result<Option<u64>> {
39        Ok(None)
40    }
41}
42
43#[async_trait]
44pub trait PrunedHeightStorage: Sized {
45    async fn load_pruned_height(&mut self) -> anyhow::Result<Option<u64>> {
46        Ok(None)
47    }
48
49    async fn load_state_pruned_height(&mut self) -> anyhow::Result<Option<u64>> {
50        Ok(None)
51    }
52}
53
54#[async_trait]
55pub trait PrunedHeightDataSource: Sized {
56    async fn load_pruned_height(&self) -> anyhow::Result<Option<u64>> {
57        Ok(None)
58    }
59
60    async fn load_state_pruned_height(&self) -> anyhow::Result<Option<u64>> {
61        Ok(None)
62    }
63}
64
65pub trait PrunerConfig {
66    fn set_pruning_config(&mut self, _cfg: PrunerCfg) {}
67    fn get_pruning_config(&self) -> Option<PrunerCfg> {
68        None
69    }
70}
71
72impl PrunerCfg {
73    pub fn new() -> Self {
74        Default::default()
75    }
76
77    pub fn validate(&self) -> anyhow::Result<()> {
78        if let Some(pruning_threshold) = self.pruning_threshold
79            && pruning_threshold == 0
80        {
81            bail!("pruning_threshold must be greater than 0 or set to None")
82        }
83
84        if self.max_usage > 10000 {
85            bail!("max_usage must be less than or equal to 10000")
86        }
87
88        Ok(())
89    }
90
91    pub fn with_state_tables(mut self, state_tables: Vec<String>) -> Self {
92        self.state_tables = state_tables;
93        self
94    }
95
96    pub fn with_pruning_threshold(mut self, pruning_threshold: u64) -> Self {
97        self.pruning_threshold = Some(pruning_threshold);
98        self
99    }
100
101    pub fn with_minimum_retention(mut self, minimum_retention: Duration) -> Self {
102        self.minimum_retention = minimum_retention;
103        self
104    }
105
106    pub fn with_state_minimum_retention(mut self, state_minimum_retention: Duration) -> Self {
107        self.state_minimum_retention = state_minimum_retention;
108        self
109    }
110
111    pub fn with_target_retention(mut self, target_retention: Duration) -> Self {
112        self.target_retention = target_retention;
113        self
114    }
115
116    pub fn with_state_target_retention(mut self, state_target_retention: Duration) -> Self {
117        self.state_target_retention = state_target_retention;
118        self
119    }
120
121    pub fn with_batch_size(mut self, batch_size: u64) -> Self {
122        self.batch_size = batch_size;
123        self
124    }
125
126    pub fn with_max_usage(mut self, max_usage: u16) -> Self {
127        self.max_usage = max_usage;
128        self
129    }
130
131    pub fn with_interval(mut self, interval: Duration) -> Self {
132        self.interval = interval;
133        self
134    }
135
136    pub fn with_incremental_vacuum_pages(mut self, pages: u64) -> Self {
137        self.incremental_vacuum_pages = pages;
138        self
139    }
140
141    /// Disk space threshold (in bytes).
142    ///
143    /// If the disk usage exceeds this threshold, pruning of data starts from
144    /// the oldest data and continues until the disk usage falls below `MAX_USAGE
145    /// or until the oldest data is younger than `MINIMUM_RETENTION`
146    pub fn pruning_threshold(&self) -> Option<u64> {
147        self.pruning_threshold
148    }
149
150    /// Minimum data retention period
151    ///
152    /// Data younger than this is never pruned, regardless of disk usage.
153    pub fn minimum_retention(&self) -> Duration {
154        self.minimum_retention
155    }
156
157    /// Target data retention period
158    ///
159    /// This is the ideal period for which data should be retained
160    /// data younger than this and older than `MINIMUM_RETENTION` may be pruned if disk usage exceeds the `pruning_threshold`.
161    pub fn target_retention(&self) -> Duration {
162        self.target_retention
163    }
164
165    /// Minimum state retention period
166    ///
167    /// State younger than this is never pruned, regardless of disk usage.
168    pub fn state_minimum_retention(&self) -> Duration {
169        self.state_minimum_retention
170    }
171
172    /// Target state retention period
173    ///
174    /// This is the ideal period for which state should be retained
175    /// state younger than this and older than `STATE_MINIMUM_RETENTION` may be pruned if disk usage exceeds the `pruning_threshold`.
176    pub fn state_target_retention(&self) -> Duration {
177        self.state_target_retention
178    }
179
180    /// Number of blocks to remove in a single pruning operation.
181    pub fn batch_size(&self) -> u64 {
182        self.batch_size
183    }
184
185    /// Maximum disk usage (in basis points).
186    ///
187    /// Pruning stops once the disk usage falls below this value, even if
188    /// some data older than the `MINIMUM_RETENTION` remains. Values range
189    /// from 0 (0%) to 10000 (100%).
190    pub fn max_usage(&self) -> u16 {
191        self.max_usage
192    }
193
194    /// Pruning interval
195    pub fn interval(&self) -> Duration {
196        self.interval
197    }
198
199    /// pages to remove from freelist during SQLite vacuuming
200    pub fn incremental_vacuum_pages(&self) -> u64 {
201        self.incremental_vacuum_pages
202    }
203
204    /// State tables to prune
205    pub fn state_tables(&self) -> &[String] {
206        &self.state_tables
207    }
208}
209
210impl Default for PrunerCfg {
211    fn default() -> Self {
212        Self {
213            // 3 TB
214            pruning_threshold: Some(3 * 10_u64.pow(12)),
215            // 1 day
216            minimum_retention: Duration::from_secs(24 * 3600),
217            // 7 days
218            target_retention: Duration::from_secs(7 * 24 * 3600),
219            // 1 day
220            state_minimum_retention: Duration::from_secs(24 * 3600),
221            // 7 days
222            state_target_retention: Duration::from_secs(7 * 24 * 3600),
223            batch_size: 1000,
224            // 80%
225            max_usage: 8000,
226            // 1.5 hour
227            interval: Duration::from_secs(5400),
228            // 8000 pages
229            incremental_vacuum_pages: 8000,
230            state_tables: Vec::new(),
231        }
232    }
233}
hotshot_query_service/data_source/storage/pruning.rs

hotshot_query_service/data_source/storage/
pruning.rs
