1use std::sync::Arc;
4
5use ::light_client::{state::LightClientOptions, storage::LightClientSqliteOptions};
6use anyhow::{Context, bail};
7use clap::Parser;
8use espresso_telemetry as telemetry;
9use espresso_types::{
10 BlockMerkleTree, PubKey, SeqTypes,
11 v0::traits::{EventConsumer, NullEventConsumer, PersistenceOptions, SequencerPersistence},
12 v0_3::RewardMerkleTreeV1,
13 v0_4::RewardMerkleTreeV2,
14};
15use futures::{
16 channel::oneshot,
17 future::{BoxFuture, Future},
18};
19use hotshot_query_service::{
20 ApiState as AppState, Error,
21 data_source::{ExtensibleDataSource, MetricsDataSource},
22 status::{self, HasMetrics, UpdateStatusData},
23};
24use hotshot_types::traits::{
25 metrics::{Metrics, NoMetrics},
26 network::ConnectedNetwork,
27};
28use jf_merkle_tree_compat::MerkleTreeScheme;
29use process_metrics::ProcessMetrics;
30use tide_disco::{Api, App, Url, listener::RateLimitListener, method::ReadState};
31use vbs::version::StaticVersionType;
32
33use super::{
34 ApiState, StorageState,
35 data_source::{
36 CatchupDataSource, HotShotConfigDataSource, NodeStateDataSource, Provider,
37 PruningDataSource, SequencerDataSource, StateSignatureDataSource, SubmitDataSource,
38 provider,
39 },
40 endpoints, fs, light_client, sql,
41 state::NodeApiStateImpl,
42 update::ApiEventConsumer,
43};
44use crate::{
45 SequencerApiVersion,
46 api::{LightClientProvider, endpoints::RewardMerkleTreeVersion},
47 catchup::CatchupStorage,
48 context::{SequencerContext, TaskList},
49 options::PublicNodeConfig,
50 persistence,
51 request_response::data_source::Storage as RequestResponseStorage,
52 state::update_state_storage_loop,
53};
54
55#[derive(Clone, Debug)]
56pub struct Options {
57 pub http: Http,
58 pub query: Option<Query>,
59 pub submit: Option<Submit>,
60 pub status: Option<Status>,
61 pub catchup: Option<Catchup>,
62 pub config: Option<Config>,
63 pub hotshot_events: Option<HotshotEvents>,
64 pub explorer: Option<Explorer>,
65 pub light_client: Option<LightClient>,
66 pub storage_fs: Option<persistence::fs::Options>,
67 pub storage_sql: Option<persistence::sql::Options>,
68 pub public_node_config: Option<Box<PublicNodeConfig>>,
69}
70
71impl From<Http> for Options {
72 fn from(http: Http) -> Self {
73 Self {
74 http,
75 query: None,
76 submit: None,
77 status: None,
78 catchup: None,
79 config: None,
80 hotshot_events: None,
81 explorer: None,
82 light_client: None,
83 storage_fs: None,
84 storage_sql: None,
85 public_node_config: None,
86 }
87 }
88}
89
90impl Options {
91 pub fn with_port(port: u16) -> Self {
93 Http::with_port(port).into()
94 }
95
96 pub fn query_sql(mut self, query: Query, storage: persistence::sql::Options) -> Self {
98 self.query = Some(query);
99 self.storage_sql = Some(storage);
100 self
101 }
102
103 pub fn query_fs(mut self, query: Query, storage: persistence::fs::Options) -> Self {
105 self.query = Some(query);
106 self.storage_fs = Some(storage);
107 self
108 }
109
110 pub fn submit(mut self, opt: Submit) -> Self {
112 self.submit = Some(opt);
113 self
114 }
115
116 pub fn status(mut self, opt: Status) -> Self {
118 self.status = Some(opt);
119 self
120 }
121
122 pub fn catchup(mut self, opt: Catchup) -> Self {
124 self.catchup = Some(opt);
125 self
126 }
127
128 pub fn config(mut self, opt: Config) -> Self {
130 self.config = Some(opt);
131 self
132 }
133
134 pub fn public_node_config(mut self, c: PublicNodeConfig) -> Self {
138 self.public_node_config = Some(Box::new(c));
139 self
140 }
141
142 pub fn hotshot_events(mut self, opt: HotshotEvents) -> Self {
144 self.hotshot_events = Some(opt);
145 self
146 }
147
148 pub fn explorer(mut self, opt: Explorer) -> Self {
150 self.explorer = Some(opt);
151 self
152 }
153
154 pub fn light_client(mut self, opt: LightClient) -> Self {
156 self.light_client = Some(opt);
157 self
158 }
159
160 pub fn has_query_module(&self) -> bool {
162 self.query.is_some() && (self.storage_fs.is_some() || self.storage_sql.is_some())
163 }
164
165 pub async fn serve<N, P, F>(mut self, init_context: F) -> anyhow::Result<SequencerContext<N, P>>
171 where
172 N: ConnectedNetwork<PubKey>,
173 P: SequencerPersistence,
174 F: FnOnce(
175 Box<dyn Metrics>,
176 Box<dyn EventConsumer>,
177 Option<RequestResponseStorage>,
178 ) -> BoxFuture<'static, anyhow::Result<SequencerContext<N, P>>>,
179 {
180 let (send_ctx, recv_ctx) = oneshot::channel();
184 let state = ApiState::new(async move {
185 recv_ctx
186 .await
187 .expect("context initialized and sent over channel")
188 });
189 let mut tasks = TaskList::default();
190
191 #[allow(clippy::type_complexity)]
194 let (metrics, consumer, storage): (
195 Box<dyn Metrics>,
196 Box<dyn EventConsumer>,
197 Option<RequestResponseStorage>,
198 ) = if let Some(query_opt) = self.query.take() {
199 if let Some(opt) = self.storage_sql.take() {
200 self.init_with_query_module_sql(
201 query_opt,
202 opt,
203 state,
204 &mut tasks,
205 SequencerApiVersion::instance(),
206 )
207 .await?
208 } else if let Some(opt) = self.storage_fs.take() {
209 self.init_with_query_module_fs(
210 query_opt,
211 opt,
212 state,
213 &mut tasks,
214 SequencerApiVersion::instance(),
215 )
216 .await?
217 } else {
218 bail!("query module requested but not storage provided");
219 }
220 } else if self.status.is_some() {
221 let ds = MetricsDataSource::default();
225 let metrics = ds.populate_metrics();
226 telemetry::set_registry(Arc::new(ds.metrics().registry().clone()));
227 tasks.spawn("process_metrics", ProcessMetrics::new(ds.metrics()).run());
228 let mut app = App::<_, Error>::with_state(AppState::from(ExtensibleDataSource::new(
229 ds,
230 state.clone(),
231 )));
232
233 register_api("status", &mut app, move |ver| {
235 status::define_api(&Default::default(), SequencerApiVersion::instance(), ver)
236 .context("failed to define status api")
237 })?;
238
239 self.init_hotshot_modules(&mut app)?;
240
241 if self.hotshot_events.is_some() {
243 self.init_hotshot_events_module(&mut app)?;
244 }
245
246 tasks.spawn(
247 "API server",
248 self.listen(self.http.port, app, SequencerApiVersion::instance()),
249 );
250
251 if self.http.axum_port.is_some() {
254 tracing::warn!("Axum reward API not available in status-only mode");
255 }
256
257 if self.http.tonic_port.is_some() {
258 tracing::warn!("gRPC reward API not available in status-only mode");
259 }
260
261 (metrics, Box::new(NullEventConsumer), None)
262 } else {
263 let mut app = App::<_, Error>::with_state(AppState::from(state.clone()));
270
271 self.init_hotshot_modules(&mut app)?;
272
273 if self.hotshot_events.is_some() {
275 self.init_hotshot_events_module(&mut app)?;
276 }
277
278 tasks.spawn(
279 "API server",
280 self.listen(self.http.port, app, SequencerApiVersion::instance()),
281 );
282
283 (Box::new(NoMetrics), Box::new(NullEventConsumer), None)
284 };
285
286 let ctx = init_context(metrics, consumer, storage.clone()).await?;
287 send_ctx
288 .send(ctx.clone())
289 .ok()
290 .context("API server exited without receiving context")?;
291 Ok(ctx.with_task_list(tasks))
292 }
293
294 async fn init_app_modules<N, P, D>(
295 &self,
296 ds: D,
297 state: ApiState<N, P>,
298 bind_version: SequencerApiVersion,
299 ) -> anyhow::Result<(
300 Box<dyn Metrics>,
301 Arc<StorageState<N, P, D>>,
302 App<AppState<StorageState<N, P, D>>, Error>,
303 )>
304 where
305 N: ConnectedNetwork<PubKey>,
306 P: SequencerPersistence,
307 D: SequencerDataSource + CatchupStorage + PruningDataSource + Send + Sync + 'static,
308 {
309 let metrics = ds.populate_metrics();
310 telemetry::set_registry(Arc::new(ds.metrics().registry().clone()));
313 let ds = Arc::new(ExtensibleDataSource::new(ds, state.clone()));
314 let api_state: endpoints::AvailState<N, P, D> = ds.clone().into();
315 let mut app = App::<_, Error>::with_state(api_state);
316
317 register_api("status", &mut app, move |ver| {
319 status::define_api(&Default::default(), SequencerApiVersion::instance(), ver)
320 .context("failed to define status api")
321 })?;
322
323 register_api("availability", &mut app, move |ver| {
330 endpoints::availability(ver).context("failed to define availability api")
331 })?;
332
333 register_api("node", &mut app, move |ver| {
334 endpoints::node(ver).context("failed to define node api")
335 })?;
336
337 register_api("token", &mut app, move |ver| {
338 endpoints::token(ver).context("failed to define token api")
339 })?;
340
341 if self.submit.is_some() {
343 register_api("submit", &mut app, move |ver| {
344 endpoints::submit::<_, _, _, SequencerApiVersion>(ver)
345 .context("failed to define submit api")
346 })?;
347 }
348
349 tracing::info!("initializing catchup API");
350
351 register_api("catchup", &mut app, move |ver| {
352 endpoints::catchup(bind_version, ver).context("failed to define catchup api")
353 })?;
354
355 register_api("state-signature", &mut app, move |ver| {
356 endpoints::state_signature(bind_version, ver)
357 .context("failed to define state signature api")
358 })?;
359
360 if self.config.is_some() {
361 let node_cfg = self.public_node_config.as_deref().cloned();
362 register_api("config", &mut app, move |ver| {
363 endpoints::config(bind_version, ver, node_cfg.clone())
364 .context("failed to define config api")
365 })?;
366 }
367 Ok((metrics, ds, app))
368 }
369
370 async fn init_with_query_module_fs<N, P>(
371 &self,
372 query_opt: Query,
373 mod_opt: persistence::fs::Options,
374 state: ApiState<N, P>,
375 tasks: &mut TaskList,
376 bind_version: SequencerApiVersion,
377 ) -> anyhow::Result<(
378 Box<dyn Metrics>,
379 Box<dyn EventConsumer>,
380 Option<RequestResponseStorage>,
381 )>
382 where
383 N: ConnectedNetwork<PubKey>,
384 P: SequencerPersistence,
385 {
386 let ds = <fs::DataSource as SequencerDataSource>::create(
387 mod_opt,
388 provider(
389 query_opt.peers,
390 &state,
391 query_opt.light_client,
392 query_opt.light_client_db,
393 )
394 .await?,
395 false,
396 )
397 .await?;
398
399 let inner_storage = ds.inner();
401
402 tasks.spawn("process_metrics", ProcessMetrics::new(ds.metrics()).run());
403
404 let (metrics, ds, mut app) = self
405 .init_app_modules(ds, state.clone(), bind_version)
406 .await?;
407
408 if self.hotshot_events.is_some() {
410 self.init_hotshot_events_module(&mut app)?;
411 }
412
413 tasks.spawn("API server", self.listen(self.http.port, app, bind_version));
414
415 if self.http.axum_port.is_some() {
418 tracing::warn!("Axum reward API not available with filesystem storage");
419 }
420
421 if self.http.tonic_port.is_some() {
422 tracing::warn!("gRPC reward API not available with filesystem storage");
423 }
424
425 Ok((
426 metrics,
427 Box::new(ApiEventConsumer::from(ds)),
428 Some(RequestResponseStorage::Fs(inner_storage)),
429 ))
430 }
431
432 async fn init_with_query_module_sql<N, P>(
433 self,
434 query_opt: Query,
435 mod_opt: persistence::sql::Options,
436 state: ApiState<N, P>,
437 tasks: &mut TaskList,
438 bind_version: SequencerApiVersion,
439 ) -> anyhow::Result<(
440 Box<dyn Metrics>,
441 Box<dyn EventConsumer>,
442 Option<RequestResponseStorage>,
443 )>
444 where
445 N: ConnectedNetwork<PubKey>,
446 P: SequencerPersistence,
447 {
448 let mut provider = Provider::default();
449
450 let db_provider = mod_opt.clone().create().await?;
453 provider = provider
454 .with_block_provider(db_provider.clone())
455 .with_vid_common_provider(db_provider);
456 provider = provider.with_provider(
458 LightClientProvider::new(
459 query_opt.peers,
460 state.clone(),
461 query_opt.light_client,
462 query_opt.light_client_db,
463 )
464 .await?,
465 );
466
467 let ds = sql::DataSource::create(mod_opt.clone(), provider, false).await?;
468 let inner_storage = ds.inner();
469 tasks.spawn("process_metrics", ProcessMetrics::new(ds.metrics()).run());
470 let (metrics, ds, mut app) = self
471 .init_app_modules(ds, state.clone(), bind_version)
472 .await?;
473
474 if self.explorer.is_some() {
475 register_api("explorer", &mut app, move |ver| {
476 endpoints::explorer(ver).context("failed to define explorer api")
477 })?;
478 }
479
480 register_api("database", &mut app, move |ver| {
482 endpoints::database::<_, SequencerApiVersion>(ver)
483 .context("failed to define database api")
484 })?;
485
486 register_api("block-state", &mut app, move |ver| {
489 endpoints::merklized_state::<N, P, _, BlockMerkleTree, 3>(ver)
490 .context("failed to define block-state api")
491 })?;
492
493 register_api("fee-state", &mut app, move |ver| {
496 endpoints::fee::<_, SequencerApiVersion>(ver).context("failed to define fee-state api")
497 })?;
498
499 register_api("reward-state", &mut app, move |ver| {
500 endpoints::reward::<
501 _,
502 SequencerApiVersion,
503 RewardMerkleTreeV1,
504 { RewardMerkleTreeV1::ARITY },
505 >(ver, RewardMerkleTreeVersion::V1)
506 .context("failed to define reward-state api")
507 })?;
508
509 register_api("reward-state-v2", &mut app, move |ver| {
511 endpoints::reward::<
512 _,
513 SequencerApiVersion,
514 RewardMerkleTreeV2,
515 { RewardMerkleTreeV2::ARITY },
516 >(ver, RewardMerkleTreeVersion::V2)
517 .context("failed to define reward-state api")
518 })?;
519
520 let get_node_state = {
521 let state = state.clone();
522 async move { state.node_state().await.clone() }
523 };
524 tasks.spawn(
525 "merklized state storage update loop",
526 update_state_storage_loop(ds.clone(), get_node_state),
527 );
528
529 if self.hotshot_events.is_some() {
531 self.init_hotshot_events_module(&mut app)?;
532 }
533
534 if self.light_client.is_some() {
536 register_api("light-client", &mut app, move |ver| {
537 light_client::define_api::<_, SequencerApiVersion>(Default::default(), ver)
538 .context("failed to define light client api")
539 })?;
540 }
541
542 tasks.spawn(
543 "API server",
544 self.listen(self.http.port, app, SequencerApiVersion::instance()),
545 );
546
547 if let Some(axum_port) = self.http.axum_port {
549 let ds_for_axum = ds.clone();
550 let env_vars = endpoints::get_public_env_vars().unwrap_or_default();
551 let node_cfg = self.public_node_config.as_deref().cloned();
552 tasks.spawn("Axum API server", async move {
553 let state = NodeApiStateImpl::new(ds_for_axum)
554 .with_env_vars(env_vars)
555 .with_public_node_config(node_cfg);
556 if let Err(e) = espresso_api::serve_axum(axum_port, state).await {
557 tracing::error!("Axum server error: {}", e);
558 }
559 });
560 }
561
562 if let Some(tonic_port) = self.http.tonic_port {
563 let ds_for_tonic = ds.clone();
564 tasks.spawn("Tonic gRPC server", async move {
565 let state = NodeApiStateImpl::new(ds_for_tonic);
566 if let Err(e) = espresso_api::serve_tonic(tonic_port, state).await {
567 tracing::error!("Tonic gRPC server error: {}", e);
568 }
569 });
570 }
571
572 Ok((
573 metrics,
574 Box::new(ApiEventConsumer::from(ds)),
575 Some(RequestResponseStorage::Sql(inner_storage)),
576 ))
577 }
578
579 fn init_hotshot_modules<N, P, S>(&self, app: &mut App<S, Error>) -> anyhow::Result<()>
585 where
586 S: 'static + Send + Sync + ReadState,
587 P: SequencerPersistence,
588 S::State: Send
589 + Sync
590 + SubmitDataSource<N, P>
591 + StateSignatureDataSource<N>
592 + NodeStateDataSource
593 + CatchupDataSource
594 + HotShotConfigDataSource,
595 N: ConnectedNetwork<PubKey>,
596 {
597 let bind_version = SequencerApiVersion::instance();
598 if self.submit.is_some() {
600 register_api("submit", app, move |ver| {
601 endpoints::submit::<_, _, _, SequencerApiVersion>(ver)
602 .context("failed to define submit api")
603 })?;
604 }
605
606 if self.catchup.is_some() {
608 tracing::info!("initializing state API");
609
610 register_api("catchup", app, move |ver| {
611 endpoints::catchup(bind_version, ver).context("failed to define catchup api")
612 })?;
613 }
614
615 register_api("state-signature", app, move |ver| {
616 endpoints::state_signature(bind_version, ver)
617 .context("failed to define state signature api")
618 })?;
619
620 if self.config.is_some() {
621 let node_cfg = self.public_node_config.as_deref().cloned();
622 register_api("config", app, move |ver| {
623 endpoints::config(bind_version, ver, node_cfg.clone())
624 .context("failed to define config api")
625 })?;
626 }
627
628 Ok(())
629 }
630
631 fn init_hotshot_events_module<S>(&self, app: &mut App<S, Error>) -> anyhow::Result<()>
636 where
637 S: 'static + Send + Sync + ReadState,
638 S::State: Send + Sync + hotshot_events_service::events_source::EventsSource<SeqTypes>,
639 {
640 tracing::info!("Initializing HotShot events API at /hotshot-events");
641 register_api("hotshot-events", app, move |ver| {
642 hotshot_events_service::events::define_api::<_, _, SequencerApiVersion>(
643 &hotshot_events_service::events::Options::default(),
644 ver,
645 )
646 .with_context(|| "failed to define the HotShot events API")
647 })?;
648
649 Ok(())
650 }
651
652 fn listen<S, E, ApiVer>(
653 &self,
654 port: u16,
655 app: App<S, E>,
656 bind_version: ApiVer,
657 ) -> impl Future<Output = anyhow::Result<()>> + use<S, E, ApiVer>
658 where
659 S: Send + Sync + 'static,
660 E: Send + Sync + tide_disco::Error,
661 ApiVer: StaticVersionType + 'static,
662 {
663 let max_connections = self.http.max_connections;
664
665 async move {
666 if let Some(limit) = max_connections {
667 app.serve(RateLimitListener::with_port(port, limit), bind_version)
668 .await?;
669 } else {
670 app.serve(format!("0.0.0.0:{port}"), bind_version).await?;
671 }
672 Ok(())
673 }
674 }
675}
676
677#[derive(Parser, Clone, Copy, Debug)]
682pub struct Http {
683 #[clap(long, env = "ESPRESSO_NODE_API_PORT", default_value = "8080")]
685 pub port: u16,
686
687 #[clap(long, env = "ESPRESSO_NODE_API_MAX_CONNECTIONS")]
693 pub max_connections: Option<usize>,
694
695 #[clap(long, env = "ESPRESSO_NODE_AXUM_PORT")]
697 pub axum_port: Option<u16>,
698
699 #[clap(long, env = "ESPRESSO_NODE_TONIC_PORT")]
701 pub tonic_port: Option<u16>,
702}
703
704impl Http {
705 pub fn with_port(port: u16) -> Self {
707 Self {
708 port,
709 max_connections: None,
710 axum_port: None,
711 tonic_port: None,
712 }
713 }
714}
715
716#[derive(Parser, Clone, Copy, Debug, Default)]
718pub struct Submit;
719
720#[derive(Parser, Clone, Copy, Debug, Default)]
722pub struct Status;
723
724#[derive(Parser, Clone, Copy, Debug, Default)]
726pub struct Catchup;
727
728#[derive(Parser, Clone, Copy, Debug, Default)]
730pub struct Config;
731
732#[derive(Parser, Clone, Debug, Default)]
734pub struct Query {
735 #[clap(long, env = "ESPRESSO_NODE_API_PEERS", value_delimiter = ',')]
737 pub peers: Vec<Url>,
738
739 #[clap(flatten)]
741 pub light_client: LightClientOptions,
742
743 #[clap(flatten)]
745 pub light_client_db: LightClientSqliteOptions,
746}
747
748#[cfg(test)]
749impl Query {
750 pub fn test() -> Self {
751 Self::default()
752 }
753}
754
755#[derive(Parser, Clone, Copy, Debug, Default)]
757pub struct State;
758
759#[derive(Parser, Clone, Copy, Debug, Default)]
761pub struct HotshotEvents;
762
763#[derive(Parser, Clone, Copy, Debug, Default)]
765pub struct Explorer;
766
767#[derive(Parser, Clone, Copy, Debug, Default)]
769pub struct LightClient;
770
771fn register_api<E, S, F, ModuleError, ModuleVersion>(
773 path: &'static str,
774 app: &mut App<S, E>,
775 f: F,
776) -> anyhow::Result<()>
777where
778 S: 'static + Send + Sync,
779 E: Send + Sync + 'static + tide_disco::Error + From<ModuleError>,
780 ModuleError: Send + Sync + 'static,
781 ModuleVersion: StaticVersionType + 'static,
782 F: Fn(semver::Version) -> anyhow::Result<Api<S, ModuleError, ModuleVersion>>,
783{
784 let v0 = "0.0.1".parse().unwrap();
785 let v1 = "1.1.0".parse().unwrap();
786 let result1 = f(v0)?;
787 let result2 = f(v1)?;
788
789 app.register_module(path, result1)?;
790 app.register_module(path, result2)?;
791
792 Ok(())
793}