Public declaration syntax from forge-rs/crates/forge-web/src/config.rs
Original source SHA-256: 3fd9f95e5a6d2752b8fbd8bd12c2b52bcd34b9c44f8214c7347d30924e3c6b64
Function bodies and constant values are omitted. This is not the complete implementation.

Source line 35
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct WebSubstrateConfig {
/// Maximum response body size in bytes.

///

/// Responses exceeding this limit will return

/// [`WebError::ContentTooLarge`](crate::error::WebError::ContentTooLarge).

///

/// Default: 10,485,760 (10 MB).

pub max_fetch_size_bytes: u64,
/// Maximum number of HTTP redirects to follow per request.

///

/// Exceeding this limit returns

/// [`WebError::RedirectLimitExceeded`](crate::error::WebError::RedirectLimitExceeded).

///

/// Default: 5.

pub max_redirects: u32,
/// Request timeout in milliseconds.

///

/// If the server does not respond within this duration, the request

/// returns [`WebError::Timeout`](crate::error::WebError::Timeout).

///

/// Default: 30,000 (30 seconds).

pub request_timeout_ms: u64,
/// IP address ranges to block for SSRF protection.

///

/// Each entry is a CIDR notation string (e.g., `10.0.0.0/8`). Requests

/// whose resolved IP falls within any of these ranges will return

/// [`WebError::SsrfBlocked`](crate::error::WebError::SsrfBlocked).

///

/// Default: private (RFC 1918), loopback, link-local, and IPv6 private ranges.

pub blocked_ip_ranges: Vec<String>,
/// The `User-Agent` header sent with all outgoing requests.

///

/// Default: `"forge-web/0.1 (+https://github.com/l1fe-labs/forge)"`.

pub user_agent: String,
/// Whether to respect `robots.txt` directives when crawling.

///

/// When `true`, the substrate will fetch and honor `robots.txt` rules

/// before crawling a site. Disabling this is only appropriate for

/// authorized internal crawling.

///

/// Default: `true`.

pub respect_robots_txt: bool
}