stygian_graph/domain/
policy.rs

1//! Single robots policy value type — closes guide failure mode #9.
2//!
3//! Recon and production paths must agree on the policy applied to every
4//! URL they touch. The guide calls this out as the ninth failure mode of
5//! agent-built scrapers: _"Two robots.txt policies, never reconciled.
6//! Exploration ignores it; the deliverable obeys it."_
7//!
8//! This module exposes a single [`RobotsPolicy`](crate::domain::policy::RobotsPolicy) enum plus its decision
9//! type [`RobotsDecision`](crate::domain::policy::RobotsDecision). Both recon and production consume the same
10//! value through [`RobotsPolicyGuard`](crate::ports::robots_policy::RobotsPolicyGuard)
11//! (in `ports/robots_policy.rs`), so
12//! the policy choice can be asserted at pipeline-build time and again at
13//! execute time.
14
15use std::fmt;
16use std::str::FromStr;
17
18use serde::{Deserialize, Serialize};
19
20use super::error::StygianError;
21
22/// How a pipeline treats `robots.txt` across recon and production.
23///
24/// The default is [`RobotsPolicy::Obey`] — honouring robots.txt is the
25/// only choice that's safe to ship without a written authorisation.
26///
27/// # Example
28///
29/// ```
30/// use stygian_graph::domain::policy::RobotsPolicy;
31/// use std::str::FromStr;
32///
33/// let p: RobotsPolicy = "obey".parse().unwrap();
34/// assert_eq!(p, RobotsPolicy::Obey);
35/// assert_eq!(p.to_string(), "obey");
36/// ```
37#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Hash, Serialize, Deserialize)]
38#[serde(rename_all = "snake_case")]
39pub enum RobotsPolicy {
40    /// Honour `robots.txt`. Refuse to fetch URLs the guard forbids.
41    ///
42    /// The canonical choice — anything else requires explicit operator
43    /// consent and audit.
44    #[default]
45    Obey,
46
47    /// Ignore `robots.txt` but record every ignored URL with the
48    /// reason. Surfaced through the pipeline run report.
49    IgnoreWithAudit,
50
51    /// Ignore `robots.txt` without record. Escape hatch only.
52    IgnoreSilently,
53}
54
55impl fmt::Display for RobotsPolicy {
56    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
57        let s = match self {
58            Self::Obey => "obey",
59            Self::IgnoreWithAudit => "ignore_with_audit",
60            Self::IgnoreSilently => "ignore_silently",
61        };
62        f.write_str(s)
63    }
64}
65
66impl FromStr for RobotsPolicy {
67    type Err = StygianError;
68
69    fn from_str(s: &str) -> Result<Self, Self::Err> {
70        match s.trim() {
71            "obey" | "Obey" | "OBEY" => Ok(Self::Obey),
72            "ignore_with_audit" | "IgnoreWithAudit" => Ok(Self::IgnoreWithAudit),
73            "ignore_silently" | "IgnoreSilently" => Ok(Self::IgnoreSilently),
74            other => Err(StygianError::Config(
75                super::error::ConfigError::InvalidValue {
76                    key: "robots_policy".to_string(),
77                    reason: format!(
78                        "unknown variant '{other}' (expected one of: \
79                         obey, ignore_with_audit, ignore_silently)"
80                    ),
81                },
82            )),
83        }
84    }
85}
86
87/// The verdict returned by a [`RobotsPolicyGuard`](crate::ports::robots_policy::RobotsPolicyGuard)
88/// for a single URL.
89#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
90#[serde(rename_all = "snake_case")]
91pub enum RobotsDecision {
92    /// The guard has no opinion — caller should treat this as "no signal".
93    ///
94    /// A real guard returns this for URLs outside any rule it tracks
95    /// (e.g. a non-HTTP scheme, or a host the guard has no data for).
96    Unknown,
97
98    /// The URL is permitted under the current policy.
99    Allow {
100        /// Human-readable reason — usually the matched rule, or
101        /// "no rule matched" when the guard has no robots.txt data.
102        reason: String,
103    },
104
105    /// The URL is forbidden under the current policy.
106    Forbid {
107        /// Human-readable reason — usually the matched `Disallow` rule.
108        reason: String,
109    },
110}
111
112impl RobotsDecision {
113    /// `true` when the guard is explicitly permitting the URL.
114    #[must_use]
115    pub const fn is_allowed(&self) -> bool {
116        matches!(self, Self::Allow { .. })
117    }
118
119    /// `true` when the guard is explicitly forbidding the URL.
120    #[must_use]
121    pub const fn is_forbidden(&self) -> bool {
122        matches!(self, Self::Forbid { .. })
123    }
124
125    /// `true` when the guard has no opinion either way.
126    #[must_use]
127    pub const fn is_unknown(&self) -> bool {
128        matches!(self, Self::Unknown)
129    }
130}
131
132#[cfg(test)]
133#[allow(
134    clippy::unwrap_used,
135    clippy::expect_used,
136    clippy::panic,
137    clippy::indexing_slicing
138)]
139mod tests {
140    use super::*;
141
142    #[test]
143    fn default_is_obey() {
144        assert_eq!(RobotsPolicy::default(), RobotsPolicy::Obey);
145    }
146
147    #[test]
148    fn round_trip_display_fromstr() {
149        for policy in [
150            RobotsPolicy::Obey,
151            RobotsPolicy::IgnoreWithAudit,
152            RobotsPolicy::IgnoreSilently,
153        ] {
154            assert_eq!(RobotsPolicy::from_str(&policy.to_string()).unwrap(), policy);
155        }
156    }
157
158    #[test]
159    fn fromstr_accepts_camel_case_aliases() {
160        assert_eq!(
161            RobotsPolicy::from_str("IgnoreWithAudit").unwrap(),
162            RobotsPolicy::IgnoreWithAudit
163        );
164        assert_eq!(
165            RobotsPolicy::from_str("IgnoreSilently").unwrap(),
166            RobotsPolicy::IgnoreSilently
167        );
168    }
169
170    #[test]
171    fn fromstr_rejects_unknown_variant() {
172        let err = RobotsPolicy::from_str("always_obey").unwrap_err();
173        let msg = format!("{err}");
174        assert!(
175            msg.contains("unknown variant"),
176            "msg should explain the unknown variant: {msg}"
177        );
178        assert!(
179            msg.contains("always_obey"),
180            "msg should echo the bad input: {msg}"
181        );
182    }
183
184    #[test]
185    fn serialize_round_trip_json() {
186        let json = serde_json::to_string(&RobotsPolicy::IgnoreWithAudit).unwrap();
187        assert_eq!(json, "\"ignore_with_audit\"");
188        let back: RobotsPolicy = serde_json::from_str(&json).unwrap();
189        assert_eq!(back, RobotsPolicy::IgnoreWithAudit);
190    }
191
192    #[test]
193    fn decision_helpers_classify_correctly() {
194        let allow = RobotsDecision::Allow {
195            reason: "no rule matched".to_string(),
196        };
197        assert!(allow.is_allowed());
198        assert!(!allow.is_forbidden());
199        assert!(!allow.is_unknown());
200
201        let forbid = RobotsDecision::Forbid {
202            reason: "Disallow: /private".to_string(),
203        };
204        assert!(forbid.is_forbidden());
205        assert!(!forbid.is_allowed());
206
207        assert!(RobotsDecision::Unknown.is_unknown());
208    }
209}
stygian_graph/domain/policy.rs

stygian_graph/domain/
policy.rs
