Current section

Files

Jump to
rbt_weave crates weave-content src cache.rs
Raw

crates/weave-content/src/cache.rs

use std::collections::HashMap;
use std::path::Path;
use std::sync::RwLock;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use crate::verifier::CheckStatus;
/// Cache TTL for `ok` results: 90 days. Working URLs rarely break.
const OK_TTL_SECS: u64 = 90 * 24 * 60 * 60;
/// Cache TTL for `rate_limited` results: 7 days. URL is valid, just throttled.
const RATE_LIMITED_TTL_SECS: u64 = 7 * 24 * 60 * 60;
/// Cache TTL for `warn` results: 1 day. Transient issues worth rechecking soon.
const WARN_TTL_SECS: u64 = 24 * 60 * 60;
/// Maximum cache entries to prevent unbounded growth.
const MAX_CACHE_ENTRIES: usize = 500_000;
/// A single cached URL check result.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheEntry {
pub status: String,
pub checked_at: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub http_status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub detail: Option<String>,
}
/// Return the TTL for a given status string, or `None` if the status
/// should never be cached (always re-checked).
fn ttl_for_status(status: &str) -> Option<u64> {
match status {
"ok" => Some(OK_TTL_SECS),
"rate_limited" => Some(RATE_LIMITED_TTL_SECS),
"warn" => Some(WARN_TTL_SECS),
// error and suspicious are always re-checked
_ => None,
}
}
/// Verify cache backed by a JSON file.
///
/// Thread-safe via interior `RwLock`: concurrent readers, exclusive writer.
/// Load once at startup, call `get` / `put` from any thread, save once at end.
#[derive(Debug)]
pub struct VerifyCache {
path: String,
entries: RwLock<HashMap<String, CacheEntry>>,
}
impl VerifyCache {
/// Create an empty in-memory cache (no file backing).
pub fn empty() -> Self {
Self {
path: String::new(),
entries: RwLock::new(HashMap::new()),
}
}
/// Load cache from file, or create empty if file doesn't exist or is invalid.
///
/// # Errors
///
/// Returns an error if the file exists but cannot be read (permissions, etc.).
/// Invalid JSON is treated as empty cache (not an error).
pub fn load(path: &str) -> Result<Self, String> {
let entries = if Path::new(path).exists() {
let content = std::fs::read_to_string(path)
.map_err(|e| format!("failed to read cache file {path}: {e}"))?;
let map: HashMap<String, CacheEntry> =
serde_json::from_str(&content).unwrap_or_default();
eprintln!("cache: loaded {} entries from {path}", map.len());
map
} else {
HashMap::new()
};
Ok(Self {
path: path.to_string(),
entries: RwLock::new(entries),
})
}
/// Check if a URL has a valid (non-expired) cache entry.
/// Returns `None` if the URL should be re-checked.
///
/// Acquires a read lock — multiple threads can call this concurrently.
pub fn get(&self, url: &str) -> Option<CacheEntry> {
let entries = self.entries.read().ok()?;
let entry = entries.get(url)?;
let now = now_secs();
let ttl = ttl_for_status(&entry.status)?;
if now.saturating_sub(entry.checked_at) > ttl {
return None;
}
Some(entry.clone())
}
/// Record a check result in the cache.
///
/// Acquires a write lock — blocks concurrent readers/writers briefly.
pub fn put(&self, url: &str, status: CheckStatus, detail: Option<&str>) {
let Ok(mut entries) = self.entries.write() else {
return;
};
// Enforce boundary
if entries.len() >= MAX_CACHE_ENTRIES && !entries.contains_key(url) {
return;
}
entries.insert(
url.to_string(),
CacheEntry {
status: status.to_string(),
checked_at: now_secs(),
http_status: extract_http_status(detail),
detail: detail.map(String::from),
},
);
}
/// Number of entries currently in the cache.
pub fn len(&self) -> usize {
self.entries.read().map_or(0, |e| e.len())
}
/// Whether the cache is empty.
pub fn is_empty(&self) -> bool {
self.entries.read().map_or(true, |e| e.is_empty())
}
/// Save cache to file atomically (write tmp + rename).
/// Prunes expired entries before writing.
///
/// # Errors
///
/// Returns an error if the file cannot be written.
pub fn save(&self) -> Result<(), String> {
if self.path.is_empty() {
return Ok(());
}
let entries = self
.entries
.read()
.map_err(|e| format!("cache lock poisoned: {e}"))?;
// Prune expired entries before saving
let now = now_secs();
let pruned: HashMap<&String, &CacheEntry> = entries
.iter()
.filter(|(_, e)| {
ttl_for_status(&e.status).is_some_and(|ttl| now.saturating_sub(e.checked_at) <= ttl)
})
.collect();
let before = entries.len();
let after = pruned.len();
let json = serde_json::to_string_pretty(&pruned)
.map_err(|e| format!("failed to serialize cache: {e}"))?;
// Atomic write: tmp file + rename
let tmp_path = format!("{}.tmp", self.path);
std::fs::write(&tmp_path, json)
.map_err(|e| format!("failed to write cache tmp file {tmp_path}: {e}"))?;
std::fs::rename(&tmp_path, &self.path)
.map_err(|e| format!("failed to rename cache file: {e}"))?;
if before == after {
eprintln!("cache: saved {after} entries");
} else {
eprintln!(
"cache: saved {after} entries (pruned {} expired)",
before - after
);
}
Ok(())
}
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| d.as_secs())
}
/// Try to extract HTTP status code from detail string like "HTTP 404 Not Found".
fn extract_http_status(detail: Option<&str>) -> Option<u16> {
let detail = detail?;
let rest = detail.strip_prefix("HTTP ")?;
let code_str: String = rest.chars().take_while(char::is_ascii_digit).collect();
code_str.parse().ok()
}
#[cfg(test)]
mod tests {
use super::*;
fn empty_cache() -> VerifyCache {
VerifyCache::empty()
}
#[test]
fn cache_put_and_get_ok() {
let cache = empty_cache();
cache.put("https://example.com", CheckStatus::Ok, None);
assert!(cache.get("https://example.com").is_some());
}
#[test]
fn cache_error_always_rechecked() {
let cache = empty_cache();
cache.put("https://example.com", CheckStatus::Error, Some("HTTP 404"));
assert!(cache.get("https://example.com").is_none());
}
#[test]
fn cache_suspicious_always_rechecked() {
let cache = empty_cache();
cache.put(
"https://example.com",
CheckStatus::Suspicious,
Some("HTTP 403 Forbidden"),
);
assert!(cache.get("https://example.com").is_none());
}
#[test]
fn cache_ok_expired_after_90_days() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://old.com".into(),
CacheEntry {
status: "ok".into(),
checked_at: now_secs() - OK_TTL_SECS - 1,
http_status: None,
detail: None,
},
);
}
assert!(cache.get("https://old.com").is_none());
}
#[test]
fn cache_ok_valid_within_90_days() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://recent.com".into(),
CacheEntry {
status: "ok".into(),
checked_at: now_secs() - OK_TTL_SECS + 3600,
http_status: None,
detail: None,
},
);
}
assert!(cache.get("https://recent.com").is_some());
}
#[test]
fn cache_rate_limited_expires_after_7_days() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://throttled.com".into(),
CacheEntry {
status: "rate_limited".into(),
checked_at: now_secs() - RATE_LIMITED_TTL_SECS - 1,
http_status: Some(429),
detail: Some("HTTP 429".into()),
},
);
}
assert!(cache.get("https://throttled.com").is_none());
}
#[test]
fn cache_rate_limited_valid_within_7_days() {
let cache = empty_cache();
cache.put(
"https://example.com",
CheckStatus::RateLimited,
Some("HTTP 429 Too Many Requests"),
);
assert!(cache.get("https://example.com").is_some());
}
#[test]
fn cache_warn_expires_after_1_day() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://flaky.com".into(),
CacheEntry {
status: "warn".into(),
checked_at: now_secs() - WARN_TTL_SECS - 1,
http_status: None,
detail: Some("timeout".into()),
},
);
}
assert!(cache.get("https://flaky.com").is_none());
}
#[test]
fn cache_warn_valid_within_1_day() {
let cache = empty_cache();
cache.put("https://example.com", CheckStatus::Warn, Some("timeout"));
assert!(cache.get("https://example.com").is_some());
}
#[test]
fn cache_unknown_url_returns_none() {
let cache = empty_cache();
assert!(cache.get("https://unknown.com").is_none());
}
#[test]
fn extract_http_status_parses() {
assert_eq!(extract_http_status(Some("HTTP 404 Not Found")), Some(404));
assert_eq!(extract_http_status(Some("HTTP 200")), Some(200));
assert_eq!(extract_http_status(Some("timeout")), None);
assert_eq!(extract_http_status(None), None);
}
#[test]
fn cache_boundary_enforced() {
let cache = empty_cache();
for i in 0..MAX_CACHE_ENTRIES {
cache.put(&format!("https://example.com/{i}"), CheckStatus::Ok, None);
}
// One more should be rejected
cache.put("https://overflow.com", CheckStatus::Ok, None);
assert!(cache.get("https://overflow.com").is_none());
assert_eq!(cache.len(), MAX_CACHE_ENTRIES);
}
#[test]
fn cache_update_existing_within_boundary() {
let cache = empty_cache();
for i in 0..MAX_CACHE_ENTRIES {
cache.put(&format!("https://example.com/{i}"), CheckStatus::Ok, None);
}
// Updating existing should work even at capacity
cache.put(
"https://example.com/0",
CheckStatus::Error,
Some("HTTP 500"),
);
assert_eq!(cache.len(), MAX_CACHE_ENTRIES);
}
#[test]
fn cache_concurrent_read_write() {
use std::sync::Arc;
use std::thread;
let cache = Arc::new(empty_cache());
let mut handles = Vec::new();
// Spawn writers
for i in 0..10 {
let c = Arc::clone(&cache);
handles.push(thread::spawn(move || {
for j in 0..100 {
c.put(
&format!("https://example.com/{i}/{j}"),
CheckStatus::Ok,
None,
);
}
}));
}
// Spawn readers
for _ in 0..10 {
let c = Arc::clone(&cache);
handles.push(thread::spawn(move || {
for _ in 0..100 {
let _ = c.get("https://example.com/0/0");
}
}));
}
for h in handles {
h.join().expect("thread panicked");
}
// All 1000 entries should be present
assert_eq!(cache.len(), 1000);
}
#[test]
fn ttl_tiers_are_correct() {
assert_eq!(ttl_for_status("ok"), Some(90 * 24 * 60 * 60));
assert_eq!(ttl_for_status("rate_limited"), Some(7 * 24 * 60 * 60));
assert_eq!(ttl_for_status("warn"), Some(24 * 60 * 60));
assert_eq!(ttl_for_status("error"), None);
assert_eq!(ttl_for_status("suspicious"), None);
}
}