Packages
rbt_weave
0.2.65
0.2.67
0.2.66
0.2.65
0.2.64
0.2.63
0.2.62
0.2.61
0.2.60
0.2.59
0.2.58
0.2.57
0.2.56
0.2.55
0.2.54
0.2.53
0.2.52
0.2.51
0.2.50
0.2.49
0.2.48
0.2.47
0.2.46
0.2.45
0.2.44
0.2.43
0.2.42
0.2.41
0.2.40
0.2.39
0.2.38
0.2.37
0.2.36
0.2.35
0.2.34
0.2.33
0.2.32
0.2.31
0.2.30
0.2.29
0.2.28
0.2.27
0.2.26
0.2.25
0.2.24
0.2.23
0.2.22
0.2.21
0.2.20
0.2.19
0.2.18
0.2.17
0.2.16
0.2.15
0.2.14
0.2.13
0.2.12
0.2.11
0.2.10
0.2.9
0.2.8
0.2.7
0.2.6
0.2.5
0.2.4
0.2.3
0.2.2
0.2.1
0.2.0
0.1.0
Graph conflict detection and image processing NIFs for OSINT knowledge graphs, powered by Rust.
Current section
Files
Jump to
Current section
Files
crates/weave-content/src/cache.rs
use std::collections::HashMap;
use std::path::Path;
use std::sync::RwLock;
use std::time::{SystemTime, UNIX_EPOCH};
use serde::{Deserialize, Serialize};
use crate::verifier::CheckStatus;
/// Cache TTL for `ok` results: 90 days. Working URLs rarely break.
const OK_TTL_SECS: u64 = 90 * 24 * 60 * 60;
/// Cache TTL for `rate_limited` results: 7 days. URL is valid, just throttled.
const RATE_LIMITED_TTL_SECS: u64 = 7 * 24 * 60 * 60;
/// Cache TTL for `warn` results: 1 day. Transient issues worth rechecking soon.
const WARN_TTL_SECS: u64 = 24 * 60 * 60;
/// Maximum cache entries to prevent unbounded growth.
const MAX_CACHE_ENTRIES: usize = 500_000;
/// A single cached URL check result.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct CacheEntry {
pub status: String,
pub checked_at: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub http_status: Option<u16>,
#[serde(skip_serializing_if = "Option::is_none")]
pub detail: Option<String>,
}
/// Return the TTL for a given status string, or `None` if the status
/// should never be cached (always re-checked).
fn ttl_for_status(status: &str) -> Option<u64> {
match status {
"ok" => Some(OK_TTL_SECS),
"rate_limited" => Some(RATE_LIMITED_TTL_SECS),
"warn" => Some(WARN_TTL_SECS),
// error and suspicious are always re-checked
_ => None,
}
}
/// Verify cache backed by a JSON file.
///
/// Thread-safe via interior `RwLock`: concurrent readers, exclusive writer.
/// Load once at startup, call `get` / `put` from any thread, save once at end.
#[derive(Debug)]
pub struct VerifyCache {
path: String,
entries: RwLock<HashMap<String, CacheEntry>>,
}
impl VerifyCache {
/// Create an empty in-memory cache (no file backing).
pub fn empty() -> Self {
Self {
path: String::new(),
entries: RwLock::new(HashMap::new()),
}
}
/// Load cache from file, or create empty if file doesn't exist or is invalid.
///
/// # Errors
///
/// Returns an error if the file exists but cannot be read (permissions, etc.).
/// Invalid JSON is treated as empty cache (not an error).
pub fn load(path: &str) -> Result<Self, String> {
let entries = if Path::new(path).exists() {
let content = std::fs::read_to_string(path)
.map_err(|e| format!("failed to read cache file {path}: {e}"))?;
let map: HashMap<String, CacheEntry> =
serde_json::from_str(&content).unwrap_or_default();
eprintln!("cache: loaded {} entries from {path}", map.len());
map
} else {
HashMap::new()
};
Ok(Self {
path: path.to_string(),
entries: RwLock::new(entries),
})
}
/// Check if a URL has a valid (non-expired) cache entry.
/// Returns `None` if the URL should be re-checked.
///
/// Acquires a read lock — multiple threads can call this concurrently.
pub fn get(&self, url: &str) -> Option<CacheEntry> {
let entries = self.entries.read().ok()?;
let entry = entries.get(url)?;
let now = now_secs();
let ttl = ttl_for_status(&entry.status)?;
if now.saturating_sub(entry.checked_at) > ttl {
return None;
}
Some(entry.clone())
}
/// Record a check result in the cache.
///
/// Acquires a write lock — blocks concurrent readers/writers briefly.
pub fn put(&self, url: &str, status: CheckStatus, detail: Option<&str>) {
let Ok(mut entries) = self.entries.write() else {
return;
};
// Enforce boundary
if entries.len() >= MAX_CACHE_ENTRIES && !entries.contains_key(url) {
return;
}
entries.insert(
url.to_string(),
CacheEntry {
status: status.to_string(),
checked_at: now_secs(),
http_status: extract_http_status(detail),
detail: detail.map(String::from),
},
);
}
/// Number of entries currently in the cache.
pub fn len(&self) -> usize {
self.entries.read().map_or(0, |e| e.len())
}
/// Whether the cache is empty.
pub fn is_empty(&self) -> bool {
self.entries.read().map_or(true, |e| e.is_empty())
}
/// Save cache to file atomically (write tmp + rename).
/// Prunes expired entries before writing.
///
/// # Errors
///
/// Returns an error if the file cannot be written.
pub fn save(&self) -> Result<(), String> {
if self.path.is_empty() {
return Ok(());
}
let entries = self
.entries
.read()
.map_err(|e| format!("cache lock poisoned: {e}"))?;
// Prune expired entries before saving
let now = now_secs();
let pruned: HashMap<&String, &CacheEntry> = entries
.iter()
.filter(|(_, e)| {
ttl_for_status(&e.status).is_some_and(|ttl| now.saturating_sub(e.checked_at) <= ttl)
})
.collect();
let before = entries.len();
let after = pruned.len();
let json = serde_json::to_string_pretty(&pruned)
.map_err(|e| format!("failed to serialize cache: {e}"))?;
// Atomic write: tmp file + rename
let tmp_path = format!("{}.tmp", self.path);
std::fs::write(&tmp_path, json)
.map_err(|e| format!("failed to write cache tmp file {tmp_path}: {e}"))?;
std::fs::rename(&tmp_path, &self.path)
.map_err(|e| format!("failed to rename cache file: {e}"))?;
if before == after {
eprintln!("cache: saved {after} entries");
} else {
eprintln!(
"cache: saved {after} entries (pruned {} expired)",
before - after
);
}
Ok(())
}
}
fn now_secs() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_or(0, |d| d.as_secs())
}
/// Try to extract HTTP status code from detail string like "HTTP 404 Not Found".
fn extract_http_status(detail: Option<&str>) -> Option<u16> {
let detail = detail?;
let rest = detail.strip_prefix("HTTP ")?;
let code_str: String = rest.chars().take_while(char::is_ascii_digit).collect();
code_str.parse().ok()
}
#[cfg(test)]
mod tests {
use super::*;
fn empty_cache() -> VerifyCache {
VerifyCache::empty()
}
#[test]
fn cache_put_and_get_ok() {
let cache = empty_cache();
cache.put("https://example.com", CheckStatus::Ok, None);
assert!(cache.get("https://example.com").is_some());
}
#[test]
fn cache_error_always_rechecked() {
let cache = empty_cache();
cache.put("https://example.com", CheckStatus::Error, Some("HTTP 404"));
assert!(cache.get("https://example.com").is_none());
}
#[test]
fn cache_suspicious_always_rechecked() {
let cache = empty_cache();
cache.put(
"https://example.com",
CheckStatus::Suspicious,
Some("HTTP 403 Forbidden"),
);
assert!(cache.get("https://example.com").is_none());
}
#[test]
fn cache_ok_expired_after_90_days() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://old.com".into(),
CacheEntry {
status: "ok".into(),
checked_at: now_secs() - OK_TTL_SECS - 1,
http_status: None,
detail: None,
},
);
}
assert!(cache.get("https://old.com").is_none());
}
#[test]
fn cache_ok_valid_within_90_days() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://recent.com".into(),
CacheEntry {
status: "ok".into(),
checked_at: now_secs() - OK_TTL_SECS + 3600,
http_status: None,
detail: None,
},
);
}
assert!(cache.get("https://recent.com").is_some());
}
#[test]
fn cache_rate_limited_expires_after_7_days() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://throttled.com".into(),
CacheEntry {
status: "rate_limited".into(),
checked_at: now_secs() - RATE_LIMITED_TTL_SECS - 1,
http_status: Some(429),
detail: Some("HTTP 429".into()),
},
);
}
assert!(cache.get("https://throttled.com").is_none());
}
#[test]
fn cache_rate_limited_valid_within_7_days() {
let cache = empty_cache();
cache.put(
"https://example.com",
CheckStatus::RateLimited,
Some("HTTP 429 Too Many Requests"),
);
assert!(cache.get("https://example.com").is_some());
}
#[test]
fn cache_warn_expires_after_1_day() {
let cache = empty_cache();
{
let mut entries = cache.entries.write().unwrap();
entries.insert(
"https://flaky.com".into(),
CacheEntry {
status: "warn".into(),
checked_at: now_secs() - WARN_TTL_SECS - 1,
http_status: None,
detail: Some("timeout".into()),
},
);
}
assert!(cache.get("https://flaky.com").is_none());
}
#[test]
fn cache_warn_valid_within_1_day() {
let cache = empty_cache();
cache.put("https://example.com", CheckStatus::Warn, Some("timeout"));
assert!(cache.get("https://example.com").is_some());
}
#[test]
fn cache_unknown_url_returns_none() {
let cache = empty_cache();
assert!(cache.get("https://unknown.com").is_none());
}
#[test]
fn extract_http_status_parses() {
assert_eq!(extract_http_status(Some("HTTP 404 Not Found")), Some(404));
assert_eq!(extract_http_status(Some("HTTP 200")), Some(200));
assert_eq!(extract_http_status(Some("timeout")), None);
assert_eq!(extract_http_status(None), None);
}
#[test]
fn cache_boundary_enforced() {
let cache = empty_cache();
for i in 0..MAX_CACHE_ENTRIES {
cache.put(&format!("https://example.com/{i}"), CheckStatus::Ok, None);
}
// One more should be rejected
cache.put("https://overflow.com", CheckStatus::Ok, None);
assert!(cache.get("https://overflow.com").is_none());
assert_eq!(cache.len(), MAX_CACHE_ENTRIES);
}
#[test]
fn cache_update_existing_within_boundary() {
let cache = empty_cache();
for i in 0..MAX_CACHE_ENTRIES {
cache.put(&format!("https://example.com/{i}"), CheckStatus::Ok, None);
}
// Updating existing should work even at capacity
cache.put(
"https://example.com/0",
CheckStatus::Error,
Some("HTTP 500"),
);
assert_eq!(cache.len(), MAX_CACHE_ENTRIES);
}
#[test]
fn cache_concurrent_read_write() {
use std::sync::Arc;
use std::thread;
let cache = Arc::new(empty_cache());
let mut handles = Vec::new();
// Spawn writers
for i in 0..10 {
let c = Arc::clone(&cache);
handles.push(thread::spawn(move || {
for j in 0..100 {
c.put(
&format!("https://example.com/{i}/{j}"),
CheckStatus::Ok,
None,
);
}
}));
}
// Spawn readers
for _ in 0..10 {
let c = Arc::clone(&cache);
handles.push(thread::spawn(move || {
for _ in 0..100 {
let _ = c.get("https://example.com/0/0");
}
}));
}
for h in handles {
h.join().expect("thread panicked");
}
// All 1000 entries should be present
assert_eq!(cache.len(), 1000);
}
#[test]
fn ttl_tiers_are_correct() {
assert_eq!(ttl_for_status("ok"), Some(90 * 24 * 60 * 60));
assert_eq!(ttl_for_status("rate_limited"), Some(7 * 24 * 60 * 60));
assert_eq!(ttl_for_status("warn"), Some(24 * 60 * 60));
assert_eq!(ttl_for_status("error"), None);
assert_eq!(ttl_for_status("suspicious"), None);
}
}