Current section
Files
Jump to
Current section
Files
lib/common_crawl/warc.ex
defmodule CommonCrawl.WARC do
@moduledoc """
Common Crawl .warc file download and parsing
"""
@base_url Application.compile_env(:common_crawl, :base_url, "https://data.commoncrawl.org/")
@receive_timeout Application.compile_env(:common_crawl, :receive_timeout, 120_000)
@doc """
Fetches a segment of the WARC file.
## Examples
iex> CommonCrawl.WARC.get_segment(
...> "crawl-data/CC-MAIN-2024-51/segments/1733066125982.36/warc/CC-MAIN-20241214181735-20241214211735-00433.warc.gz",
...> 36544657,
...> 1223
...> )
## Return Value
{:ok,
%{
warc: "WARC/1.0\\r\\nWARC-Type: response\\r\\nWARC-Date: 2024...",
headers: "HTTP/1.1 200 OK\\r\\nContent-Type: text/html...",
response: "<!doctype html>\\n<html>\\n<head>\\n<title>Example..."
}}
"""
@spec get_segment(String.t(), integer(), integer(), keyword()) ::
{:ok, %{warc: String.t(), headers: String.t(), response: String.t()}} | {:error, any()}
def get_segment(filename, offset, length, opts \\ [])
when is_binary(filename) and is_integer(offset) and is_integer(length) and length > 0 do
uri = URI.merge(@base_url, filename)
headers =
Enum.concat(opts[:headers] || [], [{"Range", "bytes=#{offset}-#{offset + length - 1}"}])
opts =
opts
|> Keyword.put(:headers, headers)
|> Keyword.put_new(:receive_timeout, @receive_timeout)
case Req.get(uri, opts) do
{:ok, %{body: body}} ->
case parse_response_body(body) do
[warc, headers, response] ->
{:ok, %{warc: warc, headers: headers, response: response}}
other ->
{:error, {:no_headers_or_response, other}}
end
{:error, reason} ->
{:error, reason}
end
end
@spec parse_response_body(binary()) :: [String.t()]
defp parse_response_body(bin) do
bin
|> String.trim()
|> String.split("\r\n\r\n", parts: 3)
end
end