Packages
kreuzcrawl
0.2.0
0.3.0
0.3.0-rc.88
0.3.0-rc.87
0.3.0-rc.86
0.3.0-rc.85
0.3.0-rc.84
0.3.0-rc.83
0.3.0-rc.82
0.3.0-rc.81
0.3.0-rc.80
0.3.0-rc.79
0.3.0-rc.77
0.3.0-rc.72
0.3.0-rc.71
0.3.0-rc.70
0.3.0-rc.55
0.3.0-rc.51
0.3.0-rc.50
0.3.0-rc.49
0.3.0-rc.48
0.3.0-rc.47
0.3.0-rc.46
0.3.0-rc.45
0.3.0-rc.44
0.3.0-rc.43
0.3.0-rc.42
0.3.0-rc.37
0.3.0-rc.19
0.3.0-rc.18
0.3.0-rc.17
0.3.0-rc.15
0.3.0-rc.14
0.3.0-rc.13
0.3.0-rc.12
0.3.0-rc.11
0.3.0-rc.10
0.3.0-rc.9
0.3.0-rc.8
0.3.0-rc.7
0.3.0-rc.3
0.3.0-rc.2
0.2.0
0.1.2
0.1.1
0.1.0-rc.9
0.1.0-rc.8
0.1.0-rc.7
0.1.0-rc.6
0.1.0-rc.5
0.1.0-rc.2
High-performance web crawling engine
Current section
Files
Jump to
Current section
Files
lib/kreuzcrawl.ex
# This file is auto-generated by alef. DO NOT EDIT.
defmodule Kreuzcrawl do
@moduledoc "High-level API for kreuzcrawl."
@doc "Create a new crawl engine with the given configuration."
@spec create_engine() :: {:ok, reference()} | {:error, String.t()}
def create_engine do
Kreuzcrawl.Native.create_engine(nil)
end
@doc "Create a new crawl engine with the given configuration."
@spec create_engine(String.t() | nil) :: {:ok, reference()} | {:error, String.t()}
def create_engine(config) do
Kreuzcrawl.Native.create_engine(config)
end
@doc "Scrape a single URL, returning extracted page data."
@spec scrape_async(reference(), String.t()) :: {:ok, String.t() | nil} | {:error, String.t()}
def scrape_async(engine, url) do
Kreuzcrawl.Native.scrape_async(engine, url)
end
@doc "Crawl a website starting from `url`, following links up to the configured depth."
@spec crawl_async(reference(), String.t()) :: {:ok, String.t() | nil} | {:error, String.t()}
def crawl_async(engine, url) do
Kreuzcrawl.Native.crawl_async(engine, url)
end
@doc "Discover all pages on a website by following links and sitemaps."
@spec map_urls_async(reference(), String.t()) :: {:ok, String.t() | nil} | {:error, String.t()}
def map_urls_async(engine, url) do
Kreuzcrawl.Native.map_urls_async(engine, url)
end
@doc "Scrape multiple URLs concurrently."
@spec batch_scrape_async(reference(), [String.t()]) :: [String.t() | nil]
def batch_scrape_async(engine, urls) do
Kreuzcrawl.Native.batch_scrape_async(engine, urls)
end
@doc "Crawl multiple seed URLs concurrently, each following links to configured depth."
@spec batch_crawl_async(reference(), [String.t()]) :: [String.t() | nil]
def batch_crawl_async(engine, urls) do
Kreuzcrawl.Native.batch_crawl_async(engine, urls)
end
@doc "Method"
@spec contentconfig_default() :: String.t() | nil
def contentconfig_default do
Kreuzcrawl.Native.contentconfig_default()
end
@doc "Method"
@spec browserconfig_default() :: String.t() | nil
def browserconfig_default do
Kreuzcrawl.Native.browserconfig_default()
end
@doc "Method"
@spec crawlconfig_default() :: String.t() | nil
def crawlconfig_default do
Kreuzcrawl.Native.crawlconfig_default()
end
@doc "Validate the configuration, returning an error if any values are invalid."
@spec crawlconfig_validate(map()) :: {:ok, nil} | {:error, String.t()}
def crawlconfig_validate(obj) do
Kreuzcrawl.Native.crawlconfig_validate(obj)
end
@doc "Returns the count of unique normalized URLs encountered during crawling."
@spec crawlresult_unique_normalized_urls(map()) :: non_neg_integer()
def crawlresult_unique_normalized_urls(obj) do
Kreuzcrawl.Native.crawlresult_unique_normalized_urls(obj)
end
end