Packages

API client library for any MediaWiki-based site, such as Wikipedia or Wikimedia Commons. Provides the Action API, realtime feed ingestion, and ORES scoring.

Current section

Files

Jump to
mediawiki_client lib site_matrix.ex
Raw

lib/site_matrix.ex

defmodule Wiki.SiteMatrix do
@moduledoc """
Retrieves sites from a wiki farm with the SiteMatrix extension installed.
"""
@metawiki_api "https://meta.wikimedia.org/w/api.php"
alias Wiki.Error
@type client_options :: [
Wiki.Action.client_option()
| {:api, binary()}
| {:key_fun, (Wiki.SiteMatrix.Spec.t() -> any())}
]
defmodule Spec do
@moduledoc """
Container for a single site.
"""
@type t :: %__MODULE__{
base_url: String.t(),
closed: boolean(),
dbname: String.t(),
dir: String.t(),
lang: String.t(),
name: String.t(),
opts: Wiki.SiteMatrix.client_options(),
private: boolean(),
project: String.t()
}
@enforce_keys [:base_url]
defstruct [
:base_url,
:closed,
:dbname,
:dir,
:lang,
:name,
:private,
:project,
opts: []
]
end
@doc """
## Options
- `api` - Action API URL for a site participating in the farm.
Defaults to #{@metawiki_api}.
- `key_fun` - Function on a site spec, returning the key to build a lookup with,
later passed as the `key` argument to `get`.
Defaults to returning the site's `:dbname` (eg. "enwiki").
"""
@opaque sitematrix_state :: %{
api: binary(),
sites: %{binary() => Spec.t()}
}
@spec new(client_options()) :: sitematrix_state()
def new(opts \\ []) do
api = opts[:api] || @metawiki_api
key_fun = opts[:key_fun] || fn site -> site.dbname end
%{
api: api,
sites:
do_get_all(api, opts)
|> Map.new(fn site -> {key_fun.(site), site} end)
}
end
@doc """
Get all sites for a wiki farm.
## Arguments
* `sitematrix` - Result of Sitematrix.new()
## Return value
List of site specifications.
"""
@spec get_all(sitematrix_state()) :: {:ok, [Spec.t()]} | {:error, any}
def get_all(sitematrix) do
{:ok, Map.values(sitematrix.sites)}
end
@spec action_client(binary(), client_options()) :: Wiki.Action.Session.t()
defp action_client(api, opts) do
api
|> Wiki.Action.new(opts)
end
@spec do_get_all(binary(), client_options()) :: [Spec.t()]
defp do_get_all(api, opts) do
action_client(api, opts)
|> fetch_sitematrix()
|> Enum.map(&site_spec/1)
end
defp site_spec(site) do
%Spec{
base_url: site["url"],
closed: site["closed"] == true,
dbname: site["dbname"],
dir: site["dir"] || "ltr",
lang: site["lang"],
name: site["name"] || site["sitename"],
private: site["private"] == true,
project: site["code"]
}
end
defp fetch_sitematrix(client) do
client
|> Wiki.Action.stream(
action: :sitematrix,
smsiteprop: [:code, :dbname, :lang, :sitename, :url]
)
|> Enum.flat_map(fn response ->
flatten_sitematrix(response["sitematrix"])
end)
end
defp flatten_sitematrix(all) do
(all
|> Map.drop(["count", "specials"])
|> flatten_language_wikis()) ++
(all["specials"] || [])
end
defp flatten_language_wikis(language_sites) do
language_sites
|> Map.delete("specials")
|> Map.values()
|> Enum.flat_map(fn group ->
group_name = group["localname"] || group["name"] || group["code"]
group["site"]
|> Enum.map(fn site ->
site
|> Map.merge(%{
"dir" => group["dir"],
# FIXME: Lacking translation and grammar
"name" => group_name <> " " <> site["sitename"]
})
end)
end)
end
@doc """
Look up a single site by key
## Arguments
* `sitematrix` - Result of Sitematrix.new()
* `key` - The dbname (eg. "enwiki") or value of another key if specified as an
option to new().
## Return value
Site spec or error
"""
@spec get(sitematrix_state(), String.t()) :: {:ok, Spec.t()} | {:error, any}
def get(sitematrix, key) do
Map.fetch(sitematrix.sites, key)
|> case do
:error -> {:error, %Error{message: "Site #{key} not found."}}
x -> x
end
end
@doc """
Assertive variant of `get`.
"""
@spec get!(sitematrix_state(), String.t()) :: Spec.t()
def get!(sitematrix, key) do
case get(sitematrix, key) do
{:ok, site} -> site
{:error, error} -> raise error
end
end
@doc """
Get the Action API for a known site
```elixir
Wiki.SiteMatrix.new()
|> Wiki.SiteMatrix.get("enwiki")
|> Wiki.SiteMatrix.action_api()
# "https://en.wikipedia.org/w/api.php"
```
As a convenience, the site can be referenced as a bare string, in which case
it will be looked up in the default Wikimedia farm. Note that this will be
uncached and so inappropriate for most production use.
```elixir
Wiki.SiteMatrix.action_api("dewiki")
```
## Arguments
* `site` - Populated site structure.
## Return value
Calculated Action API.
"""
@spec action_api(String.t() | Spec.t()) :: String.t()
def action_api(site)
def action_api(site) when is_binary(site) do
# FIXME: Globally cached site matrix for this case
new()
|> get!(site)
|> action_api()
end
def action_api(site) do
site.base_url <>
"/w/api.php"
end
end