Packages
API client library for any MediaWiki-based site, such as Wikipedia or Wikimedia Commons. Provides the Action API, realtime feed ingestion, and ORES scoring.
Current section
Files
Jump to
Current section
Files
lib/enterprise.ex
defmodule Wiki.Enterprise.Session do
@moduledoc """
API state container which is returned during login and is then passed to all
methods. Note that the access token will expire in 24h so long-running
processes will need to reauthenticate.
"""
@type t :: %__MODULE__{
__client__: Req.Request.t(),
refresh_token: binary(),
access_token: binary()
}
defstruct [:__client__, :refresh_token, :access_token]
end
defmodule Wiki.Enterprise do
@moduledoc """
Client for the Wikimedia Enterprise API
Docs: https://enterprise.wikimedia.com/docs/
These APIs are only provided by the Enterprise organization and hit two
endpoints: one for authentication and the other for information retrieval.
An account is required.
For the moment, only the Snapshot API verbs will be implemented.
"""
@auth_base "https://auth.enterprise.wikimedia.com"
@api_base "https://api.enterprise.wikimedia.com"
@default_timeout_ms 3_600_000
@type client_option ::
{:access_token, binary()}
| {:debug, true}
| {:timeout, integer()}
| {:user_agent, binary()}
@type client_options :: [client_option()]
alias Wiki.Enterprise.Session
alias Wiki.{ReqDebug, Util}
@doc """
Example:
```
{:ok, api} = Wiki.Enterprise.new(
username: System.get_env("ENTERPRISE_USERNAME"),
refresh_token: System.get_env("ENTERPRISE_REFRESH_TOKEN"))
```
"""
@spec new(client_options) :: {:ok, Session.t()} | {:error, term()}
def new(opts) do
{refresh_token, opts} = Keyword.pop!(opts, :refresh_token)
{username, opts} = Keyword.pop!(opts, :username)
with {:ok, access_token} <- get_tokens(username, refresh_token, opts) do
{:ok,
%Session{
__client__: client(opts ++ [access_token: access_token]),
refresh_token: refresh_token,
access_token: access_token
}}
end
end
@doc """
Lists information about available snapshots, namespaces, and chunks.
Example:
```
Wiki.Enterprise.list_available_snapshots(api)
```
"""
@spec list_available_snapshots(Session.t()) :: list()
def list_available_snapshots(api) do
{:ok, response} = Req.get(api.__client__, url: "/v2/snapshots")
response.body
end
@spec get_tokens(binary(), binary(), client_options) :: {:ok, binary()} | {:error, term()}
defp get_tokens(username, refresh_token, opts) do
with {:ok, %Req.Response{status: 200} = response} <-
Req.post(client(opts),
url: "#{@auth_base}/v1/token-refresh",
json: %{
username: username,
refresh_token: refresh_token
}
),
{:ok, token} <- Map.fetch(response.body, "access_token") do
{:ok, token}
else
{:ok, %Req.Response{body: %{"message" => message}}} ->
{:error, message}
{:ok, %Req.Response{status: status}} ->
{:error, "auth HTTP error status #{status}"}
_ ->
{:error, "unknown error"}
end
end
@spec client(client_options()) :: Req.Request.t()
defp client(opts) do
(Keyword.drop(opts, [:access_token, :debug, :timeout, :user_agent]) ++
[
base_url: @api_base,
receive_timeout: @default_timeout_ms,
user_agent: opts[:user_agent] || Util.default_user_agent()
])
|> opts_maybe_authenticated(opts[:access_token])
|> Req.new()
|> ReqProxy.attach()
|> step_prevent_redundant_auth()
|> ReqDebug.attach(debug: opts[:debug])
end
defp opts_maybe_authenticated(opts, nil), do: opts
defp opts_maybe_authenticated(opts, access_token) when is_binary(access_token),
do: opts ++ [auth: {:bearer, access_token}]
defp step_prevent_redundant_auth(request) do
Req.Request.append_request_steps(request,
no_redundant_auth: fn request ->
with %{query: query} when is_binary(query) <- URI.parse(request.url),
%{"X-Amz-Algorithm" => _} <- URI.decode_query(query) do
Req.Request.delete_header(request, "authorization")
else
_ -> request
end
end
)
end
@doc """
Get a list of chunk files for a given snapshot, along with the chunk metadata.
The order of chunks doesn't matter, each is a self-contained file.
Example:
```
Wiki.Enterprise.list_snapshot_chunks(api, "aawiki_namespace_0")
```
"""
@spec list_snapshot_chunks(Session.t(), binary()) :: list()
def list_snapshot_chunks(api, snapshot_id) do
{:ok, response} = Req.get(api.__client__, url: "/v2/snapshots/#{snapshot_id}/chunks")
response.body
end
@doc """
Retrieve a snapshot chunk as a binary .tar.gz
Example:
```
ndjson_tgz = Wiki.Enterprise.get_snapshot_chunk(api, "aawiki_namespace_0", "aawiki_namespace_0_chunk_0")
```
"""
@spec get_snapshot_chunk(Session.t(), binary(), binary()) :: binary()
def get_snapshot_chunk(api, snapshot_id, chunk_id) do
{:ok, result} =
Req.get(
api.__client__,
url: "/v2/snapshots/#{snapshot_id}/chunks/#{chunk_id}/download"
# TODO: req doesn't handle streaming decompression yet, we should look
# carefully at memory usage.
# See https://github.com/wojtekmach/req/issues/521
)
result.body
end
end