Current section
Files
Jump to
Current section
Files
src/scrapbook@page.erl
-module(scrapbook@page).
-compile([no_auto_import, nowarn_unused_vars, nowarn_unused_function, nowarn_nomatch]).
-export([get_event_collection/1, construct_event_items_requests/2, scrape_event_items_responses/1, construct_event_collection_request_from_url/2, construct_past_event_collection_request_from_name/2, construct_upcoming_event_collection_request_from_name/2, scrape_event_collection_response/1, to_json/1]).
-export_type([page_event_item/0, page_event_collection/0]).
-if(?OTP_RELEASE >= 27).
-define(MODULEDOC(Str), -moduledoc(Str)).
-define(DOC(Str), -doc(Str)).
-else.
-define(MODULEDOC(Str), -compile([])).
-define(DOC(Str), -compile([])).
-endif.
?MODULEDOC(
" # Examples\n"
"\n"
" You can use this module to create HTTP requests and parse HTTP responses,\n"
" which allows you to use this module in all targets.\n"
"\n"
" ## Event collection\n"
"\n"
" ```gleam\n"
" import gleam/httpc\n"
" import gleam/result\n"
" import scrapbook/page\n"
"\n"
" // define a const with user agent\n"
"\n"
" fn event_collection() -> Result(page.PageEventCollection, Nil) {\n"
" use req <- result.try(result.replace_error(\n"
" page.construct_past_event_collection_request_from_name(\n"
" \"CodeSyncGlobal\",\n"
" user_agent,\n"
" ),\n"
" Nil,\n"
" ))\n"
" use resp <- result.try(result.replace_error(httpc.send(req), Nil))\n"
" result.replace_error(page.scrape_event_collection_response(resp), Nil)\n"
" }\n"
"\n"
" pub fn main() -> Nil {\n"
" let assert Ok(event_collection) = event_collection()\n"
" echo event_collection\n"
" Nil\n"
" }\n"
" ```\n"
"\n"
" ## Event collection items (events)\n"
"\n"
" ```gleam\n"
" import gleam/http/request.{type Request}\n"
" import gleam/http/response.{type Response}\n"
" import gleam/httpc\n"
" import gleam/result\n"
" import scrapbook/event\n"
" import scrapbook/page\n"
"\n"
" // define a const with user agent\n"
"\n"
" // fn event_collection()\n"
"\n"
" fn do_send_many(\n"
" reqs: List(Request(String)),\n"
" resps: List(Response(String)),\n"
" ) -> Result(List(Response(String)), Nil) {\n"
" case reqs {\n"
" [] -> Ok(resps)\n"
" [first, ..rest] -> {\n"
" use resp <- result.try(result.replace_error(httpc.send(first), Nil))\n"
" do_send_many(rest, [resp, ..resps])\n"
" }\n"
" }\n"
" }\n"
"\n"
" fn send_many(reqs: List(Request(String))) -> Result(List(Response(String)), Nil) {\n"
" do_send_many(reqs, [])\n"
" }\n"
"\n"
" fn event_items(\n"
" event_collection: page.PageEventCollection,\n"
" ) -> Result(List(event.EventData), Nil) {\n"
" use reqs <- result.try(result.replace_error(\n"
" page.construct_event_items_requests(event_collection.items, user_agent),\n"
" Nil,\n"
" ))\n"
" use resps <- result.try(send_many(reqs))\n"
" result.replace_error(page.scrape_event_items_responses(resps), Nil)\n"
" }\n"
"\n"
" pub fn main() -> Nil {\n"
" // first, get the page's event collection like shown above\n"
" let assert Ok(event_collection) = event_collection()\n"
" // then, based on the event collection, send requests for each event item\n"
" let assert Ok(event_items) = event_items(event_collection)\n"
" echo event_items\n"
" Nil\n"
" }\n"
" ```\n"
).
-type page_event_item() :: {page_event_item, binary(), binary(), boolean()}.
-type page_event_collection() :: {page_event_collection,
binary(),
integer(),
list(page_event_item())}.
-file("src/scrapbook/page.gleam", 109).
-spec encode_page_event_item(page_event_item()) -> gleam@json:json().
encode_page_event_item(Page_event_item) ->
{page_event_item, Id, Name, Canceled} = Page_event_item,
gleam@json:object(
[{<<"id"/utf8>>, gleam@json:string(Id)},
{<<"name"/utf8>>, gleam@json:string(Name)},
{<<"canceled"/utf8>>, gleam@json:bool(Canceled)}]
).
-file("src/scrapbook/page.gleam", 118).
-spec encode_page_event_collection(page_event_collection()) -> gleam@json:json().
encode_page_event_collection(Page_event_collection) ->
{page_event_collection, Name, Count, Items} = Page_event_collection,
gleam@json:object(
[{<<"name"/utf8>>, gleam@json:string(Name)},
{<<"count"/utf8>>, gleam@json:int(Count)},
{<<"items"/utf8>>,
gleam@json:array(Items, fun encode_page_event_item/1)}]
).
-file("src/scrapbook/page.gleam", 131).
-spec event_item_decoder() -> gleam@dynamic@decode:decoder(page_event_item()).
event_item_decoder() ->
gleam@dynamic@decode:field(
<<"id"/utf8>>,
{decoder, fun gleam@dynamic@decode:decode_string/1},
fun(Id) ->
gleam@dynamic@decode:field(
<<"name"/utf8>>,
{decoder, fun gleam@dynamic@decode:decode_string/1},
fun(Name) ->
gleam@dynamic@decode:field(
<<"is_canceled"/utf8>>,
{decoder, fun gleam@dynamic@decode:decode_bool/1},
fun(Canceled) ->
gleam@dynamic@decode:success(
{page_event_item, Id, Name, Canceled}
)
end
)
end
)
end
).
-file("src/scrapbook/page.gleam", 139).
-spec event_items_decoder() -> gleam@dynamic@decode:decoder(list(page_event_item())).
event_items_decoder() ->
gleam@dynamic@decode:list(
gleam@dynamic@decode:at(
[<<"node"/utf8>>, <<"node"/utf8>>],
event_item_decoder()
)
).
-file("src/scrapbook/page.gleam", 144).
?DOC(" Get a collection of page event items from the given HTML string.\n").
-spec get_event_collection(binary()) -> {ok, page_event_collection()} |
{error, list(scrapbook@error:property_error())}.
get_event_collection(Html) ->
Decoder = begin
gleam@dynamic@decode:field(
<<"name"/utf8>>,
{decoder, fun gleam@dynamic@decode:decode_string/1},
fun(Name) ->
gleam@dynamic@decode:then(
gleam@dynamic@decode:at(
[<<"items"/utf8>>, <<"count"/utf8>>],
{decoder, fun gleam@dynamic@decode:decode_int/1}
),
fun(Count) ->
gleam@dynamic@decode:then(
gleam@dynamic@decode:at(
[<<"pageItems"/utf8>>, <<"edges"/utf8>>],
event_items_decoder()
),
fun(Items) ->
gleam@dynamic@decode:success(
{page_event_collection, Name, Count, Items}
)
end
)
end
)
end
)
end,
scrapbook@internal@html:get_property(
Html,
<<"collection"/utf8>>,
Decoder,
fun(_) -> true end
).
-file("src/scrapbook/page.gleam", 161).
-spec do_construct_event_items_requests(
list(page_event_item()),
binary(),
list(gleam@http@request:request(binary()))
) -> {ok, list(gleam@http@request:request(binary()))} |
{error, scrapbook@error:scrape_error()}.
do_construct_event_items_requests(Items, User_agent, Reqs) ->
case Items of
[] ->
{ok, Reqs};
[First | Rest] ->
gleam@result:'try'(
scrapbook@event:construct_event_request_from_id(
erlang:element(2, First),
User_agent
),
fun(Event_request) ->
do_construct_event_items_requests(
Rest,
User_agent,
[Event_request | Reqs]
)
end
)
end.
-file("src/scrapbook/page.gleam", 182).
?DOC(" Construct the list of requests for the page's event items.\n").
-spec construct_event_items_requests(list(page_event_item()), binary()) -> {ok,
list(gleam@http@request:request(binary()))} |
{error, scrapbook@error:scrape_error()}.
construct_event_items_requests(Items, User_agent) ->
do_construct_event_items_requests(Items, User_agent, []).
-file("src/scrapbook/page.gleam", 189).
-spec do_scrape_event_items_responses(
list(gleam@http@response:response(binary())),
list(scrapbook@event:event_data())
) -> {ok, list(scrapbook@event:event_data())} |
{error, scrapbook@error:scrape_error()}.
do_scrape_event_items_responses(Resps, Events) ->
case Resps of
[] ->
{ok, Events};
[First | Rest] ->
gleam@result:'try'(
scrapbook@event:scrape_event_response(First),
fun(Event) ->
do_scrape_event_items_responses(Rest, [Event | Events])
end
)
end.
-file("src/scrapbook/page.gleam", 204).
?DOC(
" Scrape the list of events from the responses received for previously\n"
" generated page event item requests.\n"
).
-spec scrape_event_items_responses(list(gleam@http@response:response(binary()))) -> {ok,
list(scrapbook@event:event_data())} |
{error, scrapbook@error:scrape_error()}.
scrape_event_items_responses(Resps) ->
do_scrape_event_items_responses(Resps, []).
-file("src/scrapbook/page.gleam", 211).
?DOC(" Construct the request for the page's event collection with a given URL.\n").
-spec construct_event_collection_request_from_url(binary(), binary()) -> {ok,
gleam@http@request:request(binary())} |
{error, scrapbook@error:scrape_error()}.
construct_event_collection_request_from_url(Url, User_agent) ->
gleam@result:'try'(
gleam@result:replace_error(
scrapbook@internal@page@url:format_url(Url),
url_error
),
fun(Url@1) ->
case scrapbook@internal@http:construct_request(Url@1, User_agent) of
{ok, Req} ->
{ok, Req};
{error, Err} ->
{error, {request_error, Err}}
end
end
).
-file("src/scrapbook/page.gleam", 227).
?DOC(
" Construct the request for the page's past events collection with a given\n"
" name.\n"
).
-spec construct_past_event_collection_request_from_name(binary(), binary()) -> {ok,
gleam@http@request:request(binary())} |
{error, scrapbook@error:scrape_error()}.
construct_past_event_collection_request_from_name(Name, User_agent) ->
gleam@result:'try'(
gleam@result:replace_error(
scrapbook@internal@page@url:name_to_past_url(Name),
url_error
),
fun(Url) ->
case scrapbook@internal@http:construct_request(Url, User_agent) of
{ok, Req} ->
{ok, Req};
{error, Err} ->
{error, {request_error, Err}}
end
end
).
-file("src/scrapbook/page.gleam", 243).
?DOC(
" Construct the request for the page's upcoming events collection with a given\n"
" name.\n"
).
-spec construct_upcoming_event_collection_request_from_name(binary(), binary()) -> {ok,
gleam@http@request:request(binary())} |
{error, scrapbook@error:scrape_error()}.
construct_upcoming_event_collection_request_from_name(Name, User_agent) ->
gleam@result:'try'(
gleam@result:replace_error(
scrapbook@internal@page@url:name_to_upcoming_url(Name),
url_error
),
fun(Url) ->
case scrapbook@internal@http:construct_request(Url, User_agent) of
{ok, Req} ->
{ok, Req};
{error, Err} ->
{error, {request_error, Err}}
end
end
).
-file("src/scrapbook/page.gleam", 259).
?DOC(
" Scrape the page event collection from the response received for a previously\n"
" generated page event collection request.\n"
).
-spec scrape_event_collection_response(gleam@http@response:response(binary())) -> {ok,
page_event_collection()} |
{error, scrapbook@error:scrape_error()}.
scrape_event_collection_response(Resp) ->
case get_event_collection(erlang:element(4, Resp)) of
{ok, Event} ->
{ok, Event};
{error, Err} ->
{error, {property_error, Err}}
end.
-file("src/scrapbook/page.gleam", 269).
?DOC(" Encode the scraped page event collection into a JSON object.\n").
-spec to_json(page_event_collection()) -> gleam@json:json().
to_json(Page_event_collection) ->
encode_page_event_collection(Page_event_collection).