Current section
4 Versions
Jump to
Current section
4 Versions
Compare versions
3
files changed
+12
additions
-6
deletions
| @@ -8,4 +8,4 @@ | |
| 8 8 | {<<"links">>,[{<<"Github">>,<<"https://github.com/AntoineGagne/robots">>}]}. |
| 9 9 | {<<"name">>,<<"robots">>}. |
| 10 10 | {<<"requirements">>,[]}. |
| 11 | - {<<"version">>,<<"1.1.1">>}. |
| 11 | + {<<"version">>,<<"1.1.2">>}. |
| @@ -1,6 +1,6 @@ | |
| 1 1 | {application,robots, |
| 2 2 | [{description,"A parser for robots.txt."}, |
| 3 | - {vsn,"1.1.1"}, |
| 3 | + {vsn,"1.1.2"}, |
| 4 4 | {registered,[]}, |
| 5 5 | {applications,[kernel,stdlib]}, |
| 6 6 | {env,[]}, |
| @@ -134,7 +134,7 @@ handle_line(Line) -> | |
| 134 134 | sort_rules(_, Value = {allowed, all}) -> |
| 135 135 | Value; |
| 136 136 | sort_rules(_, {Allowed, Disallowed}) -> |
| 137 | - Compare = fun(R1, R2) -> not (R1 =< R2) end, |
| 137 | + Compare = fun(R1, R2) -> R1 > R2 end, |
| 138 138 | {lists:sort(Compare, Allowed), lists:sort(Compare, Disallowed)}; |
| 139 139 | sort_rules(sitemap, Value) -> |
| 140 140 | Value. |
| @@ -143,17 +143,23 @@ sort_rules(sitemap, Value) -> | |
| 143 143 | trim(String) -> |
| 144 144 | string:trim(String, both). |
| 145 145 | |
| 146 | - -spec build_rules({binary(), binary()}, {[agent()], boolean(), rules_index()}) -> |
| 147 | - {[agent()], boolean(), rules_index()}. |
| 146 | + -spec build_rules({binary(), binary()}, {[agent()], IsFirstAgent, rules_index()}) -> |
| 147 | + {[agent()], IsFirstAgent, rules_index()} |
| 148 | + when |
| 149 | + IsFirstAgent :: boolean(). |
| 148 150 | build_rules({<<"user-agent">>, RawAgent}, {Agents, false, RulesIndex}) -> |
| 149 151 | Reversed = to_agent(RawAgent), |
| 150 152 | {[Reversed | Agents], false, RulesIndex}; |
| 151 153 | build_rules({<<"user-agent">>, RawAgent}, {_Agents, true, RulesIndex}) -> |
| 152 154 | Reversed = to_agent(RawAgent), |
| 153 155 | {[Reversed], false, RulesIndex}; |
| 156 | + build_rules({<<"allow">>, <<>>}, {Agents, _, RulesIndex}) -> |
| 157 | + {Agents, true, RulesIndex}; |
| 154 158 | build_rules({<<"allow">>, Rule}, {Agents, _, RulesIndex}) -> |
| 155 159 | {_, UpdatedIndex} = lists:foldl(fun update_index/2, {{allowed, Rule}, RulesIndex}, Agents), |
| 156 160 | {Agents, true, UpdatedIndex}; |
| 161 | + build_rules({<<"disallow">>, <<>>}, {Agents, _, RulesIndex}) -> |
| 162 | + {Agents, true, RulesIndex}; |
| 157 163 | build_rules({<<"disallow">>, Rule}, {Agents, _, RulesIndex}) -> |
| 158 164 | {_, UpdatedIndex} = lists:foldl(fun update_index/2, {{disallowed, Rule}, RulesIndex}, Agents), |
| 159 165 | {Agents, true, UpdatedIndex}; |
| @@ -278,7 +284,7 @@ nested_wildcard_test_() -> | |
| 278 284 | ?_assertNot(match(<<"/windows.PHP">>, Rule)) |
| 279 285 | ]. |
| 280 286 | |
| 281 | - nested_wilcard_with_ending_test_() -> |
| 287 | + nested_wildcard_with_ending_test_() -> |
| 282 288 | Rule = <<"/*.php$">>, |
| 283 289 | [ |
| 284 290 | ?_assert(match(<<"/filename.php">>, Rule)), |