Current section
20 Versions
Jump to
Current section
20 Versions
Compare versions
11
files changed
+337
additions
-82
deletions
| @@ -1,5 +1,12 @@ | |
| 1 1 | # Changelog |
| 2 2 | |
| 3 | + ## v0.5.1 (2025-12-19) |
| 4 | + |
| 5 | + ### Enhancements |
| 6 | + |
| 7 | + * Improved support for `order_by` + `limit` queries. |
| 8 | + * Caller may provide query to run on assign_ready for SchemaMetadata watches. |
| 9 | + |
| 3 10 | ## v0.5.0 (2025-07-08) |
| 4 11 | |
| 5 12 | ### Enhancements |
| @@ -2,7 +2,7 @@ | |
| 2 2 | [{<<"GitHub">>, |
| 3 3 | <<"https://github.com/foundationdb-beam/ecto_foundationdb">>}]}. |
| 4 4 | {<<"name">>,<<"ecto_foundationdb">>}. |
| 5 | - {<<"version">>,<<"0.5.0">>}. |
| 5 | + {<<"version">>,<<"0.5.1">>}. |
| 6 6 | {<<"description">>,<<"FoundationDB adapter for Ecto">>}. |
| 7 7 | {<<"elixir">>,<<"~> 1.15">>}. |
| 8 8 | {<<"app">>,<<"ecto_foundationdb">>}. |
| @@ -253,6 +253,76 @@ defmodule Ecto.Adapters.FoundationDB do | |
| 253 253 | We encourage you to test execution times of your expected workload before deciding to create an index. |
| 254 254 | Remember: since your data is already partitioned into tenants, you're effectively getting 1 index for free. |
| 255 255 | |
| 256 | + ### Query behavior |
| 257 | + |
| 258 | + In the implementation of Layer queries, we choose to provide the following guarantee to the developer: |
| 259 | + |
| 260 | + > A query is valid only if it can be achieved with a single Get or a single GetRange. |
| 261 | + |
| 262 | + This ensures that your application's queries will always be efficient. It allows you to construct your own |
| 263 | + complex query orchestration, but only when you need it. |
| 264 | + |
| 265 | + For example, a query with an `or` condition is not possible in EctoFDB, because to satisfy the `or`, we |
| 266 | + would need to issue 2 GetRange operations for a single query. |
| 267 | + |
| 268 | + ```elixir |
| 269 | + from(c in City, where: c.country == "Sweden", or_where: c.country == "Brazil") |
| 270 | + |> Repo.all(prefix: tenant) |
| 271 | + # =x Error |
| 272 | + ``` |
| 273 | + |
| 274 | + Instead, we want the database to do both GetRange at the same time with `async_all`. |
| 275 | + |
| 276 | + ```elixir |
| 277 | + Repo.transactional(tenant, fn -> |
| 278 | + [ |
| 279 | + Repo.async_all(from(c in City, where: c.country == "Sweden")), |
| 280 | + Repo.async_all(from(c in City, where: c.country == "Brazil")), |
| 281 | + ] |
| 282 | + |> Repo.await() |
| 283 | + |> List.flatten() |
| 284 | + end) |
| 285 | + ``` |
| 286 | + |
| 287 | + The trade-off that we choose to accept is that the expressiveness of the queries we support is limited |
| 288 | + compared to SQL databases. But you can develop with confidence that your queries will always be |
| 289 | + efficient. |
| 290 | + |
| 291 | + #### Using `order_by` and `limit` in the same query |
| 292 | + |
| 293 | + EctoFDB will always attempt to return query result that has the ordering applied first and then the limit. If it's |
| 294 | + unable to do so, then an exception is raised. Remember: a key design decision of our Layer is that a query is valid |
| 295 | + only if it can be achieved with a single Get or GetRange. |
| 296 | + |
| 297 | + `order_by` and `limit` will work as expected as long as one of the following conditions is true: |
| 298 | + |
| 299 | + 1. An index is not being queried AND the `order_by` is defined on the primary key field. |
| 300 | + 2. The fields of the selected index match with the fields in the `order_by` clause. |
| 301 | + |
| 302 | + Examples: |
| 303 | + |
| 304 | + ```elixir |
| 305 | + opts = [prefix: tenant, limit: 10] |
| 306 | + |
| 307 | + # order_by primary key field |
| 308 | + Repo.all( |
| 309 | + from(c in City, order_by: c.id), |
| 310 | + opts |
| 311 | + ) |
| 312 | + |
| 313 | + # where+order_by indexed field (`[:country]` is an index) |
| 314 | + Repo.all( |
| 315 | + from(c in City, where: c.country == "Sweden", order_by: [asc: c.country]), |
| 316 | + opts |
| 317 | + ) |
| 318 | + |
| 319 | + # where+order_by index subset (`[:continent, :incorporated_date]` is an index) |
| 320 | + Repo.all( |
| 321 | + from(c in City, where: c.country == "Europe", order_by: [asc: c.country, desc: c.incorporated_date]), |
| 322 | + opts |
| 323 | + ) |
| 324 | + ``` |
| 325 | + |
| 256 326 | ### Custom indexes |
| 257 327 | |
| 258 328 | As data is inserted, updated, deleted, and queried, the Indexer callbacks (via behaviour `EctoFoundationDB.Indexer`) |
| @@ -4,6 +4,8 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterAssigns do | |
| 4 4 | alias EctoFoundationDB.Indexer.SchemaMetadata |
| 5 5 | alias EctoFoundationDB.Layer.Tx |
| 6 6 | |
| 7 | + alias Ecto.Adapters.FoundationDB |
| 8 | + |
| 7 9 | def assign_ready(_module, repo, futures, ready_refs, options) when is_list(ready_refs) do |
| 8 10 | Tx.transactional(options[:prefix], fn _tx -> |
| 9 11 | {assign_futures_rev, futures} = filter_ready(repo, futures, ready_refs, options) |
| @@ -39,7 +41,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterAssigns do | |
| 39 41 | {nil, futures} |
| 40 42 | |
| 41 43 | {future, futures} -> |
| 42 | - {schema, query, watch_options, new_watch_fn} = Future.result(future) |
| 44 | + {schema, kind, watch_options, new_watch_fn} = Future.result(future) |
| 43 45 | |
| 44 46 | if not Keyword.has_key?(watch_options, :label) do |
| 45 47 | raise """ |
| @@ -52,7 +54,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterAssigns do | |
| 52 54 | """ |
| 53 55 | end |
| 54 56 | |
| 55 | - case query do |
| 57 | + case kind do |
| 56 58 | {:pk, pk} -> |
| 57 59 | async_get(repo, futures, schema, pk, watch_options, options, new_watch_fn) |
| 58 60 | |
| @@ -66,10 +68,13 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterAssigns do | |
| 66 68 | defp async_get(repo, futures, schema, id, watch_options, options, new_watch_fn) do |
| 67 69 | label = watch_options[:label] |
| 68 70 | |
| 71 | + tenant = options[:prefix] |
| 72 | + |
| 69 73 | Tx.transactional(options[:prefix], fn _tx -> |
| 70 74 | assign_future = |
| 71 75 | repo.async_get(schema, id, options) |
| 72 76 | |> Future.apply(fn struct_or_nil -> |
| 77 | + struct_or_nil = usetenant(struct_or_nil, tenant) |
| 73 78 | new_future = maybe_new_watch(struct_or_nil, watch_options, options, new_watch_fn) |
| 74 79 | |
| 75 80 | {[{label, struct_or_nil}], new_future} |
| @@ -81,11 +86,14 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterAssigns do | |
| 81 86 | |
| 82 87 | defp async_all(repo, futures, schema, watch_options, options, new_watch_fn) do |
| 83 88 | label = watch_options[:label] |
| 89 | + query = watch_options[:query] || schema |
| 90 | + tenant = options[:prefix] |
| 84 91 | |
| 85 | - Tx.transactional(options[:prefix], fn _tx -> |
| 92 | + Tx.transactional(tenant, fn _tx -> |
| 86 93 | assign_future = |
| 87 | - repo.async_all(schema, options) |
| 94 | + repo.async_all(query, options) |
| 88 95 | |> Future.apply(fn result -> |
| 96 | + result = usetenant(result, tenant) |
| 89 97 | new_future = maybe_new_watch(result, watch_options, options, new_watch_fn) |
| 90 98 | |
| 91 99 | {[{label, result}], new_future} |
| @@ -95,6 +103,10 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterAssigns do | |
| 95 103 | end) |
| 96 104 | end |
| 97 105 | |
| 106 | + defp usetenant(nil, _tenant), do: nil |
| 107 | + defp usetenant(list, tenant) when is_list(list), do: Enum.map(list, &usetenant(&1, tenant)) |
| 108 | + defp usetenant(struct, tenant), do: FoundationDB.usetenant(struct, tenant) |
| 109 | + |
| 98 110 | defp maybe_new_watch(result, watch_options, options, new_watch_fn) do |
| 99 111 | if Keyword.get(options, :watch?, false) do |
| 100 112 | new_watch_fn.(result, watch_options) |
| @@ -4,6 +4,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 4 4 | |
| 5 5 | alias Ecto.Adapters.FoundationDB |
| 6 6 | alias EctoFoundationDB.Exception.IncorrectTenancy |
| 7 | + alias EctoFoundationDB.Exception.Unsupported |
| 7 8 | alias EctoFoundationDB.Future |
| 8 9 | alias EctoFoundationDB.Layer.Fields |
| 9 10 | alias EctoFoundationDB.Layer.Ordering |
| @@ -14,17 +15,17 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 14 15 | alias EctoFoundationDB.Tenant |
| 15 16 | |
| 16 17 | @impl Ecto.Adapter.Queryable |
| 17 | - def prepare( |
| 18 | - operation, |
| 19 | - query = %Ecto.Query{ |
| 20 | - order_bys: order_bys, |
| 21 | - limit: limit |
| 22 | - } |
| 23 | - ) do |
| 24 | - ordering_fn = Ordering.get_ordering_fn(order_bys) |
| 18 | + def prepare(operation, query) do |
| 19 | + %Ecto.Query{ |
| 20 | + from: %Ecto.Query.FromExpr{source: {_source, schema}}, |
| 21 | + order_bys: order_bys, |
| 22 | + limit: limit |
| 23 | + } = query |
| 24 | + |
| 25 | + {ordering, ordering_fn} = Ordering.get_ordering_fn(schema, order_bys) |
| 25 26 | limit = get_limit(limit) |
| 26 27 | limit_fn = if limit == nil, do: & &1, else: &Stream.take(&1, limit) |
| 27 | - {:nocache, {operation, query, {limit, limit_fn}, %{}, ordering_fn}} |
| 28 | + {:nocache, {operation, query, {limit, limit_fn}, %{}, ordering, ordering_fn}} |
| 28 29 | end |
| 29 30 | |
| 30 31 | @impl Ecto.Adapter.Queryable |
| @@ -39,7 +40,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 39 40 | select: %Ecto.Query.SelectExpr{ |
| 40 41 | fields: select_fields |
| 41 42 | } |
| 42 | - }, {_limit, limit_fn}, %{}, ordering_fn}}, |
| 43 | + }, {_limit, limit_fn}, %{}, ordering, ordering_fn}}, |
| 43 44 | params, |
| 44 45 | options |
| 45 46 | ) do |
| @@ -51,7 +52,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 51 52 | _ -> |
| 52 53 | {context, query = %Ecto.Query{prefix: tenant}} = assert_tenancy!(query, adapter_opts) |
| 53 54 | |
| 54 | - future = execute_all(tenant, adapter_meta, context, query, params, options) |
| 55 | + future = execute_all(tenant, adapter_meta, context, query, params, ordering, options) |
| 55 56 | |
| 56 57 | future = |
| 57 58 | Future.apply(future, fn {objs, _continuation} -> |
| @@ -74,13 +75,13 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 74 75 | query = %Ecto.Query{ |
| 75 76 | from: %Ecto.Query.FromExpr{source: {source, schema}}, |
| 76 77 | wheres: wheres |
| 77 | - }, {nil, _limit_fn}, %{}, _ordering_fn}}, |
| 78 | + }, {nil, _limit_fn}, %{}, _ordering, _ordering_fn}}, |
| 78 79 | params, |
| 79 80 | _options |
| 80 81 | ) do |
| 81 82 | {context, %Ecto.Query{prefix: tenant}} = assert_tenancy!(query, adapter_opts) |
| 82 83 | |
| 83 | - plan = QueryPlan.get(tenant, source, schema, context, wheres, [], params) |
| 84 | + plan = QueryPlan.get(tenant, source, schema, context, wheres, [], params, []) |
| 84 85 | num = Query.delete(tenant, adapter_meta, plan) |
| 85 86 | |
| 86 87 | {num, []} |
| @@ -90,7 +91,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 90 91 | adapter_meta = %{opts: adapter_opts}, |
| 91 92 | _query_meta, |
| 92 93 | _query_cache = |
| 93 | - {:nocache, {:update_all, query, {nil, _limit_fn}, %{}, _ordering_fn}}, |
| 94 | + {:nocache, {:update_all, query, {nil, _limit_fn}, %{}, _ordering, _ordering_fn}}, |
| 94 95 | params, |
| 95 96 | _options |
| 96 97 | ) do |
| @@ -107,7 +108,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 107 108 | adapter_meta = %{opts: adapter_opts}, |
| 108 109 | _query_meta, |
| 109 110 | _query_cache = |
| 110 | - {:nocache, {:all, query, {nil, _limit_fn}, %{}, _ordering_fn}}, |
| 111 | + {:nocache, {:all, query, {nil, _limit_fn}, %{}, [], _ordering_fn}}, |
| 111 112 | params, |
| 112 113 | options |
| 113 114 | ) do |
| @@ -117,6 +118,19 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 117 118 | |> stream_all(adapter_meta, context, query, params, options) |
| 118 119 | end |
| 119 120 | |
| 121 | + def stream( |
| 122 | + _adapter_meta, |
| 123 | + _query_meta, |
| 124 | + _query_cache = |
| 125 | + {:nocache, {:all, _query, {nil, _limit_fn}, %{}, _ordering, _ordering_fn}}, |
| 126 | + _params, |
| 127 | + _options |
| 128 | + ) do |
| 129 | + raise Unsupported, """ |
| 130 | + Stream ordering is not supported. |
| 131 | + """ |
| 132 | + end |
| 133 | + |
| 120 134 | def execute_all_range(_module, _repo, queryable, id_s, id_e, {adapter_meta, options}) do |
| 121 135 | %{opts: adapter_opts} = adapter_meta |
| 122 136 | |
| @@ -191,6 +205,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 191 205 | wheres: wheres |
| 192 206 | }, |
| 193 207 | params, |
| 208 | + ordering, |
| 194 209 | options |
| 195 210 | ) do |
| 196 211 | # Steps: |
| @@ -203,7 +218,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 203 218 | # 3. Use :erlfdb.get, :erlfdb.get_range |
| 204 219 | # 4. Post-get filtering (Remove :not_found, remove index conflicts, ) |
| 205 220 | # 5. Arrange fields based on the select input |
| 206 | - plan = QueryPlan.get(tenant, source, schema, context, wheres, [], params) |
| 221 | + plan = QueryPlan.get(tenant, source, schema, context, wheres, [], params, ordering) |
| 207 222 | |
| 208 223 | Query.all(tenant, adapter_meta, plan, options) |
| 209 224 | end |
| @@ -235,7 +250,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 235 250 | }, |
| 236 251 | params |
| 237 252 | ) do |
| 238 | - plan = QueryPlan.get(tenant, source, schema, context, wheres, updates, params) |
| 253 | + plan = QueryPlan.get(tenant, source, schema, context, wheres, updates, params, []) |
| 239 254 | Query.update(tenant, adapter_meta, plan, options) |
| 240 255 | end |
| 241 256 | |
| @@ -264,7 +279,7 @@ defmodule Ecto.Adapters.FoundationDB.EctoAdapterQueryable do | |
| 264 279 | end |
| 265 280 | |
| 266 281 | start_fun = fn -> |
| 267 | - plan = QueryPlan.get(tenant, source, schema, context, wheres, [], params) |
| 282 | + plan = QueryPlan.get(tenant, source, schema, context, wheres, [], params, []) |
| 268 283 | |
| 269 284 | %{ |
| 270 285 | adapter_meta: adapter_meta, |
Loading more files…