Current section
11 Versions
Jump to
Current section
11 Versions
Compare versions
7
files changed
+170
additions
-104
deletions
| @@ -1,28 +1,36 @@ | |
| 1 1 | # aws_gleam_s3 |
| 2 2 | |
| 3 | - Typed Gleam client for AWS s3. Auto-generated from the |
| 4 | - upstream Smithy model in [aws-gleam](https://github.com/Ulberg/aws-gleam). |
| 3 | + Typed Gleam client for Amazon S3. The generated client comes from the upstream |
| 4 | + Smithy model in [aws-gleam](https://github.com/Ulberg/aws-gleam); this package |
| 5 | + also includes S3-specific helpers under `aws/s3/` for streaming and multipart |
| 6 | + upload. |
| 5 7 | |
| 6 8 | ```gleam |
| 7 9 | import aws/services/s3 |
| 8 10 | |
| 9 11 | pub fn main() { |
| 10 12 | let assert Ok(client) = s3.new() |
| 11 | - // ... typed ops, e.g. s3.<op>(client, input) |
| 13 | + // Call generated operations with typed input records. |
| 14 | + s3.shutdown(client) |
| 12 15 | } |
| 13 16 | ``` |
| 14 17 | |
| 15 | - Depends on [`aws_gleam_runtime`](https://hex.pm/packages/aws_gleam_runtime) |
| 16 | - for SigV4 signing, credentials, endpoint resolution, retry, and |
| 17 | - the protocol codecs. Each AWS service ships as a separate hex |
| 18 | - package so consumers only compile the services they import; the |
| 19 | - SDK's full set of ~409 generated services lives at |
| 20 | - <https://github.com/Ulberg/aws-gleam/tree/main/services>. |
| 18 | + Use `s3.new_with(settings, endpoint_params)` for explicit `aws/config.Settings` |
| 19 | + and S3 endpoint-rule-set parameters such as `UseFIPS`, `UseDualStack`, and |
| 20 | + `ForcePathStyle`. |
| 21 | + |
| 22 | + Depends on [`aws_gleam_runtime`](https://hex.pm/packages/aws_gleam_runtime) for |
| 23 | + signing, credentials, endpoint resolution, retry, transport, streaming, and |
| 24 | + protocol codecs. |
| 21 25 | |
| 22 26 | ## Documentation |
| 23 27 | |
| 24 28 | Full docs at <https://hexdocs.pm/aws_gleam_s3>. |
| 25 29 | |
| 30 | + ## Source |
| 31 | + |
| 32 | + <https://github.com/Ulberg/aws-gleam/tree/main/services/s3> |
| 33 | + |
| 26 34 | ## License |
| 27 35 | |
| 28 36 | Apache 2.0. See LICENSE. |
| @@ -1,8 +1,8 @@ | |
| 1 1 | name = "aws_gleam_s3" |
| 2 | - version = "1.4.0" |
| 2 | + version = "1.4.1" |
| 3 3 | target = "erlang" |
| 4 4 | |
| 5 | - description = "Typed Gleam client for Amazon S3. Generated from the upstream Smithy model + hand-written multipart-upload helpers under aws/s3/. Depends on aws_runtime for SigV4 signing, the credentials chain, endpoint resolution, retry, and the protocol codecs." |
| 5 | + description = "Typed Gleam client for Amazon S3. Generated from the upstream Smithy model with S3 helpers under aws/s3/. Depends on aws_gleam_runtime." |
| 6 6 | |
| 7 7 | licences = ["Apache-2.0"] |
| 8 8 | repository = { type = "github", user = "Ulberg", repo = "aws-gleam" } |
| @@ -11,7 +11,7 @@ links = [ | |
| 11 11 | ] |
| 12 12 | |
| 13 13 | [dependencies] |
| 14 | - aws_gleam_runtime = ">= 1.4.0" |
| 14 | + aws_gleam_runtime = ">= 1.4.1" |
| 15 15 | gleam_stdlib = ">= 0.40.0 and < 2.0.0" |
| 16 16 | gleam_otp = ">= 1.0.0 and < 2.0.0" |
| 17 17 | gleam_erlang = ">= 1.0.0 and < 2.0.0" |
| @@ -1,7 +1,7 @@ | |
| 1 1 | {<<"name">>, <<"aws_gleam_s3"/utf8>>}. |
| 2 2 | {<<"app">>, <<"aws_gleam_s3"/utf8>>}. |
| 3 | - {<<"version">>, <<"1.4.0"/utf8>>}. |
| 4 | - {<<"description">>, <<"Typed Gleam client for Amazon S3. Generated from the upstream Smithy model + hand-written multipart-upload helpers under aws/s3/. Depends on aws_runtime for SigV4 signing, the credentials chain, endpoint resolution, retry, and the protocol codecs."/utf8>>}. |
| 3 | + {<<"version">>, <<"1.4.1"/utf8>>}. |
| 4 | + {<<"description">>, <<"Typed Gleam client for Amazon S3. Generated from the upstream Smithy model with S3 helpers under aws/s3/. Depends on aws_gleam_runtime."/utf8>>}. |
| 5 5 | {<<"licenses">>, [<<"Apache-2.0"/utf8>>]}. |
| 6 6 | {<<"build_tools">>, [<<"gleam"/utf8>>]}. |
| 7 7 | {<<"links">>, [ |
| @@ -12,33 +12,33 @@ | |
| 12 12 | {<<"aws_gleam_runtime"/utf8>>, [ |
| 13 13 | {<<"app">>, <<"aws_gleam_runtime"/utf8>>}, |
| 14 14 | {<<"optional">>, false}, |
| 15 | - {<<"requirement">>, <<">= 1.4.0"/utf8>>} |
| 16 | - ]}, |
| 17 | - {<<"gleam_httpc"/utf8>>, [ |
| 18 | - {<<"app">>, <<"gleam_httpc"/utf8>>}, |
| 19 | - {<<"optional">>, false}, |
| 20 | - {<<"requirement">>, <<">= 5.0.0 and < 6.0.0"/utf8>>} |
| 21 | - ]}, |
| 22 | - {<<"gleam_stdlib"/utf8>>, [ |
| 23 | - {<<"app">>, <<"gleam_stdlib"/utf8>>}, |
| 24 | - {<<"optional">>, false}, |
| 25 | - {<<"requirement">>, <<">= 0.40.0 and < 2.0.0"/utf8>>} |
| 15 | + {<<"requirement">>, <<">= 1.4.1"/utf8>>} |
| 26 16 | ]}, |
| 27 17 | {<<"gleam_otp"/utf8>>, [ |
| 28 18 | {<<"app">>, <<"gleam_otp"/utf8>>}, |
| 29 19 | {<<"optional">>, false}, |
| 30 20 | {<<"requirement">>, <<">= 1.0.0 and < 2.0.0"/utf8>>} |
| 31 21 | ]}, |
| 32 | - {<<"gleam_http"/utf8>>, [ |
| 33 | - {<<"app">>, <<"gleam_http"/utf8>>}, |
| 22 | + {<<"gleam_stdlib"/utf8>>, [ |
| 23 | + {<<"app">>, <<"gleam_stdlib"/utf8>>}, |
| 34 24 | {<<"optional">>, false}, |
| 35 | - {<<"requirement">>, <<">= 4.0.0 and < 5.0.0"/utf8>>} |
| 25 | + {<<"requirement">>, <<">= 0.40.0 and < 2.0.0"/utf8>>} |
| 36 26 | ]}, |
| 37 27 | {<<"gleam_json"/utf8>>, [ |
| 38 28 | {<<"app">>, <<"gleam_json"/utf8>>}, |
| 39 29 | {<<"optional">>, false}, |
| 40 30 | {<<"requirement">>, <<">= 2.0.0 and < 4.0.0"/utf8>>} |
| 41 31 | ]}, |
| 32 | + {<<"gleam_http"/utf8>>, [ |
| 33 | + {<<"app">>, <<"gleam_http"/utf8>>}, |
| 34 | + {<<"optional">>, false}, |
| 35 | + {<<"requirement">>, <<">= 4.0.0 and < 5.0.0"/utf8>>} |
| 36 | + ]}, |
| 37 | + {<<"gleam_httpc"/utf8>>, [ |
| 38 | + {<<"app">>, <<"gleam_httpc"/utf8>>}, |
| 39 | + {<<"optional">>, false}, |
| 40 | + {<<"requirement">>, <<">= 5.0.0 and < 6.0.0"/utf8>>} |
| 41 | + ]}, |
| 42 42 | {<<"gleam_erlang"/utf8>>, [ |
| 43 43 | {<<"app">>, <<"gleam_erlang"/utf8>>}, |
| 44 44 | {<<"optional">>, false}, |
| @@ -51,6 +51,7 @@ | |
| 51 51 | <<"gleam.toml"/utf8>>, |
| 52 52 | <<"include/aws@s3@transfer_CompleteFailed.hrl"/utf8>>, |
| 53 53 | <<"include/aws@s3@transfer_CreateFailed.hrl"/utf8>>, |
| 54 | + <<"include/aws@s3@transfer_InvalidPartSize.hrl"/utf8>>, |
| 54 55 | <<"include/aws@s3@transfer_UploadOptions.hrl"/utf8>>, |
| 55 56 | <<"include/aws@s3@transfer_UploadPartFailed.hrl"/utf8>>, |
| 56 57 | <<"include/aws@s3@transfer_UploadResult.hrl"/utf8>>, |
| @@ -0,0 +1 @@ | |
| 1 | + -record(invalid_part_size, {part_size_bytes :: integer()}). |
| @@ -38,13 +38,16 @@ import gleam/result | |
| 38 38 | /// `MissingUploadId` fires if S3's `CreateMultipartUpload` response |
| 39 39 | /// arrives without an `upload_id` (should never happen in |
| 40 40 | /// production, but the wire-type is `Option(String)` so we surface |
| 41 | - /// it explicitly rather than `assert`ing). |
| 41 | + /// it explicitly rather than `assert`ing). `InvalidPartSize` rejects |
| 42 | + /// caller-supplied part sizes outside S3's documented bounds before |
| 43 | + /// any multipart-upload request reaches the bucket. |
| 42 44 | pub type Error { |
| 43 45 | CreateFailed(cause: s3.CreateMultipartUploadError) |
| 44 46 | UploadPartFailed(part_number: Int, cause: s3.UploadPartError) |
| 45 47 | CompleteFailed(cause: s3.CompleteMultipartUploadError) |
| 46 48 | MissingUploadId |
| 47 49 | EmptyBody |
| 50 | + InvalidPartSize(part_size_bytes: Int) |
| 48 51 | } |
| 49 52 | |
| 50 53 | /// Result of a successful multipart upload. `upload_id` is exposed |
| @@ -140,6 +143,12 @@ pub fn with_max_concurrency(opts: UploadOptions, n: Int) -> UploadOptions { | |
| 140 143 | /// per-part round trips but raise outstanding-request memory. |
| 141 144 | pub const default_part_size_bytes: Int = 5_242_880 |
| 142 145 | |
| 146 | + /// S3's maximum size for any individual multipart-upload part |
| 147 | + /// (5 GiB). `upload` / `upload_from_stream` reject larger |
| 148 | + /// `part_size_bytes` values up front so callers get a typed SDK |
| 149 | + /// error instead of a later S3 service error. |
| 150 | + pub const max_part_size_bytes: Int = 5_368_709_120 |
| 151 | + |
| 143 152 | /// S3's hard cap on parts per multipart upload. Past 10,000 the |
| 144 153 | /// `Complete` call returns `InvalidArgument` regardless of total |
| 145 154 | /// size, so `part_size_for` scales `part_size_bytes` up for large |
| @@ -184,6 +193,10 @@ pub fn part_size_for(total_bytes: Int) -> Int { | |
| 184 193 | /// silenced — the caller already has the more interesting error |
| 185 194 | /// from the step that failed. |
| 186 195 | /// |
| 196 | + /// `part_size_bytes` must be between |
| 197 | + /// `default_part_size_bytes` (5 MiB) and `max_part_size_bytes` |
| 198 | + /// (5 GiB), inclusive. Invalid values return |
| 199 | + /// `Error(InvalidPartSize(part_size_bytes))` before any HTTP work. |
| 187 200 | /// An empty body returns `Error(EmptyBody)`; S3 rejects empty |
| 188 201 | /// multipart uploads with `EntityTooSmall`, so we short-circuit |
| 189 202 | /// before the create round trip. |
| @@ -217,6 +230,7 @@ pub fn upload_with_options( | |
| 217 230 | part_size_bytes part_size_bytes: Int, |
| 218 231 | options options: UploadOptions, |
| 219 232 | ) -> Result(UploadResult, Error) { |
| 233 | + use _ <- result.try(validate_part_size(part_size_bytes)) |
| 220 234 | case bit_array.byte_size(body) { |
| 221 235 | 0 -> Error(EmptyBody) |
| 222 236 | _ -> |
| @@ -240,10 +254,15 @@ pub fn upload_with_options( | |
| 240 254 | /// |
| 241 255 | /// Today both `StreamingBody` representations (Buffered / Chunked) |
| 242 256 | /// hold their full bytes in memory, so this variant doesn't yet |
| 243 | - /// reduce peak memory vs `upload(buffer_to_bit_array(body), ...)`. |
| 257 | + /// reduce peak memory vs `upload(streaming.to_bit_array(body), ...)`. |
| 244 258 | /// Once `StreamingBody` grows a lazy `Source(...)` variant (file |
| 245 259 | /// handles, generators), this path picks up true bounded-memory |
| 246 260 | /// streaming for free. |
| 261 | + /// |
| 262 | + /// `part_size_bytes` has the same validation as `upload`: it must |
| 263 | + /// be between 5 MiB and 5 GiB, inclusive, or the function returns |
| 264 | + /// `Error(InvalidPartSize(part_size_bytes))` before reading chunks |
| 265 | + /// or creating the multipart upload. |
| 247 266 | pub fn upload_from_stream( |
| 248 267 | client client: s3.Client, |
| 249 268 | bucket bucket: String, |
| @@ -271,6 +290,7 @@ pub fn upload_from_stream_with_options( | |
| 271 290 | part_size_bytes part_size_bytes: Int, |
| 272 291 | options options: UploadOptions, |
| 273 292 | ) -> Result(UploadResult, Error) { |
| 293 | + use _ <- result.try(validate_part_size(part_size_bytes)) |
| 274 294 | let parts = rechunk_to_parts(body, part_size_bytes) |
| 275 295 | case list.is_empty(parts) { |
| 276 296 | True -> Error(EmptyBody) |
| @@ -278,6 +298,13 @@ pub fn upload_from_stream_with_options( | |
| 278 298 | } |
| 279 299 | } |
| 280 300 | |
| 301 | + fn validate_part_size(part_size: Int) -> Result(Nil, Error) { |
| 302 | + case part_size >= default_part_size_bytes && part_size <= max_part_size_bytes { |
| 303 | + True -> Ok(Nil) |
| 304 | + False -> Error(InvalidPartSize(part_size_bytes: part_size)) |
| 305 | + } |
| 306 | + } |
| 307 | + |
| 281 308 | fn coordinate( |
| 282 309 | client: s3.Client, |
| 283 310 | bucket: String, |
Loading more files…