iree_tokenizers

defmodule IREETokenizers.MixProject do
  use Mix.Project

  @source_url "https://github.com/goodhamgupta/iree_tokenizers"
  @version "0.2.0"

  def project do
    [
      app: :iree_tokenizers,
      version: @version,
      elixir: "~> 1.19",
      start_permanent: Mix.env() == :prod,
      deps: deps(),
      package: package(),
      docs: docs(),
      description:
        "Fast Hugging Face tokenizer.json and OpenAI tiktoken bindings for Elixir via the IREE tokenizer runtime"
    ]
  end

  def application do
    [
      extra_applications: [:logger, :inets, :ssl, :public_key]
    ]
  end

  def cli do
    [
      preferred_envs: [
        docs: :docs,
        "hex.publish": :docs
      ]
    ]
  end

  defp deps do
    [
      {:castore, "~> 0.1 or ~> 1.0"},
      {:ex_doc, "~> 0.38", only: :docs, runtime: false},
      {:jason, "~> 1.4"},
      {:rustler, "~> 0.37.3", optional: true, runtime: false},
      {:rustler_precompiled, "~> 0.8"},
      {:tokenizers, "~> 0.5.1", only: :test}
    ]
  end

  defp package do
    [
      licenses: ["Apache-2.0"],
      maintainers: ["Shubham Gupta"],
      links: %{"GitHub" => @source_url},
      files: package_files()
    ]
  end

  defp docs do
    [
      main: "IREE.Tokenizers",
      source_ref: "v#{@version}",
      source_url: @source_url,
      groups_for_modules: [
        Tokenization: [
          IREE.Tokenizers.Tokenizer,
          IREE.Tokenizers.Encoding,
          IREE.Tokenizers.Encoding.Transformation,
          IREE.Tokenizers.Model,
          IREE.Tokenizers.Model.BPE,
          IREE.Tokenizers.Model.WordPiece,
          IREE.Tokenizers.Model.Unigram,
          IREE.Tokenizers.EncodeStream,
          IREE.Tokenizers.DecodeStream
        ],
        Other: [
          IREE.Tokenizers.HTTPClient
        ]
      ]
    ]
  end

  defp package_files do
    [
      ".formatter.exs",
      ".tool-versions",
      "LICENSE",
      "README.md",
      "mix.exs",
      "mix.lock",
      "scripts/update_iree_bundle.sh",
      "lib/iree/tokenizers.ex",
      "lib/iree/tokenizers/decode_stream.ex",
      "lib/iree/tokenizers/encode_stream.ex",
      "lib/iree/tokenizers/encoding.ex",
      "lib/iree/tokenizers/encoding/transformation.ex",
      "lib/iree/tokenizers/component_registry.ex",
      "lib/iree/tokenizers/http_client.ex",
      "lib/iree/tokenizers/model.ex",
      "lib/iree/tokenizers/model/bpe.ex",
      "lib/iree/tokenizers/model/unigram.ex",
      "lib/iree/tokenizers/model/wordpiece.ex",
      "lib/iree/tokenizers/native.ex",
      "lib/iree/tokenizers/tokenizer.ex",
      "native/iree_tokenizers_native/Cargo.lock",
      "native/iree_tokenizers_native/Cargo.toml",
      "native/iree_tokenizers_native/build.rs",
      "native/iree_tokenizers_native/sources/base_sources.txt",
      "native/iree_tokenizers_native/sources/tokenizer_sources.txt",
      "native/iree_tokenizers_native/src/error.rs",
      "native/iree_tokenizers_native/src/ffi.rs",
      "native/iree_tokenizers_native/src/lib.rs",
      "native/iree_tokenizers_native/src/stream.rs",
      "native/iree_tokenizers_native/src/tokenizer.rs",
      "native/iree_tokenizers_native/vendor/IREE_COMMIT",
      "native/iree_tokenizers_native/vendor/iree_tokenizer_src/IREE-LICENSE"
    ] ++
      Path.wildcard("native/iree_tokenizers_native/vendor/iree_tokenizer_src/iree/**/*.{c,h}") ++
      Path.wildcard("test/fixtures/*")
  end
end