diff --git a/.gitignore b/.gitignore index 4d53d71..6c5cafc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ # If you run "mix test --cover", coverage assets end up here. /cover/ +# ExUnit tmp_dir scratch space. +/tmp/ + # The directory Mix downloads your dependencies sources to. /deps/ @@ -21,4 +24,3 @@ erl_crash.dump # Ignore package tarball (built via "mix hex.build"). exatomvm-*.tar - diff --git a/README.md b/README.md index 8a3ac2a..e170b12 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,68 @@ Example: Compiling 1 file (.ex) Generated my_project app +### The `atomvm.esp32.build` task + +The `atomvm.esp32.build` task builds an AtomVM ESP32 firmware image from source, with Elixir support enabled, using either a local ESP-IDF installation or the ESP-IDF Docker image. The resulting flashable image is written under `_build/atomvm_images/` and can be flashed with `mix atomvm.esp32.install`. + +If no AtomVM source is supplied, the task clones the AtomVM `main` branch automatically. Use `--atomvm-path` to build from a local checkout or `--atomvm-url`/`--ref` to build from a specific git source. + +#### Requirements + +* Erlang/OTP 27 or later, Elixir 1.18 or later, and Git +* **Without Docker:** CMake (3.13+), Ninja (preferred) or Make, and ESP-IDF (v5.5.4 or later recommended) +* **With Docker (`--use-docker`):** Docker. Note that Docker build support requires AtomVM `main` from Jan 2, 2026 or later; earlier AtomVM versions must be built with a local ESP-IDF toolchain. + +#### Options + +| Option | Default | Description | +|--------|---------|-------------| +| `--atomvm-path` | - | Path to a local AtomVM repository (overrides `--atomvm-url` if both are given) | +| `--atomvm-url` | `https://github.com/atomvm/AtomVM` | Git URL to clone AtomVM from | +| `--ref` | `main` | Git reference to check out: branch, tag, commit SHA, or PR (e.g. `pr/1234` or `pull/1234/head`) | +| `--chip` | `esp32` | Target chip(s), comma-separated for multiple (`esp32`, `esp32s2`, `esp32s3`, `esp32c2`, `esp32c3`, `esp32c6`, `esp32h2`, `esp32p4`) | +| `--idf-path` | `idf.py` | Path to the `idf.py` executable | +| `--use-docker` | `false` | Use the ESP-IDF Docker image instead of a local installation | +| `--idf-version` | `v5.5.4` | ESP-IDF version for the Docker image | +| `--clean` | `false` | Clean the build directory before building | +| `--mbedtls-prefix` | - | Path to a custom MbedTLS installation (falls back to the `MBEDTLS_PREFIX` env var) | +| `--partition-table` | - | Path to custom partition table CSV file (falls back to `custom_partitions.csv` in project root) | + +#### Custom partition table + +You can explicitly specify a custom partition table file with the `--partition-table` option: + +```shell +mix atomvm.esp32.build --partition-table path/to/partitions.csv +``` + +If the `--partition-table` option is not provided but the root of your Mix project contains `custom_partitions.csv`, it is used as the default partition table for the build. + +The custom partition file is copied into the AtomVM ESP32 platform tree only while the build runs — so Docker builds see it through the mounted AtomVM source tree — and the original partition table is restored afterwards, leaving the AtomVM checkout clean. + +> Note. When a custom partition table is used (either via `--partition-table` or default `custom_partitions.csv`), the task automatically forces a clean ESP32 platform build so CMake regenerates the partition layout — you do not need to pass `--clean` yourself. + +#### Examples + + # Build for the default esp32 chip (clones AtomVM main automatically) + shell$ mix atomvm.esp32.build + + # Build from a local AtomVM checkout for a specific chip, cleaning first + shell$ mix atomvm.esp32.build --atomvm-path /path/to/AtomVM --chip esp32s3 --clean + + # Build for multiple chips in one run + shell$ mix atomvm.esp32.build --chip esp32,esp32s3,esp32c6 + + # Build from a pull request + shell$ mix atomvm.esp32.build --ref pr/1234 + + # Build using Docker (clones AtomVM main automatically) + shell$ mix atomvm.esp32.build --use-docker --chip esp32s3 --clean + +When combining `--use-docker` with a local `--atomvm-path`, the path is bind-mounted into the container, so it must be a real path (absolute, or relative starting with `./` or `../`) rather than a bare name, which Docker would treat as a named volume. For example, if AtomVM is checked out next to your project: + + shell$ mix atomvm.esp32.build --use-docker --atomvm-path ../AtomVM --chip esp32s3 + ### The `atomvm.esp32.flash` task The `atomvm.esp32.flash` task is used to flash your application to a micro-controller and executed by the AtomVM virtual machine. diff --git a/lib/esp32_custom_partitions.ex b/lib/esp32_custom_partitions.ex new file mode 100644 index 0000000..f065214 --- /dev/null +++ b/lib/esp32_custom_partitions.ex @@ -0,0 +1,150 @@ +defmodule ExAtomVM.Esp32CustomPartitions do + @moduledoc false + + @custom_partitions_csv "custom_partitions.csv" + @atomvm_elixir_partitions_csv "partitions-elixir.csv" + + # Returns {:ok, path} if a custom partition table file is resolved, or :error. + @doc false + def custom_partitions_path(nil) do + path = Path.join(File.cwd!(), @custom_partitions_csv) + if File.exists?(path), do: {:ok, path}, else: :error + end + + @doc false + def custom_partitions_path(user_provided_path) do + path = Path.expand(user_provided_path) + if File.exists?(path), do: {:ok, path}, else: :error + end + + # Validates the project's custom_partitions.csv (if present) up front, before + # any expensive build work, so an invalid file fails fast. + @doc false + def validate_custom_partitions(user_provided_path) do + cond do + not is_nil(user_provided_path) and not File.exists?(Path.expand(user_provided_path)) -> + {:error, "Partition table file does not exist: #{user_provided_path}"} + + true -> + case custom_partitions_path(user_provided_path) do + :error -> :ok + {:ok, path} -> validate_partition_file(path) + end + end + end + + @doc false + def with_custom_partitions(platform_dir, partition_table, fun) do + case custom_partitions_path(partition_table) do + :error -> + fun.() + + {:ok, source_path} -> + copy_custom_partitions(source_path, platform_dir, fun) + end + end + + @doc false + def copy_custom_partitions(source_path, platform_dir, fun) do + dest_path = Path.join(platform_dir, @atomvm_elixir_partitions_csv) + + if same_file?(source_path, dest_path) do + IO.puts("Using custom ESP32 partition table: #{source_path}") + fun.() + else + case snapshot_file(dest_path) do + {:ok, original} -> + source_filename = Path.basename(source_path) + + IO.puts("Copying #{source_filename} to #{dest_path} for this build...") + + try do + case File.cp(source_path, dest_path) do + :ok -> + fun.() + + {:error, reason} -> + {:error, "Failed to copy #{source_filename}: #{:file.format_error(reason)}"} + end + after + restore_file(dest_path, original) + end + + {:error, reason} -> + {:error, + "Failed to read existing #{@atomvm_elixir_partitions_csv}: #{:file.format_error(reason)}"} + end + end + end + + # Returns {:ok, {:content, binary}} when the file exists, {:ok, :missing} when + # it does not, or {:error, reason} if it exists but cannot be read. + @doc false + def snapshot_file(path) do + case File.read(path) do + {:ok, content} -> {:ok, {:content, content}} + {:error, :enoent} -> {:ok, :missing} + {:error, reason} -> {:error, reason} + end + end + + @doc false + def restore_file(path, {:content, content}) do + case File.write(path, content) do + :ok -> + :ok + + {:error, reason} -> + IO.puts( + "Warning: failed to restore #{path}: #{:file.format_error(reason)} " <> + "(AtomVM checkout may be left modified)" + ) + end + end + + @doc false + def restore_file(path, :missing) do + case File.rm(path) do + :ok -> + :ok + + {:error, :enoent} -> + :ok + + {:error, reason} -> + IO.puts( + "Warning: failed to remove temporary #{path}: #{:file.format_error(reason)} " <> + "(AtomVM checkout may be left modified)" + ) + end + end + + @doc false + def same_file?(left, right) do + Path.expand(left) == Path.expand(right) or + with {:ok, l} <- File.stat(left), + {:ok, r} <- File.stat(right) do + l.inode != 0 and + {l.major_device, l.minor_device, l.inode} == + {r.major_device, r.minor_device, r.inode} + else + _ -> false + end + end + + defp validate_partition_file(path) do + case File.stat(path) do + {:ok, %File.Stat{type: :regular, size: 0}} -> + {:error, "#{Path.basename(path)} is empty"} + + {:ok, %File.Stat{type: :regular}} -> + :ok + + {:ok, _stat} -> + {:error, "#{Path.basename(path)} exists but is not a regular file"} + + {:error, reason} -> + {:error, "cannot read #{Path.basename(path)}: #{:file.format_error(reason)}"} + end + end +end diff --git a/lib/mix/tasks/esp32.build.ex b/lib/mix/tasks/esp32.build.ex index 20ca29c..ba2bced 100644 --- a/lib/mix/tasks/esp32.build.ex +++ b/lib/mix/tasks/esp32.build.ex @@ -7,8 +7,8 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do ## Requirements **General requirements** - * Erlang/OTP (25 or later) - * Elixir (1.16 or later) + * Erlang/OTP (27 or later) + * Elixir (1.18 or later) * Git **Without Docker:** @@ -32,6 +32,10 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do * `--idf-version` - ESP-IDF version for Docker image (default: v5.5.4) * `--clean` - Clean build directory before building * `--mbedtls-prefix` - Path to custom MbedTLS installation (optional, falls back to MBEDTLS_PREFIX env var) + * `--partition-table` - Path to custom partition table CSV file (optional, defaults to custom_partitions.csv if present) + + If `--partition-table` is provided, or if your Mix project root contains `custom_partitions.csv`, + it will be used as the ESP32 partition table for the build. ## Examples @@ -70,6 +74,7 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do """ use Mix.Task + alias ExAtomVM.Esp32CustomPartitions @shortdoc "Build AtomVM for ESP32 from source" @@ -93,7 +98,8 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do use_docker: :boolean, idf_version: :string, clean: :boolean, - mbedtls_prefix: :string + mbedtls_prefix: :string, + partition_table: :string ] ) @@ -104,6 +110,7 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do use_docker = Keyword.get(opts, :use_docker, false) idf_version = Keyword.get(opts, :idf_version, @default_idf_version) clean = Keyword.get(opts, :clean, false) + partition_table = Keyword.get(opts, :partition_table) chips = opts @@ -145,9 +152,22 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do """) - with :ok <- check_esp_idf(idf_path, use_docker, idf_version), + with :ok <- Esp32CustomPartitions.validate_custom_partitions(partition_table), + :ok <- check_esp_idf(idf_path, use_docker, idf_version), :ok <- check_escript(), :ok <- ExAtomVM.AtomVMBuilder.build_generic_unix(atomvm_path, mbedtls_prefix, clean) do + custom_partitions? = + match?({:ok, _}, Esp32CustomPartitions.custom_partitions_path(partition_table)) + + if custom_partitions? and not clean do + {:ok, path} = Esp32CustomPartitions.custom_partitions_path(partition_table) + filename = Path.basename(path) + + IO.puts( + "#{filename} detected; forcing clean ESP32 platform build so partition metadata is regenerated..." + ) + end + results = chips |> Enum.with_index(1) @@ -156,9 +176,17 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do IO.puts("\n━━━ Building chip #{index}/#{length(chips)}: #{chip} ━━━\n") end - force_clean = index > 1 or clean - - case build_atomvm(atomvm_path, chip, idf_path, idf_version, use_docker, force_clean) do + force_clean = clean or index > 1 or custom_partitions? + + case build_atomvm( + atomvm_path, + chip, + idf_path, + idf_version, + use_docker, + force_clean, + partition_table + ) do {:ok, src_img} -> img = save_image(src_img) {chip, :ok, img} @@ -283,7 +311,7 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do end end - defp build_atomvm(atomvm_path, chip, idf_path, idf_version, use_docker, clean) do + defp build_atomvm(atomvm_path, chip, idf_path, idf_version, use_docker, clean, partition_table) do build_dir = Path.join([atomvm_path, "src", "platforms", "esp32", "build"]) platform_dir = Path.join([atomvm_path, "src", "platforms", "esp32"]) @@ -317,55 +345,57 @@ defmodule Mix.Tasks.Atomvm.Esp32.Build do File.cp!(dependencies_lock, dest_path) end - if clean and File.dir?(build_dir) do - IO.puts("Cleaning build directory...") - ExAtomVM.AtomVMBuilder.clean_dir(build_dir) - end - - IO.puts("Configuring build for #{chip}...") - - {_output, status} = - run_idf_command( - use_docker, - idf_version, - atomvm_path, - platform_dir, - idf_path, - idf_set_target_args(chip) - ) + Esp32CustomPartitions.with_custom_partitions(platform_dir, partition_table, fn -> + if clean and File.dir?(build_dir) do + IO.puts("Cleaning build directory...") + ExAtomVM.AtomVMBuilder.clean_dir(build_dir) + end - case status do - 0 -> - IO.puts("Building AtomVM... (this may take several minutes)") - - {_output, build_status} = - run_idf_command( - use_docker, - idf_version, - atomvm_path, - platform_dir, - idf_path, - idf_build_args() - ) + IO.puts("Configuring build for #{chip}...") + + {_output, status} = + run_idf_command( + use_docker, + idf_version, + atomvm_path, + platform_dir, + idf_path, + idf_set_target_args(chip) + ) + + case status do + 0 -> + IO.puts("Building AtomVM... (this may take several minutes)") + + {_output, build_status} = + run_idf_command( + use_docker, + idf_version, + atomvm_path, + platform_dir, + idf_path, + idf_build_args() + ) - case build_status do - 0 -> - copy_dependencies_lock(platform_dir) + case build_status do + 0 -> + copy_dependencies_lock(platform_dir) - create_flashable_image( - Path.expand(atomvm_path), - Path.expand(build_dir), - chip, - use_docker - ) + create_flashable_image( + Path.expand(atomvm_path), + Path.expand(build_dir), + chip, + use_docker + ) - _status -> - {:error, "Build failed"} - end + _status -> + {:error, "Build failed"} + end - _status -> - {:error, "Failed to set target chip"} - end + _status -> + {:error, "Failed to set target chip"} + end + end) end defp idf_set_target_args(chip) do diff --git a/test/esp32_custom_partitions_test.exs b/test/esp32_custom_partitions_test.exs new file mode 100644 index 0000000..8e34b6b --- /dev/null +++ b/test/esp32_custom_partitions_test.exs @@ -0,0 +1,151 @@ +defmodule ExAtomVM.Esp32CustomPartitionsTest do + use ExUnit.Case, async: false + + import ExUnit.CaptureIO + + alias ExAtomVM.Esp32CustomPartitions + + @moduletag :tmp_dir + + setup %{tmp_dir: tmp_dir} do + tmp_dir = Path.expand(tmp_dir) + tmp_dir_parent = Path.dirname(tmp_dir) + + on_exit(fn -> + File.rm_rf(tmp_dir) + File.rmdir(tmp_dir_parent) + end) + + {:ok, tmp_dir: tmp_dir} + end + + test "custom_partitions_path/1 auto-discovers default file in cwd", %{tmp_dir: tmp_dir} do + File.cd!(tmp_dir, fn -> + assert Esp32CustomPartitions.custom_partitions_path(nil) == :error + + path = Path.join(tmp_dir, "custom_partitions.csv") + File.write!(path, "custom table") + + assert Esp32CustomPartitions.custom_partitions_path(nil) == {:ok, path} + end) + end + + test "validate_custom_partitions/1 rejects missing, empty, and directory paths", %{ + tmp_dir: tmp_dir + } do + missing_path = Path.join(tmp_dir, "missing.csv") + + assert {:error, "Partition table file does not exist: " <> _} = + Esp32CustomPartitions.validate_custom_partitions(missing_path) + + File.cd!(tmp_dir, fn -> + File.write!("custom_partitions.csv", "") + + assert {:error, "custom_partitions.csv is empty"} = + Esp32CustomPartitions.validate_custom_partitions(nil) + end) + + directory_path = Path.join(tmp_dir, "partitions.csv") + File.mkdir!(directory_path) + + assert {:error, "partitions.csv exists but is not a regular file"} = + Esp32CustomPartitions.validate_custom_partitions(directory_path) + end + + test "validate_custom_partitions/1 accepts a non-empty custom partition table", %{ + tmp_dir: tmp_dir + } do + path = Path.join(tmp_dir, "partitions.csv") + File.write!(path, "# Name, Type, SubType, Offset, Size\n") + + assert Esp32CustomPartitions.validate_custom_partitions(path) == :ok + end + + test "copy_custom_partitions/3 copies source and restores existing destination", %{ + tmp_dir: tmp_dir + } do + platform_dir = Path.join(tmp_dir, "platform") + File.mkdir_p!(platform_dir) + + source_path = Path.join(tmp_dir, "my_partitions.csv") + dest_path = Path.join(platform_dir, "partitions-elixir.csv") + + File.write!(source_path, "custom table") + File.write!(dest_path, "original table") + + output = + capture_io(fn -> + assert Esp32CustomPartitions.copy_custom_partitions(source_path, platform_dir, fn -> + assert File.read!(dest_path) == "custom table" + :ok + end) == :ok + end) + + assert output =~ "Copying my_partitions.csv" + assert File.read!(dest_path) == "original table" + end + + test "copy_custom_partitions/3 removes destination when it did not exist before copying", %{ + tmp_dir: tmp_dir + } do + platform_dir = Path.join(tmp_dir, "platform") + File.mkdir_p!(platform_dir) + + source_path = Path.join(tmp_dir, "custom_partitions.csv") + dest_path = Path.join(platform_dir, "partitions-elixir.csv") + + File.write!(source_path, "custom table") + + capture_io(fn -> + assert Esp32CustomPartitions.copy_custom_partitions(source_path, platform_dir, fn -> + assert File.read!(dest_path) == "custom table" + :ok + end) == :ok + end) + + refute File.exists?(dest_path) + end + + test "copy_custom_partitions/3 reports copy failures with the source filename", %{ + tmp_dir: tmp_dir + } do + platform_dir = Path.join(tmp_dir, "platform") + File.mkdir_p!(platform_dir) + + source_path = Path.join(tmp_dir, "missing_partitions.csv") + + capture_io(fn -> + assert {:error, message} = + Esp32CustomPartitions.copy_custom_partitions(source_path, platform_dir, fn -> + flunk("callback should not run when copying fails") + end) + + assert message =~ "Failed to copy missing_partitions.csv" + end) + end + + test "restore_file/2 restores content and removes previously missing files", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "partitions-elixir.csv") + + File.write!(path, "temporary") + assert Esp32CustomPartitions.restore_file(path, {:content, "original"}) == :ok + assert File.read!(path) == "original" + + assert Esp32CustomPartitions.restore_file(path, :missing) == :ok + refute File.exists?(path) + end + + test "same_file?/2 detects path equality and inode equality", %{tmp_dir: tmp_dir} do + path = Path.join(tmp_dir, "partitions.csv") + linked_path = Path.join(tmp_dir, "linked_partitions.csv") + other_path = Path.join(tmp_dir, "other_partitions.csv") + + File.write!(path, "table") + File.write!(other_path, "other") + File.ln!(path, linked_path) + + assert Esp32CustomPartitions.same_file?(path, path) + assert Esp32CustomPartitions.same_file?(path, linked_path) + refute Esp32CustomPartitions.same_file?(path, other_path) + end +end