Add recompile-free extraction via persisted module attributes

CHANGELOG.md

@@ -4,6 +4,8 @@
 
   * Parallelize per-locale merging in `mix gettext.merge`.
 
+  * Add experimental `mix gettext.generate` task: messages are persisted as module attributes during normal compilation (for backends with `automatic_extraction: true` in the application environment, for example `config :gettext, MyApp.Gettext, automatic_extraction: true` in `config/dev.exs`) and read back from the compiled BEAM files, so extraction no longer needs to force-recompile the project.
+
 ## v1.0.2
 
   * Only skip manifest removal on Elixir v1.19.3+

lib/gettext/compiler.ex

@@ -61,6 +61,8 @@ defmodule Gettext.Compiler do
         Gettext.ExtractorAgent.add_backend(__MODULE__)
       end
 
+      Gettext.Extractor.persist_backend_marker(__MODULE__)
+
       # These are the two functions we generate inside the backend.
 
       @impl Gettext.Backend

lib/gettext/extractor.ex

@@ -21,6 +21,9 @@ defmodule Gettext.Extractor do
 
   @extracted_messages_flag "elixir-autogen"
 
+  @persisted_messages_attribute :__gettext_messages__
+  @persisted_backend_attribute :__gettext_backend_module__
+
   @new_pot_comment String.split(
                      """
                      # This file is a PO Template file.
@@ -69,14 +72,20 @@ defmodule Gettext.Extractor do
   Note that this function doesn't perform any operation on the filesystem.
   """
   @spec extract(
-          Macro.Env.t(),
+          Macro.Env.t() | {file :: String.t(), line :: pos_integer},
           backend :: module,
           domain :: binary | :default,
           msgctxt :: binary,
           id :: binary | {binary, binary},
           extracted_comments :: [binary]
         ) :: :ok
+  def extract(caller_or_location, backend, domain, msgctxt, id, extracted_comments)
+
   def extract(%Macro.Env{} = caller, backend, domain, msgctxt, id, extracted_comments) do
+    extract({caller.file, caller.line}, backend, domain, msgctxt, id, extracted_comments)
+  end
+
+  def extract({file, line}, backend, domain, msgctxt, id, extracted_comments) do
     format_flag = backend.__gettext__(:interpolation).message_format()
 
     domain =
@@ -89,15 +98,158 @@ defmodule Gettext.Extractor do
       create_message_struct(
         id,
         msgctxt,
-        caller.file,
-        caller.line,
+        file,
+        line,
         extracted_comments,
         format_flag
       )
 
     ExtractorAgent.add_message(backend, domain, message)
   end
 
+  @doc """
+  Tells whether gettext macros should persist the messages they extract as
+  module attributes in the module that is currently being compiled.
+
+  This is true when there is a module being compiled (`env.module` is not
+  `nil`) and the `backend` has automatic extraction enabled, that is, when the
+  application environment holds:
+
+      config :gettext, MyApp.Gettext, automatic_extraction: true
+
+  This is read from the application environment (not `Mix`), so it is `false`
+  by default and outside of Mix (for example, when modules are compiled at
+  runtime in a release).
+  """
+  @spec persisting_to_attributes?(Macro.Env.t(), backend :: module) :: boolean
+  def persisting_to_attributes?(%Macro.Env{module: nil}, _backend), do: false
+  def persisting_to_attributes?(%Macro.Env{}, backend), do: automatic_extraction?(backend)
+
+  # Tells whether the given backend has automatic extraction enabled in the
+  # application environment (read from the :gettext app, not Mix).
+  defp automatic_extraction?(backend) when is_atom(backend) do
+    Application.get_env(:gettext, backend, [])
+    |> Keyword.get(:automatic_extraction, false) == true
+  end
+
+  @doc """
+  Persists a message in a module attribute of the module currently being
+  compiled, so that it can be read back from the compiled BEAM file by
+  `fill_from_compiled_beams/1` without recompiling the module.
+  """
+  @spec persist_message(
+          Macro.Env.t(),
+          backend :: module,
+          domain :: binary | :default,
+          msgctxt :: binary,
+          id :: binary | {binary, binary},
+          extracted_comments :: [binary]
+        ) :: :ok
+  def persist_message(%Macro.Env{module: module} = env, backend, domain, msgctxt, id, comments)
+      when not is_nil(module) do
+    if not Module.has_attribute?(module, @persisted_messages_attribute) do
+      Module.register_attribute(module, @persisted_messages_attribute,
+        accumulate: true,
+        persist: true
+      )
+    end
+
+    {msgid, msgid_plural} =
+      case id do
+        {msgid, msgid_plural} -> {msgid, msgid_plural}
+        msgid when is_binary(msgid) -> {msgid, nil}
+      end
+
+    Module.put_attribute(module, @persisted_messages_attribute, %{
+      backend: backend,
+      domain: domain,
+      msgctxt: msgctxt,
+      msgid: msgid,
+      msgid_plural: msgid_plural,
+      file: env.file,
+      line: env.line,
+      comments: comments
+    })
+
+    :ok
+  end
+
+  @doc """
+  Persists a marker attribute in the Gettext backend module currently being
+  compiled, so that `fill_from_compiled_beams/1` can find all backends
+  without recompiling.
+  """
+  @spec persist_backend_marker(backend :: module) :: :ok
+  def persist_backend_marker(backend) when is_atom(backend) do
+    if automatic_extraction?(backend) do
+      Module.register_attribute(backend, @persisted_backend_attribute, persist: true)
+      Module.put_attribute(backend, @persisted_backend_attribute, true)
+    end
+
+    :ok
+  end
+
+  @doc """
+  Reads messages and backends persisted as module attributes from all the
+  BEAM files in `compile_path` and stores them in the extractor agent, as if
+  they had just been extracted during compilation.
+
+  Returns `{backends_found, messages_found}`.
+  """
+  @spec fill_from_compiled_beams(Path.t()) ::
+          {backends_found :: non_neg_integer, messages_found :: non_neg_integer}
+  def fill_from_compiled_beams(compile_path) do
+    compile_path
+    |> Path.join("*.beam")
+    |> Path.wildcard()
+    |> Enum.reduce({0, 0}, fn beam_path, {backends_acc, messages_acc} ->
+      {backends, messages} = fill_from_beam(beam_path)
+      {backends_acc + backends, messages_acc + messages}
+    end)
+  end
+
+  defp fill_from_beam(beam_path) do
+    case :beam_lib.chunks(String.to_charlist(beam_path), [:attributes]) do
+      {:ok, {module, [attributes: attributes]}} ->
+        backend? = attributes[@persisted_backend_attribute] == [true]
+
+        if backend? do
+          ExtractorAgent.add_backend(module)
+        end
+
+        entries = attributes[@persisted_messages_attribute] || []
+        messages = Enum.count(entries, &fill_message_from_entry/1)
+
+        {if(backend?, do: 1, else: 0), messages}
+
+      {:error, :beam_lib, _reason} ->
+        {0, 0}
+    end
+  end
+
+  # Entries are validated structurally so that a malformed attribute (for
+  # example, one written by an unrelated module that happens to use the same
+  # attribute name) is skipped instead of crashing the whole scan.
+  defp fill_message_from_entry(%{
+         backend: backend,
+         domain: domain,
+         msgctxt: msgctxt,
+         msgid: msgid,
+         msgid_plural: msgid_plural,
+         file: file,
+         line: line,
+         comments: comments
+       })
+       when is_atom(backend) and is_binary(msgid) and
+              (is_binary(msgid_plural) or is_nil(msgid_plural)) and
+              is_binary(file) and is_integer(line) and is_list(comments) do
+    id = if msgid_plural, do: {msgid, msgid_plural}, else: msgid
+    extract({file, line}, backend, domain, msgctxt, id, comments)
+    true
+  end
+
+  defp fill_message_from_entry(_malformed), do: false
+
   @doc """
   Returns a list of POT files based on the results of the extraction.
 

lib/gettext/macros.ex

@@ -596,16 +596,7 @@ defmodule Gettext.Macros do
     msgid = expand_to_binary(msgid, "msgid", env)
     msgctxt = expand_to_binary(msgctxt, "msgctxt", env)
 
-    if Extractor.extracting?() do
-      Extractor.extract(
-        env,
-        backend,
-        domain,
-        msgctxt,
-        msgid,
-        get_and_flush_extracted_comments()
-      )
-    end
+    extract_message(env, backend, domain, msgctxt, msgid)
 
     msgid
   end
@@ -617,20 +608,33 @@ defmodule Gettext.Macros do
     msgctxt = expand_to_binary(msgctxt, "msgctxt", env)
     msgid_plural = expand_to_binary(msgid_plural, "msgid_plural", env)
 
-    if Extractor.extracting?() do
-      Extractor.extract(
-        env,
-        backend,
-        domain,
-        msgctxt,
-        {msgid, msgid_plural},
-        get_and_flush_extracted_comments()
-      )
-    end
+    extract_message(env, backend, domain, msgctxt, {msgid, msgid_plural})
 
     {msgid, msgid_plural}
   end
 
+  defp extract_message(env, backend, domain, msgctxt, id) do
+    # Extractor.extracting?() returns nil (not false) when the extractor
+    # agent is not running, such as when compiling a dependency that uses
+    # Gettext while the :gettext application is not started.
+    extracting? = Extractor.extracting?()
+    persisting? = Extractor.persisting_to_attributes?(env, backend)
+
+    if extracting? || persisting? do
+      comments = get_and_flush_extracted_comments()
+
+      if extracting? do
+        Extractor.extract(env, backend, domain, msgctxt, id, comments)
+      end
+
+      if persisting? do
+        Extractor.persist_message(env, backend, domain, msgctxt, id, comments)
+      end
+    end
+
+    :ok
+  end
+
   defp singular_extract_and_translate(env, backend, domain, msgctxt, msgid, bindings) do
     domain = expand_domain(domain, env)
     msgid = extract_singular_translation(env, backend, domain, msgctxt, msgid)

lib/mix/tasks/gettext.extract.ex

@@ -55,6 +55,13 @@ defmodule Mix.Tasks.Gettext.Extract do
   mix gettext.extract --merge --no-fuzzy
   ```
 
+  ## Extraction Without Recompiling (Experimental)
+
+  This task always **force-recompiles** the whole project, because extraction
+  happens during the expansion of the Gettext macros. If you would rather
+  generate the POT files without a force-recompile, see `mix gettext.generate`,
+  which reads the messages back from the compiled BEAM files instead.
+
   """
 
   @switches [merge: :boolean, check_up_to_date: :boolean]
@@ -66,7 +73,14 @@ defmodule Mix.Tasks.Gettext.Extract do
     mix_config = Mix.Project.config()
     {opts, _} = OptionParser.parse!(args, switches: @switches)
     pot_files = extract(mix_config[:app], mix_config[:gettext] || [])
+    process(pot_files, opts, args)
+  end
 
+  # Shared by `mix gettext.extract` and `mix gettext.generate`:
+  # both compute `pot_files` (their only difference) and then write, check, or
+  # merge them in exactly the same way.
+  @doc false
+  def process(pot_files, opts, args) do
     if opts[:check_up_to_date] do
       run_up_to_date_check(pot_files)
     else

lib/mix/tasks/gettext.generate.ex

@@ -0,0 +1,85 @@
+defmodule Mix.Tasks.Gettext.Generate do
+  use Mix.Task
+  @recursive true
+
+  @shortdoc "Generates POT files from messages persisted during compilation"
+
+  @moduledoc """
+  Generates POT files without force-recompiling the project (experimental).
+
+  ```bash
+  mix gettext.generate [OPTIONS]
+  ```
+
+  Unlike `mix gettext.extract`, which force-recompiles the whole project so
+  that the Gettext macros run again and re-extract the messages, this task
+  generates the POT files from messages that were already extracted during the
+  normal compilation. It compiles the project normally (a no-op when it is
+  already compiled) and then reads the messages back from the persisted module
+  attributes in the compiled BEAM files, so it avoids the cost of a
+  force-recompile.
+
+  For this to work, messages must have been persisted as module attributes
+  during normal compilation. This happens automatically when the backend has
+  automatic extraction enabled in the application environment, which you
+  typically set in `config/dev.exs` so it stays off in `:prod`:
+
+      # config/dev.exs
+      config :gettext, MyApp.Gettext, automatic_extraction: true
+
+  Since the attributes are only persisted when `automatic_extraction` is
+  enabled (so not in `:prod`), release artifacts are unaffected.
+
+  This task accepts the same `--merge` and `--check-up-to-date` options as
+  `mix gettext.extract`, and forwards any other options to
+  `Mix.Tasks.Gettext.Merge`:
+
+  ```bash
+  mix gettext.generate --merge --no-fuzzy
+  mix gettext.generate --check-up-to-date
+  ```
+
+  """
+
+  @switches [merge: :boolean, check_up_to_date: :boolean]
+
+  @impl true
+  def run(args) do
+    Application.ensure_all_started(:gettext)
+    _ = Mix.Project.get!()
+    mix_config = Mix.Project.config()
+    {opts, _} = OptionParser.parse!(args, switches: @switches)
+    pot_files = generate(mix_config[:app], mix_config[:gettext] || [])
+    Mix.Tasks.Gettext.Extract.process(pot_files, opts, args)
+  end
+
+  defp generate(app, gettext_config) do
+    # The messages are extracted and persisted by the normal compilation; here
+    # we just make sure that has happened. This is a no-op when the project is
+    # already compiled.
+    Mix.Task.run("compile", [])
+
+    {backends, messages} =
+      Gettext.Extractor.fill_from_compiled_beams(Mix.Project.compile_path())
+
+    if backends == 0 and messages == 0 do
+      Mix.raise("""
+      mix gettext.generate found no persisted Gettext messages \
+      or backends in #{Path.relative_to_cwd(Mix.Project.compile_path())}.
+
+      Messages are persisted to module attributes during normal compilation only \
+      when the backend has automatic extraction enabled in the application \
+      environment, for example in config/dev.exs:
+
+          config :gettext, MyApp.Gettext, automatic_extraction: true
+
+      If you just enabled this or updated Gettext, force a recompile so that \
+      up-to-date modules get their attributes written:
+
+          mix compile --force
+      """)
+    end
+
+    Gettext.Extractor.pot_files(app, gettext_config)
+  end
+end