feat(docworker, tdk): Support Document Template Locale

.cspell/dictionary.txt

@@ -33,6 +33,25 @@ gridline
 gridlines
 pofile
 mofile
+polib
+pybabel
+gettext
+ngettext
+pgettext
+npgettext
+msgid
+msgids
+msgstr
+msgctxt
+notrimmed
+reindenting
+reindented
+urlize
+unparseable
+msginit
+localedir
+nplurals
+pluralize
 awatch
 naturalsize
 ustar
@@ -126,3 +145,8 @@ heighta
 xurl
 nosniff
 IPPROTO
+
+# Czech strings used in translation tests
+Ahoj
+Nazdar
+Svete

packages/dsw-database/CHANGELOG.md

@@ -7,6 +7,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- `language` and `pot_file_ready` fields of `document_template` and a method to update the POT file flag
+
 ### Changed
 
 - Read the `project` table column `knowledge_model_package_uuid` (renamed in the backend)

packages/dsw-database/dsw/database/database.py

@@ -50,6 +50,8 @@ class Database:
     UPDATE_DOCUMENT_FINISHED = ('UPDATE document SET finished_at = %s, state = %s, '
                                 'file_name = %s, content_type = %s, worker_log = %s, '
                                 'file_size = %s WHERE uuid = %s;')
+    UPDATE_DOCUMENT_TEMPLATE_POT_READY = ('UPDATE document_template SET pot_file_ready = %s '
+                                          'WHERE uuid = %s AND tenant_uuid = %s;')
     SELECT_TEMPLATE = ('SELECT * FROM document_template '
                        'WHERE uuid = %s AND tenant_uuid = %s LIMIT 1;')
     SELECT_TEMPLATE_FORMATS = ('SELECT * FROM document_template_format '
@@ -382,6 +384,22 @@ def update_document_finished(
             )
             return cursor.rowcount == 1
 
+    @tenacity.retry(
+        reraise=True,
+        wait=tenacity.wait_exponential(multiplier=RETRY_QUERY_MULTIPLIER),
+        stop=tenacity.stop_after_attempt(RETRY_QUERY_TRIES),
+        before=tenacity.before_log(LOG, logging.DEBUG),
+        after=tenacity.after_log(LOG, logging.DEBUG),
+    )
+    def update_document_template_pot_file_ready(self, *, template_uuid: str,
+                                                tenant_uuid: str, ready: bool) -> bool:
+        with self.conn_query.new_cursor() as cursor:
+            cursor.execute(
+                query=self.UPDATE_DOCUMENT_TEMPLATE_POT_READY,
+                params=(ready, template_uuid, tenant_uuid),
+            )
+            return cursor.rowcount == 1
+
     @tenacity.retry(
         reraise=True,
         wait=tenacity.wait_exponential(multiplier=RETRY_QUERY_MULTIPLIER),

packages/dsw-database/dsw/database/model.py

@@ -109,6 +109,8 @@ class DBDocumentTemplate:
     created_at: datetime
     updated_at: datetime
     tenant_uuid: str
+    language: str = 'en'
+    pot_file_ready: bool = False
 
     @property
     def is_draft(self):
@@ -144,6 +146,8 @@ def from_dict_row(data: dict) -> DBDocumentTemplate:
             created_at=data['created_at'],
             updated_at=data['updated_at'],
             tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)),
+            language=data.get('language', 'en'),
+            pot_file_ready=data.get('pot_file_ready', False),
         )
 
 

packages/dsw-document-worker/CHANGELOG.md

@@ -7,10 +7,28 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 
+### Added
+
+- Document template locales: a `.po` file attached to a document template version is applied at render time, so language is a parameter of document generation instead of a property of the format
+- Generation of the POT file with translatable strings (new `generatePotFile` command function), stored in S3 and flagged by `document_template.pot_file_ready`
+- `document.language` and `document.locale` in the document context
+- Translations are available to all steps via `Step.before_render` and the `gettext` / `ngettext` / `pgettext` helpers (see [Translations](./support/Translations.md))
+
 ### Changed
 
+- Update to DT metamodel 18.3
+- The `jinja2.ext.i18n` extension is always enabled for Jinja-powered steps
 - `extras.project` (and the deprecated `extras.questionnaire`) provide `knowledge_model_package_uuid` instead of `knowledge_package_uuid`
 
+### Fixed
+
+- The `policy.urlize.extra_schemes` step option is applied to the `urlize.extra_schemes` Jinja policy instead of overwriting `truncate.leeway`
+
+### Removed
+
+- The `policy.ext.i18n.trimmed` step option; `{% trans %}` blocks are now always trimmed, since the trimming decides the `msgid` and the POT file is per document template while the option was per format. Use `{% trans notrimmed %}` where the whitespace matters.
+- The experimental `i18n-dir`, `i18n-domain` and `i18n-lang` step options; they are now ignored and a template still using them renders untranslated. Use document template locales instead.
+
 
 ## [4.34.0]
 

packages/dsw-document-worker/README.md

@@ -26,6 +26,7 @@ DSW Document Worker technical documentation for template development:
 * [Document Context](./support/DocumentContext.md)
 * [Jinja Filters](./support/JinjaFilters.md)
 * [Jinja Tests](./support/JinjaTests.md)
+* [Translations](./support/Translations.md)
 
 ## Docker
 

packages/dsw-document-worker/dsw/document_worker/consts.py

@@ -5,6 +5,7 @@
 
 CMD_CHANNEL = 'doc_worker'
 CMD_COMPONENT = 'doc_worker'
+CMD_FUNCTION_GENERATE_POT_FILE = 'generatePotFile'
 COMPONENT_NAME = 'Document Worker'
 DEFAULT_ENCODING = 'utf-8'
 EXIT_SUCCESS = 0
@@ -13,8 +14,23 @@
 PLUGINS_ENTRYPOINT = 'dsw_document_worker_plugins'
 PROG_NAME = 'docworker'
 
+JINJA_EXTENSIONS = ('jinja2.ext.do', 'jinja2.ext.loopcontrols')
+JINJA_FILE_EXTENSIONS = ('.j2', '.jinja', '.jinja2', '.jnj')
+
+# Rendering and POT extraction must agree on this: the msgid of a {% trans %}
+# block depends on it, and the POT file is per document template while a step
+# option would be per format.
+JINJA_I18N_TRIMMED = True
+
+DEFAULT_LANGUAGE = 'en'
+DEFAULT_LOCALE_DOMAIN = 'default'
+LOCALE_PO_FILE_NAME = 'translation.po'
+LOCALE_MO_FILE_NAME = 'translation.mo'
+LOCALE_STAMP_FILE_NAME = 'updated_at'
+LOCALES_CACHE_DIR = '.locales'
+
 CURRENT_METAMODEL_MAJOR = 18
-CURRENT_METAMODEL_MINOR = 2
+CURRENT_METAMODEL_MINOR = 3
 
 try:
     __version__ = version(PACKAGE_NAME)

packages/dsw-document-worker/dsw/document_worker/model/context.py

@@ -1726,26 +1726,54 @@ def load(data: dict, **options):
         )
 
 
+class DocumentTemplateLocale:
+
+    def __init__(self, *, uuid: str, name: str, code: str,
+                 created_at: datetime, updated_at: datetime):
+        self.uuid = uuid
+        self.name = name
+        self.code = code
+        self.created_at = created_at
+        self.updated_at = updated_at
+
+    @staticmethod
+    def load(data: dict, **options):
+        return DocumentTemplateLocale(
+            uuid=data['uuid'],
+            name=data['name'],
+            code=data['code'],
+            created_at=_datetime(data['createdAt']),
+            updated_at=_datetime(data['updatedAt']),
+        )
+
+
 class Document:
 
     def __init__(self, *, uuid: str, name: str, document_template_uuid: str, format_uuid: str,
-                 created_by: User | None, created_at: datetime):
+                 created_by: User | None, created_at: datetime, language: str | None,
+                 locale: DocumentTemplateLocale | None):
         self.uuid = uuid
         self.name = name
         self.document_template_uuid = document_template_uuid
         self.format_uuid = format_uuid
         self.created_by = created_by
         self.created_at = created_at
+        self.language = language
+        self.locale = locale
 
     @staticmethod
     def load(data: dict, **options):
+        locale_data = data.get('locale')
         return Document(
             uuid=data['uuid'],
             name=data['name'],
             document_template_uuid=data['documentTemplateUuid'],
             format_uuid=data['formatUuid'],
             created_by=User.load(data['createdBy'], **options),
             created_at=_datetime(data['createdAt']),
+            language=data.get('language'),
+            locale=(DocumentTemplateLocale.load(locale_data, **options)
+                    if locale_data is not None else None),
         )
 
 

packages/dsw-document-worker/dsw/document_worker/plugins/specs.py

@@ -28,6 +28,12 @@ def provide_steps() -> dict[str, type[Step]]:
     `Step` class in the current implementation (use correct dsw-document-worker as
     a dependency).
 
+    Before the format pipeline runs, every step gets `before_render(render_ctx)` called
+    with the `RenderContext` of the document. The base implementation stores it, so a
+    step that does not override it still has `self.translations`, `self.language` and
+    the `self.gettext` / `self.ngettext` / `self.pgettext` helpers available. A step
+    that overrides `before_render` must call `super().before_render(render_ctx)`.
+
     :return: a dictionary of steps that the plugin can execute
     """
     return {}
@@ -59,6 +65,12 @@ def enrich_jinja_env(jinja_env: Environment, options: dict[str, str]) -> None:
     all steps that are subclass of the `JinjaPoweredStep` (mainly the `jinja` step
     implemented in class `Jinja2Step`).
 
+    The `jinja2.ext.i18n` extension is always enabled and its `gettext`, `ngettext`,
+    `pgettext` and `npgettext` globals are re-installed before every rendering. The
+    plugin must not provide globals with these names; to work with translations, it
+    should override `Step.before_render` instead.
+
     :param jinja_env: the Jinja environment to enrich
     :param options: the options provided to the step
     """
+