Process lifecycle fixes and a running-config dump

.github/workflows/build.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         # only use oldest and newest version for the lint step
-        python-version: ["3.10", "3.14"]
+        python-version: ["3.13", "3.14"]
 
     steps:
 
@@ -70,7 +70,7 @@ jobs:
     strategy:
       max-parallel: 4
       matrix:
-        python-version: ["3.10", "3.14"]
+        python-version: ["3.13", "3.14"]
 
     steps:
       - id: checkout-code

Makefile

@@ -26,6 +26,10 @@ mypy:
 systest:
 	$(PYTHON) -m pytest -vv systest/
 
+.PHONY: systest-on-podman
+systest-on-podman:
+	./scripts/systest-on-podman
+
 py-egg:
 	VERSION=$(shell git describe --tags) $(PYTHON) setup.py bdist_egg
 

README.rst

@@ -40,7 +40,7 @@ For a source install the dependency `python-systemd <https://github.com/systemd/
 to be installed through your distribution's package manager (The PyPI `systemd` package is not the
 same!).
 
-journalpump requires Python 3.4 or newer.
+journalpump requires Python 3.13 or 3.14.
 
 
 Installation
@@ -91,9 +91,9 @@ configuration file.
 ``journalpump`` is the main process that should be run under systemd or
 supervisord.
 
-While journalpump is running it may be useful to read the JSON state file
-that will be created as ``journalpump_state.json`` to the current working
-directory.  The JSON state file is human readable and should give an
+While journalpump is running it may be useful to read the JSON state
+file named by ``json_state_file_path``, if that setting is present.
+The JSON state file is human readable and should give an
 understandable description of the current state of the journalpump.
 
 
@@ -103,13 +103,16 @@ Example::
 
   {
       "log_level": "INFO",
+      "msg_buffer_max_length": 50000,
+      "msg_buffer_max_bytes": 5242880,
       "field_filters": {
          ...
       },
       "unit_log_levels": {
          ...
       },
       "json_state_file_path": "/var/lib/journalpump/journalpump_state.json",
+      "write_running_config": true,
       "readers": {
          ...
       },
@@ -124,10 +127,22 @@ Example::
   }
 
 
-``json_state_file_path`` (default ``"journalpump_state.json"``)
+``json_state_file_path`` (default unset)
 
 Location of a JSON state file which describes the state of the
-journalpump process.
+journalpump process.  If omitted, journalpump does not persist
+reader and sender cursors.
+
+``write_running_config`` (default unset / false)
+
+If true, after each successful config load journalpump writes the
+configuration it is using.  If omitted or false, it writes nothing.
+
+``json_running_config_path`` (default ``$RUNTIME_DIRECTORY/running_config.json``)
+
+Path used when ``write_running_config`` is true.  If omitted,
+``RUNTIME_DIRECTORY`` must be set.  journalpump creates the directory
+with mode ``0700`` and the file with mode ``0600``.
 
 ``statsd`` (default ``null``)
 
@@ -146,6 +161,16 @@ Metrics sending follows the `Telegraf spec`_.
 
 Determines log level of journalpump. `Available log levels <https://docs.python.org/3/library/logging.html#logging-levels>`_.
 
+``msg_buffer_max_length`` (default ``50000``)
+
+How many journal entries to hold at most in each sender's memory
+buffer. Applies to every reader.
+
+``msg_buffer_max_bytes`` (default ``5242880``)
+
+How many bytes of serialized journal entries to hold at most in each
+sender's memory buffer (5 MiB). Applies to every reader.
+
 Field filter configuration
 ==========================
 
@@ -304,11 +329,6 @@ defines the value to match against.  Currently only equality is allowed.
 Note this means if you specify ``match_key`` and not ``match_value``, then the reader
 will match all entries that do not contain the ``match_key``.
 
-``msg_buffer_max_length`` (default ``50000``)
-
-How many journal entries to read at most into a memory buffer from
-which the journalpump feeds the configured logsender.
-
 ``journal_path`` (default ``null``)
 
 Path to the directory containing journal files if you want to override the

debian/control

@@ -3,9 +3,9 @@ Maintainer: Hannu Valtonen <hannu.valtonen@ohmu.fi>
 Section: python
 Priority: optional
 Build-Depends: debhelper (>= 8), dh-python, dh-systemd,
- python3-all (>= 3.7), python3-setuptools,
+ python3-all (>= 3.13), python3-setuptools,
  python3-requests, python3-systemd
-X-Python3-Version: >= 3.7
+X-Python3-Version: >= 3.13
 Standards-Version: 3.9.4
 Homepage: https://github.com/aiven/journalpump
 

journalpump.unit

@@ -9,6 +9,8 @@ Restart=always
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/bin/journalpump /var/lib/journalpump/journalpump.json
 WorkingDirectory=/var/lib/journalpump
+RuntimeDirectory=journalpump
+RuntimeDirectoryMode=0700
 
 [Install]
 WantedBy=multi-user.target

journalpump/__main__.py

@@ -3,6 +3,8 @@
 # This file is under the Apache License, Version 2.0.
 # See the file `LICENSE` for details.
 
-from . import journalpump
+from .cli import main
 
-journalpump.JournalPump.run_exit()
+import sys
+
+sys.exit(main())

journalpump/cli.py

@@ -0,0 +1,12 @@
+# Copyright 2015, Aiven, https://aiven.io/
+#
+# This file is under the Apache License, Version 2.0.
+# See the file `LICENSE` for details.
+
+from .journalpump import JournalPump
+
+import sys
+
+
+def main() -> int | None:
+    return JournalPump.main(sys.argv[1:])

journalpump/daemon.py

@@ -47,6 +47,7 @@ def __init__(
         self.require_config = require_config
         self.reload_config()
         self.running = True
+        self.reload_requested = False
 
         signal.signal(signal.SIGHUP, self.sighup)
         signal.signal(signal.SIGINT, self.sigterm)
@@ -69,8 +70,17 @@ def configure_logging(self) -> None:
             logging.root.setLevel(self.log_level)
 
     def sighup(self, signum: int, frame: FrameType | None) -> None:  # pylint: disable=unused-argument
-        self.log.info("Received SIGHUP, reloading config")
-        self.reload_config()
+        self.log.info("Received SIGHUP")
+        self.reload_requested = True
+
+    def reload_if_requested(self) -> None:
+        if not self.reload_requested:
+            return
+        self.reload_requested = False
+        try:
+            self.reload_config()
+        except Exception:  # pylint: disable=broad-except
+            self.log.exception("Reloading configuration failed")
 
     def sigterm(self, signum: int, frame: FrameType | None) -> None:  # pylint: disable=unused-argument
         self.log.info(
@@ -129,7 +139,3 @@ def main(cls, args: list[str]) -> int | None:
         finally:
             if exe:
                 exe.cleanup()
-
-    @classmethod
-    def run_exit(cls) -> None:
-        sys.exit(cls.main(sys.argv[1:]))