From 5142b9495e322799b9e353421877094775514a28 Mon Sep 17 00:00:00 2001 From: Ardiea Date: Fri, 16 Oct 2026 15:21:38 -0400 Subject: [PATCH] fix(concourse): ship worker journald units to Loki The worker AMI never installed concourse_logs.yaml: it was added inside the web-only branch of deploy.py, so vector on workers ran with host metrics and no log source. The spot-watch, lifecycle-hook, preflight and drain lines it was written to retain never left the instance, and the ASG churn diagnosis had to be reconstructed from CloudTrail. Plain-text lines (the helper scripts, the reaper, some concourse.service output) also landed without an environment field, because it was only set when .message parsed as JSON. Set it unconditionally, and record which node type emitted a line since both now share application=concourse. Co-Authored-By: Claude Opus 5.5 --- src/bilder/images/concourse/deploy.py | 7 ++++--- .../images/concourse/templates/vector/concourse_logs.yaml | 7 +++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/bilder/images/concourse/deploy.py b/src/bilder/images/concourse/deploy.py index 89fa053db0..9c486bc4b1 100644 --- a/src/bilder/images/concourse/deploy.py +++ b/src/bilder/images/concourse/deploy.py @@ -286,6 +286,10 @@ consul_configuration = {Path("00-default.json"): ConsulConfig()} vector_config = VectorConfig() +# Both node types: workers carry the spot/lifecycle/preflight helper units. +vector_config.configuration_templates[ + TEMPLATES_DIRECTORY.joinpath("vector", "concourse_logs.yaml") +] = {} if concourse_config._node_type == CONCOURSE_WEB_NODE_TYPE: # noqa: SLF001 @@ -351,9 +355,6 @@ traefik_config_changed = configure_traefik(traefik_config) # Add vector configurations specific to concourse web nodes - vector_config.configuration_templates[ - TEMPLATES_DIRECTORY.joinpath("vector", "concourse_logs.yaml") - ] = {} vector_config.configuration_templates[ TEMPLATES_DIRECTORY.joinpath("vector", "concourse_metrics.yaml") ] = {"concourse_prometheus_port": concourse_config.prometheus_bind_port} diff --git a/src/bilder/images/concourse/templates/vector/concourse_logs.yaml b/src/bilder/images/concourse/templates/vector/concourse_logs.yaml index 06372f8826..fdb4069425 100644 --- a/src/bilder/images/concourse/templates/vector/concourse_logs.yaml +++ b/src/bilder/images/concourse/templates/vector/concourse_logs.yaml @@ -19,9 +19,12 @@ transforms: if event != null { .,err = merge(., event) .@timestamp = .timestamp - .labels = ["concourse", "${ENVIRONMENT}"] - .environment = "${ENVIRONMENT}" } + .labels = ["concourse", "${ENVIRONMENT}"] + .environment = "${ENVIRONMENT}" + # concourse-web or concourse-worker; the sink's application label is + # shared by both node types. + .node_application = "${APPLICATION}" drop_unwanted_logs: type: remap