Attempt to catch crashes with backtrace script

Author: alan-george-lk

.github/workflows/ci.yml

@@ -57,6 +57,7 @@ jobs:
               - client-sdk-rust/**
               - cmake/**
               - .token_helpers/**
+              - scripts/run-with-backtrace.sh
               - CMakeLists.txt
               - CMakePresets.json
               - build*

.github/workflows/tests.yml

@@ -117,6 +117,7 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: full
   # Disable Cargo incremental artifacts. The Rust FFI is built once per
   # submodule SHA and cached whole (see the cargo target cache below), so
   # incremental dirs add nothing but bloat the cached target/ directory and
@@ -220,7 +221,7 @@ jobs:
             libabsl-dev \
             libcurl4-openssl-dev \
             libwayland-dev libdecor-0-dev \
-            jq
+            gdb jq
 
       - name: Install deps (macOS)
         if: runner.os == 'macOS'
@@ -285,6 +286,16 @@ jobs:
           LLVM_VERSION=$(llvm-config --version | cut -d. -f1)
           echo "LIBCLANG_PATH=/usr/lib/llvm-${LLVM_VERSION}/lib" >> "$GITHUB_ENV"
 
+      - name: Configure Unix crash diagnostics
+        if: runner.os != 'Windows'
+        shell: bash
+        run: |
+          echo "LIVEKIT_CRASH_DIAGNOSTICS=1" >> "$GITHUB_ENV"
+          if [[ "$RUNNER_OS" == "Linux" ]]; then
+            echo "LIVEKIT_CORE_DIR=${RUNNER_TEMP}" >> "$GITHUB_ENV"
+            sudo sysctl -w "kernel.core_pattern=${RUNNER_TEMP}/livekit-core.%p"
+          fi
+
       # ---------- Build (tests on, examples off) ----------
       - name: Build tests (Unix)
         if: runner.os != 'Windows'
@@ -309,7 +320,8 @@ jobs:
         timeout-minutes: ${{ inputs.unit_timeout_minutes }}
         shell: bash
         run: |
-          ${{ env.BUILD_DIR }}/bin/livekit_unit_tests \
+          bash scripts/run-with-backtrace.sh \
+            ${{ env.BUILD_DIR }}/bin/livekit_unit_tests \
             --gtest_repeat=${{ inputs.unit_repeat }} \
             --gtest_brief=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/unit-test-results.xml
@@ -383,7 +395,8 @@ jobs:
           set -euo pipefail
           source .token_helpers/set_data_track_test_tokens.bash
           # TODO(BOT-477) Temporarily disable platform audio integration tests in CI until instability is resolved
-          ${{ env.BUILD_DIR }}/bin/livekit_integration_tests \
+          bash scripts/run-with-backtrace.sh \
+            ${{ env.BUILD_DIR }}/bin/livekit_integration_tests \
             --gtest_filter='-PlatformAudioIntegrationTest.*' \
             --gtest_repeat=${{ inputs.integration_repeat }} \
             --gtest_recreate_environments_when_repeating=1 \
@@ -398,7 +411,8 @@ jobs:
         run: |
           set -euo pipefail
           source .token_helpers/set_data_track_test_tokens.bash
-          ${{ env.BUILD_DIR }}/bin/livekit_stress_tests \
+          bash scripts/run-with-backtrace.sh \
+            ${{ env.BUILD_DIR }}/bin/livekit_stress_tests \
             --gtest_repeat=${{ inputs.stress_repeat }} \
             --gtest_recreate_environments_when_repeating=1 \
             --gtest_output=xml:${{ env.BUILD_DIR }}/stress-test-results.xml

scripts/run-with-backtrace.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 LiveKit
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Runs a command under platform-appropriate crash diagnostics and prints a
+# native backtrace if it crashes. Linux core files are inspected with GDB;
+# macOS commands run under LLDB. The command's original exit code is preserved.
+set -euo pipefail
+
+if (($# == 0)); then
+  echo "Usage: $0 COMMAND [ARG ...]" >&2
+  exit 2
+fi
+
+platform="$(uname -s)"
+
+# Keep this wrapper transparent on Windows and in unconfigured environments.
+if [[ "${platform}" != "Linux" && "${platform}" != "Darwin" ]] || [[ "${LIVEKIT_CRASH_DIAGNOSTICS:-}" != "1" ]]; then
+  exec "$@"
+fi
+
+if [[ "${platform}" == "Darwin" ]]; then
+  set +e
+  lldb --batch \
+    -o run \
+    -o 'script import lldb, os; os._exit(lldb.debugger.GetSelectedTarget().GetProcess().GetExitStatus())' \
+    -k "process status" \
+    -k "thread backtrace all" \
+    -k 'script import lldb, os; t = lldb.debugger.GetSelectedTarget().GetProcess().GetSelectedThread(); os._exit(128 + t.GetStopReasonDataAtIndex(0))' \
+    -- "$@"
+  exit_code=$?
+  set -e
+  exit "${exit_code}"
+fi
+
+if [[ -z "${LIVEKIT_CORE_DIR:-}" ]]; then
+  echo "LIVEKIT_CORE_DIR must be set when Linux crash diagnostics are enabled." >&2
+  exit 2
+fi
+
+mkdir -p "${LIVEKIT_CORE_DIR}"
+rm -f "${LIVEKIT_CORE_DIR}"/livekit-core.*
+ulimit -c unlimited
+
+set +e
+"$@"
+exit_code=$?
+set -e
+
+shopt -s nullglob
+core_files=("${LIVEKIT_CORE_DIR}"/livekit-core.*)
+
+if ((${#core_files[@]} > 0)); then
+  for core_file in "${core_files[@]}"; do
+    if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+      echo "::group::Native crash backtrace ($(basename "${core_file}"))"
+    fi
+
+    timeout 120s gdb --batch --quiet \
+      -ex "set pagination off" \
+      -ex "info threads" \
+      -ex "thread apply all backtrace" \
+      "$1" "${core_file}" || true
+
+    if [[ -n "${GITHUB_ACTIONS:-}" ]]; then
+      echo "::endgroup::"
+    fi
+    rm -f "${core_file}"
+  done
+elif ((exit_code >= 128)); then
+  echo "::warning::Test process exited from a signal (${exit_code}), but no core file was produced."
+fi
+
+exit "${exit_code}"