From 91bca4d4e8a74df136c8f12d8e5575b69b97fb81 Mon Sep 17 00:00:00 2001 From: r266-tech <233881301+r266-tech@users.noreply.github.com> Date: Mon, 10 Aug 2026 00:35:09 +0000 Subject: [PATCH 1/5] [FEATURE] Allow opt-in optimization of SQLite databases larger than 100 MiB --- bin/optimize.sh | 19 +++- lib/optimize/tasks.sh | 195 ++++++++++++++++++++++++++++++++++++----- tests/optimize.bats | 83 +++++++++++++++++- tests/optimize_db.bats | 131 ++++++++++++++++++++++++++- 4 files changed, 394 insertions(+), 34 deletions(-) diff --git a/bin/optimize.sh b/bin/optimize.sh index 9b37901f..ef88ba79 100755 --- a/bin/optimize.sh +++ b/bin/optimize.sh @@ -193,16 +193,26 @@ main() { # Set current command for operation logging export MOLE_CURRENT_COMMAND="optimize" - local health_json - for arg in "$@"; do + local health_json database_max_size_seen=0 + while [[ $# -gt 0 ]]; do + local arg="$1" case "$arg" in "--help" | "-h") show_optimize_help + echo " --database-max-size SIZE Allow curated SQLite databases up to SIZE (maximum 1GiB)" exit 0 ;; "--debug") export MO_DEBUG=1 ;; + "--database-max-size") + shift + if [[ "$database_max_size_seen" -eq 1 || $# -eq 0 ]] || ! optimize_set_database_max_size "$1"; then + echo "Use 'mo optimize --help' for supported options." >&2 + exit 1 + fi + database_max_size_seen=1 + ;; "--dry-run") export MOLE_DRY_RUN=1 ;; @@ -216,6 +226,7 @@ main() { exit 1 ;; esac + shift done log_operation_session_start "optimize" @@ -317,4 +328,6 @@ main() { optimize_outcomes_succeeded } -main "$@" +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/lib/optimize/tasks.sh b/lib/optimize/tasks.sh index 351d58e6..7ddf5306 100644 --- a/lib/optimize/tasks.sh +++ b/lib/optimize/tasks.sh @@ -16,7 +16,107 @@ source "$_MOLE_OPTIMIZE_TASKS_DIR/outcomes.sh" # Config constants (override via env). readonly MOLE_TM_THIN_TIMEOUT=180 readonly MOLE_TM_THIN_VALUE=9999999999 -readonly MOLE_SQLITE_MAX_SIZE=104857600 # 100MB +readonly MOLE_SQLITE_DEFAULT_MAX_SIZE=104857600 +readonly MOLE_SQLITE_HARD_MAX_SIZE=1073741824 +readonly MOLE_SQLITE_ARITH_MAX=9000000000000000000 +MOLE_SQLITE_MAX_SIZE="$MOLE_SQLITE_DEFAULT_MAX_SIZE" +MOLE_SQLITE_MAX_SIZE_DISPLAY="100MiB" + +optimize_decimal_leq() { + local value="$1" limit="$2" value_length limit_length + value_length=${#value} + limit_length=${#limit} + if ((value_length < limit_length)); then + return 0 + fi + if ((value_length > limit_length)); then + return 1 + fi + [[ "$value" == "$limit" || "$value" < "$limit" ]] +} + +optimize_set_database_max_size() { + local requested="${1:-}" amount unit bytes + [[ "$requested" =~ ^[1-9][0-9]*(KiB|MiB|GiB)$ ]] || { + echo "Invalid --database-max-size: use a positive value with units up to 1GiB." >&2 + return 1 + } + unit="${requested##*[0-9]}" + amount="${requested%$unit}" + case "$unit" in + KiB) + optimize_decimal_leq "$amount" 1048576 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2; return 1; } + bytes=$((amount * 1024)) + ;; + MiB) + optimize_decimal_leq "$amount" 1024 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2; return 1; } + bytes=$((amount * 1048576)) + ;; + GiB) + optimize_decimal_leq "$amount" 1 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2; return 1; } + bytes="$MOLE_SQLITE_HARD_MAX_SIZE" + ;; + esac + MOLE_SQLITE_MAX_SIZE="$bytes" + MOLE_SQLITE_MAX_SIZE_DISPLAY="$requested" + export MOLE_SQLITE_MAX_SIZE MOLE_SQLITE_MAX_SIZE_DISPLAY +} + +optimize_sqlite_timeout() { + local base="$1" cap="$2" timeout + timeout=$((base * MOLE_SQLITE_MAX_SIZE / MOLE_SQLITE_DEFAULT_MAX_SIZE)) + ((timeout < base)) && timeout="$base" + ((timeout > cap)) && timeout="$cap" + echo "$timeout" +} + +optimize_sqlite_retry_size() { + local size="${1:-0}" retry + [[ "$size" =~ ^[1-9][0-9]*$ ]] || return 1 + awk -v size="$size" -v limit="$MOLE_SQLITE_HARD_MAX_SIZE" 'BEGIN { exit !(size + 0 <= limit + 0) }' || return 1 + retry=$(( (size + 1048575) / 1048576 )) + ((retry < 100)) && retry=100 + ((retry > 1024)) && return 1 + echo "${retry}MiB" +} + +optimize_sqlite_log() { + log_operation "optimize" "$2" "$1" "original_size=$3 final_size=$4${5:+ $5}" +} + +optimize_get_free_space_kb_for_path() { + local db_path="$1" target available_kb df_output df_status=0 + target=$(dirname -- "$db_path") + df_output=$(LC_ALL=C run_with_timeout "$MOLE_TIMEOUT_QUICK_DETECT_SEC" df -Pk "$target" 2> /dev/null) || df_status=$? + [[ $df_status -eq 0 ]] || return "$df_status" + available_kb=$(printf '%s\n' "$df_output" | awk 'NR==2 {print $4}') + [[ "$available_kb" =~ ^[0-9]+$ ]] || return 1 + printf '%s\n' "$available_kb" +} + +optimize_sqlite_reclaimable_bytes() { + local freelist_count="$1" page_size="$2" file_size="$3" + [[ "$freelist_count" =~ ^[0-9]+$ && "$page_size" =~ ^[0-9]+$ && "$file_size" =~ ^[0-9]+$ ]] || return 1 + LC_ALL=C awk -v value="$freelist_count" -v limit="$MOLE_SQLITE_ARITH_MAX" 'BEGIN { exit !(value + 0 <= limit + 0) }' || return 1 + LC_ALL=C awk -v value="$page_size" 'BEGIN { exit !(value + 0 <= 65536) }' || return 1 + if ((freelist_count > MOLE_SQLITE_ARITH_MAX / page_size)); then + printf '%s\n' "$MOLE_SQLITE_ARITH_MAX" + else + printf '%s\n' "$((freelist_count * page_size))" + fi +} + +optimize_sqlite_probe_metrics_valid() { + local page_count="$1" freelist_count="$2" page_size="$3" + LC_ALL=C awk -v pages="$page_count" -v freelist="$freelist_count" -v size="$page_size" \ + -v max="$MOLE_SQLITE_ARITH_MAX" \ + 'BEGIN { + exit !(pages ~ /^[0-9]+$/ && freelist ~ /^[0-9]+$/ && size ~ /^[0-9]+$/ && + pages + 0 > 0 && freelist + 0 <= pages + 0 && + pages + 0 <= max + 0 && freelist + 0 <= max + 0 && + size + 0 > 0 && size + 0 <= 65536) + }' +} # Dry-run aware output. opt_msg() { @@ -601,6 +701,7 @@ opt_sqlite_vacuum() { # Paths held back only by the size ceiling (issue #1367): never claim # "all already optimized" when this list is non-empty. local -a policy_skipped_paths=() + local -a policy_skipped_reclaimable=() for pattern in "${db_paths[@]}"; do while IFS= read -r db_file; do @@ -614,62 +715,104 @@ opt_sqlite_vacuum() { *) continue ;; esac - # Skip large DBs (>100MB). local file_size file_size=$(get_file_size "$db_file") - if [[ "$file_size" -gt "$MOLE_SQLITE_MAX_SIZE" ]]; then - policy_skipped=$((policy_skipped + 1)) - policy_skipped_paths+=("$db_file") + if [[ ! "$file_size" =~ ^[0-9]+$ ]]; then + failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "0" "0" "size_probe" continue fi - # Skip if freelist is tiny (already compact). - local page_info="" - local page_status=0 - page_info=$(run_with_timeout "$MOLE_TIMEOUT_MEDIUM_PROBE_SEC" sqlite3 "$db_file" "PRAGMA page_count; PRAGMA freelist_count;" 2> /dev/null) || page_status=$? + # Probe reclaimability before applying the configured size policy. + local page_info="" page_status=0 probe_timeout + probe_timeout=$(optimize_sqlite_timeout "$MOLE_TIMEOUT_MEDIUM_PROBE_SEC" 30) + page_info=$(run_with_timeout "$probe_timeout" sqlite3 "$db_file" "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" 2> /dev/null) || page_status=$? if [[ $page_status -ne 0 ]]; then failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "probe status=$page_status" continue fi - local page_count="" - local freelist_count="" + local page_count="" freelist_count="" page_size="" page_count="${page_info%%$'\n'*}" if [[ "$page_info" == *$'\n'* ]]; then freelist_count="${page_info#*$'\n'}" freelist_count="${freelist_count%%$'\n'*}" + page_size="${page_info#*$'\n'}" + page_size="${page_size#*$'\n'}" + page_size="${page_size%%$'\n'*}" fi - if [[ "$page_count" =~ ^[0-9]+$ && "$freelist_count" =~ ^[0-9]+$ && "$page_count" -gt 0 ]]; then - if ((freelist_count * 100 < page_count * 5)); then - already_optimal=$((already_optimal + 1)) + if ! optimize_sqlite_probe_metrics_valid "$page_count" "$freelist_count" "$page_size"; then + failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "probe_data" + continue + fi + if awk -v freelist="$freelist_count" -v pages="$page_count" \ + 'BEGIN { exit !(freelist + 0 < pages + 0 && freelist * 100 < pages * 5) }'; then + already_optimal=$((already_optimal + 1)) + optimize_sqlite_log "$db_file" "UNCHANGED" "$file_size" "$file_size" "already_optimal" + continue + fi + if [[ "$file_size" -gt "$MOLE_SQLITE_MAX_SIZE" ]]; then + local reclaimable_bytes + reclaimable_bytes=$(optimize_sqlite_reclaimable_bytes "$freelist_count" "$page_size" "$file_size" 2> /dev/null) || { + failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "reclaimable_probe" continue - fi + } + policy_skipped=$((policy_skipped + 1)) + policy_skipped_paths+=("$db_file") + policy_skipped_reclaimable+=("$reclaimable_bytes") + optimize_sqlite_log "$db_file" "SKIPPED" "$file_size" "$file_size" "size_limit=$MOLE_SQLITE_MAX_SIZE_DISPLAY" + continue fi # Verify integrity before VACUUM. if [[ "${MOLE_DRY_RUN:-0}" != "1" ]]; then - local integrity_check="" - local integrity_status=0 - integrity_check=$(run_with_timeout "$MOLE_TIMEOUT_PKG_LIST_SEC" sqlite3 "$db_file" "PRAGMA integrity_check;" 2> /dev/null) || integrity_status=$? + local integrity_check="" integrity_status=0 integrity_timeout + integrity_timeout=$(optimize_sqlite_timeout "$MOLE_TIMEOUT_PKG_LIST_SEC" 60) + integrity_check=$(run_with_timeout "$integrity_timeout" sqlite3 "$db_file" "PRAGMA integrity_check;" 2> /dev/null) || integrity_status=$? if [[ $integrity_status -ne 0 || "$integrity_check" != "ok" ]]; then failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "integrity" + continue + fi + local free_space_kb free_space_status=0 + free_space_kb=$(optimize_get_free_space_kb_for_path "$db_file" 2> /dev/null) || free_space_status=$? + if [[ $free_space_status -ne 0 ]]; then + echo -e " ${YELLOW}${ICON_WARNING}${NC} Unable to verify temporary space for ${db_file/#$HOME/~}" + failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "temporary_space_probe status=$free_space_status" + continue + fi + if ! awk -v free="$free_space_kb" -v size="$file_size" \ + 'BEGIN { exit !(free + 0 >= (size * 2 + 1023) / 1024) }'; then + echo -e " ${YELLOW}${ICON_WARNING}${NC} Insufficient temporary space for ${db_file/#$HOME/~}" + failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "insufficient_temporary_space" continue fi fi local exit_code=0 if [[ "${MOLE_DRY_RUN:-0}" != "1" ]]; then - run_with_timeout "$MOLE_TIMEOUT_PKG_CLEANUP_SEC" sqlite3 "$db_file" "VACUUM;" 2> /dev/null || exit_code=$? + local vacuum_timeout + vacuum_timeout=$(optimize_sqlite_timeout "$MOLE_TIMEOUT_PKG_CLEANUP_SEC" 120) + run_with_timeout "$vacuum_timeout" sqlite3 "$db_file" "VACUUM;" 2> /dev/null || exit_code=$? if [[ $exit_code -eq 0 ]]; then vacuumed=$((vacuumed + 1)) + optimize_sqlite_log "$db_file" "REBUILT" "$file_size" "$(get_file_size "$db_file")" "vacuum" elif [[ $exit_code -eq 124 ]]; then timed_out=$((timed_out + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "timeout" else failed=$((failed + 1)) + optimize_sqlite_log "$db_file" "FAILED" "$file_size" "$file_size" "vacuum" fi else vacuumed=$((vacuumed + 1)) + optimize_sqlite_log "$db_file" "DRY_RUN" "$file_size" "$file_size" "would_rebuild" fi done < <(compgen -G "$pattern" || true) done @@ -699,16 +842,22 @@ opt_sqlite_vacuum() { fi if [[ $policy_skipped -gt 0 ]]; then - opt_msg "Skipped $policy_skipped databases over the 100 MB safety limit" - local skipped_path skipped_size skipped_display - for skipped_path in "${policy_skipped_paths[@]}"; do + opt_msg "Skipped $policy_skipped databases over the ${MOLE_SQLITE_MAX_SIZE_DISPLAY} safety limit" + local skipped_path skipped_size skipped_display skipped_reclaimable retry_size index + for ((index = 0; index < ${#policy_skipped_paths[@]}; index++)); do + skipped_path="${policy_skipped_paths[$index]}" skipped_size=$(get_file_size "$skipped_path" 2> /dev/null || echo 0) if [[ "$skipped_size" =~ ^[0-9]+$ && "$skipped_size" -gt 0 ]]; then skipped_display=$(bytes_to_human "$skipped_size") else skipped_display="unknown size" fi - echo -e " ${GRAY}${ICON_SUBLIST}${NC} ${skipped_path/#$HOME/~} ยท ${skipped_display}" + echo -e " ${GRAY}${ICON_SUBLIST}${NC} ${skipped_path/#$HOME/~} | ${skipped_display}" + skipped_reclaimable="${policy_skipped_reclaimable[$index]:-0}" + echo -e " ${GRAY}Reclaimable: $(bytes_to_human "$skipped_reclaimable") | limit: ${MOLE_SQLITE_MAX_SIZE_DISPLAY}${NC}" + if retry_size=$(optimize_sqlite_retry_size "$skipped_size"); then + echo -e " ${GRAY}Retry: mo optimize --database-max-size ${retry_size}${NC}" + fi done fi diff --git a/tests/optimize.bats b/tests/optimize.bats index 435dd84b..3d89214c 100644 --- a/tests/optimize.bats +++ b/tests/optimize.bats @@ -490,7 +490,7 @@ run_with_timeout() { } sqlite3() { case "$2" in - "PRAGMA page_count; PRAGMA freelist_count;") printf '100\n10\n' ;; + "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;") printf '100\n10\n4096\n' ;; "PRAGMA integrity_check;") echo "ok" ;; "VACUUM;") [[ "$1" == *"chat.db" ]] ;; esac @@ -520,8 +520,8 @@ file() { echo "SQLite 3.x database"; } get_file_size() { echo 1; } run_with_timeout() { shift - if [[ "$3" == "PRAGMA page_count; PRAGMA freelist_count;" ]]; then - printf '100\n10\n' + if [[ "$3" == "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" ]]; then + printf '100\n10\n4096\n' return 0 fi return 7 @@ -548,6 +548,7 @@ touch "$db" pgrep() { return 1; } file() { echo "SQLite 3.x database"; } get_file_size() { echo $((MOLE_SQLITE_MAX_SIZE + 1)); } +run_with_timeout() { printf '100000\n10000\n4096\n'; } export -f pgrep file get_file_size execute_optimization sqlite_vacuum @@ -555,7 +556,81 @@ execute_optimization sqlite_vacuum EOF [[ "$status" -eq 0 ]] || { echo "$output"; return 1; } - [[ "$output" == *"Skipped 1 databases over the 100 MB safety limit"* ]] || return 1 + [[ "$output" == *"Skipped 1 databases over the 100MiB safety limit"* ]] || return 1 +} + +@test "mo optimize accepts the bounded database size option at the entrypoint" { + run env HOME="$HOME" MOLE_TEST_NO_AUTH=1 "$PROJECT_ROOT/bin/optimize.sh" --database-max-size 300MiB --help + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"--database-max-size SIZE"* ]] || return 1 + [[ "$output" == *"maximum 1GiB"* ]] || return 1 +} + +@test "mo optimize rejects invalid database size options at the entrypoint" { + local value + for value in "" 0MiB 300 2GiB -1MiB; do + if [[ -n "$value" ]]; then + run env HOME="$HOME" MOLE_TEST_NO_AUTH=1 "$PROJECT_ROOT/bin/optimize.sh" --database-max-size "$value" + else + run env HOME="$HOME" MOLE_TEST_NO_AUTH=1 "$PROJECT_ROOT/bin/optimize.sh" --database-max-size + fi + [ "$status" -ne 0 ] || { echo "accepted invalid value: ${value:-missing}"; return 1; } + [[ "$output" == *"Use 'mo optimize --help'"* ]] || return 1 + done +} + +@test "mo optimize rejects duplicate database size options at the entrypoint" { + run env HOME="$HOME" MOLE_TEST_NO_AUTH=1 "$PROJECT_ROOT/bin/optimize.sh" \ + --database-max-size 200MiB --database-max-size 300MiB + + [ "$status" -ne 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"Use 'mo optimize --help'"* ]] || return 1 +} + +@test "mo optimize propagates database size option through the curated dispatch" { + run env HOME="$HOME/sqlite-entrypoint" PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/bin/optimize.sh" + +generate_health_json() { printf '%s\n' '{"memory_used_gb":0,"optimizations":{}}'; } +show_system_health() { :; } +run_optimize_diagnostics() { :; } +load_whitelist() { CURRENT_WHITELIST_PATTERNS=(); } +announce_action() { :; } + +# Keep the catalog's readonly action list intact while exercising main's +# argument parsing and dispatch path. Only the curated database task runs its +# real handler; the other catalog entries complete as unchanged. +execute_optimization() { + local action="$1" + optimize_task_start + if [[ "$action" == "sqlite_vacuum" ]]; then + opt_sqlite_vacuum + else + optimize_task_result "$MOLE_OPTIMIZE_OUTCOME_UNCHANGED" + fi + optimize_task_finish "$action" +} + +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +get_file_size() { echo 209715200; } +should_protect_path() { return 1; } +run_with_timeout() { + [[ "$4" == "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" ]] || return 64 + printf '100000\n10000\n4096\n' +} + +main --database-max-size 300MiB --dry-run --debug +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"Would apply 1 optimizations"* ]] || return 1 + [[ "$output" != *"Skipped 1 databases over the 100MiB safety limit"* ]] || return 1 } @test "optimize does not auto-fix Gatekeeper anymore" { diff --git a/tests/optimize_db.bats b/tests/optimize_db.bats index bec986f5..0bd09235 100644 --- a/tests/optimize_db.bats +++ b/tests/optimize_db.bats @@ -236,10 +236,13 @@ get_file_size() { echo 1024; } should_protect_path() { return 1; } run_with_timeout() { case "$4" in - "PRAGMA page_count; PRAGMA freelist_count;") printf '100\n10\n' ;; + "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;") printf '100\n10\n4096\n' ;; "PRAGMA integrity_check;") echo "ok" ;; "VACUUM;") echo "VACUUM_CALLED" ;; - *) return 64 ;; + *) + [[ "$2" == "df" ]] || return 64 + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/mock 2000000 100000 1900000 5%% /\n' + ;; esac } @@ -267,7 +270,7 @@ file() { echo "SQLite 3.x database"; } get_file_size() { echo 209715200; } should_protect_path() { return 1; } bytes_to_human() { echo "200.0MB"; } -run_with_timeout() { echo "UNEXPECTED_SQLITE"; return 0; } +run_with_timeout() { printf '100000\n10000\n4096\n'; } execute_optimization sqlite_vacuum EOF @@ -277,8 +280,128 @@ EOF return 1 } [[ "$output" == *"No databases compacted"* ]] || return 1 - [[ "$output" == *"100 MB safety limit"* ]] || return 1 + [[ "$output" == *"100MiB safety limit"* ]] || return 1 [[ "$output" == *"Messages/chat.db"* ]] || return 1 [[ "$output" != *"All databases already optimized"* ]] || return 1 [[ "$output" != *"UNEXPECTED_SQLITE"* ]] || return 1 } + +@test "SQLite size override is bounded and unit-bearing" { + run env PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +optimize_set_database_max_size 300MiB +[[ "$MOLE_SQLITE_MAX_SIZE" -eq 314572800 ]] || exit 1 +[[ "$MOLE_SQLITE_MAX_SIZE_DISPLAY" == "300MiB" ]] || exit 1 +for value in 0MiB 300 2GiB -1MiB 999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999999KiB; do + if optimize_set_database_max_size "$value"; then + exit 1 + fi +done +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } +} + +@test "SQLite size override allows a reclaimable large curated database" { + run env HOME="$HOME/sqlite-opt-in" PROJECT_ROOT="$PROJECT_ROOT" MOLE_DRY_RUN=1 /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +optimize_set_database_max_size 300MiB +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +get_file_size() { echo 209715200; } +should_protect_path() { return 1; } +run_with_timeout() { printf '100000\n10000\n4096\n'; } +execute_optimization sqlite_vacuum +[[ "$(optimize_outcome_count applied)" == "1" ]] || exit 1 +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"Optimized 1 databases"* ]] || return 1 +} + +@test "SQLite compact large database is classified before size policy" { + run env HOME="$HOME/sqlite-compact-large" PROJECT_ROOT="$PROJECT_ROOT" MOLE_DRY_RUN=1 /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +get_file_size() { echo 209715200; } +should_protect_path() { return 1; } +run_with_timeout() { printf '100000\n100\n4096\n'; } +execute_optimization sqlite_vacuum +[[ "$(optimize_outcome_count unchanged)" == "1" ]] || exit 1 +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"Already optimal for 1 databases"* ]] || return 1 +} + +@test "SQLite extreme PRAGMA values fail closed without arithmetic overflow" { + run env HOME="$HOME/sqlite-extreme" PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +get_file_size() { echo 209715200; } +should_protect_path() { return 1; } +run_with_timeout() { printf '1\n999999999999999999999\n4096\n'; } +execute_optimization sqlite_vacuum +[[ "$(optimize_outcome_count failed)" == "1" ]] || exit 1 +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"Failed on 1 databases"* ]] || return 1 +} + +@test "SQLite temporary-space probe failures are reported separately" { + local mode + for mode in timeout malformed low; do + run env HOME="$HOME/sqlite-space-$mode" PROJECT_ROOT="$PROJECT_ROOT" SPACE_MODE="$mode" /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +get_file_size() { echo 1024; } +should_protect_path() { return 1; } +run_with_timeout() { + case "$4" in + "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;") printf '100\n10\n4096\n' ;; + "PRAGMA integrity_check;") echo ok ;; + *) + if [[ "$SPACE_MODE" == timeout ]]; then return 124; fi + if [[ "$SPACE_MODE" == malformed ]]; then printf 'bad\n'; return 0; fi + printf 'Filesystem 1024-blocks Used Available Capacity Mounted on\n/dev/mock 2000 1999 1 99%% /\n' + ;; + esac +} +execute_optimization sqlite_vacuum +[[ "$(optimize_outcome_count failed)" == "1" ]] || exit 1 +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + if [[ "$mode" == low ]]; then + [[ "$output" == *"Insufficient temporary space"* ]] || return 1 + else + [[ "$output" == *"Unable to verify temporary space"* ]] || return 1 + fi + done +} From 83a41a2659ac7e44630d81fe3bcc264c402ac464 Mon Sep 17 00:00:00 2001 From: r266-tech <233881301+r266-tech@users.noreply.github.com> Date: Mon, 10 Aug 2026 15:20:43 +0000 Subject: [PATCH 2/5] [FEATURE] Allow opt-in optimization of SQLite databases larger than 100 MiB --- lib/optimize/tasks.sh | 31 ++++++++++++++++++------- tests/optimize_db.bats | 52 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 8 deletions(-) diff --git a/lib/optimize/tasks.sh b/lib/optimize/tasks.sh index 7ddf5306..e8d09ce9 100644 --- a/lib/optimize/tasks.sh +++ b/lib/optimize/tasks.sh @@ -45,15 +45,24 @@ optimize_set_database_max_size() { amount="${requested%$unit}" case "$unit" in KiB) - optimize_decimal_leq "$amount" 1048576 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2; return 1; } + optimize_decimal_leq "$amount" 1048576 || { + echo "Invalid --database-max-size: maximum is 1GiB." >&2 + return 1 + } bytes=$((amount * 1024)) ;; MiB) - optimize_decimal_leq "$amount" 1024 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2; return 1; } + optimize_decimal_leq "$amount" 1024 || { + echo "Invalid --database-max-size: maximum is 1GiB." >&2 + return 1 + } bytes=$((amount * 1048576)) ;; GiB) - optimize_decimal_leq "$amount" 1 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2; return 1; } + optimize_decimal_leq "$amount" 1 || { + echo "Invalid --database-max-size: maximum is 1GiB." >&2 + return 1 + } bytes="$MOLE_SQLITE_HARD_MAX_SIZE" ;; esac @@ -74,7 +83,7 @@ optimize_sqlite_retry_size() { local size="${1:-0}" retry [[ "$size" =~ ^[1-9][0-9]*$ ]] || return 1 awk -v size="$size" -v limit="$MOLE_SQLITE_HARD_MAX_SIZE" 'BEGIN { exit !(size + 0 <= limit + 0) }' || return 1 - retry=$(( (size + 1048575) / 1048576 )) + retry=$(((size + 1048575) / 1048576)) ((retry < 100)) && retry=100 ((retry > 1024)) && return 1 echo "${retry}MiB" @@ -715,11 +724,17 @@ opt_sqlite_vacuum() { *) continue ;; esac - local file_size - file_size=$(get_file_size "$db_file") - if [[ ! "$file_size" =~ ^[0-9]+$ ]]; then + local file_size="" size_status=0 + file_size=$(get_file_size "$db_file") || size_status=$? + if [[ $size_status -ne 0 || ! "$file_size" =~ ^[0-9]+$ ]]; then + if [[ $size_status -eq 130 ]]; then + if [[ "$spinner_started" == "true" ]]; then + stop_inline_spinner + fi + return 130 + fi failed=$((failed + 1)) - optimize_sqlite_log "$db_file" "FAILED" "0" "0" "size_probe" + optimize_sqlite_log "$db_file" "FAILED" "0" "0" "size_probe status=$size_status" continue fi diff --git a/tests/optimize_db.bats b/tests/optimize_db.bats index 0bd09235..9559e06c 100644 --- a/tests/optimize_db.bats +++ b/tests/optimize_db.bats @@ -264,6 +264,9 @@ source "$PROJECT_ROOT/lib/optimize/tasks.sh" db="$HOME/Library/Messages/chat.db" mkdir -p "$(dirname "$db")" touch "$db" +db2="$HOME/Library/Safari/History.db" +mkdir -p "$(dirname "$db2")" +touch "$db2" pgrep() { return 1; } file() { echo "SQLite 3.x database"; } # 200 MiB > MOLE_SQLITE_MAX_SIZE (100 MiB). @@ -347,6 +350,55 @@ EOF [[ "$output" == *"Already optimal for 1 databases"* ]] || return 1 } +@test "SQLite file size probe failures are reported without aborting" { + run env HOME="$HOME/sqlite-size-failure" PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +get_file_size() { return 124; } +should_protect_path() { return 1; } +execute_optimization sqlite_vacuum +[[ "$(optimize_outcome_count failed)" == "1" ]] || exit 1 +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } + [[ "$output" == *"Failed on 1 databases"* ]] || return 1 +} + +@test "SQLite size probe interruption propagates and stops the task" { + run env HOME="$HOME/sqlite-size-interrupted" PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' +set -euo pipefail +source "$PROJECT_ROOT/lib/core/common.sh" +source "$PROJECT_ROOT/lib/optimize/tasks.sh" +db="$HOME/Library/Messages/chat.db" +mkdir -p "$(dirname "$db")" +touch "$db" +db2="$HOME/Library/Safari/History.db" +mkdir -p "$(dirname "$db2")" +touch "$db2" +pgrep() { return 1; } +file() { echo "SQLite 3.x database"; } +should_protect_path() { return 1; } +calls_log="$HOME/size-probe-calls" +get_file_size() { + printf '%s\n' "$1" >> "$calls_log" + return 130 +} +rc=0 +optimize_task_start +opt_sqlite_vacuum || rc=$? +[[ "$rc" -eq 130 ]] || exit 1 +[[ "$(wc -l < "$calls_log")" -eq 1 ]] || exit 1 +EOF + + [ "$status" -eq 0 ] || { echo "$output"; return 1; } +} + @test "SQLite extreme PRAGMA values fail closed without arithmetic overflow" { run env HOME="$HOME/sqlite-extreme" PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' set -euo pipefail From 44d5a61fb789902e6d22aec48d8b18181d70fa31 Mon Sep 17 00:00:00 2001 From: r266-tech <233881301+r266-tech@users.noreply.github.com> Date: Tue, 11 Aug 2026 02:19:44 +0000 Subject: [PATCH 3/5] [FEATURE] Allow opt-in optimization of SQLite databases larger than 100 MiB --- lib/optimize/tasks.sh | 15 ++++++++------- tests/optimize.bats | 3 ++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/optimize/tasks.sh b/lib/optimize/tasks.sh index e8d09ce9..4af55ec5 100644 --- a/lib/optimize/tasks.sh +++ b/lib/optimize/tasks.sh @@ -36,29 +36,30 @@ optimize_decimal_leq() { } optimize_set_database_max_size() { - local requested="${1:-}" amount unit bytes + local requested="${1:-}" amount bytes [[ "$requested" =~ ^[1-9][0-9]*(KiB|MiB|GiB)$ ]] || { echo "Invalid --database-max-size: use a positive value with units up to 1GiB." >&2 return 1 } - unit="${requested##*[0-9]}" - amount="${requested%$unit}" - case "$unit" in - KiB) + case "$requested" in + *KiB) + amount="${requested%KiB}" optimize_decimal_leq "$amount" 1048576 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2 return 1 } bytes=$((amount * 1024)) ;; - MiB) + *MiB) + amount="${requested%MiB}" optimize_decimal_leq "$amount" 1024 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2 return 1 } bytes=$((amount * 1048576)) ;; - GiB) + *GiB) + amount="${requested%GiB}" optimize_decimal_leq "$amount" 1 || { echo "Invalid --database-max-size: maximum is 1GiB." >&2 return 1 diff --git a/tests/optimize.bats b/tests/optimize.bats index 3d89214c..896d60f1 100644 --- a/tests/optimize.bats +++ b/tests/optimize.bats @@ -621,7 +621,8 @@ file() { echo "SQLite 3.x database"; } get_file_size() { echo 209715200; } should_protect_path() { return 1; } run_with_timeout() { - [[ "$4" == "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" ]] || return 64 + local sql="${4:-}" + [[ "$sql" == "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" ]] || return 64 printf '100000\n10000\n4096\n' } From 5af141e0037c85ed09faf3079efc8899dc758fe8 Mon Sep 17 00:00:00 2001 From: r266-tech <233881301+r266-tech@users.noreply.github.com> Date: Tue, 11 Aug 2026 04:29:33 +0000 Subject: [PATCH 4/5] [FEATURE] Allow opt-in optimization of SQLite databases larger than 100 MiB --- tests/optimize.bats | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/optimize.bats b/tests/optimize.bats index 896d60f1..0f4d1405 100644 --- a/tests/optimize.bats +++ b/tests/optimize.bats @@ -620,6 +620,7 @@ pgrep() { return 1; } file() { echo "SQLite 3.x database"; } get_file_size() { echo 209715200; } should_protect_path() { return 1; } +sqlite3() { :; } run_with_timeout() { local sql="${4:-}" [[ "$sql" == "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" ]] || return 64 From 08298cfb0264376ac3712e9479935d7f82ed8261 Mon Sep 17 00:00:00 2001 From: r266-tech <233881301+r266-tech@users.noreply.github.com> Date: Tue, 11 Aug 2026 08:30:08 +0000 Subject: [PATCH 5/5] [FEATURE] Allow opt-in optimization of SQLite databases larger than 100 MiB --- tests/optimize.bats | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/optimize.bats b/tests/optimize.bats index 0f4d1405..22ceef6c 100644 --- a/tests/optimize.bats +++ b/tests/optimize.bats @@ -589,7 +589,7 @@ EOF } @test "mo optimize propagates database size option through the curated dispatch" { - run env HOME="$HOME/sqlite-entrypoint" PROJECT_ROOT="$PROJECT_ROOT" /bin/bash --noprofile --norc <<'EOF' + run env HOME="$HOME/sqlite-entrypoint" PROJECT_ROOT="$PROJECT_ROOT" NO_COLOR=1 /bin/bash --noprofile --norc <<'EOF' set -euo pipefail source "$PROJECT_ROOT/bin/optimize.sh" @@ -620,11 +620,27 @@ pgrep() { return 1; } file() { echo "SQLite 3.x database"; } get_file_size() { echo 209715200; } should_protect_path() { return 1; } -sqlite3() { :; } +bc() { :; } +sqlite3() { + local sql="${2:-}" + case "$sql" in + "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;") + printf '100000\n10000\n4096\n' + ;; + "PRAGMA integrity_check;") printf 'ok\n' ;; + "VACUUM;") : ;; + *) : ;; + esac +} run_with_timeout() { local sql="${4:-}" - [[ "$sql" == "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;" ]] || return 64 - printf '100000\n10000\n4096\n' + case "$sql" in + "PRAGMA page_count; PRAGMA freelist_count; PRAGMA page_size;") + printf '100000\n10000\n4096\n' + ;; + "PRAGMA integrity_check;" | "VACUUM;") : ;; + *) : ;; + esac } main --database-max-size 300MiB --dry-run --debug