Skip to content

fix(engine): migration deadlocks #5357

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mrkaye97 merged 1 commit into from
Oct 28, 2026
Merged

fix(engine): migration deadlocks #5357

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 16 additions & 15 deletions cmd/hatchet-migrate/migrate/migrations/20261027194324_v1_0_182_olap_aggregates.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
-- +goose NO TRANSACTION
-- +goose Up
-- +goose StatementBegin
CREATE TABLE v1_agg_task_statuses_minute_olap (
CREATE TABLE IF NOT EXISTS v1_agg_task_statuses_minute_olap (
tenant_id UUID NOT NULL,
bucket TIMESTAMPTZ NOT NULL,
workflow_id UUID NOT NULL,
Expand All @@ -13,7 +14,7 @@ CREATE TABLE v1_agg_task_statuses_minute_olap (
PRIMARY KEY (tenant_id, bucket, workflow_id)
) PARTITION BY RANGE (bucket);

CREATE TABLE v1_agg_task_events_minute_olap (
CREATE TABLE IF NOT EXISTS v1_agg_task_events_minute_olap (
tenant_id UUID NOT NULL,
bucket TIMESTAMPTZ NOT NULL,
workflow_id UUID NOT NULL,
Expand All @@ -25,14 +26,14 @@ CREATE TABLE v1_agg_task_events_minute_olap (
PRIMARY KEY (tenant_id, bucket, workflow_id)
) PARTITION BY RANGE (bucket);

CREATE TABLE v1_agg_events_minute_olap (
CREATE TABLE IF NOT EXISTS v1_agg_events_minute_olap (
tenant_id UUID NOT NULL,
bucket TIMESTAMPTZ NOT NULL,
event_count BIGINT NOT NULL DEFAULT 0,
PRIMARY KEY (tenant_id, bucket)
) PARTITION BY RANGE (bucket);

CREATE TABLE v1_agg_task_statuses_minute_delta_olap (
CREATE TABLE IF NOT EXISTS v1_agg_task_statuses_minute_delta_olap (
id BIGINT GENERATED ALWAYS AS IDENTITY,
tenant_id UUID NOT NULL,
bucket TIMESTAMPTZ NOT NULL,
Expand All @@ -46,7 +47,7 @@ CREATE TABLE v1_agg_task_statuses_minute_delta_olap (
PRIMARY KEY (id, tenant_id)
) PARTITION BY HASH (tenant_id);

CREATE TABLE v1_agg_task_events_minute_delta_olap (
CREATE TABLE IF NOT EXISTS v1_agg_task_events_minute_delta_olap (
id BIGINT GENERATED ALWAYS AS IDENTITY,
tenant_id UUID NOT NULL,
bucket TIMESTAMPTZ NOT NULL,
Expand All @@ -59,17 +60,17 @@ CREATE TABLE v1_agg_task_events_minute_delta_olap (
PRIMARY KEY (id, tenant_id)
) PARTITION BY HASH (tenant_id);

CREATE TABLE v1_agg_events_minute_delta_olap (
CREATE TABLE IF NOT EXISTS v1_agg_events_minute_delta_olap (
id BIGINT GENERATED ALWAYS AS IDENTITY,
tenant_id UUID NOT NULL,
bucket TIMESTAMPTZ NOT NULL,
event_count INTEGER NOT NULL,
PRIMARY KEY (id, tenant_id)
) PARTITION BY HASH (tenant_id);

CREATE INDEX v1_agg_task_statuses_minute_delta_olap_idx ON v1_agg_task_statuses_minute_delta_olap (tenant_id, bucket);
CREATE INDEX v1_agg_task_events_minute_delta_olap_idx ON v1_agg_task_events_minute_delta_olap (tenant_id, bucket);
CREATE INDEX v1_agg_events_minute_delta_olap_idx ON v1_agg_events_minute_delta_olap (tenant_id, bucket);
CREATE INDEX IF NOT EXISTS v1_agg_task_statuses_minute_delta_olap_idx ON v1_agg_task_statuses_minute_delta_olap (tenant_id, bucket);
CREATE INDEX IF NOT EXISTS v1_agg_task_events_minute_delta_olap_idx ON v1_agg_task_events_minute_delta_olap (tenant_id, bucket);
CREATE INDEX IF NOT EXISTS v1_agg_events_minute_delta_olap_idx ON v1_agg_events_minute_delta_olap (tenant_id, bucket);

CREATE OR REPLACE FUNCTION claim_agg_deltas(
delta_table_row anyelement,
Expand Down Expand Up @@ -232,37 +233,37 @@ BEGIN
RETURN NULL;
END;
$$;
-- +goose StatementEnd

CREATE TRIGGER v1_agg_task_statuses_insert_trigger
CREATE OR REPLACE TRIGGER v1_agg_task_statuses_insert_trigger
AFTER INSERT ON v1_runs_olap
REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT
EXECUTE FUNCTION v1_agg_task_statuses_insert_fn();

CREATE TRIGGER v1_agg_task_statuses_update_trigger
CREATE OR REPLACE TRIGGER v1_agg_task_statuses_update_trigger
Comment on lines +238 to +244

@greptile-apps greptile-apps Bot Oct 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Status counts miss updates

If an existing engine keeps writing during migration, installing the three v1_runs_olap triggers in separate transactions can leave counts wrong. After the insert trigger commits but before the update trigger commits, a run can be inserted as QUEUED and then finish. The insert adds a queued count, but the update produces no matching change, so the run stays counted as queued. A failure while creating the update trigger extends this window.

The compactor only adds recorded changes; it cannot repair the missed update. Keep these three triggers in one transaction while installing triggers for other tables separately.

AFTER UPDATE ON v1_runs_olap
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
FOR EACH STATEMENT
EXECUTE FUNCTION v1_agg_task_statuses_update_fn();

CREATE TRIGGER v1_agg_task_statuses_delete_trigger
CREATE OR REPLACE TRIGGER v1_agg_task_statuses_delete_trigger
AFTER DELETE ON v1_runs_olap
REFERENCING OLD TABLE AS old_rows
FOR EACH STATEMENT
EXECUTE FUNCTION v1_agg_task_statuses_delete_fn();

CREATE TRIGGER v1_agg_task_events_insert_trigger
CREATE OR REPLACE TRIGGER v1_agg_task_events_insert_trigger
AFTER INSERT ON v1_task_events_olap
REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT
EXECUTE FUNCTION v1_agg_task_events_insert_fn();

CREATE TRIGGER v1_agg_events_insert_trigger
CREATE OR REPLACE TRIGGER v1_agg_events_insert_trigger
AFTER INSERT ON v1_events_olap
REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT
EXECUTE FUNCTION v1_agg_events_insert_fn();
-- +goose StatementEnd

-- +goose Down
-- +goose StatementBegin
Expand Down
Loading