diff --git a/README.md b/README.md index 053f6ae..918ea01 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,37 @@ confirmation as something it gets a handful of tries at per user: `createSessionFromToken()` in `playground.php` shows the shape of that handling. +### Queued work + +Asking for an asynchronous function run is two steps on the server: the +execution row is written, and then the job is published to the message queue the +function workers read. If the publish throws, the row is already there and the +caller is answered `500 general_unknown` — the generic code every unhandled +server error folds into, which says nothing about how far the server got. + +[appwrite/appwrite#15510](https://github.com/appwrite/appwrite/pull/15510) +removed the common cause of that failure. A pooled connection to the message +broker that nothing had read since the broker closed it — during a rolling +restart, where each node announces itself and then drops the sockets parked on +it — was written into before anything read it, so the first publish after a +restart threw. The server now acts on whatever the broker already sent before it +writes. Nothing in the SDK changed; a publish already in flight when a close +lands is still ambiguous, so the `500` stays possible, just rare. + +What a caller has to hold up is not retrying into it. If the row was written, +a retry queues a *second* run and the function does its work twice. +`createExecution()` takes no caller-chosen execution ID, so there is nothing to +retry *as* — but it does take the path the function is called on, and the server +stores that, query string included, as the execution's `requestPath`. Tag the +attempt there and `listExecutions()` will say what it left behind: + +* nothing carries the tag — the attempt left no trace, so retrying is safe; +* something does — the run is registered; queueing another would double it. One + still `waiting` long afterwards lost its publish, and replacing it is a + decision to make, not a retry. + +`createAsyncExecution()` in `playground.php` shows that handling. + ## Tests `composer test` runs the playground end to end against a stub Appwrite @@ -76,8 +107,11 @@ every call succeeded and hit the endpoint it was supposed to (`tests/smoke.php`), and then drives the abuse-limited confirmation routes with an API key to check that wrong guesses are counted for it, that the count is per user, and that the `429` reaches the caller in a shape it can act on -(`tests/code-guesses.php`). No Appwrite instance and no network access are -needed. +(`tests/code-guesses.php`). It then runs the playground twice more against a +stub told to fail one async execution, with and without recording the execution +first, to check that the ambiguous `500` is reconciled rather than blindly +retried (`tests/queue-restart.php`). No Appwrite instance and no network access +are needed. ## Contributing diff --git a/composer.json b/composer.json index 7d25ac7..e325003 100644 --- a/composer.json +++ b/composer.json @@ -26,7 +26,7 @@ "php": ">=8.5", "ext-curl": "*", "ext-json": "*", - "appwrite/appwrite": "^32.2.0" + "appwrite/appwrite": "^32.3.0" }, "config": { "optimize-autoloader": true, @@ -35,7 +35,8 @@ "scripts": { "test": [ "@php tests/smoke.php", - "@php tests/code-guesses.php" + "@php tests/code-guesses.php", + "@php tests/queue-restart.php" ] }, "support": { diff --git a/playground.php b/playground.php index 531174f..15a0f91 100644 --- a/playground.php +++ b/playground.php @@ -7,7 +7,9 @@ use Appwrite\Enums\Runtime; use Appwrite\ID; use Appwrite\InputFile; +use Appwrite\Models\Execution; use Appwrite\Permission; +use Appwrite\Query; use Appwrite\Role; use Appwrite\Services\Account; use Appwrite\Services\Functions; @@ -27,6 +29,11 @@ $fileId = ""; $functionId = ""; +/** + * How many times `createAsyncExecution()` will queue a run before giving up. + */ +const ENQUEUE_ATTEMPTS = 3; + $tablesDB = new TablesDB($client); $storage = new Storage($client); $functions = new Functions($client); @@ -582,7 +589,66 @@ functionId: $functionId, } /** - * Create async execution + * Find the execution a tagged attempt left behind, if the server got that far. + * + * `createExecution()` takes no caller-chosen execution ID, so there is nothing + * to look an attempt up by. What it does take is the path the function will be + * called on, query string included, and the server stores that on the execution + * as `requestPath` — so a tag put in the query comes back from + * `listExecutions()` and names the attempt that wrote it. + * + * @see https://appwrite.io/docs/references/cloud/server-php/functions#listExecutions + * @throws Exception + */ +function findTaggedExecution(string $tag): ?Execution +{ + global $functions, $functionId; + + $executions = $functions->listExecutions( + $functionId, + queries: [ + Query::orderDesc('$createdAt'), + Query::limit(25), + ] + ); + + foreach ($executions->executions as $execution) { + if (\str_contains($execution->requestPath, $tag)) { + return $execution; + } + } + + return null; +} + +/** + * Create async execution, and survive the queue behind it blinking. + * + * Queueing a run is two server-side steps: the execution row is written, and + * then the job is published to the message queue the function workers read. A + * publish that throws leaves the row behind and reaches the caller as + * `500 general_unknown` — the generic code every unhandled server error folds + * into, which says nothing about how far the server got. + * + * [appwrite/appwrite#15510](https://github.com/appwrite/appwrite/pull/15510) + * (CLOUD-3RCB) removed the common cause. A pooled connection to the message + * broker that nothing had read since the broker closed it — during a rolling + * restart, where each node announces itself and then drops the sockets parked + * on it — was written into before anything read it, so the first publish after + * a restart threw. The server now acts on whatever the broker already sent + * before it writes, and reconnects while doing that is still safe. A publish + * already in flight when the close lands stays ambiguous, so this remains a + * `500` a caller can meet — just rarely. + * + * Retrying such a `500` blindly is what hurts: if the row was written, the + * retry queues a *second* run and the function does its work twice. So tag the + * attempt, and when the answer is ambiguous ask what was recorded: + * + * - nothing carries the tag — the attempt left no trace, so retrying it cannot + * duplicate anything; + * - something does — the run is already registered and queueing another would + * double it. One still at `waiting` long afterwards is a run whose publish + * was lost; replacing it is a decision for the caller, not a blind retry. * * @see https://appwrite.io/docs/references/cloud/server-php/functions#createExecution * @throws Exception @@ -591,14 +657,54 @@ function createAsyncExecution(): array { global $functions, $functionId; - $response = $functions->createExecution( - functionId: $functionId, - body: "", - async: true, - xpath: "/", - method: ExecutionMethod::GET(), - headers: [], - ); + $tag = 'attempt=' . \bin2hex(\random_bytes(8)); + $response = null; + + for ($attempt = 1; $attempt <= ENQUEUE_ATTEMPTS; $attempt++) { + try { + $response = $functions->createExecution( + functionId: $functionId, + body: "", + async: true, + xpath: "/?{$tag}", + method: ExecutionMethod::GET(), + headers: [], + ); + + break; + } catch (AppwriteException $e) { + // Anything the server could name — a missing function, a build that + // is not ready — is an answer, not an ambiguity. Only a 5xx leaves + // the question of how far it got. + if ($e->getCode() < 500) { + throw $e; + } + + $recorded = findTaggedExecution($tag); + + if ($recorded !== null) { + return [ + 'call' => 'api.createExecution', + 'response' => [ + 'queued' => false, + 'code' => $e->getCode(), + 'executionId' => $recorded->id, + 'status' => (string) $recorded->status, + 'message' => 'The execution was recorded before the failure. ' + . 'Queueing another would run the function twice.', + ], + ]; + } + + if ($attempt === ENQUEUE_ATTEMPTS) { + throw $e; + } + + // Back off: a broker losing a node is over in seconds, and the + // attempt left nothing behind to collide with. + sleep($attempt); + } + } // wait for 2 seconds to ensure execution is finished sleep(2); diff --git a/tests/queue-restart.php b/tests/queue-restart.php new file mode 100644 index 0000000..d2ca75a --- /dev/null +++ b/tests/queue-restart.php @@ -0,0 +1,242 @@ + $output, 2 => $output], + $pipes, + $root, + ['PLAYGROUND_ENDPOINT' => $stub->endpoint] + \getenv() + ); + + if (!\is_resource($playground)) { + throw new RuntimeException('Could not start playground.php'); + } + + \proc_close($playground); + \rewind($output); + $printed = (string) \stream_get_contents($output); + \fclose($output); + + return $printed; +} + +/** + * Every request the playground made that was about executing a function, in + * order. Creating a deployment or listing functions is not one of them. + * + * @return list + */ +function executionRequests(Stub $stub): array +{ + return \array_values(\array_filter( + $stub->requests(), + static fn (array $request): bool => \str_contains($request['path'], '/executions') + )); +} + +/** + * The tag `createAsyncExecution()` put in the path of its attempts. There is + * one per playground run, which is what makes a retry recognisable as the same + * piece of work rather than a second one. + * + * @param list $requests + * @return list + */ +function attemptTags(array $requests): array +{ + $tags = []; + + foreach ($requests as $request) { + if (($request['async'] ?? false) !== true) { + continue; + } + + if (\preg_match('/attempt=[0-9a-f]{16}/', (string) $request['executionPath'], $matches) === 1) { + $tags[] = $matches[0]; + } + } + + return $tags; +} + +/** + * The execution requests named by what they ask for rather than by their path, + * so a whole run reads as a sequence. + * + * @param list $requests + * @return list + */ +function executionSequence(array $requests): array +{ + return \array_map(static function (array $request): string { + if ($request['method'] === 'POST') { + return ($request['async'] ?? false) === true ? 'create async' : 'create sync'; + } + + return \str_ends_with($request['path'], '/executions') ? 'list' : 'get'; + }, $requests); +} + +echo "A failed queue publish that recorded nothing is retried\n"; + +$stub = Stub::start([ + 'STUB_EXECUTIONS_FAIL' => '1', + 'STUB_EXECUTIONS_RECORD_ON_FAILURE' => '0', +]); + +try { + $output = runPlayground($stub); + $requests = executionRequests($stub); + $tags = attemptTags($requests); + + $sequence = executionSequence($requests); + + check( + 'the failure was reconciled and only then retried (got ' . \implode(', ', $sequence) . ')', + $sequence === ['create sync', 'create async', 'list', 'create async', 'get'] + ); + check( + 'both attempts carried the same tag, so they are one piece of work', + \count($tags) === 2 && $tags[0] === $tags[1] + ); + check( + 'the playground did not report the run as unqueued', + !\str_contains($output, '[queued] =>') + ); + + if ($failures !== []) { + \fwrite(STDERR, "\n--- playground.php output ---\n{$output}\n"); + \fwrite(STDERR, '--- stub server log ---' . "\n" . $stub->errorLog() . "\n"); + } +} finally { + $stub->stop(); +} + +echo "A failed queue publish that recorded the run is not retried\n"; + +$stub = Stub::start([ + 'STUB_EXECUTIONS_FAIL' => '1', + 'STUB_EXECUTIONS_RECORD_ON_FAILURE' => '1', +]); + +try { + $output = runPlayground($stub); + $requests = executionRequests($stub); + + $sequence = executionSequence($requests); + + check( + 'the run was attempted once and not queued again (got ' . \implode(', ', $sequence) . ')', + $sequence === ['create sync', 'create async', 'list'] + ); + check( + 'the caller was told the run was not queued', + \str_contains($output, '[queued] => ') + ); + check( + 'it was handed the execution that was left behind, still waiting', + \str_contains($output, '[status] => waiting') + ); + check( + 'the ambiguous answer was reported as the 500 it was', + \str_contains($output, '[code] => 500') + ); + check( + 'the playground still finished the rest of its calls', + \str_contains($output, 'api.deleteFunction') + ); + + if ($failures !== []) { + \fwrite(STDERR, "\n--- playground.php output ---\n{$output}\n"); + \fwrite(STDERR, '--- stub server log ---' . "\n" . $stub->errorLog() . "\n"); + } +} finally { + $stub->stop(); +} + +if ($failures !== []) { + \fwrite(STDERR, "FAILED\n"); + + foreach ($failures as $failure) { + \fwrite(STDERR, " - {$failure}\n"); + } + + exit(1); +} + +echo "OK — an ambiguous queue failure is reconciled, not blindly retried\n"; diff --git a/tests/stub-server.php b/tests/stub-server.php index 308a8f6..1282c7d 100644 --- a/tests/stub-server.php +++ b/tests/stub-server.php @@ -9,12 +9,20 @@ * endpoints it means to and that the SDK can hydrate what comes back — without * needing a running Appwrite. * - * The routes that confirm a secret are the ones with state behind them. They - * count wrong guesses per project, route and user in the file named by - * `STUB_STATE`, the way the server keys those abuse limits, and answer `429` - * once a user's share of attempts is spent. The count is kept for every caller, - * an API key holder included, which is what appwrite/appwrite#15470 fixed. See + * A few routes have state behind them, kept in the file named by `STUB_STATE`. + * + * The routes that confirm a secret count wrong guesses per project, route and + * user, the way the server keys those abuse limits, and answer `429` once a + * user's share of attempts is spent. The count is kept for every caller, an API + * key holder included, which is what appwrite/appwrite#15470 fixed. See * `tests/code-guesses.php`. + * + * The execution routes record what they create, so a list can be read back. + * `STUB_EXECUTIONS_FAIL` makes that many async executions answer `500` — the + * code an Appwrite server gives when publishing the job to its message queue + * throws — and `STUB_EXECUTIONS_RECORD_ON_FAILURE` decides whether the + * execution row is written before that answer. Those are the two halves of the + * ambiguity a caller is left to resolve. See `tests/queue-restart.php`. */ declare(strict_types=1); @@ -33,22 +41,6 @@ $path = \parse_url($_SERVER['REQUEST_URI'], PHP_URL_PATH) ?: '/'; $project = $_SERVER['HTTP_X_APPWRITE_PROJECT'] ?? ''; -$log = \getenv('STUB_LOG'); -if (\is_string($log) && $log !== '') { - \file_put_contents( - $log, - \json_encode([ - 'method' => $method, - 'path' => $path, - 'project' => $project, - // Whether the caller held an API key, which used to exempt it from - // the limits on the routes below. - 'keyed' => ($_SERVER['HTTP_X_APPWRITE_KEY'] ?? '') !== '', - ]) . "\n", - FILE_APPEND | LOCK_EX - ); -} - /** * Answer a request and stop. * @@ -84,48 +76,81 @@ function body(): array } /** - * Read the counters, run `$change` over them and write back what it returns. + * Is this a request that asks for a function to be executed? + */ +function isExecutionRequest(string $method, string $path): bool +{ + return $method === 'POST' && \preg_match('#^/v1/functions/[^/]+/executions$#', $path) === 1; +} + +$log = \getenv('STUB_LOG'); +if (\is_string($log) && $log !== '') { + \file_put_contents( + $log, + \json_encode([ + 'method' => $method, + 'path' => $path, + 'project' => $project, + // Whether the caller held an API key, which used to exempt it from + // the limits on the routes below. + 'keyed' => ($_SERVER['HTTP_X_APPWRITE_KEY'] ?? '') !== '', + // The path a function execution asks for, which carries the tag + // `tests/queue-restart.php` recognises one attempt by across + // retries. Null for every other route. + 'executionPath' => isExecutionRequest($method, $path) + ? (string) (body()['path'] ?? '/') + : null, + 'async' => isExecutionRequest($method, $path) + ? (bool) (body()['async'] ?? false) + : null, + ]) . "\n", + FILE_APPEND | LOCK_EX + ); +} + +/** + * Read the stub's state, run `$change` over it and write back what it returns. * * Nothing here looks at whether the caller holds an API key: the limits on the * routes that confirm a secret count for every caller. * - * @param callable(array): array $change - * @return array the counters after the change + * @param callable(array): array $change + * @return array the state after the change */ -function counters(callable $change): array +function state(callable $change): array { - $state = \getenv('STUB_STATE'); + $file = \getenv('STUB_STATE'); - if (!\is_string($state) || $state === '') { + if (!\is_string($file) || $file === '') { return $change([]); } - $handle = \fopen($state, 'c+'); + $handle = \fopen($file, 'c+'); if ($handle === false) { respond(500, ['message' => 'Stub could not open its state', 'type' => 'general_unknown', 'code' => 500]); } \flock($handle, LOCK_EX); - $counts = $change(\json_decode(\stream_get_contents($handle) ?: '{}', true) ?: []); + $changed = $change(\json_decode(\stream_get_contents($handle) ?: '{}', true) ?: []); \ftruncate($handle, 0); \rewind($handle); - \fwrite($handle, (string) \json_encode($counts)); + \fwrite($handle, (string) \json_encode($changed)); \fflush($handle); \flock($handle, LOCK_UN); \fclose($handle); - return $counts; + return $changed; } function spent(string $counter): int { - return counters(static fn (array $counts): array => $counts)[$counter] ?? 0; + return state(static fn (array $counts): array => $counts)[$counter] ?? 0; } function spend(string $counter): void { - counters(static function (array $counts) use ($counter): array { + state(static function (array $counts) use ($counter): array { $counts[$counter] = ($counts[$counter] ?? 0) + 1; return $counts; @@ -197,6 +222,80 @@ function confirmSecret(string $project, string $path, array $success, bool $coun ], countEveryAttempt: true); } +/** + * Write down an execution the way the server does before it publishes the job. + * + * Only `requestPath` and `status` carry anything here: the path is where a + * caller's tag rides, and the status is what tells a run that was queued from + * one left behind by a publish that never happened. + * + * @return array the recorded execution + */ +function recordExecution(string $requestPath, string $status): array +{ + $execution = [ + '$id' => 'playground-execution', + '$createdAt' => \date('c'), + 'trigger' => 'http', + 'status' => $status, + 'requestMethod' => 'GET', + 'requestPath' => $requestPath, + ]; + + state(static function (array $state) use ($execution): array { + $state['executions'][] = $execution; + + return $state; + }); + + return $execution; +} + +if (isExecutionRequest($method, $path)) { + $async = (bool) (body()['async'] ?? false); + $requestPath = (string) (body()['path'] ?? '/'); + + /** + * How many async executions answer `500`, and whether the execution row is + * written before that answer. A synchronous execution never reaches the + * queue, so it is never one of them. + */ + $failures = (int) (\getenv('STUB_EXECUTIONS_FAIL') ?: '0'); + $recordOnFailure = \getenv('STUB_EXECUTIONS_RECORD_ON_FAILURE') === '1'; + + if ($async && $failures > 0) { + $attempt = state(static function (array $state): array { + $state['asyncAttempts'] = ($state['asyncAttempts'] ?? 0) + 1; + + return $state; + })['asyncAttempts']; + + if ($attempt <= $failures) { + if ($recordOnFailure) { + // The row is written, and then publishing the job throws. The + // run is registered and will never happen, and the caller is + // told only that something went wrong. + recordExecution($requestPath, 'waiting'); + } + + respond(500, [ + 'message' => 'Server Error', + 'type' => 'general_unknown', + 'code' => 500, + ]); + } + } + + respond(201, recordExecution($requestPath, $async ? 'waiting' : 'completed')); +} + +if ($method === 'GET' && \preg_match('#^/v1/functions/[^/]+/executions$#', $path) === 1) { + // Newest first, which is the order the playground's query asks for. + $executions = \array_reverse(state(static fn (array $state): array => $state)['executions'] ?? []); + + respond(200, ['total' => \count($executions), 'executions' => $executions]); +} + /** * @var array|null}> * [method, path pattern, status, body] — `*` matches one path segment. @@ -228,7 +327,7 @@ function confirmSecret(string $project, string $path, array $success, bool $coun ['DELETE', '/v1/functions/*', 204, null], ['POST', '/v1/functions/*/deployments', 202, ['$id' => 'playground-deployment', 'status' => 'ready']], ['GET', '/v1/functions/*/executions/*', 200, ['$id' => 'playground-execution', 'trigger' => 'http', 'status' => 'completed']], - ['POST', '/v1/functions/*/executions', 201, ['$id' => 'playground-execution', 'trigger' => 'http', 'status' => 'completed']], + // Creating and listing executions is handled above, where the state is. // Reached only once the JWT-only call at the end of the playground's // `$methods` list is uncommented. diff --git a/tests/stub.php b/tests/stub.php index cf20005..9f3e11d 100644 --- a/tests/stub.php +++ b/tests/stub.php @@ -55,7 +55,11 @@ private static function tempFile(string $prefix): string return $file; } - public static function start(): self + /** + * @param array $env extra environment for the stub server, + * used to ask it for a behaviour a test needs + */ + public static function start(array $env = []): self { $root = \dirname(__DIR__); $port = self::freePort(); @@ -72,7 +76,7 @@ public static function start(): self [1 => ['file', '/dev/null', 'w'], 2 => $errors], $pipes, $root, - ['STUB_LOG' => $requestLog, 'STUB_STATE' => $stateFile] + \getenv() + ['STUB_LOG' => $requestLog, 'STUB_STATE' => $stateFile] + $env + \getenv() ); if (!\is_resource($process)) { @@ -106,7 +110,8 @@ public static function start(): self /** * The requests the stub has served so far, oldest first. * - * @return list + * @return list */ public function requests(): array {