feat: in-process nightly scheduler + pause at window end
Hook registers a 15-minute schedule that starts/resumes the backfill only inside the maintenance window (no external cron or token). The backfill pauses when the window closes and resumes next night. UI shows paused state and last-run errors.
This commit is contained in:
parent
18db52a47c
commit
ee39633ad7
6 changed files with 60 additions and 8 deletions
|
|
@ -88,6 +88,14 @@ pacing).
|
|||
- `state.jsonl` = append-only completed `{file_id,preset_key,format}` → resume.
|
||||
- All state is **file-backed** (PM2 cluster, 8 workers — never in-memory).
|
||||
|
||||
## Nightly scheduling
|
||||
|
||||
- The hook registers `schedule('*/15 * * * *')` — **in-process**, no external cron/token. Every tick it
|
||||
checks enabled + maintenance window + no active run, then starts/resumes the backfill.
|
||||
- The backfill **pauses** when the window closes (`run.paused = true`) and resumes next night.
|
||||
- `scheduleSynchronizedJob` makes the cron cluster-safe; the `run.lock` file is a second guard.
|
||||
- `Run now` in the admin module ignores the window (`ignoreWindow: true`).
|
||||
|
||||
## Gotchas learned the hard way
|
||||
|
||||
- **Directus coerces env values**: `"true"` arrives as boolean `true`, `"1"` as number. Env parsing must
|
||||
|
|
|
|||
10
README.md
10
README.md
|
|
@ -110,12 +110,12 @@ The admin module includes a full API reference with curl examples.
|
|||
|
||||
## Scheduling
|
||||
|
||||
Scheduled runs are window-bound; an external cron can trigger them:
|
||||
The extension schedules itself **inside Directus** — no external cron, no token:
|
||||
|
||||
```bash
|
||||
0 * * * * curl -fsS -X POST http://localhost:8055/image-previews/run \
|
||||
-H 'Authorization: Bearer <admin-token>' -H 'Content-Type: application/json' -d '{}'
|
||||
```
|
||||
- every 15 minutes it checks whether it is inside the maintenance window;
|
||||
- if enabled, inside the window, and no run is active, it starts/resumes the backfill;
|
||||
- the backfill **pauses when the window closes** and resumes the next night (it is resumable);
|
||||
- once all tasks are done, it stays idle until the next day.
|
||||
|
||||
`Run now` in the admin module ignores the window.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,11 @@
|
|||
import { defineHook } from '@directus/extensions-sdk';
|
||||
import { runBackfill } from '../lib/backfill';
|
||||
import { recordOp } from '../lib/diagnostics';
|
||||
import { buildRegistry, isImageType, loadCustomPresets } from '../lib/presets';
|
||||
import { readSettings } from '../lib/settings';
|
||||
import { readRun } from '../lib/state';
|
||||
import { warmFile } from '../lib/warmer';
|
||||
import { isInWindow } from '../lib/window';
|
||||
|
||||
interface UploadMeta {
|
||||
key?: string;
|
||||
|
|
@ -10,13 +13,19 @@ interface UploadMeta {
|
|||
payload?: { type?: string | null; collection?: string };
|
||||
}
|
||||
|
||||
export default defineHook(({ action }, { services, getSchema, env, logger }) => {
|
||||
export default defineHook(({ action, schedule }, { services, database, getSchema, env, logger }) => {
|
||||
action('files.upload', (meta) => {
|
||||
void warmUpload(meta as UploadMeta).catch((error) =>
|
||||
logger.error(error, 'image-previews: upload warm failed'),
|
||||
);
|
||||
});
|
||||
|
||||
schedule('*/15 * * * *', () => {
|
||||
void scheduledBackfill().catch((error) =>
|
||||
logger.error(error, 'image-previews: scheduled run failed'),
|
||||
);
|
||||
});
|
||||
|
||||
async function warmUpload(meta: UploadMeta): Promise<void> {
|
||||
const settings = await readSettings(env);
|
||||
|
||||
|
|
@ -53,4 +62,27 @@ export default defineHook(({ action }, { services, getSchema, env, logger }) =>
|
|||
|
||||
if (settings.debug) logger.info({ fileId, summary }, 'image-previews: warmed upload');
|
||||
}
|
||||
|
||||
async function scheduledBackfill(): Promise<void> {
|
||||
const settings = await readSettings(env);
|
||||
|
||||
if (!settings.enabled) return;
|
||||
if (!isInWindow(settings.windowStart, settings.windowEnd)) return;
|
||||
|
||||
const run = await readRun(env);
|
||||
|
||||
if (run?.running) return;
|
||||
|
||||
const finishedToday = run?.finishedAt
|
||||
? new Date(run.finishedAt).toDateString() === new Date().toDateString()
|
||||
: false;
|
||||
|
||||
if (finishedToday && run && run.done >= run.total) return;
|
||||
|
||||
logger.info('image-previews: scheduled backfill starting');
|
||||
|
||||
void runBackfill({ services, database, getSchema, env, logger }, {}).catch((error) =>
|
||||
logger.error(error, 'image-previews: scheduled backfill failed'),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
|
|
|||
|
|
@ -89,6 +89,7 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions =
|
|||
rate: 0,
|
||||
eta: null,
|
||||
cancelled: false,
|
||||
paused: false,
|
||||
};
|
||||
|
||||
await writeRun(run, deps.env);
|
||||
|
|
@ -119,6 +120,11 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions =
|
|||
if (!Array.isArray(files) || files.length === 0) break;
|
||||
|
||||
for (const file of files) {
|
||||
if (!options.ignoreWindow && !isInWindow(settings.windowStart, settings.windowEnd)) {
|
||||
run.paused = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (await isCancelled(deps.env)) {
|
||||
run.cancelled = true;
|
||||
break;
|
||||
|
|
@ -209,7 +215,7 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions =
|
|||
await writeRun(run, deps.env);
|
||||
}
|
||||
|
||||
if (run.cancelled || files.length < PAGE_SIZE) break;
|
||||
if (run.cancelled || run.paused || files.length < PAGE_SIZE) break;
|
||||
page += 1;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -47,5 +47,6 @@ export interface RunState {
|
|||
rate: number;
|
||||
eta: number | null;
|
||||
cancelled: boolean;
|
||||
paused?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,9 @@
|
|||
<v-notice v-if="status && !status.enabled" type="warning">
|
||||
Preview generation is disabled. Enable it in Configuration below.
|
||||
</v-notice>
|
||||
<v-notice v-if="status?.run?.error" type="danger">
|
||||
Last run error: {{ status.run.error }}
|
||||
</v-notice>
|
||||
|
||||
<section class="ip__card">
|
||||
<header class="ip__head">
|
||||
|
|
@ -35,7 +38,7 @@
|
|||
|
||||
<p class="ip__hint">
|
||||
{{ status?.enabled ? 'enabled' : 'disabled' }} ·
|
||||
{{ status?.inWindow ? 'inside maintenance window' : 'outside maintenance window' }} ·
|
||||
{{ status?.run?.paused ? 'paused (window closed)' : status?.inWindow ? 'inside maintenance window' : 'outside maintenance window' }} ·
|
||||
next window {{ status?.nextWindowHuman ?? '—' }}
|
||||
</p>
|
||||
</section>
|
||||
|
|
@ -228,6 +231,8 @@ interface RunState {
|
|||
etaHuman: string;
|
||||
startedAtHuman: string | null;
|
||||
finishedAtHuman: string | null;
|
||||
paused?: boolean;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
interface Status {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue