feat: in-process nightly scheduler + pause at window end
Hook registers a 15-minute schedule that starts/resumes the backfill only inside the maintenance window (no external cron or token). The backfill pauses when the window closes and resumes next night. UI shows paused state and last-run errors.
This commit is contained in:
parent
18db52a47c
commit
ee39633ad7
6 changed files with 60 additions and 8 deletions
|
|
@ -88,6 +88,14 @@ pacing).
|
||||||
- `state.jsonl` = append-only completed `{file_id,preset_key,format}` → resume.
|
- `state.jsonl` = append-only completed `{file_id,preset_key,format}` → resume.
|
||||||
- All state is **file-backed** (PM2 cluster, 8 workers — never in-memory).
|
- All state is **file-backed** (PM2 cluster, 8 workers — never in-memory).
|
||||||
|
|
||||||
|
## Nightly scheduling
|
||||||
|
|
||||||
|
- The hook registers `schedule('*/15 * * * *')` — **in-process**, no external cron/token. Every tick it
|
||||||
|
checks enabled + maintenance window + no active run, then starts/resumes the backfill.
|
||||||
|
- The backfill **pauses** when the window closes (`run.paused = true`) and resumes next night.
|
||||||
|
- `scheduleSynchronizedJob` makes the cron cluster-safe; the `run.lock` file is a second guard.
|
||||||
|
- `Run now` in the admin module ignores the window (`ignoreWindow: true`).
|
||||||
|
|
||||||
## Gotchas learned the hard way
|
## Gotchas learned the hard way
|
||||||
|
|
||||||
- **Directus coerces env values**: `"true"` arrives as boolean `true`, `"1"` as number. Env parsing must
|
- **Directus coerces env values**: `"true"` arrives as boolean `true`, `"1"` as number. Env parsing must
|
||||||
|
|
|
||||||
10
README.md
10
README.md
|
|
@ -110,12 +110,12 @@ The admin module includes a full API reference with curl examples.
|
||||||
|
|
||||||
## Scheduling
|
## Scheduling
|
||||||
|
|
||||||
Scheduled runs are window-bound; an external cron can trigger them:
|
The extension schedules itself **inside Directus** — no external cron, no token:
|
||||||
|
|
||||||
```bash
|
- every 15 minutes it checks whether it is inside the maintenance window;
|
||||||
0 * * * * curl -fsS -X POST http://localhost:8055/image-previews/run \
|
- if enabled, inside the window, and no run is active, it starts/resumes the backfill;
|
||||||
-H 'Authorization: Bearer <admin-token>' -H 'Content-Type: application/json' -d '{}'
|
- the backfill **pauses when the window closes** and resumes the next night (it is resumable);
|
||||||
```
|
- once all tasks are done, it stays idle until the next day.
|
||||||
|
|
||||||
`Run now` in the admin module ignores the window.
|
`Run now` in the admin module ignores the window.
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,8 +1,11 @@
|
||||||
import { defineHook } from '@directus/extensions-sdk';
|
import { defineHook } from '@directus/extensions-sdk';
|
||||||
|
import { runBackfill } from '../lib/backfill';
|
||||||
import { recordOp } from '../lib/diagnostics';
|
import { recordOp } from '../lib/diagnostics';
|
||||||
import { buildRegistry, isImageType, loadCustomPresets } from '../lib/presets';
|
import { buildRegistry, isImageType, loadCustomPresets } from '../lib/presets';
|
||||||
import { readSettings } from '../lib/settings';
|
import { readSettings } from '../lib/settings';
|
||||||
|
import { readRun } from '../lib/state';
|
||||||
import { warmFile } from '../lib/warmer';
|
import { warmFile } from '../lib/warmer';
|
||||||
|
import { isInWindow } from '../lib/window';
|
||||||
|
|
||||||
interface UploadMeta {
|
interface UploadMeta {
|
||||||
key?: string;
|
key?: string;
|
||||||
|
|
@ -10,13 +13,19 @@ interface UploadMeta {
|
||||||
payload?: { type?: string | null; collection?: string };
|
payload?: { type?: string | null; collection?: string };
|
||||||
}
|
}
|
||||||
|
|
||||||
export default defineHook(({ action }, { services, getSchema, env, logger }) => {
|
export default defineHook(({ action, schedule }, { services, database, getSchema, env, logger }) => {
|
||||||
action('files.upload', (meta) => {
|
action('files.upload', (meta) => {
|
||||||
void warmUpload(meta as UploadMeta).catch((error) =>
|
void warmUpload(meta as UploadMeta).catch((error) =>
|
||||||
logger.error(error, 'image-previews: upload warm failed'),
|
logger.error(error, 'image-previews: upload warm failed'),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
schedule('*/15 * * * *', () => {
|
||||||
|
void scheduledBackfill().catch((error) =>
|
||||||
|
logger.error(error, 'image-previews: scheduled run failed'),
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
async function warmUpload(meta: UploadMeta): Promise<void> {
|
async function warmUpload(meta: UploadMeta): Promise<void> {
|
||||||
const settings = await readSettings(env);
|
const settings = await readSettings(env);
|
||||||
|
|
||||||
|
|
@ -53,4 +62,27 @@ export default defineHook(({ action }, { services, getSchema, env, logger }) =>
|
||||||
|
|
||||||
if (settings.debug) logger.info({ fileId, summary }, 'image-previews: warmed upload');
|
if (settings.debug) logger.info({ fileId, summary }, 'image-previews: warmed upload');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function scheduledBackfill(): Promise<void> {
|
||||||
|
const settings = await readSettings(env);
|
||||||
|
|
||||||
|
if (!settings.enabled) return;
|
||||||
|
if (!isInWindow(settings.windowStart, settings.windowEnd)) return;
|
||||||
|
|
||||||
|
const run = await readRun(env);
|
||||||
|
|
||||||
|
if (run?.running) return;
|
||||||
|
|
||||||
|
const finishedToday = run?.finishedAt
|
||||||
|
? new Date(run.finishedAt).toDateString() === new Date().toDateString()
|
||||||
|
: false;
|
||||||
|
|
||||||
|
if (finishedToday && run && run.done >= run.total) return;
|
||||||
|
|
||||||
|
logger.info('image-previews: scheduled backfill starting');
|
||||||
|
|
||||||
|
void runBackfill({ services, database, getSchema, env, logger }, {}).catch((error) =>
|
||||||
|
logger.error(error, 'image-previews: scheduled backfill failed'),
|
||||||
|
);
|
||||||
|
}
|
||||||
});
|
});
|
||||||
|
|
|
||||||
|
|
@ -89,6 +89,7 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions =
|
||||||
rate: 0,
|
rate: 0,
|
||||||
eta: null,
|
eta: null,
|
||||||
cancelled: false,
|
cancelled: false,
|
||||||
|
paused: false,
|
||||||
};
|
};
|
||||||
|
|
||||||
await writeRun(run, deps.env);
|
await writeRun(run, deps.env);
|
||||||
|
|
@ -119,6 +120,11 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions =
|
||||||
if (!Array.isArray(files) || files.length === 0) break;
|
if (!Array.isArray(files) || files.length === 0) break;
|
||||||
|
|
||||||
for (const file of files) {
|
for (const file of files) {
|
||||||
|
if (!options.ignoreWindow && !isInWindow(settings.windowStart, settings.windowEnd)) {
|
||||||
|
run.paused = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
if (await isCancelled(deps.env)) {
|
if (await isCancelled(deps.env)) {
|
||||||
run.cancelled = true;
|
run.cancelled = true;
|
||||||
break;
|
break;
|
||||||
|
|
@ -209,7 +215,7 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions =
|
||||||
await writeRun(run, deps.env);
|
await writeRun(run, deps.env);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (run.cancelled || files.length < PAGE_SIZE) break;
|
if (run.cancelled || run.paused || files.length < PAGE_SIZE) break;
|
||||||
page += 1;
|
page += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -47,5 +47,6 @@ export interface RunState {
|
||||||
rate: number;
|
rate: number;
|
||||||
eta: number | null;
|
eta: number | null;
|
||||||
cancelled: boolean;
|
cancelled: boolean;
|
||||||
|
paused?: boolean;
|
||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -4,6 +4,9 @@
|
||||||
<v-notice v-if="status && !status.enabled" type="warning">
|
<v-notice v-if="status && !status.enabled" type="warning">
|
||||||
Preview generation is disabled. Enable it in Configuration below.
|
Preview generation is disabled. Enable it in Configuration below.
|
||||||
</v-notice>
|
</v-notice>
|
||||||
|
<v-notice v-if="status?.run?.error" type="danger">
|
||||||
|
Last run error: {{ status.run.error }}
|
||||||
|
</v-notice>
|
||||||
|
|
||||||
<section class="ip__card">
|
<section class="ip__card">
|
||||||
<header class="ip__head">
|
<header class="ip__head">
|
||||||
|
|
@ -35,7 +38,7 @@
|
||||||
|
|
||||||
<p class="ip__hint">
|
<p class="ip__hint">
|
||||||
{{ status?.enabled ? 'enabled' : 'disabled' }} ·
|
{{ status?.enabled ? 'enabled' : 'disabled' }} ·
|
||||||
{{ status?.inWindow ? 'inside maintenance window' : 'outside maintenance window' }} ·
|
{{ status?.run?.paused ? 'paused (window closed)' : status?.inWindow ? 'inside maintenance window' : 'outside maintenance window' }} ·
|
||||||
next window {{ status?.nextWindowHuman ?? '—' }}
|
next window {{ status?.nextWindowHuman ?? '—' }}
|
||||||
</p>
|
</p>
|
||||||
</section>
|
</section>
|
||||||
|
|
@ -228,6 +231,8 @@ interface RunState {
|
||||||
etaHuman: string;
|
etaHuman: string;
|
||||||
startedAtHuman: string | null;
|
startedAtHuman: string | null;
|
||||||
finishedAtHuman: string | null;
|
finishedAtHuman: string | null;
|
||||||
|
paused?: boolean;
|
||||||
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface Status {
|
interface Status {
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue