feat: pre-generate Directus image previews
Bundle extension (upload hook + API endpoint + admin module) that warms Directus image transform cache in-process with an explicit acceptFormat, so the generated file matches the browser's cache key byte-for-byte. - resumable, window-aware backfill with progress + cancel - system + custom presets, effective-format reporting, shadow warnings - storage breakdown (previews vs originals), human-readable API fields - diagnostics: resolve-plan prediction, per-file generate, ops ring buffer - isolated dev/staging compose stack + deploy/rollback scripts Verified on Directus 11.17.4: 40/40 tests, hash-match gate passed, idempotent backfill, custom fixed-format presets, error surfacing.
This commit is contained in:
commit
c8f8bbc5ac
42 changed files with 10005 additions and 0 deletions
312
src/endpoint/index.ts
Normal file
312
src/endpoint/index.ts
Normal file
|
|
@ -0,0 +1,312 @@
|
|||
import { defineEndpoint } from '@directus/extensions-sdk';
|
||||
import type { Request, Response, NextFunction } from 'express';
|
||||
import { runBackfill, requestCancel } from '../lib/backfill';
|
||||
import { computePlan, getOps, recordOp } from '../lib/diagnostics';
|
||||
import { humanBytes, humanDuration, humanMs, toIso } from '../lib/format';
|
||||
import { buildRegistry, effectiveFormatsForPreset, formatsForPreset, loadCustomPresets } from '../lib/presets';
|
||||
import { mergeSettings, readSettings, writeSettings } from '../lib/settings';
|
||||
import { isLocked, readRun } from '../lib/state';
|
||||
import { previewExists, storageUsage } from '../lib/storage';
|
||||
import { SYSTEM_PRESETS } from '../lib/system-presets';
|
||||
import { isInWindow, nextWindow } from '../lib/window';
|
||||
import { warmFile } from '../lib/warmer';
|
||||
|
||||
const FILE_FIELDS = [
|
||||
'id',
|
||||
'filename_disk',
|
||||
'type',
|
||||
'width',
|
||||
'height',
|
||||
'focal_point_x',
|
||||
'focal_point_y',
|
||||
'storage',
|
||||
];
|
||||
|
||||
function asyncHandler(handler: (req: Request, res: Response) => Promise<void>) {
|
||||
return (req: Request, res: Response, next: NextFunction) => {
|
||||
handler(req, res).catch(next);
|
||||
};
|
||||
}
|
||||
|
||||
function bucket(bytes: number, count: number) {
|
||||
return { bytes, count, bytesHuman: humanBytes(bytes) };
|
||||
}
|
||||
|
||||
export default defineEndpoint({
|
||||
id: 'image-previews',
|
||||
handler: (router, context) => {
|
||||
const { services, database, getSchema, env, logger } = context;
|
||||
|
||||
router.use((req: Request, res: Response, next: NextFunction) => {
|
||||
const accountability = (req as Request & { accountability?: { admin?: boolean } }).accountability;
|
||||
|
||||
if (accountability?.admin === true) return next();
|
||||
|
||||
res.status(403).json({ error: 'admin access required' });
|
||||
});
|
||||
|
||||
router.get(
|
||||
'/status',
|
||||
asyncHandler(async (_req, res) => {
|
||||
const settings = await readSettings(env);
|
||||
const run = await readRun(env);
|
||||
const next = nextWindow(settings.windowStart, settings.windowEnd);
|
||||
|
||||
res.json({
|
||||
enabled: settings.enabled,
|
||||
window: { start: settings.windowStart, end: settings.windowEnd },
|
||||
inWindow: isInWindow(settings.windowStart, settings.windowEnd),
|
||||
nextWindow: next?.toISOString() ?? null,
|
||||
nextWindowHuman: next?.toLocaleString() ?? null,
|
||||
locked: await isLocked(env),
|
||||
run: run
|
||||
? {
|
||||
...run,
|
||||
startedAtIso: toIso(run.startedAt),
|
||||
finishedAtIso: toIso(run.finishedAt),
|
||||
startedAtHuman: run.startedAt ? new Date(run.startedAt).toLocaleString() : null,
|
||||
finishedAtHuman: run.finishedAt ? new Date(run.finishedAt).toLocaleString() : null,
|
||||
etaHuman: humanDuration(run.eta),
|
||||
rateHuman: `${Math.round(run.rate)} items/min`,
|
||||
}
|
||||
: null,
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
'/config',
|
||||
asyncHandler(async (_req, res) => {
|
||||
res.json(await readSettings(env));
|
||||
}),
|
||||
);
|
||||
|
||||
router.put(
|
||||
'/config',
|
||||
asyncHandler(async (req, res) => {
|
||||
try {
|
||||
const current = await readSettings(env);
|
||||
const merged = mergeSettings(current, req.body ?? {});
|
||||
const saved = await writeSettings(merged, env);
|
||||
res.json(saved);
|
||||
} catch (error) {
|
||||
res.status(400).json({ error: error instanceof Error ? error.message : String(error) });
|
||||
}
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
'/presets',
|
||||
asyncHandler(async (_req, res) => {
|
||||
const schema = await getSchema();
|
||||
const settings = await readSettings(env);
|
||||
const custom = await loadCustomPresets(new services['SettingsService']({ schema }));
|
||||
const presets = buildRegistry(custom, null);
|
||||
const systemKeys = new Set(SYSTEM_PRESETS.map((preset) => preset.key));
|
||||
const shadowedKeys = custom
|
||||
.filter((preset) => systemKeys.has(preset.key))
|
||||
.map((preset) => preset.key);
|
||||
|
||||
res.json({
|
||||
shadowedKeys,
|
||||
presets: presets.map((preset) => ({
|
||||
key: preset.key,
|
||||
format: preset.format ?? null,
|
||||
effectiveFormats: effectiveFormatsForPreset(preset, settings.formats),
|
||||
system: systemKeys.has(preset.key),
|
||||
enabled: settings.presetKeys === null || settings.presetKeys.includes(preset.key),
|
||||
})),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
'/run',
|
||||
asyncHandler(async (req, res) => {
|
||||
const settings = await readSettings(env);
|
||||
const ignoreWindow = req.body?.ignoreWindow === true;
|
||||
|
||||
if (!settings.enabled) {
|
||||
res.status(409).json({ error: 'disabled' });
|
||||
return;
|
||||
}
|
||||
|
||||
if (!ignoreWindow && !isInWindow(settings.windowStart, settings.windowEnd)) {
|
||||
const next = nextWindow(settings.windowStart, settings.windowEnd);
|
||||
res.status(409).json({
|
||||
error: 'outside-window',
|
||||
nextWindow: next?.toISOString() ?? null,
|
||||
nextWindowHuman: next?.toLocaleString() ?? null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (await isLocked(env)) {
|
||||
res.status(409).json({ error: 'already-running' });
|
||||
return;
|
||||
}
|
||||
|
||||
const presetKeys = Array.isArray(req.body?.presets) ? (req.body.presets as string[]) : undefined;
|
||||
|
||||
void runBackfill(
|
||||
{ services, database, getSchema, env, logger },
|
||||
{ presetKeys, ignoreWindow },
|
||||
).catch((error) => logger.error(error, 'image-previews: run failed'));
|
||||
|
||||
res.status(202).json({ started: true });
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
'/run/cancel',
|
||||
asyncHandler(async (_req, res) => {
|
||||
await requestCancel(env);
|
||||
res.json({ cancelling: true });
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
'/disk-usage',
|
||||
asyncHandler(async (_req, res) => {
|
||||
const usage = await storageUsage(env);
|
||||
|
||||
res.json({
|
||||
previews: bucket(usage.previews.bytes, usage.previews.count),
|
||||
originals: bucket(usage.originals.bytes, usage.originals.count),
|
||||
total: bucket(usage.total.bytes, usage.total.count),
|
||||
source: usage.source,
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
'/ops',
|
||||
asyncHandler(async (req, res) => {
|
||||
const limit = Number(req.query.limit) || 50;
|
||||
|
||||
res.json({
|
||||
ops: getOps(limit).map((op) => ({
|
||||
...op,
|
||||
tsIso: toIso(op.ts),
|
||||
tsHuman: new Date(op.ts).toLocaleString(),
|
||||
msHuman: humanMs(op.ms),
|
||||
})),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
router.post(
|
||||
'/generate',
|
||||
asyncHandler(async (req, res) => {
|
||||
const fileId = req.body?.fileId;
|
||||
|
||||
if (typeof fileId !== 'string' || fileId.length === 0) {
|
||||
res.status(400).json({ error: 'fileId is required' });
|
||||
return;
|
||||
}
|
||||
|
||||
const settings = await readSettings(env);
|
||||
const schema = await getSchema();
|
||||
const custom = await loadCustomPresets(new services['SettingsService']({ schema }));
|
||||
const presetKeys = Array.isArray(req.body?.keys) ? (req.body.keys as string[]) : settings.presetKeys;
|
||||
const formats = Array.isArray(req.body?.formats) ? (req.body.formats as string[]) : settings.formats;
|
||||
const presets = buildRegistry(custom, presetKeys);
|
||||
|
||||
const items = new services['ItemsService']('directus_files', { schema });
|
||||
|
||||
if (req.body?.dryRun === true) {
|
||||
const file = await items.readOne(fileId, { fields: FILE_FIELDS });
|
||||
const plans = [];
|
||||
|
||||
for (const preset of presets) {
|
||||
for (const format of formatsForPreset(preset, formats)) {
|
||||
const plan = await computePlan(file, preset, format);
|
||||
plans.push({
|
||||
key: preset.key,
|
||||
format,
|
||||
effectiveFormat: plan.format,
|
||||
filename: plan.filename,
|
||||
exists: await previewExists(env, file.storage ?? null, plan.filename),
|
||||
transforms: plan.transforms,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ dryRun: true, fileId, plans });
|
||||
return;
|
||||
}
|
||||
|
||||
const assets = new services['AssetsService']({ schema });
|
||||
const started = Date.now();
|
||||
const summary = await warmFile(assets, fileId, presets, {
|
||||
concurrency: settings.concurrency,
|
||||
formats,
|
||||
onResult: (result) =>
|
||||
recordOp({
|
||||
ts: Date.now(),
|
||||
fileId,
|
||||
key: result.key,
|
||||
format: result.format,
|
||||
ok: result.ok,
|
||||
ms: result.ms,
|
||||
error: result.error,
|
||||
}),
|
||||
});
|
||||
|
||||
res.json({
|
||||
fileId,
|
||||
ms: Date.now() - started,
|
||||
msHuman: humanMs(Date.now() - started),
|
||||
...summary,
|
||||
results: summary.results.map((result) => ({ ...result, msHuman: humanMs(result.ms) })),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
router.get(
|
||||
'/resolve-plan',
|
||||
asyncHandler(async (req, res) => {
|
||||
const fileId = typeof req.query.fileId === 'string' ? req.query.fileId : '';
|
||||
const key = typeof req.query.key === 'string' ? req.query.key : '';
|
||||
|
||||
if (!fileId || !key) {
|
||||
res.status(400).json({ error: 'fileId and key are required' });
|
||||
return;
|
||||
}
|
||||
|
||||
const settings = await readSettings(env);
|
||||
const schema = await getSchema();
|
||||
const custom = await loadCustomPresets(new services['SettingsService']({ schema }));
|
||||
const presets = buildRegistry(custom, null);
|
||||
const preset = presets.find((candidate) => candidate.key === key);
|
||||
|
||||
if (!preset) {
|
||||
const systemKeys = SYSTEM_PRESETS.map((candidate) => candidate.key);
|
||||
const shadowed = custom.some((candidate) => candidate.key === key);
|
||||
res.status(404).json({
|
||||
error: `unknown preset: ${key}`,
|
||||
reason: shadowed && systemKeys.includes(key) ? 'shadowed-by-system' : 'not-found',
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
const items = new services['ItemsService']('directus_files', { schema });
|
||||
const file = await items.readOne(fileId, { fields: FILE_FIELDS });
|
||||
const plans = [];
|
||||
|
||||
for (const format of formatsForPreset(preset, settings.formats)) {
|
||||
const plan = await computePlan(file, preset, format);
|
||||
plans.push({
|
||||
format,
|
||||
effectiveFormat: plan.format,
|
||||
filename: plan.filename,
|
||||
exists: await previewExists(env, file.storage ?? null, plan.filename),
|
||||
transforms: plan.transforms,
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ fileId, key, plans });
|
||||
}),
|
||||
);
|
||||
},
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue