feat: pre-generate Directus image previews
Bundle extension (upload hook + API endpoint + admin module) that warms Directus image transform cache in-process with an explicit acceptFormat, so the generated file matches the browser's cache key byte-for-byte. - resumable, window-aware backfill with progress + cancel - system + custom presets, effective-format reporting, shadow warnings - storage breakdown (previews vs originals), human-readable API fields - diagnostics: resolve-plan prediction, per-file generate, ops ring buffer - isolated dev/staging compose stack + deploy/rollback scripts Verified on Directus 11.17.4: 40/40 tests, hash-match gate passed, idempotent backfill, custom fixed-format presets, error surfacing.
This commit is contained in:
commit
c8f8bbc5ac
42 changed files with 10005 additions and 0 deletions
206
src/lib/backfill.ts
Normal file
206
src/lib/backfill.ts
Normal file
|
|
@ -0,0 +1,206 @@
|
|||
import { recordOp } from './diagnostics';
|
||||
import { buildRegistry, effectiveFormatsForPreset, isImageType, loadCustomPresets, tasksPerFile } from './presets';
|
||||
import { readSettings } from './settings';
|
||||
import {
|
||||
acquireLock,
|
||||
appendCompleted,
|
||||
clearCancel,
|
||||
completedKeyOf,
|
||||
isCancelled,
|
||||
loadCompleted,
|
||||
readRun,
|
||||
releaseLock,
|
||||
requestCancel,
|
||||
writeRun,
|
||||
} from './state';
|
||||
import type { RunState } from './types';
|
||||
import { errorMessage, warmFile } from './warmer';
|
||||
import { isInWindow } from './window';
|
||||
|
||||
export interface BackfillDeps {
|
||||
services: Record<string, any>;
|
||||
database: any;
|
||||
getSchema: () => Promise<any>;
|
||||
env: Record<string, any>;
|
||||
logger: any;
|
||||
}
|
||||
|
||||
export interface BackfillOptions {
|
||||
presetKeys?: string[] | null;
|
||||
ignoreWindow?: boolean;
|
||||
}
|
||||
|
||||
export interface BackfillOutcome {
|
||||
status: 'completed' | 'skipped' | 'failed';
|
||||
reason?: string;
|
||||
run?: RunState;
|
||||
}
|
||||
|
||||
const PAGE_SIZE = 100;
|
||||
|
||||
let inProcessRunning = false;
|
||||
|
||||
async function countImageFiles(database: any): Promise<number> {
|
||||
try {
|
||||
const rows = await database('directus_files').where('type', 'like', 'image/%').count({ count: '*' });
|
||||
const row = Array.isArray(rows) ? rows[0] : rows;
|
||||
return Number(row?.count) || 0;
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = {}): Promise<BackfillOutcome> {
|
||||
if (inProcessRunning) return { status: 'skipped', reason: 'already-running' };
|
||||
|
||||
const settings = await readSettings(deps.env);
|
||||
|
||||
if (!settings.enabled) return { status: 'skipped', reason: 'disabled' };
|
||||
|
||||
if (!options.ignoreWindow && !isInWindow(settings.windowStart, settings.windowEnd)) {
|
||||
return { status: 'skipped', reason: 'outside-window' };
|
||||
}
|
||||
|
||||
if (!(await acquireLock(deps.env))) return { status: 'skipped', reason: 'already-running' };
|
||||
|
||||
inProcessRunning = true;
|
||||
await clearCancel(deps.env);
|
||||
|
||||
try {
|
||||
const schema = await deps.getSchema();
|
||||
const settingsService = new deps.services['SettingsService']({ schema });
|
||||
const custom = await loadCustomPresets(settingsService);
|
||||
const presetKeys = options.presetKeys !== undefined ? options.presetKeys : settings.presetKeys;
|
||||
const presets = buildRegistry(custom, presetKeys);
|
||||
const formats = settings.formats;
|
||||
const perFile = tasksPerFile(presets, formats);
|
||||
const totalFiles = await countImageFiles(deps.database);
|
||||
|
||||
const run: RunState = {
|
||||
running: true,
|
||||
startedAt: Date.now(),
|
||||
finishedAt: null,
|
||||
total: totalFiles * perFile,
|
||||
done: 0,
|
||||
generated: 0,
|
||||
hits: 0,
|
||||
failed: 0,
|
||||
rate: 0,
|
||||
eta: null,
|
||||
cancelled: false,
|
||||
};
|
||||
|
||||
await writeRun(run, deps.env);
|
||||
|
||||
const completed = await loadCompleted(deps.env);
|
||||
const items = new deps.services['ItemsService']('directus_files', { schema });
|
||||
const assets = new deps.services['AssetsService']({ schema });
|
||||
const started = Date.now();
|
||||
let page = 1;
|
||||
|
||||
while (true) {
|
||||
const files = await items.readByQuery({
|
||||
filter: { type: { _starts_with: 'image/' } },
|
||||
fields: [ 'id', 'filename_disk', 'type', 'width', 'height', 'focal_point_x', 'focal_point_y', 'storage' ],
|
||||
limit: PAGE_SIZE,
|
||||
page,
|
||||
sort: [ 'id' ],
|
||||
});
|
||||
|
||||
if (!Array.isArray(files) || files.length === 0) break;
|
||||
|
||||
for (const file of files) {
|
||||
if (await isCancelled(deps.env)) {
|
||||
run.cancelled = true;
|
||||
break;
|
||||
}
|
||||
|
||||
if (!isImageType(file.type)) continue;
|
||||
|
||||
const pending = presets.filter((preset) =>
|
||||
effectiveFormatsForPreset(preset, formats).some(
|
||||
(format) => !completed.has(completedKeyOf(file.id, preset.key, format)),
|
||||
),
|
||||
);
|
||||
|
||||
const pendingTasks = pending.reduce(
|
||||
(total, preset) => total + effectiveFormatsForPreset(preset, formats).length,
|
||||
0,
|
||||
);
|
||||
const alreadyDone = perFile - pendingTasks;
|
||||
|
||||
if (alreadyDone > 0) {
|
||||
run.hits += alreadyDone;
|
||||
run.done += alreadyDone;
|
||||
}
|
||||
|
||||
if (pending.length === 0) {
|
||||
await writeRun(run, deps.env);
|
||||
continue;
|
||||
}
|
||||
|
||||
const summary = await warmFile(assets, file.id, pending, {
|
||||
concurrency: settings.concurrency,
|
||||
formats,
|
||||
onResult: (result) =>
|
||||
recordOp({
|
||||
ts: Date.now(),
|
||||
fileId: file.id,
|
||||
key: result.key,
|
||||
format: result.format,
|
||||
ok: result.ok,
|
||||
ms: result.ms,
|
||||
error: result.error,
|
||||
}),
|
||||
});
|
||||
|
||||
for (const result of summary.results) {
|
||||
if (result.ok) {
|
||||
completed.add(completedKeyOf(file.id, result.key, result.format));
|
||||
await appendCompleted(
|
||||
{ file_id: file.id, preset_key: result.key, format: result.format, ts: Date.now() },
|
||||
deps.env,
|
||||
);
|
||||
run.generated += 1;
|
||||
} else {
|
||||
run.failed += 1;
|
||||
}
|
||||
|
||||
run.done += 1;
|
||||
}
|
||||
|
||||
const elapsedMinutes = (Date.now() - started) / 60000;
|
||||
run.rate = elapsedMinutes > 0 ? run.done / elapsedMinutes : 0;
|
||||
run.eta = run.rate > 0 ? Math.round(((run.total - run.done) / run.rate) * 60) : null;
|
||||
await writeRun(run, deps.env);
|
||||
}
|
||||
|
||||
if (run.cancelled || files.length < PAGE_SIZE) break;
|
||||
page += 1;
|
||||
}
|
||||
|
||||
run.running = false;
|
||||
run.finishedAt = Date.now();
|
||||
await writeRun(run, deps.env);
|
||||
|
||||
return { status: 'completed', run };
|
||||
} catch (error) {
|
||||
deps.logger?.error?.(error, 'image-previews: backfill failed');
|
||||
const existing = await readRun(deps.env);
|
||||
|
||||
if (existing) {
|
||||
existing.running = false;
|
||||
existing.finishedAt = Date.now();
|
||||
existing.error = errorMessage(error);
|
||||
await writeRun(existing, deps.env);
|
||||
}
|
||||
|
||||
return { status: 'failed', reason: errorMessage(error) };
|
||||
} finally {
|
||||
inProcessRunning = false;
|
||||
await releaseLock(deps.env);
|
||||
await clearCancel(deps.env);
|
||||
}
|
||||
}
|
||||
|
||||
export { requestCancel };
|
||||
77
src/lib/diagnostics.test.ts
Normal file
77
src/lib/diagnostics.test.ts
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { computePlan, getOps, recordOp, type PlanFile } from './diagnostics';
|
||||
import type { Preset } from './types';
|
||||
|
||||
const file: PlanFile = {
|
||||
filename_disk: 'abc123.jpg',
|
||||
type: 'image/jpeg',
|
||||
width: 1200,
|
||||
height: 800,
|
||||
focal_point_x: null,
|
||||
focal_point_y: null,
|
||||
storage: 'local',
|
||||
};
|
||||
|
||||
const preset: Preset = {
|
||||
key: 'system-small-cover',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 64, height: 64, fit: 'cover' } ] ],
|
||||
};
|
||||
|
||||
function utilsWith(format: string | undefined) {
|
||||
return {
|
||||
resolvePreset: () =>
|
||||
format ? [ [ 'toFormat', format, { quality: undefined } ], [ 'resize', { width: 64 } ] ] : [ [ 'resize', { width: 64 } ] ],
|
||||
maybeExtractFormat: () => format,
|
||||
};
|
||||
}
|
||||
|
||||
test('computePlan predicts a hashed filename with the resolved extension', async () => {
|
||||
const plan = await computePlan(file, preset, 'avif', {
|
||||
utils: utilsWith('avif'),
|
||||
hashFn: () => 'deadbeef',
|
||||
});
|
||||
|
||||
assert.equal(plan.filename, 'abc123__deadbeef.avif');
|
||||
assert.equal(plan.format, 'avif');
|
||||
assert.equal(plan.transforms.length, 2);
|
||||
});
|
||||
|
||||
test('computePlan without a toFormat keeps the original extension but still hashes the resize', async () => {
|
||||
const plan = await computePlan(file, { key: 'resize-only', width: 100 }, null, {
|
||||
utils: utilsWith(undefined),
|
||||
hashFn: () => 'unused',
|
||||
});
|
||||
|
||||
assert.equal(plan.filename, 'abc123__unused.jpg');
|
||||
assert.equal(plan.format, null);
|
||||
});
|
||||
|
||||
test('computePlan passes acceptFormat into the transformation set', async () => {
|
||||
let seen: Record<string, unknown> | null = null;
|
||||
|
||||
await computePlan(file, preset, 'webp', {
|
||||
utils: {
|
||||
resolvePreset: (transformation) => {
|
||||
seen = transformation;
|
||||
return [ [ 'toFormat', 'webp', {} ] ];
|
||||
},
|
||||
maybeExtractFormat: () => 'webp',
|
||||
},
|
||||
hashFn: () => 'x',
|
||||
});
|
||||
|
||||
assert.equal(seen?.['acceptFormat'], 'webp');
|
||||
assert.ok(seen?.['transformationParams']);
|
||||
});
|
||||
|
||||
test('ops ring buffer is newest-first and bounded', () => {
|
||||
recordOp({ ts: 1, fileId: 'a', key: 'k', format: 'avif', ok: true, ms: 1 });
|
||||
recordOp({ ts: 2, fileId: 'b', key: 'k', format: 'avif', ok: false, ms: 2, error: 'x' });
|
||||
|
||||
const ops = getOps(1);
|
||||
assert.equal(ops.length, 1);
|
||||
assert.equal(ops[0]?.fileId, 'b');
|
||||
assert.equal(getOps(0).length, 0);
|
||||
});
|
||||
97
src/lib/diagnostics.ts
Normal file
97
src/lib/diagnostics.ts
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
import { basename, extname } from 'node:path';
|
||||
import objectHash from 'object-hash';
|
||||
import type { Preset } from './types';
|
||||
|
||||
export interface PlanFile {
|
||||
filename_disk: string;
|
||||
type?: string | null;
|
||||
width?: number | null;
|
||||
height?: number | null;
|
||||
focal_point_x?: number | null;
|
||||
focal_point_y?: number | null;
|
||||
storage?: string | null;
|
||||
}
|
||||
|
||||
export interface PlanResult {
|
||||
filename: string;
|
||||
format: string | null;
|
||||
transforms: unknown[];
|
||||
}
|
||||
|
||||
interface TransformUtils {
|
||||
resolvePreset(transformation: Record<string, unknown>, file: PlanFile): unknown[];
|
||||
maybeExtractFormat(transforms: unknown[]): string | undefined;
|
||||
}
|
||||
|
||||
export interface PlanDeps {
|
||||
utils?: TransformUtils;
|
||||
hashFn?: (value: unknown) => string;
|
||||
}
|
||||
|
||||
async function loadTransformUtils(): Promise<TransformUtils> {
|
||||
const importer = new Function('return import("@directus/api/utils/transformations")') as () => Promise<
|
||||
Record<string, unknown>
|
||||
>;
|
||||
const module = await importer();
|
||||
const namespace = (module['TransformationUtils'] as Record<string, unknown> | undefined) ?? module;
|
||||
|
||||
const resolvePreset = (namespace['resolvePreset'] ?? module['resolvePreset']) as TransformUtils['resolvePreset'];
|
||||
const maybeExtractFormat = (namespace['maybeExtractFormat'] ?? module['maybeExtractFormat']) as TransformUtils['maybeExtractFormat'];
|
||||
|
||||
if (typeof resolvePreset !== 'function' || typeof maybeExtractFormat !== 'function') {
|
||||
throw new Error('Unable to load Directus transformation utils');
|
||||
}
|
||||
|
||||
return { resolvePreset, maybeExtractFormat };
|
||||
}
|
||||
|
||||
export async function computePlan(
|
||||
file: PlanFile,
|
||||
preset: Preset,
|
||||
acceptFormat: string | null,
|
||||
deps: PlanDeps = {},
|
||||
): Promise<PlanResult> {
|
||||
const utils = deps.utils ?? (await loadTransformUtils());
|
||||
const hashFn = deps.hashFn ?? ((value: unknown) => objectHash(value));
|
||||
|
||||
const transformationParams: Record<string, unknown> = { ...preset };
|
||||
const transformation: Record<string, unknown> = { transformationParams };
|
||||
|
||||
if (acceptFormat) transformation['acceptFormat'] = acceptFormat;
|
||||
|
||||
const transforms = utils.resolvePreset(transformation, file);
|
||||
const suffix = transforms.length > 0 ? `__${hashFn(transforms)}` : '';
|
||||
const newFormat = utils.maybeExtractFormat(transforms);
|
||||
const originalExtension = extname(file.filename_disk);
|
||||
const base = basename(file.filename_disk, originalExtension);
|
||||
const extension = newFormat ? `.${newFormat}` : originalExtension;
|
||||
|
||||
return {
|
||||
filename: `${base}${suffix}${extension}`,
|
||||
format: newFormat ?? null,
|
||||
transforms,
|
||||
};
|
||||
}
|
||||
|
||||
export interface OpRecord {
|
||||
ts: number;
|
||||
fileId: string;
|
||||
key: string;
|
||||
format: string | null;
|
||||
ok: boolean;
|
||||
ms: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
const MAX_OPS = 200;
|
||||
const ops: OpRecord[] = [];
|
||||
|
||||
export function recordOp(op: OpRecord): void {
|
||||
ops.unshift(op);
|
||||
|
||||
if (ops.length > MAX_OPS) ops.length = MAX_OPS;
|
||||
}
|
||||
|
||||
export function getOps(limit = 50): OpRecord[] {
|
||||
return ops.slice(0, Math.max(0, Math.min(limit, MAX_OPS)));
|
||||
}
|
||||
31
src/lib/format.test.ts
Normal file
31
src/lib/format.test.ts
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { humanBytes, humanDuration, humanMs, toIso } from './format';
|
||||
|
||||
test('humanBytes formats sizes', () => {
|
||||
assert.equal(humanBytes(0), '0 B');
|
||||
assert.equal(humanBytes(512), '512 B');
|
||||
assert.equal(humanBytes(1024), '1.0 KB');
|
||||
assert.equal(humanBytes(1536), '1.5 KB');
|
||||
assert.equal(humanBytes(1024 * 1024), '1.0 MB');
|
||||
});
|
||||
|
||||
test('humanMs formats durations', () => {
|
||||
assert.equal(humanMs(250), '250 ms');
|
||||
assert.equal(humanMs(1500), '1.5 s');
|
||||
assert.equal(humanMs(65000), '1 min 5 s');
|
||||
});
|
||||
|
||||
test('humanDuration formats seconds', () => {
|
||||
assert.equal(humanDuration(null), '—');
|
||||
assert.equal(humanDuration(30), '30 s');
|
||||
assert.equal(humanDuration(90), '1 min 30 s');
|
||||
assert.equal(humanDuration(3700), '1 h 1 min');
|
||||
});
|
||||
|
||||
test('toIso returns null for empty values', () => {
|
||||
assert.equal(toIso(null), null);
|
||||
assert.equal(toIso(undefined), null);
|
||||
assert.equal(toIso(0), null);
|
||||
assert.equal(toIso(1792880701827), new Date(1792880701827).toISOString());
|
||||
});
|
||||
34
src/lib/format.ts
Normal file
34
src/lib/format.ts
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
export function humanBytes(bytes: number): string {
|
||||
if (!Number.isFinite(bytes) || bytes <= 0) return '0 B';
|
||||
|
||||
const units = [ 'B', 'KB', 'MB', 'GB', 'TB', 'PB' ];
|
||||
const exponent = Math.min(units.length - 1, Math.floor(Math.log(bytes) / Math.log(1024)));
|
||||
const value = bytes / Math.pow(1024, exponent);
|
||||
|
||||
return `${value.toFixed(exponent === 0 ? 0 : 1)} ${units[exponent]}`;
|
||||
}
|
||||
|
||||
export function humanMs(ms: number): string {
|
||||
if (!Number.isFinite(ms) || ms < 0) return '—';
|
||||
if (ms < 1000) return `${Math.round(ms)} ms`;
|
||||
if (ms < 60000) return `${(ms / 1000).toFixed(1)} s`;
|
||||
return `${Math.floor(ms / 60000)} min ${Math.round((ms % 60000) / 1000)} s`;
|
||||
}
|
||||
|
||||
export function humanDuration(seconds: number | null | undefined): string {
|
||||
if (seconds === null || seconds === undefined || !Number.isFinite(seconds) || seconds < 0) return '—';
|
||||
if (seconds < 60) return `${Math.round(seconds)} s`;
|
||||
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const rest = Math.round(seconds % 60);
|
||||
|
||||
if (minutes < 60) return rest > 0 ? `${minutes} min ${rest} s` : `${minutes} min`;
|
||||
|
||||
const hours = Math.floor(minutes / 60);
|
||||
return `${hours} h ${minutes % 60} min`;
|
||||
}
|
||||
|
||||
export function toIso(value: number | null | undefined): string | null {
|
||||
if (!value || !Number.isFinite(value)) return null;
|
||||
return new Date(value).toISOString();
|
||||
}
|
||||
63
src/lib/presets.test.ts
Normal file
63
src/lib/presets.test.ts
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { buildRegistry, effectiveFormatsForPreset, formatsForPreset, mergePresets, tasksPerFile, uniqueByKey } from './presets';
|
||||
import { SYSTEM_PRESETS } from './system-presets';
|
||||
import type { Preset } from './types';
|
||||
|
||||
test('system presets are the six expected keys', () => {
|
||||
assert.deepEqual(
|
||||
SYSTEM_PRESETS.map((preset) => preset.key),
|
||||
[
|
||||
'system-small-cover',
|
||||
'system-small-contain',
|
||||
'system-medium-cover',
|
||||
'system-medium-contain',
|
||||
'system-large-cover',
|
||||
'system-large-contain',
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test('mergePresets appends custom after system', () => {
|
||||
const custom: Preset[] = [ { key: 'custom', width: 100 } ];
|
||||
const merged = mergePresets(SYSTEM_PRESETS, custom);
|
||||
assert.equal(merged.length, SYSTEM_PRESETS.length + 1);
|
||||
assert.equal(merged[merged.length - 1]?.key, 'custom');
|
||||
});
|
||||
|
||||
test('uniqueByKey keeps the first occurrence (controller parity)', () => {
|
||||
const custom: Preset[] = [ { key: 'system-small-cover', width: 999 } ];
|
||||
const merged = uniqueByKey(mergePresets(SYSTEM_PRESETS, custom));
|
||||
const small = merged.find((preset) => preset.key === 'system-small-cover');
|
||||
assert.equal(small?.width, undefined);
|
||||
assert.equal(merged.filter((preset) => preset.key === 'system-small-cover').length, 1);
|
||||
});
|
||||
|
||||
test('buildRegistry selects by key or returns all', () => {
|
||||
const all = buildRegistry([], null);
|
||||
assert.equal(all.length, 6);
|
||||
|
||||
const selected = buildRegistry([], [ 'system-medium-cover' ]);
|
||||
assert.deepEqual(selected.map((preset) => preset.key), [ 'system-medium-cover' ]);
|
||||
});
|
||||
|
||||
test('formatsForPreset emits two variants for auto and one otherwise', () => {
|
||||
const auto: Preset = { key: 'a', format: 'auto' };
|
||||
const fixed: Preset = { key: 'b', format: 'webp' };
|
||||
const none: Preset = { key: 'c', width: 100 };
|
||||
|
||||
assert.deepEqual(formatsForPreset(auto, [ 'avif', 'webp' ]), [ 'avif', 'webp' ]);
|
||||
assert.deepEqual(formatsForPreset(fixed, [ 'avif', 'webp' ]), [ null ]);
|
||||
assert.deepEqual(formatsForPreset(none, [ 'avif', 'webp' ]), [ null ]);
|
||||
});
|
||||
|
||||
test('effectiveFormatsForPreset reports the real output format', () => {
|
||||
assert.deepEqual(effectiveFormatsForPreset({ key: 'a', format: 'auto' }, [ 'avif', 'webp' ]), [ 'avif', 'webp' ]);
|
||||
assert.deepEqual(effectiveFormatsForPreset({ key: 'b', format: 'png' }, [ 'avif', 'webp' ]), [ 'png' ]);
|
||||
assert.deepEqual(effectiveFormatsForPreset({ key: 'c', width: 100 }, [ 'avif', 'webp' ]), [ null ]);
|
||||
});
|
||||
|
||||
test('tasksPerFile counts variants across presets', () => {
|
||||
assert.equal(tasksPerFile(SYSTEM_PRESETS, [ 'avif', 'webp' ]), 12);
|
||||
assert.equal(tasksPerFile(SYSTEM_PRESETS, [ 'avif' ]), 6);
|
||||
});
|
||||
58
src/lib/presets.ts
Normal file
58
src/lib/presets.ts
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import { SYSTEM_PRESETS } from './system-presets';
|
||||
import type { Preset } from './types';
|
||||
|
||||
export function mergePresets(system: Preset[], custom: Preset[]): Preset[] {
|
||||
return [ ...system, ...custom ];
|
||||
}
|
||||
|
||||
export function uniqueByKey(presets: Preset[]): Preset[] {
|
||||
const seen = new Set<string>();
|
||||
|
||||
return presets.filter((preset) => {
|
||||
if (seen.has(preset.key)) return false;
|
||||
seen.add(preset.key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
export function selectPresets(presets: Preset[], keys: string[] | null): Preset[] {
|
||||
if (keys === null) return presets;
|
||||
|
||||
const wanted = new Set(keys);
|
||||
return presets.filter((preset) => wanted.has(preset.key));
|
||||
}
|
||||
|
||||
export function buildRegistry(custom: Preset[], keys: string[] | null): Preset[] {
|
||||
const merged = uniqueByKey(mergePresets(SYSTEM_PRESETS, custom));
|
||||
return selectPresets(merged, keys);
|
||||
}
|
||||
|
||||
export function formatsForPreset(preset: Preset, formats: string[]): (string | null)[] {
|
||||
if (preset.format === 'auto') return formats.length > 0 ? [ ...formats ] : [ 'avif', 'webp' ];
|
||||
return [ null ];
|
||||
}
|
||||
|
||||
export function effectiveFormat(preset: Preset, variant: string | null): string | null {
|
||||
return variant ?? preset.format ?? null;
|
||||
}
|
||||
|
||||
export function effectiveFormatsForPreset(preset: Preset, formats: string[]): (string | null)[] {
|
||||
return formatsForPreset(preset, formats).map((variant) => effectiveFormat(preset, variant));
|
||||
}
|
||||
|
||||
export function tasksPerFile(presets: Preset[], formats: string[]): number {
|
||||
return presets.reduce((total, preset) => total + formatsForPreset(preset, formats).length, 0);
|
||||
}
|
||||
|
||||
export function isImageType(type: unknown): boolean {
|
||||
return typeof type === 'string' && type.startsWith('image/');
|
||||
}
|
||||
|
||||
export async function loadCustomPresets(settingsService: {
|
||||
readSingleton(query?: Record<string, unknown>): Promise<Record<string, unknown> | null>;
|
||||
}): Promise<Preset[]> {
|
||||
const settings = await settingsService.readSingleton({ fields: [ 'storage_asset_presets' ] });
|
||||
const list = settings?.['storage_asset_presets'];
|
||||
|
||||
return Array.isArray(list) ? (list as Preset[]) : [];
|
||||
}
|
||||
69
src/lib/settings.test.ts
Normal file
69
src/lib/settings.test.ts
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { envDefaults, mergeSettings, validateSettings } from './settings';
|
||||
|
||||
test('envDefaults applies safe defaults', () => {
|
||||
const settings = envDefaults({});
|
||||
assert.equal(settings.enabled, true);
|
||||
assert.equal(settings.windowStart, '01:00');
|
||||
assert.equal(settings.windowEnd, '04:00');
|
||||
assert.equal(settings.concurrency, 1);
|
||||
assert.deepEqual(settings.formats, [ 'avif', 'webp' ]);
|
||||
assert.equal(settings.onUpload, true);
|
||||
assert.equal(settings.presetKeys, null);
|
||||
});
|
||||
|
||||
test('envDefaults parses overrides', () => {
|
||||
const settings = envDefaults({
|
||||
IMAGE_PREVIEWS_ENABLED: 'true',
|
||||
IMAGE_PREVIEWS_CONCURRENCY: '4',
|
||||
IMAGE_PREVIEWS_FORMATS: 'webp',
|
||||
IMAGE_PREVIEWS_PRESETS: 'system-small-cover, system-large-cover',
|
||||
IMAGE_PREVIEWS_ON_UPLOAD: '0',
|
||||
});
|
||||
|
||||
assert.equal(settings.enabled, true);
|
||||
assert.equal(settings.concurrency, 4);
|
||||
assert.deepEqual(settings.formats, [ 'webp' ]);
|
||||
assert.deepEqual(settings.presetKeys, [ 'system-small-cover', 'system-large-cover' ]);
|
||||
assert.equal(settings.onUpload, false);
|
||||
});
|
||||
|
||||
test('envDefaults tolerates Directus-coerced booleans and numbers', () => {
|
||||
const settings = envDefaults({
|
||||
IMAGE_PREVIEWS_ENABLED: true,
|
||||
IMAGE_PREVIEWS_CONCURRENCY: 3,
|
||||
IMAGE_PREVIEWS_ON_UPLOAD: false,
|
||||
});
|
||||
|
||||
assert.equal(settings.enabled, true);
|
||||
assert.equal(settings.concurrency, 3);
|
||||
assert.equal(settings.onUpload, false);
|
||||
});
|
||||
|
||||
test('mergeSettings keeps base values for absent overrides', () => {
|
||||
const base = envDefaults({});
|
||||
const merged = mergeSettings(base, { enabled: true });
|
||||
assert.equal(merged.enabled, true);
|
||||
assert.equal(merged.concurrency, base.concurrency);
|
||||
assert.deepEqual(merged.formats, base.formats);
|
||||
});
|
||||
|
||||
test('validateSettings rejects invalid concurrency', () => {
|
||||
const base = envDefaults({});
|
||||
assert.throws(() => validateSettings({ ...base, concurrency: 0 }), /concurrency/);
|
||||
assert.throws(() => validateSettings({ ...base, concurrency: 99 }), /concurrency/);
|
||||
});
|
||||
|
||||
test('validateSettings rejects invalid window and format', () => {
|
||||
const base = envDefaults({});
|
||||
assert.throws(() => validateSettings({ ...base, windowStart: '25:00' }), /windowStart/);
|
||||
assert.throws(() => validateSettings({ ...base, formats: [ 'bmp' ] }), /unsupported format/);
|
||||
});
|
||||
|
||||
test('validateSettings normalises a valid payload', () => {
|
||||
const base = envDefaults({});
|
||||
const result = validateSettings({ ...base, windowStart: '03:00', windowEnd: '05:30' });
|
||||
assert.equal(result.windowStart, '03:00');
|
||||
assert.equal(result.windowEnd, '05:30');
|
||||
});
|
||||
156
src/lib/settings.ts
Normal file
156
src/lib/settings.ts
Normal file
|
|
@ -0,0 +1,156 @@
|
|||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import { dirname, isAbsolute, join } from 'node:path';
|
||||
|
||||
export interface Settings {
|
||||
enabled: boolean;
|
||||
windowStart: string | null;
|
||||
windowEnd: string | null;
|
||||
concurrency: number;
|
||||
presetKeys: string[] | null;
|
||||
formats: string[];
|
||||
onUpload: boolean;
|
||||
debug: boolean;
|
||||
}
|
||||
|
||||
export interface EnvLike {
|
||||
[ key: string ]: unknown;
|
||||
}
|
||||
|
||||
const DEFAULT_STATE_SUBDIR = '.image-previews';
|
||||
const DEFAULT_WINDOW_START = '01:00';
|
||||
const DEFAULT_WINDOW_END = '04:00';
|
||||
|
||||
function asString(value: unknown): string | null {
|
||||
if (value === undefined || value === null) return null;
|
||||
if (typeof value === 'string') return value;
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
||||
return null;
|
||||
}
|
||||
|
||||
export function stateDir(env: EnvLike = process.env): string {
|
||||
const configured = asString(env['IMAGE_PREVIEWS_STATE_DIR']);
|
||||
|
||||
if (configured && configured.length > 0) return configured;
|
||||
|
||||
const root = asString(env['STORAGE_LOCAL_ROOT']) || '/directus/uploads';
|
||||
const absoluteRoot = isAbsolute(root) ? root : join(process.cwd(), root);
|
||||
|
||||
return join(absoluteRoot, DEFAULT_STATE_SUBDIR);
|
||||
}
|
||||
|
||||
function parseBool(value: unknown, fallback: boolean): boolean {
|
||||
const raw = asString(value);
|
||||
if (raw === null || raw.trim() === '') return fallback;
|
||||
return [ '1', 'true', 'yes', 'on' ].includes(raw.trim().toLowerCase());
|
||||
}
|
||||
|
||||
function parseList(value: unknown): string[] | null {
|
||||
const raw = asString(value);
|
||||
if (raw === null || raw.trim() === '') return null;
|
||||
|
||||
return raw
|
||||
.split(',')
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function parseNumber(value: unknown, fallback: number): number {
|
||||
const raw = asString(value);
|
||||
if (raw === null || raw.trim() === '') return fallback;
|
||||
const parsed = Number(raw);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
export function envDefaults(env: EnvLike = process.env): Settings {
|
||||
return {
|
||||
enabled: parseBool(env['IMAGE_PREVIEWS_ENABLED'], true),
|
||||
windowStart: asString(env['IMAGE_PREVIEWS_WINDOW_START']) ?? DEFAULT_WINDOW_START,
|
||||
windowEnd: asString(env['IMAGE_PREVIEWS_WINDOW_END']) ?? DEFAULT_WINDOW_END,
|
||||
concurrency: parseNumber(env['IMAGE_PREVIEWS_CONCURRENCY'], 1),
|
||||
presetKeys: parseList(env['IMAGE_PREVIEWS_PRESETS']),
|
||||
formats: parseList(env['IMAGE_PREVIEWS_FORMATS']) ?? [ 'avif', 'webp' ],
|
||||
onUpload: parseBool(env['IMAGE_PREVIEWS_ON_UPLOAD'], true),
|
||||
debug: parseBool(env['IMAGE_PREVIEWS_DEBUG'], false),
|
||||
};
|
||||
}
|
||||
|
||||
export function settingsPath(env: EnvLike = process.env): string {
|
||||
return join(stateDir(env), 'settings.json');
|
||||
}
|
||||
|
||||
export function mergeSettings(base: Settings, override: Partial<Settings>): Settings {
|
||||
return {
|
||||
...base,
|
||||
...override,
|
||||
formats: override.formats ?? base.formats,
|
||||
presetKeys: override.presetKeys !== undefined ? override.presetKeys : base.presetKeys,
|
||||
};
|
||||
}
|
||||
|
||||
export async function readSettings(env: EnvLike = process.env): Promise<Settings> {
|
||||
const defaults = envDefaults(env);
|
||||
|
||||
try {
|
||||
const raw = await readFile(settingsPath(env), 'utf8');
|
||||
const parsed = JSON.parse(raw) as Partial<Settings>;
|
||||
return validateSettings(mergeSettings(defaults, parsed));
|
||||
} catch {
|
||||
return defaults;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeSettings(settings: Settings, env: EnvLike = process.env): Promise<Settings> {
|
||||
const validated = validateSettings(settings);
|
||||
const target = settingsPath(env);
|
||||
await mkdir(dirname(target), { recursive: true });
|
||||
await writeFile(target, `${JSON.stringify(validated, null, 2)}\n`, 'utf8');
|
||||
return validated;
|
||||
}
|
||||
|
||||
const TIME_PATTERN = /^([01]\d|2[0-3]):[0-5]\d$/;
|
||||
const ALLOWED_FORMATS = [ 'avif', 'webp', 'jpg', 'jpeg', 'png' ];
|
||||
|
||||
export function validateSettings(settings: Settings): Settings {
|
||||
if (typeof settings.enabled !== 'boolean') throw new Error('enabled must be a boolean');
|
||||
if (typeof settings.onUpload !== 'boolean') throw new Error('onUpload must be a boolean');
|
||||
if (typeof settings.debug !== 'boolean') throw new Error('debug must be a boolean');
|
||||
|
||||
if (settings.windowStart !== null && !TIME_PATTERN.test(settings.windowStart)) {
|
||||
throw new Error('windowStart must be HH:MM or null');
|
||||
}
|
||||
|
||||
if (settings.windowEnd !== null && !TIME_PATTERN.test(settings.windowEnd)) {
|
||||
throw new Error('windowEnd must be HH:MM or null');
|
||||
}
|
||||
|
||||
const concurrency = Number(settings.concurrency);
|
||||
|
||||
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 32) {
|
||||
throw new Error('concurrency must be an integer between 1 and 32');
|
||||
}
|
||||
|
||||
if (!Array.isArray(settings.formats) || settings.formats.length === 0) {
|
||||
throw new Error('formats must be a non-empty array');
|
||||
}
|
||||
|
||||
for (const format of settings.formats) {
|
||||
if (!ALLOWED_FORMATS.includes(format)) throw new Error(`unsupported format: ${format}`);
|
||||
}
|
||||
|
||||
if (settings.presetKeys !== null) {
|
||||
if (!Array.isArray(settings.presetKeys) || settings.presetKeys.some((key) => typeof key !== 'string')) {
|
||||
throw new Error('presetKeys must be an array of strings or null');
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
enabled: settings.enabled,
|
||||
windowStart: settings.windowStart,
|
||||
windowEnd: settings.windowEnd,
|
||||
concurrency,
|
||||
presetKeys: settings.presetKeys,
|
||||
formats: [ ...settings.formats ],
|
||||
onUpload: settings.onUpload,
|
||||
debug: settings.debug,
|
||||
};
|
||||
}
|
||||
95
src/lib/state.test.ts
Normal file
95
src/lib/state.test.ts
Normal file
|
|
@ -0,0 +1,95 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { appendFile, mkdtemp, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { test } from 'node:test';
|
||||
import {
|
||||
acquireLock,
|
||||
appendCompleted,
|
||||
completedKeyOf,
|
||||
isCancelled,
|
||||
loadCompleted,
|
||||
readRun,
|
||||
releaseLock,
|
||||
requestCancel,
|
||||
writeRun,
|
||||
} from './state';
|
||||
import type { RunState } from './types';
|
||||
|
||||
async function withTempEnv(run: (env: Record<string, string>) => Promise<void>): Promise<void> {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'image-previews-'));
|
||||
|
||||
try {
|
||||
await run({ IMAGE_PREVIEWS_STATE_DIR: dir });
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
test('appendCompleted + loadCompleted round-trips and resumes', async () => {
|
||||
await withTempEnv(async (env) => {
|
||||
const empty = await loadCompleted(env);
|
||||
assert.equal(empty.size, 0);
|
||||
|
||||
await appendCompleted({ file_id: 'f1', preset_key: 'system-small-cover', format: 'avif', ts: 1 }, env);
|
||||
await appendCompleted({ file_id: 'f1', preset_key: 'system-small-cover', format: 'webp', ts: 2 }, env);
|
||||
|
||||
const completed = await loadCompleted(env);
|
||||
assert.equal(completed.size, 2);
|
||||
assert.ok(completed.has(completedKeyOf('f1', 'system-small-cover', 'avif')));
|
||||
assert.ok(completed.has(completedKeyOf('f1', 'system-small-cover', 'webp')));
|
||||
});
|
||||
});
|
||||
|
||||
test('loadCompleted tolerates corrupt lines', async () => {
|
||||
await withTempEnv(async (env) => {
|
||||
await appendFile(join(env['IMAGE_PREVIEWS_STATE_DIR'] as string, 'state.jsonl'), 'not-json\n', 'utf8');
|
||||
await appendCompleted({ file_id: 'f2', preset_key: 'k', format: null, ts: 1 }, env);
|
||||
|
||||
const completed = await loadCompleted(env);
|
||||
assert.equal(completed.size, 1);
|
||||
});
|
||||
});
|
||||
|
||||
test('run state round-trips', async () => {
|
||||
await withTempEnv(async (env) => {
|
||||
assert.equal(await readRun(env), null);
|
||||
|
||||
const run: RunState = {
|
||||
running: true,
|
||||
startedAt: 1,
|
||||
finishedAt: null,
|
||||
total: 10,
|
||||
done: 2,
|
||||
generated: 2,
|
||||
hits: 0,
|
||||
failed: 0,
|
||||
rate: 1,
|
||||
eta: 8,
|
||||
cancelled: false,
|
||||
};
|
||||
|
||||
await writeRun(run, env);
|
||||
const loaded = await readRun(env);
|
||||
assert.equal(loaded?.done, 2);
|
||||
assert.equal(loaded?.total, 10);
|
||||
});
|
||||
});
|
||||
|
||||
test('lock is exclusive and releasable', async () => {
|
||||
await withTempEnv(async (env) => {
|
||||
assert.equal(await acquireLock(env), true);
|
||||
assert.equal(await acquireLock(env), false);
|
||||
await releaseLock(env);
|
||||
assert.equal(await acquireLock(env), true);
|
||||
await releaseLock(env);
|
||||
});
|
||||
});
|
||||
|
||||
test('cancel flag toggles', async () => {
|
||||
await withTempEnv(async (env) => {
|
||||
assert.equal(await isCancelled(env), false);
|
||||
await requestCancel(env);
|
||||
assert.equal(await isCancelled(env), true);
|
||||
});
|
||||
});
|
||||
107
src/lib/state.ts
Normal file
107
src/lib/state.ts
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import { appendFile, mkdir, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { join } from 'node:path';
|
||||
import { stateDir, type EnvLike } from './settings';
|
||||
import type { CompletedRecord, RunState } from './types';
|
||||
|
||||
const STALE_LOCK_MS = 6 * 60 * 60 * 1000;
|
||||
|
||||
export function completedKey(record: CompletedRecord): string {
|
||||
return completedKeyOf(record.file_id, record.preset_key, record.format);
|
||||
}
|
||||
|
||||
export function completedKeyOf(fileId: string, presetKey: string, format: string | null): string {
|
||||
return `${fileId}::${presetKey}::${format ?? ''}`;
|
||||
}
|
||||
|
||||
export function stateFilePath(env: EnvLike, name: string): string {
|
||||
return join(stateDir(env), name);
|
||||
}
|
||||
|
||||
export async function ensureStateDir(env: EnvLike): Promise<void> {
|
||||
await mkdir(stateDir(env), { recursive: true });
|
||||
}
|
||||
|
||||
export async function loadCompleted(env: EnvLike): Promise<Set<string>> {
|
||||
const completed = new Set<string>();
|
||||
|
||||
try {
|
||||
const raw = await readFile(stateFilePath(env, 'state.jsonl'), 'utf8');
|
||||
|
||||
for (const line of raw.split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
|
||||
try {
|
||||
completed.add(completedKey(JSON.parse(line) as CompletedRecord));
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
return completed;
|
||||
}
|
||||
|
||||
return completed;
|
||||
}
|
||||
|
||||
export async function appendCompleted(record: CompletedRecord, env: EnvLike): Promise<void> {
|
||||
await ensureStateDir(env);
|
||||
await appendFile(stateFilePath(env, 'state.jsonl'), `${JSON.stringify(record)}\n`, 'utf8');
|
||||
}
|
||||
|
||||
export async function readRun(env: EnvLike): Promise<RunState | null> {
|
||||
try {
|
||||
return JSON.parse(await readFile(stateFilePath(env, 'run.json'), 'utf8')) as RunState;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeRun(run: RunState, env: EnvLike): Promise<void> {
|
||||
await ensureStateDir(env);
|
||||
await writeFile(stateFilePath(env, 'run.json'), `${JSON.stringify(run, null, 2)}\n`, 'utf8');
|
||||
}
|
||||
|
||||
async function lockAge(env: EnvLike): Promise<number | null> {
|
||||
try {
|
||||
const info = JSON.parse(await readFile(stateFilePath(env, 'run.lock'), 'utf8')) as { ts?: number };
|
||||
return Date.now() - (Number(info.ts) || 0);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function isLocked(env: EnvLike): Promise<boolean> {
|
||||
const age = await lockAge(env);
|
||||
return age !== null && age < STALE_LOCK_MS;
|
||||
}
|
||||
|
||||
export async function acquireLock(env: EnvLike): Promise<boolean> {
|
||||
await ensureStateDir(env);
|
||||
|
||||
if (await isLocked(env)) return false;
|
||||
|
||||
await writeFile(stateFilePath(env, 'run.lock'), JSON.stringify({ ts: Date.now(), pid: process.pid }), 'utf8');
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function releaseLock(env: EnvLike): Promise<void> {
|
||||
await rm(stateFilePath(env, 'run.lock'), { force: true });
|
||||
}
|
||||
|
||||
export async function requestCancel(env: EnvLike): Promise<void> {
|
||||
await ensureStateDir(env);
|
||||
await writeFile(stateFilePath(env, 'cancel.flag'), String(Date.now()), 'utf8');
|
||||
}
|
||||
|
||||
export async function clearCancel(env: EnvLike): Promise<void> {
|
||||
await rm(stateFilePath(env, 'cancel.flag'), { force: true });
|
||||
}
|
||||
|
||||
export async function isCancelled(env: EnvLike): Promise<boolean> {
|
||||
try {
|
||||
await stat(stateFilePath(env, 'cancel.flag'));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
42
src/lib/storage.test.ts
Normal file
42
src/lib/storage.test.ts
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { test } from 'node:test';
|
||||
import { isPreviewPath, isStatePath, storageUsage } from './storage';
|
||||
|
||||
test('isPreviewPath matches transform cache files only', () => {
|
||||
assert.equal(isPreviewPath('id__0123456789abcdef0123456789abcdef01234567.avif'), true);
|
||||
assert.equal(isPreviewPath('id__0123456789abcdef0123456789abcdef01234567.webp'), true);
|
||||
assert.equal(isPreviewPath('id.jpg'), false);
|
||||
assert.equal(isPreviewPath('state.jsonl'), false);
|
||||
});
|
||||
|
||||
test('isStatePath detects the state directory', () => {
|
||||
assert.equal(isStatePath('.image-previews/settings.json'), true);
|
||||
assert.equal(isStatePath('.image-previews'), true);
|
||||
assert.equal(isStatePath('abc.jpg'), false);
|
||||
});
|
||||
|
||||
test('storageUsage splits previews and originals, excluding state', async () => {
|
||||
const dir = await mkdtemp(join(tmpdir(), 'previews-usage-'));
|
||||
|
||||
try {
|
||||
await writeFile(join(dir, 'a__0123456789abcdef0123456789abcdef01234567.avif'), 'x'.repeat(10));
|
||||
await writeFile(join(dir, 'a.jpg'), 'y'.repeat(99));
|
||||
await mkdir(join(dir, '.image-previews'));
|
||||
await writeFile(join(dir, '.image-previews', 'settings.json'), 'z'.repeat(500));
|
||||
|
||||
const usage = await storageUsage({ STORAGE_LOCAL_ROOT: dir, STORAGE_LOCATIONS: 'local' });
|
||||
|
||||
assert.equal(usage.source, 'local');
|
||||
assert.equal(usage.previews.count, 1);
|
||||
assert.equal(usage.previews.bytes, 10);
|
||||
assert.equal(usage.originals.count, 1);
|
||||
assert.equal(usage.originals.bytes, 99);
|
||||
assert.equal(usage.total.count, 2);
|
||||
assert.equal(usage.total.bytes, 109);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
162
src/lib/storage.ts
Normal file
162
src/lib/storage.ts
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
import { readdir, stat as fsStat } from 'node:fs/promises';
|
||||
import { isAbsolute, join } from 'node:path';
|
||||
import type { EnvLike } from './settings';
|
||||
|
||||
const PREVIEW_PATTERN = /__[0-9a-f]{8,}\.[a-z0-9]+$/i;
|
||||
const STATE_DIR_NAME = '.image-previews';
|
||||
|
||||
function asString(value: unknown): string | null {
|
||||
if (value === undefined || value === null) return null;
|
||||
if (typeof value === 'string') return value;
|
||||
if (typeof value === 'number' || typeof value === 'boolean') return String(value);
|
||||
return null;
|
||||
}
|
||||
|
||||
export function isPreviewPath(path: string): boolean {
|
||||
return PREVIEW_PATTERN.test(path);
|
||||
}
|
||||
|
||||
export function isStatePath(path: string): boolean {
|
||||
return path === STATE_DIR_NAME || path.startsWith(`${STATE_DIR_NAME}/`) || path.includes(`/${STATE_DIR_NAME}/`);
|
||||
}
|
||||
|
||||
async function loadStorageManager(): Promise<Record<string, unknown> | null> {
|
||||
try {
|
||||
const importer = new Function('return import("@directus/api/storage/index")') as () => Promise<Record<string, unknown>>;
|
||||
const module = await importer();
|
||||
const getStorage = module['getStorage'] as (() => Promise<Record<string, unknown>>) | undefined;
|
||||
if (!getStorage) return null;
|
||||
return await getStorage();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function localStorageRoot(env: EnvLike): string {
|
||||
const root = asString(env['STORAGE_LOCAL_ROOT']) || './uploads';
|
||||
return isAbsolute(root) ? root : join(process.cwd(), root);
|
||||
}
|
||||
|
||||
export interface Bucket {
|
||||
bytes: number;
|
||||
count: number;
|
||||
}
|
||||
|
||||
export interface StorageUsage {
|
||||
previews: Bucket;
|
||||
originals: Bucket;
|
||||
total: Bucket;
|
||||
source: 'storage' | 'local';
|
||||
}
|
||||
|
||||
function emptyUsage(source: 'storage' | 'local'): StorageUsage {
|
||||
return {
|
||||
previews: { bytes: 0, count: 0 },
|
||||
originals: { bytes: 0, count: 0 },
|
||||
total: { bytes: 0, count: 0 },
|
||||
source,
|
||||
};
|
||||
}
|
||||
|
||||
function add(usage: StorageUsage, path: string, size: number): void {
|
||||
if (isPreviewPath(path)) {
|
||||
usage.previews.bytes += size;
|
||||
usage.previews.count += 1;
|
||||
} else {
|
||||
usage.originals.bytes += size;
|
||||
usage.originals.count += 1;
|
||||
}
|
||||
|
||||
usage.total.bytes += size;
|
||||
usage.total.count += 1;
|
||||
}
|
||||
|
||||
async function walkLocal(root: string, usage: StorageUsage): Promise<void> {
|
||||
let entries;
|
||||
|
||||
try {
|
||||
entries = await readdir(root, { withFileTypes: true });
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of entries) {
|
||||
const full = join(root, entry.name);
|
||||
|
||||
if (entry.isDirectory()) {
|
||||
if (entry.name === STATE_DIR_NAME) continue;
|
||||
await walkLocal(full, usage);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.isFile()) continue;
|
||||
|
||||
try {
|
||||
const info = await fsStat(full);
|
||||
add(usage, entry.name, info.size);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function storageUsage(env: EnvLike): Promise<StorageUsage> {
|
||||
const manager = await loadStorageManager();
|
||||
|
||||
if (manager) {
|
||||
const usage = emptyUsage('storage');
|
||||
|
||||
const locations = (asString(env['STORAGE_LOCATIONS']) || 'local')
|
||||
.split(',')
|
||||
.map((part) => part.trim())
|
||||
.filter(Boolean);
|
||||
|
||||
for (const location of locations) {
|
||||
try {
|
||||
const driver = (manager['location'] as (name: string) => Record<string, unknown>)(location);
|
||||
const list = (driver['list'] as () => AsyncIterable<string>)();
|
||||
|
||||
for await (const path of list) {
|
||||
if (isStatePath(path)) continue;
|
||||
|
||||
try {
|
||||
const info = (await (driver['stat'] as (p: string) => Promise<{ size: number }>)(path)) as {
|
||||
size: number;
|
||||
};
|
||||
add(usage, path, Number(info?.size) || 0);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
return usage;
|
||||
}
|
||||
|
||||
const usage = emptyUsage('local');
|
||||
await walkLocal(localStorageRoot(env), usage);
|
||||
return usage;
|
||||
}
|
||||
|
||||
export async function previewExists(env: EnvLike, location: string | null, filename: string): Promise<boolean | null> {
|
||||
const manager = await loadStorageManager();
|
||||
|
||||
if (manager && location) {
|
||||
try {
|
||||
const driver = (manager['location'] as (name: string) => Record<string, unknown>)(location);
|
||||
return await (driver['exists'] as (p: string) => Promise<boolean>)(filename);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
await fsStat(join(localStorageRoot(env), filename));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
36
src/lib/system-presets.ts
Normal file
36
src/lib/system-presets.ts
Normal file
|
|
@ -0,0 +1,36 @@
|
|||
import type { Preset } from './types';
|
||||
|
||||
export const SYSTEM_PRESETS: Preset[] = [
|
||||
{
|
||||
key: 'system-small-cover',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 64, height: 64, fit: 'cover' } ] ],
|
||||
},
|
||||
{
|
||||
key: 'system-small-contain',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 64, fit: 'contain' } ] ],
|
||||
},
|
||||
{
|
||||
key: 'system-medium-cover',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 300, height: 300, fit: 'cover' } ] ],
|
||||
},
|
||||
{
|
||||
key: 'system-medium-contain',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 300, fit: 'contain' } ] ],
|
||||
},
|
||||
{
|
||||
key: 'system-large-cover',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 800, height: 800, fit: 'cover' } ] ],
|
||||
},
|
||||
{
|
||||
key: 'system-large-contain',
|
||||
format: 'auto',
|
||||
transforms: [ [ 'resize', { width: 800, fit: 'contain' } ] ],
|
||||
},
|
||||
];
|
||||
|
||||
export const DEFAULT_FORMATS = [ 'avif', 'webp' ];
|
||||
50
src/lib/types.ts
Normal file
50
src/lib/types.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
export interface Preset {
|
||||
key: string;
|
||||
format?: string;
|
||||
quality?: number | string;
|
||||
width?: number | string;
|
||||
height?: number | string;
|
||||
fit?: string;
|
||||
withoutEnlargement?: boolean;
|
||||
focal_point_x?: number | string | null;
|
||||
focal_point_y?: number | string | null;
|
||||
transforms?: unknown[];
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface WarmResultItem {
|
||||
key: string;
|
||||
format: string | null;
|
||||
ok: boolean;
|
||||
ms: number;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface WarmSummary {
|
||||
requested: number;
|
||||
ok: number;
|
||||
failed: number;
|
||||
results: WarmResultItem[];
|
||||
}
|
||||
|
||||
export interface CompletedRecord {
|
||||
file_id: string;
|
||||
preset_key: string;
|
||||
format: string | null;
|
||||
ts: number;
|
||||
}
|
||||
|
||||
export interface RunState {
|
||||
running: boolean;
|
||||
startedAt: number | null;
|
||||
finishedAt: number | null;
|
||||
total: number;
|
||||
done: number;
|
||||
generated: number;
|
||||
hits: number;
|
||||
failed: number;
|
||||
rate: number;
|
||||
eta: number | null;
|
||||
cancelled: boolean;
|
||||
error?: string;
|
||||
}
|
||||
92
src/lib/warmer.test.ts
Normal file
92
src/lib/warmer.test.ts
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import type { Preset } from './types';
|
||||
import { warmFile, type AssetsLike } from './warmer';
|
||||
|
||||
const presets: Preset[] = [
|
||||
{ key: 'system-small-cover', format: 'auto', transforms: [ [ 'resize', { width: 64, height: 64, fit: 'cover' } ] ] },
|
||||
{ key: 'custom-fixed', format: 'webp', width: 200 },
|
||||
];
|
||||
|
||||
test('warmFile passes the full preset object and the accept variant', async () => {
|
||||
const calls: Array<Record<string, unknown>> = [];
|
||||
|
||||
const assets: AssetsLike = {
|
||||
async getAsset(_id, transformation) {
|
||||
calls.push(transformation ?? {});
|
||||
return { stream: { destroy() {} } };
|
||||
},
|
||||
};
|
||||
|
||||
const summary = await warmFile(assets, 'file-1', presets, { concurrency: 1, formats: [ 'avif', 'webp' ] });
|
||||
|
||||
assert.equal(summary.requested, 3);
|
||||
assert.equal(summary.ok, 3);
|
||||
assert.equal(calls.length, 3);
|
||||
|
||||
const autoCalls = calls.filter((call) => (call['transformationParams'] as Preset).key === 'system-small-cover');
|
||||
assert.equal(autoCalls.length, 2);
|
||||
assert.deepEqual(
|
||||
autoCalls.map((call) => call['acceptFormat']).sort(),
|
||||
[ 'avif', 'webp' ],
|
||||
);
|
||||
assert.ok((autoCalls[0]?.['transformationParams'] as Preset).transforms);
|
||||
|
||||
const fixed = calls.find((call) => (call['transformationParams'] as Preset).key === 'custom-fixed');
|
||||
assert.equal(fixed?.['acceptFormat'], undefined);
|
||||
});
|
||||
|
||||
test('warmFile isolates per-task errors', async () => {
|
||||
const assets: AssetsLike = {
|
||||
async getAsset(_id, transformation) {
|
||||
const preset = transformation?.['transformationParams'] as Preset;
|
||||
if (preset.key === 'custom-fixed') throw new Error('boom');
|
||||
return { stream: { destroy() {} } };
|
||||
},
|
||||
};
|
||||
|
||||
const summary = await warmFile(assets, 'file-1', presets, { concurrency: 1, formats: [ 'avif', 'webp' ] });
|
||||
|
||||
assert.equal(summary.ok, 2);
|
||||
assert.equal(summary.failed, 1);
|
||||
assert.equal(summary.results.find((result) => result.key === 'custom-fixed')?.error, 'boom');
|
||||
});
|
||||
|
||||
test('warmFile respects the concurrency cap', async () => {
|
||||
let active = 0;
|
||||
let maxActive = 0;
|
||||
|
||||
const assets: AssetsLike = {
|
||||
async getAsset() {
|
||||
active += 1;
|
||||
maxActive = Math.max(maxActive, active);
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
active -= 1;
|
||||
return { stream: { destroy() {} } };
|
||||
},
|
||||
};
|
||||
|
||||
await warmFile(assets, 'file-1', presets, { concurrency: 2, formats: [ 'avif', 'webp' ] });
|
||||
|
||||
assert.ok(maxActive <= 2, `maxActive=${maxActive}`);
|
||||
});
|
||||
|
||||
test('warmFile releases the returned stream', async () => {
|
||||
let destroyed = 0;
|
||||
|
||||
const assets: AssetsLike = {
|
||||
async getAsset() {
|
||||
return {
|
||||
stream: {
|
||||
destroy() {
|
||||
destroyed += 1;
|
||||
},
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
|
||||
await warmFile(assets, 'file-1', [ presets[0] as Preset ], { concurrency: 1, formats: [ 'avif' ] });
|
||||
|
||||
assert.equal(destroyed, 1);
|
||||
});
|
||||
110
src/lib/warmer.ts
Normal file
110
src/lib/warmer.ts
Normal file
|
|
@ -0,0 +1,110 @@
|
|||
import { effectiveFormat, formatsForPreset } from './presets';
|
||||
import type { Preset, WarmResultItem, WarmSummary } from './types';
|
||||
|
||||
export interface AssetsLike {
|
||||
getAsset(
|
||||
id: string,
|
||||
transformation?: Record<string, unknown>,
|
||||
range?: unknown,
|
||||
deferStream?: boolean,
|
||||
): Promise<{ stream?: unknown }>;
|
||||
}
|
||||
|
||||
export interface WarmOptions {
|
||||
concurrency: number;
|
||||
formats: string[];
|
||||
onResult?: (result: WarmResultItem) => void;
|
||||
}
|
||||
|
||||
interface WarmTask {
|
||||
preset: Preset;
|
||||
format: string | null;
|
||||
}
|
||||
|
||||
export function errorMessage(error: unknown): string {
|
||||
if (error instanceof Error) return error.message;
|
||||
if (typeof error === 'string') return error;
|
||||
|
||||
try {
|
||||
return JSON.stringify(error);
|
||||
} catch {
|
||||
return String(error);
|
||||
}
|
||||
}
|
||||
|
||||
function releaseStream(stream: unknown): void {
|
||||
const candidate = stream as { destroy?: () => void; unpipe?: () => void } | undefined;
|
||||
|
||||
if (candidate && typeof candidate.destroy === 'function') {
|
||||
candidate.destroy();
|
||||
} else if (candidate && typeof candidate.unpipe === 'function') {
|
||||
candidate.unpipe();
|
||||
}
|
||||
}
|
||||
|
||||
export async function warmFile(
|
||||
assets: AssetsLike,
|
||||
fileId: string,
|
||||
presets: Preset[],
|
||||
options: WarmOptions,
|
||||
): Promise<WarmSummary> {
|
||||
const tasks: WarmTask[] = [];
|
||||
|
||||
for (const preset of presets) {
|
||||
for (const format of formatsForPreset(preset, options.formats)) {
|
||||
tasks.push({ preset, format });
|
||||
}
|
||||
}
|
||||
|
||||
const results: WarmResultItem[] = new Array(tasks.length);
|
||||
let cursor = 0;
|
||||
|
||||
const workerCount = Math.max(1, Math.min(options.concurrency, tasks.length || 1));
|
||||
|
||||
const workers = Array.from({ length: workerCount }, async () => {
|
||||
while (cursor < tasks.length) {
|
||||
const index = cursor;
|
||||
cursor += 1;
|
||||
const task = tasks[index];
|
||||
|
||||
if (!task) break;
|
||||
|
||||
const transformation: Record<string, unknown> = { transformationParams: task.preset };
|
||||
|
||||
if (task.format) transformation['acceptFormat'] = task.format;
|
||||
|
||||
const started = Date.now();
|
||||
const outputFormat = effectiveFormat(task.preset, task.format);
|
||||
let result: WarmResultItem;
|
||||
|
||||
try {
|
||||
const asset = await assets.getAsset(fileId, transformation);
|
||||
releaseStream(asset?.stream);
|
||||
result = { key: task.preset.key, format: outputFormat, ok: true, ms: Date.now() - started };
|
||||
} catch (error) {
|
||||
result = {
|
||||
key: task.preset.key,
|
||||
format: outputFormat,
|
||||
ok: false,
|
||||
ms: Date.now() - started,
|
||||
error: errorMessage(error),
|
||||
};
|
||||
}
|
||||
|
||||
results[index] = result;
|
||||
options.onResult?.(result);
|
||||
}
|
||||
});
|
||||
|
||||
await Promise.all(workers);
|
||||
|
||||
const settled = results.filter((result): result is WarmResultItem => Boolean(result));
|
||||
const ok = settled.filter((result) => result.ok).length;
|
||||
|
||||
return {
|
||||
requested: tasks.length,
|
||||
ok,
|
||||
failed: settled.length - ok,
|
||||
results: settled,
|
||||
};
|
||||
}
|
||||
47
src/lib/window.test.ts
Normal file
47
src/lib/window.test.ts
Normal file
|
|
@ -0,0 +1,47 @@
|
|||
import assert from 'node:assert/strict';
|
||||
import { test } from 'node:test';
|
||||
import { isInWindow, nextWindow, parseMinutes } from './window';
|
||||
|
||||
function at(hours: number, minutes: number): Date {
|
||||
return new Date(2026, 9, 25, hours, minutes, 0, 0);
|
||||
}
|
||||
|
||||
test('parseMinutes parses HH:MM', () => {
|
||||
assert.equal(parseMinutes('00:00'), 0);
|
||||
assert.equal(parseMinutes('03:30'), 210);
|
||||
assert.equal(parseMinutes('23:59'), 1439);
|
||||
assert.equal(parseMinutes(null), null);
|
||||
assert.equal(parseMinutes('bad'), null);
|
||||
});
|
||||
|
||||
test('no window means always inside', () => {
|
||||
assert.equal(isInWindow(null, null, at(12, 0)), true);
|
||||
assert.equal(isInWindow('03:00', null, at(12, 0)), true);
|
||||
});
|
||||
|
||||
test('daytime window', () => {
|
||||
assert.equal(isInWindow('03:00', '05:00', at(2, 59)), false);
|
||||
assert.equal(isInWindow('03:00', '05:00', at(3, 0)), true);
|
||||
assert.equal(isInWindow('03:00', '05:00', at(4, 59)), true);
|
||||
assert.equal(isInWindow('03:00', '05:00', at(5, 0)), false);
|
||||
});
|
||||
|
||||
test('overnight window', () => {
|
||||
assert.equal(isInWindow('23:00', '02:00', at(23, 30)), true);
|
||||
assert.equal(isInWindow('23:00', '02:00', at(1, 30)), true);
|
||||
assert.equal(isInWindow('23:00', '02:00', at(12, 0)), false);
|
||||
});
|
||||
|
||||
test('nextWindow returns the upcoming start', () => {
|
||||
const next = nextWindow('03:00', '05:00', at(12, 0));
|
||||
assert.ok(next);
|
||||
assert.equal(next?.getDate(), 26);
|
||||
assert.equal(next?.getHours(), 3);
|
||||
});
|
||||
|
||||
test('nextWindow today when still before the start', () => {
|
||||
const next = nextWindow('03:00', '05:00', at(1, 0));
|
||||
assert.ok(next);
|
||||
assert.equal(next?.getDate(), 25);
|
||||
assert.equal(next?.getHours(), 3);
|
||||
});
|
||||
32
src/lib/window.ts
Normal file
32
src/lib/window.ts
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
export function parseMinutes(value: string | null): number | null {
|
||||
if (value === null) return null;
|
||||
const match = /^([01]\d|2[0-3]):([0-5]\d)$/.exec(value);
|
||||
if (!match) return null;
|
||||
return Number(match[1]) * 60 + Number(match[2]);
|
||||
}
|
||||
|
||||
export function isInWindow(start: string | null, end: string | null, now: Date = new Date()): boolean {
|
||||
const from = parseMinutes(start);
|
||||
const to = parseMinutes(end);
|
||||
|
||||
if (from === null || to === null) return true;
|
||||
if (from === to) return true;
|
||||
|
||||
const current = now.getHours() * 60 + now.getMinutes();
|
||||
|
||||
if (from < to) return current >= from && current < to;
|
||||
|
||||
return current >= from || current < to;
|
||||
}
|
||||
|
||||
export function nextWindow(start: string | null, end: string | null, now: Date = new Date()): Date | null {
|
||||
const from = parseMinutes(start);
|
||||
if (from === null || parseMinutes(end) === null) return null;
|
||||
|
||||
const next = new Date(now);
|
||||
next.setHours(Math.floor(from / 60), from % 60, 0, 0);
|
||||
|
||||
if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1);
|
||||
|
||||
return next;
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue