diff --git a/src/lib/backfill.ts b/src/lib/backfill.ts index 5891698..c688d1c 100644 --- a/src/lib/backfill.ts +++ b/src/lib/backfill.ts @@ -1,4 +1,4 @@ -import { recordOp } from './diagnostics'; +import { computePlan, recordOp } from './diagnostics'; import { buildRegistry, effectiveFormatsForPreset, isImageType, loadCustomPresets, tasksPerFile } from './presets'; import { readSettings } from './settings'; import { @@ -13,9 +13,10 @@ import { requestCancel, writeRun, } from './state'; -import type { RunState } from './types'; +import type { Preset, RunState } from './types'; import { errorMessage, warmFile } from './warmer'; import { isInWindow } from './window'; +import { listFileNames } from './storage'; export interface BackfillDeps { services: Record; @@ -93,6 +94,14 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = await writeRun(run, deps.env); const completed = await loadCompleted(deps.env); + + let existingFiles = new Set(); + try { + existingFiles = await listFileNames(deps.env); + } catch (error) { + deps.logger?.warn?.(error, 'image-previews: could not list existing files; continuing without cache detection'); + } + const items = new deps.services['ItemsService']('directus_files', { schema }); const assets = new deps.services['AssetsService']({ schema }); const started = Date.now(); @@ -139,10 +148,32 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = continue; } + const planCache = new Map(); + + const isCached = async (preset: Preset, variant: string | null): Promise => { + const cacheKey = `${preset.key}::${variant ?? ''}`; + let filename = planCache.get(cacheKey); + + try { + if (!filename) { + const plan = await computePlan(file, preset, variant); + filename = plan.filename; + planCache.set(cacheKey, filename); + } + + return existingFiles.has(filename); + } catch { + return false; + } + }; + const summary = await warmFile(assets, file.id, pending, { concurrency: settings.concurrency, formats, - onResult: (result) => + isCached, + onResult: (result) => { + if (result.cached) return; + recordOp({ ts: Date.now(), fileId: file.id, @@ -151,7 +182,8 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = ok: result.ok, ms: result.ms, error: result.error, - }), + }); + }, }); for (const result of summary.results) { @@ -161,7 +193,9 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = { file_id: file.id, preset_key: result.key, format: result.format, ts: Date.now() }, deps.env, ); - run.generated += 1; + + if (result.cached) run.hits += 1; + else run.generated += 1; } else { run.failed += 1; } diff --git a/src/lib/storage.test.ts b/src/lib/storage.test.ts index 36bd1ad..92f64d4 100644 --- a/src/lib/storage.test.ts +++ b/src/lib/storage.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import { isPreviewPath, isStatePath, storageUsage } from './storage'; +import { isPreviewPath, isStatePath, listFileNames, storageUsage } from './storage'; test('isPreviewPath matches transform cache files only', () => { assert.equal(isPreviewPath('id__0123456789abcdef0123456789abcdef01234567.avif'), true); @@ -18,6 +18,26 @@ test('isStatePath detects the state directory', () => { assert.equal(isStatePath('abc.jpg'), false); }); +test('listFileNames lists files and excludes the state dir', async () => { + const dir = await mkdtemp(join(tmpdir(), 'previews-names-')); + + try { + await writeFile(join(dir, 'a__0123456789abcdef0123456789abcdef01234567.avif'), 'x'); + await writeFile(join(dir, 'a.jpg'), 'y'); + await mkdir(join(dir, '.image-previews')); + await writeFile(join(dir, '.image-previews', 'settings.json'), 'z'); + + const names = await listFileNames({ STORAGE_LOCAL_ROOT: dir, STORAGE_LOCATIONS: 'local' }); + + assert.equal(names.size, 2); + assert.ok(names.has('a__0123456789abcdef0123456789abcdef01234567.avif')); + assert.ok(names.has('a.jpg')); + assert.equal(names.has('settings.json'), false); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test('storageUsage splits previews and originals, excluding state', async () => { const dir = await mkdtemp(join(tmpdir(), 'previews-usage-')); diff --git a/src/lib/storage.ts b/src/lib/storage.ts index c429c2d..e5b08cd 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -141,6 +141,58 @@ export async function storageUsage(env: EnvLike): Promise { return usage; } +async function walkLocalNames(root: string, names: Set): Promise { + let entries; + + try { + entries = await readdir(root, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (entry.isDirectory()) { + if (entry.name === STATE_DIR_NAME) continue; + await walkLocalNames(join(root, entry.name), names); + continue; + } + + if (entry.isFile()) names.add(entry.name); + } +} + +export async function listFileNames(env: EnvLike): Promise> { + const names = new Set(); + const manager = await loadStorageManager(); + + if (manager) { + const locations = (asString(env['STORAGE_LOCATIONS']) || 'local') + .split(',') + .map((part) => part.trim()) + .filter(Boolean); + + for (const location of locations) { + try { + const driver = (manager['location'] as (name: string) => Record)(location); + const list = (driver['list'] as () => AsyncIterable)(); + + for await (const path of list) { + if (isStatePath(path)) continue; + const base = path.split('/').pop(); + if (base) names.add(base); + } + } catch { + continue; + } + } + + return names; + } + + await walkLocalNames(localStorageRoot(env), names); + return names; +} + export async function previewExists(env: EnvLike, location: string | null, filename: string): Promise { const manager = await loadStorageManager(); diff --git a/src/lib/types.ts b/src/lib/types.ts index fe47c3c..77290f2 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -17,6 +17,7 @@ export interface WarmResultItem { format: string | null; ok: boolean; ms: number; + cached?: boolean; error?: string; } diff --git a/src/lib/warmer.test.ts b/src/lib/warmer.test.ts index a6240cd..7c05341 100644 --- a/src/lib/warmer.test.ts +++ b/src/lib/warmer.test.ts @@ -71,6 +71,47 @@ test('warmFile respects the concurrency cap', async () => { assert.ok(maxActive <= 2, `maxActive=${maxActive}`); }); +test('warmFile skips generation when isCached returns true', async () => { + let calls = 0; + + const assets: AssetsLike = { + async getAsset() { + calls += 1; + return { stream: { destroy() {} } }; + }, + }; + + const summary = await warmFile(assets, 'file-1', [ presets[0] as Preset ], { + concurrency: 1, + formats: [ 'avif' ], + isCached: () => true, + }); + + assert.equal(calls, 0); + assert.equal(summary.ok, 1); + assert.equal(summary.results[0]?.cached, true); +}); + +test('warmFile generates when isCached returns false', async () => { + let calls = 0; + + const assets: AssetsLike = { + async getAsset() { + calls += 1; + return { stream: { destroy() {} } }; + }, + }; + + const summary = await warmFile(assets, 'file-1', [ presets[0] as Preset ], { + concurrency: 1, + formats: [ 'avif' ], + isCached: () => false, + }); + + assert.equal(calls, 1); + assert.equal(summary.results[0]?.cached, false); +}); + test('warmFile releases the returned stream', async () => { let destroyed = 0; diff --git a/src/lib/warmer.ts b/src/lib/warmer.ts index 89af8b1..6ea8fae 100644 --- a/src/lib/warmer.ts +++ b/src/lib/warmer.ts @@ -13,6 +13,7 @@ export interface AssetsLike { export interface WarmOptions { concurrency: number; formats: string[]; + isCached?: (preset: Preset, format: string | null) => boolean | Promise; onResult?: (result: WarmResultItem) => void; } @@ -75,20 +76,37 @@ export async function warmFile( const started = Date.now(); const outputFormat = effectiveFormat(task.preset, task.format); + const cached = options.isCached ? await options.isCached(task.preset, task.format) : false; let result: WarmResultItem; - try { - const asset = await assets.getAsset(fileId, transformation); - releaseStream(asset?.stream); - result = { key: task.preset.key, format: outputFormat, ok: true, ms: Date.now() - started }; - } catch (error) { + if (cached) { result = { key: task.preset.key, format: outputFormat, - ok: false, + ok: true, ms: Date.now() - started, - error: errorMessage(error), + cached: true, }; + } else { + try { + const asset = await assets.getAsset(fileId, transformation); + releaseStream(asset?.stream); + result = { + key: task.preset.key, + format: outputFormat, + ok: true, + ms: Date.now() - started, + cached: false, + }; + } catch (error) { + result = { + key: task.preset.key, + format: outputFormat, + ok: false, + ms: Date.now() - started, + error: errorMessage(error), + }; + } } results[index] = result;