From 18db52a47cd9c3422e2ee2407921ad49d7e10135 Mon Sep 17 00:00:00 2001 From: Rawman Grey Date: Sun, 25 Oct 2026 13:52:28 +0300 Subject: [PATCH] fix: count pre-existing preview files as cached in backfill Pre-scan storage filenames and check each task's predicted cache file before generating. Existing previews are now counted as "already cached" and skipped, instead of being counted as remaining work and regenerated. Storage-scan and plan-computation failures fall back to generation. --- src/lib/backfill.ts | 44 ++++++++++++++++++++++++++++++---- src/lib/storage.test.ts | 22 ++++++++++++++++- src/lib/storage.ts | 52 +++++++++++++++++++++++++++++++++++++++++ src/lib/types.ts | 1 + src/lib/warmer.test.ts | 41 ++++++++++++++++++++++++++++++++ src/lib/warmer.ts | 32 +++++++++++++++++++------ 6 files changed, 179 insertions(+), 13 deletions(-) diff --git a/src/lib/backfill.ts b/src/lib/backfill.ts index 5891698..c688d1c 100644 --- a/src/lib/backfill.ts +++ b/src/lib/backfill.ts @@ -1,4 +1,4 @@ -import { recordOp } from './diagnostics'; +import { computePlan, recordOp } from './diagnostics'; import { buildRegistry, effectiveFormatsForPreset, isImageType, loadCustomPresets, tasksPerFile } from './presets'; import { readSettings } from './settings'; import { @@ -13,9 +13,10 @@ import { requestCancel, writeRun, } from './state'; -import type { RunState } from './types'; +import type { Preset, RunState } from './types'; import { errorMessage, warmFile } from './warmer'; import { isInWindow } from './window'; +import { listFileNames } from './storage'; export interface BackfillDeps { services: Record; @@ -93,6 +94,14 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = await writeRun(run, deps.env); const completed = await loadCompleted(deps.env); + + let existingFiles = new Set(); + try { + existingFiles = await listFileNames(deps.env); + } catch (error) { + deps.logger?.warn?.(error, 'image-previews: could not list existing files; continuing without cache detection'); + } + const items = new deps.services['ItemsService']('directus_files', { schema }); const assets = new deps.services['AssetsService']({ schema }); const started = Date.now(); @@ -139,10 +148,32 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = continue; } + const planCache = new Map(); + + const isCached = async (preset: Preset, variant: string | null): Promise => { + const cacheKey = `${preset.key}::${variant ?? ''}`; + let filename = planCache.get(cacheKey); + + try { + if (!filename) { + const plan = await computePlan(file, preset, variant); + filename = plan.filename; + planCache.set(cacheKey, filename); + } + + return existingFiles.has(filename); + } catch { + return false; + } + }; + const summary = await warmFile(assets, file.id, pending, { concurrency: settings.concurrency, formats, - onResult: (result) => + isCached, + onResult: (result) => { + if (result.cached) return; + recordOp({ ts: Date.now(), fileId: file.id, @@ -151,7 +182,8 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = ok: result.ok, ms: result.ms, error: result.error, - }), + }); + }, }); for (const result of summary.results) { @@ -161,7 +193,9 @@ export async function runBackfill(deps: BackfillDeps, options: BackfillOptions = { file_id: file.id, preset_key: result.key, format: result.format, ts: Date.now() }, deps.env, ); - run.generated += 1; + + if (result.cached) run.hits += 1; + else run.generated += 1; } else { run.failed += 1; } diff --git a/src/lib/storage.test.ts b/src/lib/storage.test.ts index 36bd1ad..92f64d4 100644 --- a/src/lib/storage.test.ts +++ b/src/lib/storage.test.ts @@ -3,7 +3,7 @@ import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; -import { isPreviewPath, isStatePath, storageUsage } from './storage'; +import { isPreviewPath, isStatePath, listFileNames, storageUsage } from './storage'; test('isPreviewPath matches transform cache files only', () => { assert.equal(isPreviewPath('id__0123456789abcdef0123456789abcdef01234567.avif'), true); @@ -18,6 +18,26 @@ test('isStatePath detects the state directory', () => { assert.equal(isStatePath('abc.jpg'), false); }); +test('listFileNames lists files and excludes the state dir', async () => { + const dir = await mkdtemp(join(tmpdir(), 'previews-names-')); + + try { + await writeFile(join(dir, 'a__0123456789abcdef0123456789abcdef01234567.avif'), 'x'); + await writeFile(join(dir, 'a.jpg'), 'y'); + await mkdir(join(dir, '.image-previews')); + await writeFile(join(dir, '.image-previews', 'settings.json'), 'z'); + + const names = await listFileNames({ STORAGE_LOCAL_ROOT: dir, STORAGE_LOCATIONS: 'local' }); + + assert.equal(names.size, 2); + assert.ok(names.has('a__0123456789abcdef0123456789abcdef01234567.avif')); + assert.ok(names.has('a.jpg')); + assert.equal(names.has('settings.json'), false); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test('storageUsage splits previews and originals, excluding state', async () => { const dir = await mkdtemp(join(tmpdir(), 'previews-usage-')); diff --git a/src/lib/storage.ts b/src/lib/storage.ts index c429c2d..e5b08cd 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -141,6 +141,58 @@ export async function storageUsage(env: EnvLike): Promise { return usage; } +async function walkLocalNames(root: string, names: Set): Promise { + let entries; + + try { + entries = await readdir(root, { withFileTypes: true }); + } catch { + return; + } + + for (const entry of entries) { + if (entry.isDirectory()) { + if (entry.name === STATE_DIR_NAME) continue; + await walkLocalNames(join(root, entry.name), names); + continue; + } + + if (entry.isFile()) names.add(entry.name); + } +} + +export async function listFileNames(env: EnvLike): Promise> { + const names = new Set(); + const manager = await loadStorageManager(); + + if (manager) { + const locations = (asString(env['STORAGE_LOCATIONS']) || 'local') + .split(',') + .map((part) => part.trim()) + .filter(Boolean); + + for (const location of locations) { + try { + const driver = (manager['location'] as (name: string) => Record)(location); + const list = (driver['list'] as () => AsyncIterable)(); + + for await (const path of list) { + if (isStatePath(path)) continue; + const base = path.split('/').pop(); + if (base) names.add(base); + } + } catch { + continue; + } + } + + return names; + } + + await walkLocalNames(localStorageRoot(env), names); + return names; +} + export async function previewExists(env: EnvLike, location: string | null, filename: string): Promise { const manager = await loadStorageManager(); diff --git a/src/lib/types.ts b/src/lib/types.ts index fe47c3c..77290f2 100644 --- a/src/lib/types.ts +++ b/src/lib/types.ts @@ -17,6 +17,7 @@ export interface WarmResultItem { format: string | null; ok: boolean; ms: number; + cached?: boolean; error?: string; } diff --git a/src/lib/warmer.test.ts b/src/lib/warmer.test.ts index a6240cd..7c05341 100644 --- a/src/lib/warmer.test.ts +++ b/src/lib/warmer.test.ts @@ -71,6 +71,47 @@ test('warmFile respects the concurrency cap', async () => { assert.ok(maxActive <= 2, `maxActive=${maxActive}`); }); +test('warmFile skips generation when isCached returns true', async () => { + let calls = 0; + + const assets: AssetsLike = { + async getAsset() { + calls += 1; + return { stream: { destroy() {} } }; + }, + }; + + const summary = await warmFile(assets, 'file-1', [ presets[0] as Preset ], { + concurrency: 1, + formats: [ 'avif' ], + isCached: () => true, + }); + + assert.equal(calls, 0); + assert.equal(summary.ok, 1); + assert.equal(summary.results[0]?.cached, true); +}); + +test('warmFile generates when isCached returns false', async () => { + let calls = 0; + + const assets: AssetsLike = { + async getAsset() { + calls += 1; + return { stream: { destroy() {} } }; + }, + }; + + const summary = await warmFile(assets, 'file-1', [ presets[0] as Preset ], { + concurrency: 1, + formats: [ 'avif' ], + isCached: () => false, + }); + + assert.equal(calls, 1); + assert.equal(summary.results[0]?.cached, false); +}); + test('warmFile releases the returned stream', async () => { let destroyed = 0; diff --git a/src/lib/warmer.ts b/src/lib/warmer.ts index 89af8b1..6ea8fae 100644 --- a/src/lib/warmer.ts +++ b/src/lib/warmer.ts @@ -13,6 +13,7 @@ export interface AssetsLike { export interface WarmOptions { concurrency: number; formats: string[]; + isCached?: (preset: Preset, format: string | null) => boolean | Promise; onResult?: (result: WarmResultItem) => void; } @@ -75,20 +76,37 @@ export async function warmFile( const started = Date.now(); const outputFormat = effectiveFormat(task.preset, task.format); + const cached = options.isCached ? await options.isCached(task.preset, task.format) : false; let result: WarmResultItem; - try { - const asset = await assets.getAsset(fileId, transformation); - releaseStream(asset?.stream); - result = { key: task.preset.key, format: outputFormat, ok: true, ms: Date.now() - started }; - } catch (error) { + if (cached) { result = { key: task.preset.key, format: outputFormat, - ok: false, + ok: true, ms: Date.now() - started, - error: errorMessage(error), + cached: true, }; + } else { + try { + const asset = await assets.getAsset(fileId, transformation); + releaseStream(asset?.stream); + result = { + key: task.preset.key, + format: outputFormat, + ok: true, + ms: Date.now() - started, + cached: false, + }; + } catch (error) { + result = { + key: task.preset.key, + format: outputFormat, + ok: false, + ms: Date.now() - started, + error: errorMessage(error), + }; + } } results[index] = result;