mirror of
https://github.com/shumaiOne/shumai.git
synced 2026-11-19 14:16:29 +00:00
fix(transcode): resolve RAW transcoding hang in Docker deployments (#447)
This commit is contained in:
@@ -53,6 +53,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
imagemagick \
|
||||
fonts-noto-cjk \
|
||||
libraw-bin \
|
||||
libimage-exiftool-perl \
|
||||
&& . /etc/os-release \
|
||||
&& TARGETARCH=$(dpkg --print-architecture) \
|
||||
&& FFMPEG_VERSION="8.1.2-5" \
|
||||
|
||||
@@ -7,6 +7,7 @@ import {
|
||||
decodeRawWithDcraw,
|
||||
extractAndValidateRawPreview,
|
||||
EXIF_ORIENTATION_TO_ROTATION,
|
||||
withTimeout,
|
||||
} from './raw-extract'
|
||||
|
||||
// Mock exiftool-vendored
|
||||
@@ -171,6 +172,29 @@ describe('extractEmbeddedJpeg', () => {
|
||||
expect(result!.orientation).toBeUndefined()
|
||||
result!.cleanup()
|
||||
})
|
||||
|
||||
it('skips remaining tags and aborts early when extraction times out', async () => {
|
||||
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
try {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = '20'
|
||||
// Hang on first tag
|
||||
mockExtractBinaryTag.mockImplementationOnce(
|
||||
() => new Promise((resolve) => setTimeout(resolve, 300)),
|
||||
)
|
||||
|
||||
const result = await extractEmbeddedJpeg('/path/to/photo.cr2')
|
||||
|
||||
expect(result).toBeNull()
|
||||
// Crucial: should break immediately after the first timeout instead of trying remaining 2 tags
|
||||
expect(mockExtractBinaryTag).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
if (origEnv !== undefined) {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
|
||||
} else {
|
||||
delete process.env.EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('validateExtractedJpeg', () => {
|
||||
@@ -348,6 +372,40 @@ describe('decodeRawWithDcraw', () => {
|
||||
result!.cleanup()
|
||||
})
|
||||
|
||||
it('defaults to full size without -h when EXIF metadata read times out', async () => {
|
||||
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
try {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = '20'
|
||||
mockRead.mockImplementationOnce(() => new Promise((resolve) => setTimeout(resolve, 300)))
|
||||
|
||||
let executedArgs: string[] = []
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
executedArgs = args as string[]
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
|
||||
const result = await decodeRawWithDcraw('/path/to/photo.dng')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(executedArgs).not.toContain('-h')
|
||||
result!.cleanup()
|
||||
} finally {
|
||||
if (origEnv !== undefined) {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
|
||||
} else {
|
||||
delete process.env.EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
it('returns null and cleans up when dcraw_emu exits with non-zero code', async () => {
|
||||
mockRead.mockResolvedValueOnce({ ImageWidth: 6000, ImageHeight: 4000 } as never)
|
||||
mockExecFile.mockImplementationOnce((_bin, _args, _opts, cb) => {
|
||||
@@ -573,6 +631,80 @@ describe('extractAndValidateRawPreview', () => {
|
||||
expect(result).toBeNull()
|
||||
expect(fs.existsSync(createdTiff)).toBe(false)
|
||||
})
|
||||
|
||||
it('falls back to dcraw_emu when embedded preview extraction times out', async () => {
|
||||
const origEnv = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
try {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = '20'
|
||||
// Hang on extractBinaryTag
|
||||
mockExtractBinaryTag.mockImplementationOnce(
|
||||
() => new Promise((resolve) => setTimeout(resolve, 300)),
|
||||
)
|
||||
|
||||
mockRead.mockResolvedValueOnce({
|
||||
ImageWidth: 6000,
|
||||
ImageHeight: 4000,
|
||||
Orientation: 1,
|
||||
} as never)
|
||||
|
||||
mockExecFile.mockImplementationOnce((_bin, args, _opts, cb) => {
|
||||
const tiffDest = (args as string[])[(args as string[]).indexOf('-Z') + 1]
|
||||
fs.writeFileSync(tiffDest, 'tiff-data')
|
||||
const callback = typeof _opts === 'function' ? _opts : cb
|
||||
if (callback) {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
;(callback as any)(null, { stdout: '', stderr: '' })
|
||||
}
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return {} as any
|
||||
})
|
||||
mockMetadata.mockResolvedValueOnce({ width: 6000, height: 4000 })
|
||||
|
||||
const result = await extractAndValidateRawPreview('/path/to/photo.cr2')
|
||||
|
||||
expect(result).not.toBeNull()
|
||||
expect(mockExecFile).toHaveBeenCalled()
|
||||
expect(result!.width).toBe(6000)
|
||||
expect(result!.height).toBe(4000)
|
||||
|
||||
result!.cleanup()
|
||||
} finally {
|
||||
if (origEnv !== undefined) {
|
||||
process.env.EXIFTOOL_TIMEOUT_MS = origEnv
|
||||
} else {
|
||||
delete process.env.EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe('withTimeout', () => {
|
||||
it('resolves when promise settles before timeout', async () => {
|
||||
const result = await withTimeout(Promise.resolve('ok'), 1000)
|
||||
expect(result).toBe('ok')
|
||||
})
|
||||
|
||||
it('rejects with timeout error when promise exceeds timeout', async () => {
|
||||
const hanging = new Promise((resolve) => setTimeout(resolve, 500))
|
||||
await expect(withTimeout(hanging, 20, 'TestTask')).rejects.toThrow(
|
||||
'TestTask timed out after 20ms',
|
||||
)
|
||||
})
|
||||
|
||||
it('propagates underlying promise rejection before timeout', async () => {
|
||||
const failing = Promise.reject(new Error('fail fast'))
|
||||
await expect(withTimeout(failing, 1000, 'TestTask')).rejects.toThrow('fail fast')
|
||||
})
|
||||
|
||||
it('prevents unhandled rejection if promise rejects after timeout', async () => {
|
||||
const delayedReject = new Promise((_, reject) =>
|
||||
setTimeout(() => reject(new Error('late error')), 40),
|
||||
)
|
||||
await expect(withTimeout(delayedReject, 10, 'TestTask')).rejects.toThrow(
|
||||
'TestTask timed out after 10ms',
|
||||
)
|
||||
await new Promise((resolve) => setTimeout(resolve, 60))
|
||||
})
|
||||
})
|
||||
|
||||
describe('EXIF_ORIENTATION_TO_ROTATION', () => {
|
||||
|
||||
@@ -52,6 +52,41 @@ export const EXIF_ORIENTATION_TO_ROTATION: Record<
|
||||
8: { angle: 270 }, // Rotate 270° CW
|
||||
}
|
||||
|
||||
const DEFAULT_EXIFTOOL_TIMEOUT_MS = 15_000
|
||||
|
||||
function getExifToolTimeoutMs(): number {
|
||||
const envVal = process.env.EXIFTOOL_TIMEOUT_MS
|
||||
if (envVal) {
|
||||
const parsed = parseInt(envVal, 10)
|
||||
if (!isNaN(parsed) && parsed > 0) {
|
||||
return parsed
|
||||
}
|
||||
}
|
||||
return DEFAULT_EXIFTOOL_TIMEOUT_MS
|
||||
}
|
||||
|
||||
/**
|
||||
* Wraps a promise with a timeout. If the timeout triggers first,
|
||||
* the returned promise rejects with a timeout error and attaches a no-op handler
|
||||
* to the underlying promise to prevent unhandled rejections.
|
||||
*/
|
||||
export function withTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
timeoutMs: number,
|
||||
operationName = 'Operation',
|
||||
): Promise<T> {
|
||||
let timer: ReturnType<typeof setTimeout> | undefined
|
||||
const timeoutPromise = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => {
|
||||
reject(new Error(`${operationName} timed out after ${timeoutMs}ms`))
|
||||
}, timeoutMs)
|
||||
})
|
||||
promise.catch(() => {})
|
||||
return Promise.race([promise, timeoutPromise]).finally(() => {
|
||||
if (timer) clearTimeout(timer)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempt to extract an embedded JPEG preview from a camera RAW file to a temporary file.
|
||||
*
|
||||
@@ -83,10 +118,16 @@ export async function extractEmbeddedJpeg(
|
||||
}
|
||||
}
|
||||
|
||||
const exiftoolTimeoutMs = getExifToolTimeoutMs()
|
||||
|
||||
for (const tag of tags) {
|
||||
const candidatePath = path.join(workDir, `preview-${tag}-${Date.now()}.jpg`)
|
||||
try {
|
||||
await exiftool.extractBinaryTag(tag, rawFilePath, candidatePath)
|
||||
await withTimeout(
|
||||
exiftool.extractBinaryTag(tag, rawFilePath, candidatePath),
|
||||
exiftoolTimeoutMs,
|
||||
`exiftool.extractBinaryTag(${tag})`,
|
||||
)
|
||||
if (fs.existsSync(candidatePath) && fs.statSync(candidatePath).size > 0) {
|
||||
logger.debug({ tag, rawFilePath, candidatePath }, 'Extracted embedded JPEG from RAW')
|
||||
|
||||
@@ -95,7 +136,11 @@ export async function extractEmbeddedJpeg(
|
||||
let rawWidth: number | undefined
|
||||
let rawHeight: number | undefined
|
||||
try {
|
||||
const exif = await exiftool.read(rawFilePath)
|
||||
const exif = await withTimeout(
|
||||
exiftool.read(rawFilePath),
|
||||
exiftoolTimeoutMs,
|
||||
'exiftool.read',
|
||||
)
|
||||
const rawOrientation = exif.Orientation
|
||||
if (typeof rawOrientation === 'number' && rawOrientation >= 1 && rawOrientation <= 8) {
|
||||
orientation = rawOrientation
|
||||
@@ -122,8 +167,9 @@ export async function extractEmbeddedJpeg(
|
||||
|
||||
return { previewPath: candidatePath, cleanup, orientation, rawWidth, rawHeight }
|
||||
}
|
||||
} catch {
|
||||
logger.debug({ tag, rawFilePath }, `Could not extract ${tag} from RAW file`)
|
||||
} catch (err) {
|
||||
const isTimeout = err instanceof Error && err.message.includes('timed out')
|
||||
logger.debug({ tag, rawFilePath, err, isTimeout }, `Could not extract ${tag} from RAW file`)
|
||||
try {
|
||||
if (fs.existsSync(candidatePath)) {
|
||||
fs.rmSync(candidatePath, { force: true })
|
||||
@@ -131,6 +177,13 @@ export async function extractEmbeddedJpeg(
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
if (isTimeout) {
|
||||
logger.warn(
|
||||
{ rawFilePath, tag, timeoutMs: exiftoolTimeoutMs },
|
||||
'ExifTool extraction timed out; skipping remaining tags to avoid further delays',
|
||||
)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +234,8 @@ export async function decodeRawWithDcraw(
|
||||
let rawWidth: number | undefined
|
||||
let rawHeight: number | undefined
|
||||
try {
|
||||
const exif = await exiftool.read(rawFilePath)
|
||||
const exiftoolTimeoutMs = getExifToolTimeoutMs()
|
||||
const exif = await withTimeout(exiftool.read(rawFilePath), exiftoolTimeoutMs, 'exiftool.read')
|
||||
const w =
|
||||
typeof exif.ImageWidth === 'number'
|
||||
? exif.ImageWidth
|
||||
|
||||
@@ -13,6 +13,7 @@ const depsToExtract = [
|
||||
'@temporalio/client',
|
||||
'@temporalio/worker',
|
||||
'@temporalio/workflow',
|
||||
'exiftool-vendored',
|
||||
'pdfkit',
|
||||
'prisma',
|
||||
'sharp',
|
||||
|
||||
Reference in New Issue
Block a user