How to go from video -> images saved to disk -> video?
jparismorgan opened this issue · comments
Version information
- fluent-ffmpeg version:
2.1.24 - ffmpeg version:
6.1.1 - OS: macOS
Code to reproduce
import ffmpeg from 'fluent-ffmpeg'
interface FfmpegMetadata {
fps: number
timeBase: string
bitRate: number
duration: number
frames: number
}
const getMetadata = (videoPath: string) => new Promise<FfmpegMetadata>((resolve, reject) => {
ffmpeg.ffprobe(videoPath, (err, metadata) => {
if (err) {
reject(err)
} else {
console.log('metadata', metadata)
const frameRate = metadata.streams[0].avg_frame_rate
const [numerator, denominator] = frameRate.split('/')
const fps = parseInt(numerator, 10) / parseInt(denominator, 10)
resolve({
fps,
bitRate: parseInt(metadata.streams[0].bit_rate, 10),
timeBase: metadata.streams[0].time_base,
duration: metadata.format.duration,
frames: parseInt(metadata.streams[0].nb_frames, 10)
})
}
})
})
const createVideoFromFrames = (framesDirectory: string, outputVideoPath: string, format: string, metadata: FfmpegMetadata) => new Promise<void>((resolve, reject) => {
ffmpeg().input(`${framesDirectory}/%d.png`)
.inputFPS(metadata.fps)
.output(outputVideoPath)
.outputFPS(metadata.fps)
.outputFormat(format)
.withVideoBitrate(metadata.bitRate)
.withDuration(metadata.duration)
.outputOptions(`-time_base ${metadata.timeBase}`)
.on('error', (error) => {
console.error('[ffmpeg@createVideoFromFrames@error]', error)
reject(error)
})
.on('end', () => {
resolve()
})
.run()
})
const splitFramesFromVideo = (videoPath: string, outputDirectory: string, metadata: FfmpegMetadata) => new Promise<void>((resolve, reject) => {
ffmpeg(videoPath)
// Forces ffmpeg to output an image for each frame
.toFormat('image2')
.frames(metadata.frames)
.on('error', (error) => {
console.error('[ffmpeg@splitFramesFromVideo@error]', error)
reject(error)
})
.on('end', () => {
resolve()
})
.on('stderr', (stderrLine) => {
console.log(`[ffmpeg@splitFramesFromVideo@stderr] ${stderrLine}`)
})
.on('start', (commandLine) => {
console.log(`[ffmpeg@splitFramesFromVideo@start] ${commandLine}`)
})
.on('codecData', (codecData) => {
console.log(`[ffmpeg@splitFramesFromVideo@codecData] ${codecData}`)
})
.saveToFile(`${outputDirectory}/%d.png`)
})
export const processVideo = async (inputVideoFilePath: string, inputFramesDirectory: string, outputVideoFilePath: string, fileType: FileType) => {
const metadata = await getMetadata(inputVideoFilePath)
// Split up the video into frames.
await splitFramesFromVideo(inputVideoFilePath, inputFramesDirectory, metadata)
// Then put those frames back into a video.
let type = ''
if (fileType === FileType.MP4) {
type = 'mp4'
} else if (fileType === FileType.MOV) {
type = 'mov'
}
await createVideoFromFrames(inputFramesDirectory, outputVideoFilePath, type, metadata)
}Expected results
I would like to be able to go from a .mp4 or .mov video, extract all frames, do some processing on them, and then write back to a video. I would expect the video to have identical metadata to the original video, i.e. the same duration, encoding, etc. I do not need to keep the audio track, just the video.
Observed results
My created video has a different duration, bit rate, and encoder, as you can see here.
Original video metadata:
metadata {
streams: [
{
index: 0,
codec_name: 'h264',
codec_long_name: 'H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10',
profile: 'High',
codec_type: 'video',
codec_tag_string: 'avc1',
codec_tag: '0x31637661',
width: 1080,
height: 1920,
coded_width: 1080,
coded_height: 1920,
closed_captions: 0,
film_grain: 0,
has_b_frames: 2,
sample_aspect_ratio: 'N/A',
display_aspect_ratio: 'N/A',
pix_fmt: 'yuv420p',
level: 40,
color_range: 'tv',
color_space: 'bt709',
color_transfer: 'bt709',
color_primaries: 'bt709',
chroma_location: 'left',
field_order: 'progressive',
refs: 1,
is_avc: 'true',
nal_length_size: 4,
id: '0x1',
r_frame_rate: '30000/1001',
avg_frame_rate: '27000/1001',
time_base: '1/30000',
start_pts: 0,
start_time: 0,
duration_ts: 15100,
duration: 0.503333,
bit_rate: 5660223,
max_bit_rate: 'N/A',
bits_per_raw_sample: 8,
nb_frames: 36,
nb_read_frames: 'N/A',
nb_read_packets: 'N/A',
extradata_size: 49,
tags: [Object],
disposition: [Object]
},
{
index: 1,
codec_name: 'aac',
codec_long_name: 'AAC (Advanced Audio Coding)',
profile: 'LC',
codec_type: 'audio',
codec_tag_string: 'mp4a',
codec_tag: '0x6134706d',
sample_fmt: 'fltp',
sample_rate: 48000,
channels: 2,
channel_layout: 'stereo',
bits_per_sample: 0,
initial_padding: 0,
id: '0x2',
r_frame_rate: '0/0',
avg_frame_rate: '0/0',
time_base: '1/48000',
start_pts: 0,
start_time: 0,
duration_ts: 24160,
duration: 0.503333,
bit_rate: 248416,
max_bit_rate: 'N/A',
bits_per_raw_sample: 'N/A',
nb_frames: 27,
nb_read_frames: 'N/A',
nb_read_packets: 'N/A',
extradata_size: 2,
tags: [Object],
disposition: [Object]
}
],
format: {
filename: '/Users/parismorgan/input.mp4',
nb_streams: 2,
nb_programs: 0,
format_name: 'mov,mp4,m4a,3gp,3g2,mj2',
format_long_name: 'QuickTime / MOV',
start_time: 0,
duration: 0.503333,
size: 963879,
bit_rate: 15319941,
probe_score: 100,
tags: {
major_brand: 'mp42',
minor_version: '1',
compatible_brands: 'isommp41mp42',
creation_time: '2024-02-14T01:21:12.000000Z'
}
},
chapters: []
}
Resulting video metadata:
metadata {
streams: [
{
index: 0,
codec_name: 'h264',
codec_long_name: 'H.264 / AVC / MPEG-4 AVC / MPEG-4 part 10',
profile: 'High',
codec_type: 'video',
codec_tag_string: 'avc1',
codec_tag: '0x31637661',
width: 1080,
height: 1920,
coded_width: 1080,
coded_height: 1920,
closed_captions: 0,
film_grain: 0,
has_b_frames: 2,
sample_aspect_ratio: 'N/A',
display_aspect_ratio: 'N/A',
pix_fmt: 'yuvj420p',
level: 62,
color_range: 'pc',
color_space: 'bt470bg',
color_transfer: 'unknown',
color_primaries: 'unknown',
chroma_location: 'center',
field_order: 'progressive',
refs: 1,
is_avc: 'true',
nal_length_size: 4,
id: '0x1',
r_frame_rate: '27/1',
avg_frame_rate: '14000/519',
time_base: '1/30000',
start_pts: 0,
start_time: 0,
duration_ts: 15570,
duration: 0.519,
bit_rate: 14884947,
max_bit_rate: 'N/A',
bits_per_raw_sample: 8,
nb_frames: 14,
nb_read_frames: 'N/A',
nb_read_packets: 'N/A',
extradata_size: 49,
tags: [Object],
disposition: [Object]
}
],
format: {
filename: '/Users/parismorgan/result.mp4',
nb_streams: 1,
nb_programs: 0,
format_name: 'mov,mp4,m4a,3gp,3g2,mj2',
format_long_name: 'QuickTime / MOV',
start_time: 0,
duration: 0.519,
size: 966699,
bit_rate: 14900947,
probe_score: 100,
tags: {
major_brand: 'isom',
minor_version: '512',
compatible_brands: 'isomiso2avc1mp41',
encoder: 'Lavf60.16.100'
}
},
chapters: []
}
The first issue seems to be that we only create 15 frames out of the input video, but in the metadata it says TODO. But I also tried with 'ffmpeg-extract-frames' and also got 15 frames:
await extractFrames({
input: rgbVideoFilePath,
output: `${rgbVideoDirectory}/%d.png`
})
Thank you for any help!