@inproceedings{ea0b39283da4427f9aca6f20e6ec487e,
title = "Multimodal Abstractive Summarization of Instructional Videos with Vision-Language Models",
abstract = "Multimodal video summarization requires visual features that align semantically with language generation. Traditional approaches rely on CNN features trained for object classification, which represent visual concepts as discrete categories not aligned with natural language. We propose ClipSum, a framework that leverages frozen CLIP vision-language features with explicit temporal modeling and dimension-adaptive fusion for instructional video summarization. CLIP{\textquoteright}s contrastive pre-training on 400M image-text pairs yields visual features semantically aligned with the linguistic concepts that text decoders generate, bridging the vision-language gap at the representation level. On YouCook2, ClipSum achieves 33.0\% ROUGE-1 versus 30.5\% for ResNet-152 with 4{\texttimes} lower dimensionality (512 vs. 2048), demonstrating that semantic alignment matters more than feature capacity. Frozen CLIP (33.0\%) surpasses fine-tuned CLIP (32.3\%), showing that preserving pre-trained alignment is more valuable than task-specific adaptation. GitHub Page.",
keywords = "CLIP, Cross-Modal Fusion, Instructional Video Understanding, Multimodal Abstractive Summarization, Vision-Language Models",
author = "Maham Nazir and Muhammad Aqeel and Richong Zhang and Francesco Setti",
note = "Publisher Copyright: {\textcopyright} The Author(s), under exclusive license to Springer Nature Switzerland AG 2027.; 28th International Conference on Pattern Recognition, ICPR 2026 ; Conference date: 17-08-2026 Through 22-08-2026",
year = "2027",
doi = "10.1007/978-3-032-31438-3\_46",
language = "英语",
isbn = "9783032314376",
series = "Lecture Notes in Computer Science",
publisher = "Springer Science and Business Media Deutschland GmbH",
pages = "696--710",
editor = "\{De Marsico\}, Maria and Ho, \{Tin Kam\} and Frederic Jurie and Cheng-Lin Liu and Daniel Lopresti and Ingela Nystr{\"o}m and Jean-Marc Ogier and Arun Ross and Liang Wang",
booktitle = "Pattern Recognition - 28th International Conference, ICPR 2026, Proceedings",
address = "德国",
}