SmolVLM: support video

Author: foldl

models/smollm.cpp

@@ -484,6 +484,7 @@ namespace vlm
             : BaseTokenizer::BaseTokenizer(config, encoder)
         {
             sys_prompt = "";
+            do_split = false;
         }
 
         size_t load(tokenizer::DataReader *buffer, int n_vocab) override;
@@ -496,6 +497,7 @@ namespace vlm
         int fake_token_around_image_token_id;
         int global_image_token_token_id;
         int nl_token_id;
+        bool do_split;
 
         std::vector<std::vector<int>> row_col_token_ids;
     };
@@ -616,7 +618,7 @@ namespace vlm
     public:
         typedef llama::v2::GenericConditionalGeneration<LlamaBlock> Base;
         ConditionalGeneration(const Config &config, const RuntimeConfig &runtime_config, ModelType type = ModelType::MODEL_TYPE_SMOL_VLM)
-            : ExtendEmbedding(81 * 64),
+            : ExtendEmbedding(4096),
               Base(config, runtime_config, type, config.num_key_value_heads, config.max_length, 12, false),
               visual(runtime_config)
         {
@@ -648,6 +650,17 @@ namespace vlm
             _chat_encoder.vit_loaded = visual.load(loader);
         }
 
+        void set_additional_args(const std::map<std::string, std::string> &args) override
+        {
+            Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
+            auto it = args.find("do_split");
+            if (it == args.end()) it = args.find("do-split");
+            if (it != args.end())
+            {
+                tok->do_split = it->second != "0";
+            }
+        }
+
         void before_generate(const GenerationConfig &gen_config) override
         {
             std::vector<uint8_t> buf;
@@ -667,7 +680,42 @@ namespace vlm
         Tokenizer *tok = dynamic_cast<Tokenizer *>(tokenizer);
         append_user_opening(round_idx, ids);
 
+        std::vector<std::unique_ptr<vision::VideoLoader>> videos;
+
+        // expand video into images
+        std::vector<ContentPiece> pieces;
         for (auto &piece : user.pieces)
+        {
+            if (piece.type != ContentPiece::Type::Video)
+            {
+                pieces.push_back(piece);
+                continue;
+            }
+
+            auto video = new vision::VideoLoader(piece.content.c_str(), vis_config->video.fps, vis_config->video.max_frames);
+            videos.emplace_back(video);
+            if (video->frames.size() < 1)
+                continue;
+
+            std::ostringstream oss;
+            oss << "You are provided the following series of " << utils::num2words((int)video->frames.size())
+                << " frames from a " << utils::sec2hms((float)video->frames.size() / vis_config->video.fps) << " [H:MM:SS] video.\n";
+            pieces.emplace_back(oss.str());
+
+            for (size_t i = 0; i < video->frames.size(); i++)
+            {
+                oss.str("");
+                oss << "\nFrame from " << utils::sec2ms(i / vis_config->video.fps);
+                pieces.emplace_back(oss.str());
+
+                pieces.emplace_back(video->frames[i], ContentPiece::Type::Image);
+            }
+
+            // DEFAULT_MEDIA_OUTTRO = "\n\n"
+            pieces.emplace_back("\n\n");
+        }
+
+        for (auto &piece : pieces)
         {
             if (piece.type == ContentPiece::Type::Text)
             {
@@ -688,7 +736,7 @@ namespace vlm
                 int splits_cols_num = 0;
                 int splits_rows_num = 0;
 
-                vision::image_load_split(piece.content.c_str(), splits, image_size_per_split,  image_size_per_split, splits_cols_num, splits_rows_num);
+                vision::image_load_split(piece.content.c_str(), splits, tok->do_split, image_size_per_split,  image_size_per_split, splits_cols_num, splits_rows_num);
 
                 std::vector<float> scaled;
 

src/basics.h

@@ -73,5 +73,10 @@ namespace utils
 
     std::string load_file(const char *fn);
 
+    std::string num2words(int value);
+
+    std::string sec2hms(float seconds, bool show_ms = false);
+    std::string sec2ms(float seconds, bool show_ms = false);
+
     //#define TIME_STAMP (std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch()).count())
 }

src/vectorstore.cpp

@@ -468,4 +468,107 @@ namespace utils
 
         return buffer.str();
     }
+
+    std::string num2words(int value)
+    {
+        if (value < 0)
+            return "minus " + num2words(-value);
+
+        if (value == 0)
+            return "zero";
+
+        const static std::vector<std::string> units = {"", "one", "two", "three", "four", "five", "six", "seven", "eight", "nine"};
+        const static std::vector<std::string> teens = {"ten", "eleven", "twelve", "thirteen", "fourteen", "fifteen", "sixteen", "seventeen", "eighteen", "nineteen"};
+        const static std::vector<std::string> tens = {"", "ten", "twenty", "thirty", "forty", "fifty", "sixty", "seventy", "eighty", "ninety"};
+
+        std::string result;
+
+        if (value >= 1000000000) {
+            int billion = value / 1000000000;
+            result += num2words(billion) + " billion";
+            value %= 1000000000;
+            if (value > 0) {
+                result += ", ";
+            }
+        }
+
+        if (value >= 1000000) {
+            int million = value / 1000000;
+            result += num2words(million) + " million";
+            value %= 1000000;
+            if (value > 0) {
+                result += ", ";
+            }
+        }
+
+        if (value >= 1000) {
+            int thousand = value / 1000;
+            result += num2words(thousand) + " thousand";
+            value %= 1000;
+            if (value > 0) {
+                result += ", ";
+            }
+        }
+
+        if (value >= 100) {
+            int hundred = value / 100;
+            result += units[hundred] + " hundred";
+            value %= 100;
+            if (value > 0) {
+                result += " and ";
+            }
+        }
+
+        if (value >= 20)
+        {
+            int ten = value / 10;
+            result += tens[ten];
+            value %= 10;
+
+            if (value > 0)
+            {
+                result += "-" + units[value];
+            }
+        }
+        else if (value >= 10)
+        {
+            result += teens[value - 10];
+        }
+        else if (value >= 1)
+        {
+            result += units[value];
+        }
+
+        return result;
+    }
+
+     std::string sec2hms(float seconds, bool show_ms)
+     {
+        int sec = (int)seconds;
+        int ms  = (int)((seconds - sec) * 1000000);
+        int min = sec / 60;
+        int hh  = min / 60;
+        sec %= 60;
+        min %= 60;
+        char s[100];
+        if (show_ms)
+            sprintf(s, "%d:%02d:%02d.%06d", hh, min, sec, ms);
+        else
+            sprintf(s, "%d:%02d:%02d", hh, min, sec);
+        return s;
+     }
+
+     std::string sec2ms(float seconds, bool show_ms)
+     {
+        int sec = (int)seconds;
+        int ms  = (int)((seconds - sec) * 1000000);
+        int min = sec / 60;
+        sec %= 60;
+        char s[100];
+        if (show_ms)
+            sprintf(s, "%02d:%02d.%06d", min, sec, ms);
+        else
+            sprintf(s, "%02d:%02d", min, sec);
+        return s;
+     }
 }

src/vision_process.cpp

@@ -2,6 +2,8 @@
 #include <regex>
 #include <sstream>
 #include <cmath>
+#include <filesystem>
+#include "basics.h"
 
 #if defined(_MSC_VER)
 #define popen  _popen
@@ -10,6 +12,8 @@
 
 namespace vision
 {
+    namespace fs = std::filesystem;
+
     struct Params
     {
         int pre_max_width;
@@ -186,7 +190,7 @@ namespace vision
         pclose(pp);
     }
 
-    void image_load_split(const char *fn, std::vector<image_pixels_t> &splits, const int split_width, const int split_height, int &splits_cols_num, int &splits_rows_num)
+    void image_load_split(const char *fn, std::vector<image_pixels_t> &splits, bool do_split, const int split_width, const int split_height, int &splits_cols_num, int &splits_rows_num)
     {
         splits.clear();
         splits_cols_num = 0;
@@ -204,7 +208,7 @@ namespace vision
 
         splits_rows_num =(height + split_height - 1) / split_height;
         splits_cols_num =(width  + split_width  - 1) / split_width;
-        if ((splits_rows_num > 1) || (splits_cols_num > 1))
+        if (do_split && ((splits_rows_num > 1) || (splits_cols_num > 1)))
         {
             const int optimal_height = (height + splits_rows_num - 1) / splits_rows_num;
             const int optimal_width  = (width  + splits_cols_num - 1) / splits_cols_num;
@@ -444,6 +448,44 @@ namespace vision
         }
     }
 
+    VideoLoader::VideoLoader(const char *fn, float fps, const int max_frames, const int resize_width, const int resize_height)
+    {
+        if (!fs::exists(fn))
+            return;
+
+        tmp_dir = std::tmpnam(nullptr);
+        fs::create_directories(tmp_dir);
+
+        char cmd[1024];
+        if ((resize_height <= 0) || (resize_height <= 0))
+        {
+            sprintf(cmd, "ffmpeg -loglevel error -i \"%s\" -vf \"fps=%f\" -frames:v %d \"%s\"",
+                fn, fps, max_frames, (fs::path(tmp_dir) / "%04d.jpg").string().c_str());
+        }
+        else
+        {
+            sprintf(cmd, "ffmpeg -loglevel error -i \"%s\" -vf \"scale=w=%d:h=%d:force_original_aspect_ratio=1,pad=%d:%d:(ow-iw)/2:(oh-ih)/2,fps=%f\" -frames:v %d \"%s\"",
+                fn, resize_width, resize_height, resize_width, resize_height,
+                fps, max_frames, (fs::path(tmp_dir) / "%04d.jpg").string().c_str());
+        }
+
+        int ret = std::system(cmd);
+
+        for (const auto &entry : fs::directory_iterator(tmp_dir))
+        {
+            if (entry.path().extension() == ".jpg") {
+                frames.push_back(entry.path().string());
+            }
+        }
+
+        std::sort(frames.begin(), frames.end());
+    }
+
+    VideoLoader::~VideoLoader()
+    {
+        fs::remove_all(tmp_dir);
+    }
+
     static void print_data(const std::vector<float> &pixels, const int group_size = 10, int max_elem = 100)
     {
         const size_t cnt = pixels.size();

src/vision_process.h

@@ -97,10 +97,21 @@ namespace vision
 
     void image_dimension(const char *fn, int &width, int &height);
     void image_load(const char *fn, std::vector<uint8_t> &rgb_pixels, int &width, int &height, int patch_size, PaddingMode pad = PaddingMode::No);
-    void image_load_split(const char *fn, std::vector<image_pixels_t> &splits, const int split_width, const int split_height, int &splits_cols_num, int &splits_rows_num); // splits are in natural order
+    void image_load_split(const char *fn, std::vector<image_pixels_t> &splits, bool do_split, const int split_width, const int split_height, int &splits_cols_num, int &splits_rows_num); // splits are in natural order
     void image_rescale(const std::vector<uint8_t> &rgb_pixels, std::vector<float> &scaled_rgb_pixels, float scale_factor = 1/255.0f);
     void image_normalize(std::vector<float> &rgb_pixels, const float *mean, const float *std_d);
 
+    class VideoLoader
+    {
+    public:
+        VideoLoader(const char *fn, float fps = 1.0f, const int max_frames = 10, const int resize_width = -1, const int resize_height = -1);
+        ~VideoLoader();
+    public:
+        std::vector<std::string> frames;
+    private:
+        std::string tmp_dir;
+    };
+
     // ASSUMPTION: already properly aligned to `patch_size`
     void image_arrange(const std::vector<float> &rgb_pixels, const int width, const int patch_size,
         std::vector<float> &arranged, const PatchesFormat fmt);