Merge pull request #3603 from alibaba/feature/sync

MNN:Sync: Sync 3.2.0

Author: jxt1234

docs/compile/engine.md

@@ -126,7 +126,7 @@ mkdir build && cd build && cmake .. -DCMAKE_OSX_ARCHITECTURES=arm64 && make -j8
 
 - 基于脚本编译：运行脚本并开启`MNN_ARM82`选项
 ```
-sh package_scripts/ios/buildiOS.sh "-DMNN_ARM82=true"
+sh package_scripts/ios/buildiOS.sh -DMNN_ARM82=ON
 ```
 
 ## 鸿蒙(Harmony)

docs/start/demo.md

@@ -20,9 +20,11 @@
 ### 图像实例分割
 代码位置：`demo/exec/segment.cpp`
 
-下载 deeplabv3 分割模型并转换到 mnn 模型
+下载 deeplabv3 分割模型
 [https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite](https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/deeplabv3_257_mv_gpu.tflite)
 
+使用 [模型转换工具](../tools/convert.md) 转换为 MNN 模型，转换时加上参数 --keepInputFormat=0 【把输入由NHWC转换为NC4HW4布局】
+
 ```bash
 ./segment.out model.mnn input.png result.png
 ```
@@ -95,14 +97,14 @@ flops_info: 568.792175M
 backend_info: 13
 expect 983
 output belong to class: 983
-$ python gpu_session_demo.py mobilenet_demo/mobilenet_v1.mnn mobilenet_demo/ILSVRC2012_val_00049999.JPEG
+$ python gpu_session_demo.py mobilenet_demo/mobilenet_v1.mnn mobilenet_demo/ILSVRC2012_val_00049999.JPEG 
 Testing gpu model calling method
 
 Load Cache file error.
 MNN use high precision
 Can't Find type=3 backend, use 0 instead
 Can't Find type=3 backend, use 0 instead
-Run on backendtype: 13
+Run on backendtype: 13 
 
 expect 983
 output belong to class: 983
@@ -127,7 +129,7 @@ output belong to class: 983
 #### mnist
 使用mnist数据训练模型，并测试准确率，无需下载资源，用法如下：
 ```bash
-$ pip install mnist
+$ pip install mnist 
 $ python train_mnist.py
 train loss:  2.3346531
 train loss:  0.28027835
@@ -161,7 +163,7 @@ AttributeError: module 'MNN.nn' has no attribute 'FixModule'
 #### module_save
 演示了模型权值的存储和加载
 ```bash
-$ python test_save.py
+$ python test_save.py 
 0.0004
 10
 ```
@@ -225,4 +227,3 @@ sh ../tools/script/get_model.sh
 - [视频抠图](https://github.com/DefTruth/RobustVideoMatting.lite.ai.toolkit)
 - [SuperGlue关键点匹配](https://github.com/Hanson0910/MNNSuperGlue)
 - [OCR](https://github.com/DayBreak-u/chineseocr_lite/tree/onnx/android_projects/OcrLiteAndroidMNN)
-- [Bert-VITS2-MNN](https://github.com/Voine/Bert-VITS2-MNN)

docs/transformers/llm.md

@@ -73,34 +73,48 @@ python llmexport.py \
 - 使用`--lm_quant_bit`来制定lm_head层权重的量化bit数，不指定则使用`--quant_bit`的量化bit数
 
 ### 参数
+执行 `python llmexport.py -h` 可查看参数：
 ```
-usage: llmexport.py [-h] --path PATH [--type TYPE] [--lora_path LORA_PATH] [--dst_path DST_PATH] [--test TEST] [--export EXPORT]
-                    [--quant_bit QUANT_BIT] [--quant_block QUANT_BLOCK] [--lm_quant_bit LM_QUANT_BIT]
-                    [--mnnconvert MNNCONVERT]
+usage: llmexport.py [-h] --path PATH [--type TYPE] [--tokenizer_path TOKENIZER_PATH] [--lora_path LORA_PATH]
+                    [--gptq_path GPTQ_PATH] [--dst_path DST_PATH] [--verbose] [--test TEST] [--export EXPORT]
+                    [--onnx_slim] [--quant_bit QUANT_BIT] [--quant_block QUANT_BLOCK]
+                    [--lm_quant_bit LM_QUANT_BIT] [--mnnconvert MNNCONVERT] [--ppl] [--awq] [--sym] [--seperate_embed]
+                    [--lora_split]
 
 llm_exporter
 
-options:
+optional arguments:
   -h, --help            show this help message and exit
   --path PATH           path(`str` or `os.PathLike`):
                         Can be either:
                         	- A string, the *model id* of a pretrained model like `THUDM/chatglm-6b`. [TODO]
                         	- A path to a *directory* clone from repo like `../chatglm-6b`.
   --type TYPE           type(`str`, *optional*):
                         	The pretrain llm model type.
+  --tokenizer_path TOKENIZER_PATH
+                        tokenizer path, defaut is `None` mean using `--path` value.
   --lora_path LORA_PATH
                         lora path, defaut is `None` mean not apply lora.
+  --gptq_path GPTQ_PATH
+                        gptq path, defaut is `None` mean not apply gptq.
   --dst_path DST_PATH   export onnx/mnn model to path, defaut is `./model`.
+  --verbose             Whether or not to print verbose.
   --test TEST           test model inference with query `TEST`.
   --export EXPORT       export model to an onnx/mnn model.
+  --onnx_slim           Whether or not to use onnx-slim.
   --quant_bit QUANT_BIT
                         mnn quant bit, 4 or 8, default is 4.
   --quant_block QUANT_BLOCK
-                        mnn quant block, default is 0 mean channle-wise.
+                        mnn quant block, 0 mean channle-wise, default is 128.
   --lm_quant_bit LM_QUANT_BIT
                         mnn lm_head quant bit, 4 or 8, default is `quant_bit`.
   --mnnconvert MNNCONVERT
                         local mnnconvert path, if invalid, using pymnn.
+  --ppl                 Whether or not to get all logits of input tokens.
+  --awq                 Whether or not to use awq quant.
+  --sym                 Whether or not to using symmetric quant (without zeropoint), defualt is False.
+  --seperate_embed      For lm and embed shared model, whether or not to sepearte embed to avoid quant, defualt is False, if True, embed weight will be seperate to embeddingbf16.bin.
+  --lora_split          Whether or not export lora split, defualt is False.
 ```
 
 ### 权重读取

include/MNN/MNNDefine.h

@@ -75,7 +75,7 @@ MNN_ERROR("Check failed: %s ==> %s\n", #success, #log); \
 #define STR_IMP(x) #x
 #define STR(x) STR_IMP(x)
 #define MNN_VERSION_MAJOR 3
-#define MNN_VERSION_MINOR 1
-#define MNN_VERSION_PATCH 4
+#define MNN_VERSION_MINOR 2
+#define MNN_VERSION_PATCH 0
 #define MNN_VERSION STR(MNN_VERSION_MAJOR) "." STR(MNN_VERSION_MINOR) "." STR(MNN_VERSION_PATCH)
 #endif /* MNNDefine_h */

source/backend/cpu/CPUAttention.cpp

@@ -99,14 +99,22 @@ static void pack_QK(char * pack_qk_dst, float * qk_src, int seq_len, int kv_seq_
 template <typename T>
 static void mask_QK(float * unpack_qk, int seq_len, int kv_seq_len, float mScale, float min_val, const Tensor* mask) {
     if (seq_len == 1 || mask == nullptr) {
-        for (int i = 0; i < seq_len * kv_seq_len; i++) {
+        for (int i = 0; i < kv_seq_len; i++) {
             unpack_qk[i] = unpack_qk[i] * mScale;
         }
     } else if (mask->getType() == halide_type_of<float>()) {
         // float mask
         T* fpmask_ptr = mask->host<T>();
-        for (int i = 0; i < seq_len * kv_seq_len; i++) {
-            unpack_qk[i] = unpack_qk[i] * mScale + fpmask_ptr[i];
+        int offset = kv_seq_len-seq_len;
+        for (int i=0; i<seq_len; ++i) {
+            auto unpack_qki = unpack_qk + i * kv_seq_len;
+            auto fpmask_ptri = fpmask_ptr + i * seq_len;
+            for (int j=0; j<offset; ++j) {
+                unpack_qki[j] = unpack_qki[j] * mScale;
+            }
+            for (int j=0; j<seq_len; ++j) {
+                unpack_qki[offset+j] = unpack_qki[offset+j] * mScale + fpmask_ptri[j];
+            }
         }
     } else {
         // int mask
@@ -192,7 +200,6 @@ ErrorCode CPUAttention::onExecute(const std::vector<Tensor*>& inputs, const std:
     int seq_len = query->length(1);
     if (inputs.size() > 3) {
         mask = inputs[3];
-        MNN_ASSERT(seq_len == mask->length(2));
     }
     int tileCount = UP_DIV(mNumHead, mThreadNum);
     int group_size = mNumHead / mKvNumHead;

source/backend/cpu/CPURaster.cpp

@@ -594,7 +594,7 @@ ErrorCode CPURaster::onExecute(const std::vector<Tensor *> &____inputs, const st
     }
     auto core = static_cast<CPUBackend*>(backend())->functions();
     auto output = outputs[0];
-    auto bytes = CPUBackend::getBytes(backend(), output);
+    size_t bytes = (size_t)(CPUBackend::getBytes(backend(), output));
     auto outputEleSize = static_cast<CPUBackend*>(backend())->getTensorSize(output);
     auto threadNum = static_cast<CPUBackend*>(backend())->threadNumber();
     if (mSingleConvert.type > 0) {