add disk cache

Author: GangZhuo

BaiduPCS_NET/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.0.5.0")]
-[assembly: AssemblyFileVersion("1.0.5.0")]
+[assembly: AssemblyVersion("1.0.6.0")]
+[assembly: AssemblyFileVersion("1.0.6.0")]

Sample/Sample_0_FileExplorer/Common/AppSettings.cs

@@ -54,6 +54,11 @@ public static class AppSettings
         /// </summary>
         public static bool OverWriteWhenUploadFile { get; set; }
 
+        /// <summary>
+        /// 磁盘缓存大小
+        /// </summary>
+        public static long MaxCacheSize { get; set; }
+
         public static bool Restore()
         {
             string filename = SettingsFileName;
@@ -114,6 +119,9 @@ public static bool RestoreFrom(TextReader reader)
                     case "OverWriteWhenUploadFile":
                         OverWriteWhenUploadFile = Convert.ToBoolean(value);
                         break;
+                    case "MaxCacheSize":
+                        MaxCacheSize = Convert.ToInt64(value);
+                        break;
 
                 }
             }
@@ -160,6 +168,7 @@ public static bool SaveTo(Stream stream)
                 WriteItem(writer, "AutomaticUploadMaxThreadCount", AutomaticUploadMaxThreadCount.ToString());
                 WriteItem(writer, "RetryWhenUploadFailed", RetryWhenUploadFailed.ToString());
                 WriteItem(writer, "OverWriteWhenUploadFile", OverWriteWhenUploadFile.ToString());
+                WriteItem(writer, "MaxCacheSize", MaxCacheSize.ToString());
 
                 #endregion
 

Sample/Sample_0_FileExplorer/DU/Cache.cs

@@ -0,0 +1,69 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace FileExplorer
+{
+    public class Cache
+    {
+        public long TotalSize { get; set; }
+
+        public IBigFileHelper FileHelper { get; set; }
+
+        public List<Block> BlockList { get; private set; }
+
+        public Cache()
+        {
+            TotalSize = 0L;
+            BlockList = new List<Block>();
+        }
+
+        public bool Add(long position, byte[] data, int size)
+        {
+            Block block = new Block();
+            block.Position = position;
+            block.Data = data;
+            block.Size = size;
+            BlockList.Add(block);
+            TotalSize += size;
+            return true;
+        }
+
+        public bool Flush()
+        {
+            BlockList.Sort(new BlockComparer());
+            foreach (Block block in BlockList)
+            {
+                FileHelper.Update(block.Position, block.Data, 0, block.Size);
+            }
+            return true;
+        }
+
+        public bool Reset()
+        {
+            TotalSize = 0;
+            BlockList.Clear();
+            return true;
+        }
+
+        public class Block
+        {
+            public long Position { get; set; }
+
+            public int Size { get; set; }
+
+            public byte[] Data { get; set; }
+        }
+
+        public class BlockComparer : IComparer<Block>
+        {
+            public int Compare(Block x, Block y)
+            {
+                long xpos = x == null ? 0L : x.Position;
+                long ypos = y == null ? 0L : y.Position;
+                return xpos < ypos ? -1 : (xpos > ypos ? 1 : 0);
+            }
+        }
+    }
+}

Sample/Sample_0_FileExplorer/DU/MultiThreadDownloader.cs

@@ -34,7 +34,7 @@ public class MultiThreadDownloader : Downloader
         private long taskId = 0; //本地下载任务的 ID
         private long runningThreadCount = 0; //正在运行的线程数
         private object locker = new object();
-        private object sliceFileLocker = new object();
+        private Cache cache = new Cache();
 
         public MultiThreadDownloader(BaiduPCS pcs, PcsFileInfo from, string to,
             string workfolder, int threadCount, int minSliceSize = MIN_SLICE_SIZE)
@@ -71,6 +71,11 @@ public override void Download()
                 CreateDirectory(to); //创建目录
                 CreateLocalFile(); // 如果需要则创建本地文件
                 bigfile = new BigFileHelper(to, from.size); //映射文件到内存
+                lock(cache)
+                {
+                    cache.Reset();
+                    cache.FileHelper = bigfile;
+                }
                 foreach (Slice slice in SliceList)
                 {
                     if (slice.status != SliceStatus.Successed)
@@ -201,6 +206,8 @@ private void downloadTask(object objTID)
                 string errmsg;
                 pcs = this.pcs.clone();
                 pcs.Write += onWrite;
+                Random rnd = new Random(Environment.TickCount);
+                Thread.Sleep(rnd.Next(1, 10));
                 while (tid == Interlocked.Read(ref taskId))
                 {
                     slice = popSlice();
@@ -216,6 +223,16 @@ private void downloadTask(object objTID)
                         rc = pcs.download(from.path, 0, 
                             slice.start + slice.doneSize,
                             slice.totalSize - slice.doneSize);
+                        lock (cache)
+                        {
+                            if (cache.TotalSize > 0)
+                            {
+                                if (!cache.Flush())
+                                    throw new Exception("Failed to flush cache.");
+                                cache.Reset();
+                                SliceHelper.SaveSliceList(SliceFileName, SliceList); // 保存最新的分片数据
+                            }
+                        }
                         if (rc == PcsRes.PCS_OK || slice.status == SliceStatus.Successed)
                             slice.status = SliceStatus.Successed;
                         else if (slice.status == SliceStatus.Cancelled)
@@ -288,13 +305,10 @@ private uint onWrite(BaiduPCS sender, byte[] data, uint contentlength, object us
                 size = slice.totalSize - slice.doneSize;
             if (size > 0)
             {
-                try
-                {
-                    bigfile.Update(slice.start + slice.doneSize, data, 0, (int)size);
-                }
-                catch(Exception ex)
+                lock (cache)
                 {
-                    throw;
+                    if (!cache.Add(slice.start + slice.doneSize, data, (int)size))
+                        throw new Exception("Failed to add to disk cache.");
                 }
             }
             slice.doneSize += size;
@@ -304,7 +318,16 @@ private uint onWrite(BaiduPCS sender, byte[] data, uint contentlength, object us
                 slice.status = SliceStatus.Successed;
                 size = 0;
             }
-            lock (sliceFileLocker) SliceHelper.SaveSliceList(SliceFileName, SliceList); // 保存最新的分片数据
+            lock (cache)
+            {
+                if (cache.TotalSize >= AppSettings.MaxCacheSize * 1024)
+                {
+                    if (!cache.Flush())
+                        throw new Exception("Failed to flush cache.");
+                    cache.Reset();
+                    SliceHelper.SaveSliceList(SliceFileName, SliceList); // 保存最新的分片数据
+                }
+            }
             long downloadedSize = 0;
             lock (locker) downloadedSize = DoneSize;
             ProgressEventArgs args = new ProgressEventArgs(downloadedSize, from.size);

Sample/Sample_0_FileExplorer/Forms/frmHistory.cs

@@ -448,7 +448,7 @@ private void RefreshControls()
 
         private void RefreshST()
         {
-            lblStatusST.Text = runningThreadCount.ToString() + "/" + totalThreadCount.ToString() + " threads, " +
+            lblStatusST.Text = runningThreadCount.ToString() + "/" + (totalThreadCount == 0 ? AppSettings.DownloadMaxThreadCount.ToString() : totalThreadCount.ToString()) + " threads, " +
                 lvItems.Items.Count.ToString() + " items, " +
                 Interlocked.Read(ref successCount).ToString() + " succeed";
             lblStatusST.ToolTipText = lvItems.Items.Count.ToString() + " items, "

Sample/Sample_0_FileExplorer/Forms/frmMain.cs

@@ -407,6 +407,12 @@ private void ReadAppSettings()
                 AppSettings.AutomaticUploadMaxThreadCount = true;
                 AppSettings.RetryWhenUploadFailed = false;
                 AppSettings.UploadMaxThreadCount = 1;
+                AppSettings.MaxCacheSize = 1024;
+            }
+            else
+            {
+                if (AppSettings.MaxCacheSize == 0)
+                    AppSettings.MaxCacheSize = 1024;
             }
         }