Merge pull request #32 from laorange/master

新增：url自动解析(支持BV号)

Author: sodaling

.gitignore

@@ -25,3 +25,4 @@ gen
 # Dependency directories (remove the comment below to include it)
 # vendor/
 
+/cmd/myapp.syso

Dockerfile

@@ -8,6 +8,8 @@ ENV GOPROXY="https://goproxy.io" GO111MODULE=on
 RUN go mod download
 COPY . .
 RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -o /go/bin/simple-golang-crawler cmd/start-concurrent-engine.go
+
+
 FROM alpine:3.7
 RUN apk update \
         && apk upgrade \

Makefile

@@ -6,5 +6,6 @@ build: ## Build the container
 build-nc: ## Build the container without caching
 	docker build --no-cache -t $(APP_NAME) .
 run:
-	docker run -it  -v ${CURDIR}/download:/download $(APP_NAME)
+	## docker run -it  -v ${CURDIR}/download:/download $(APP_NAME)
+	docker run -it  -v ${CURDIR}/output:/output $(APP_NAME)
 

README.md

@@ -1,16 +1,36 @@
 # FastestBibiliDownloader
 
-#### 项目地址：**[ FastestBilibiliDownloader](https://github.com/sodaling/FastestBilibiliDownloader)**
+## 原项目地址：**[ FastestBilibiliDownloader](https://github.com/sodaling/FastestBilibiliDownloader)**
+
+> 项目仅用于学习交流，请勿用于任何商业用途！
+
+## ⭐新增
+
+自动解析 **想要下载的视频网址 / UP主个人主页网址**，支持：
+
+- [x] [https://www.bilibili.com/video/**旧版的av号**/](#)，av号是以`av`开头的**一串数字**
+- [x] [https://www.bilibili.com/video/**新版的BV号**/](#)，BV号是以`BV`开头的**一串字符**
+- [x] [https://space.bilibili.com/**UP主的ID**/](#)，UP主的ID是**一串数字**
+
+![demo.png](demo.png)
+
+## ⚠较原项目的删减
+
++ 由于FFmeg拼接、转化耗时太长，故移除了 `video merge`中的功能 。下载后的视频为`.flv`格式。
+
+-----
+
+## 👍原项目说明
 
 **东半球第二快的Bilibili.com（B站）视频下载器！**
 
 如果你想下载b站某个up主的所有视频，而且要飞快的那种，那么你可以试试这个项目-.-
 
-目前提供两个视频下载方案:
+目前提供两个（三个）视频下载方案:
 
 1. 通过视频的aid,下载单个视频.
 2. 通过up主的upid(b站叫mid),下载这个up主所投稿的所有视频.
-
+3. 通过视频的BVid,下载单个视频. **(new)**
 
 
 > 特性:
@@ -26,18 +46,18 @@
 > * 当单个aid视频分了若干个part时候,或者当你选了下载up主下所有视频时候.多个视频将会同时并行下载,跑满你的网速绝对不是问题.
 > * 下载与合并视频并行处理.如果视频分了多个part,下载完成的同时就会立即合并.该视频合并处理和其他与其他下载和合并同时进行且互不影响.
 
-## 运行
-
-下载的视频会存放在运行路径下的**download**文件夹下，每个视频（aid）一个文件夹。
+### 运行
 
+下载的临时视频会存放在运行路径下的**download**文件夹下，每个视频（aid）一个文件夹，以**aid_视频标题**为文件夹名称。
+最终的视频会存放在运行路径下的**output**文件夹下，每个aid一个文件夹，以**视频标题**为文件夹名称。
 ```shell
-Please enter your id type(`aid` or `upid`) #aid是视频id，upid是up主id，可以在up主主页地址栏获得。
-Please enter your id #输入id
+go run cmd/start-concurrent-engine.go -h   # 获得参数
 ```
 
 
 
-### 使用Golang编译环境
+#### 使用Golang编译环境
+
 1. 安装Golang编译环境
 * Ubuntu
 ```shell
@@ -53,18 +73,18 @@ go env -w  GOPROXY=https://goproxy.io #使用官方代理
 2. 一次性运行FastestBibiliDownloader
 程序入口在**cmd/start-concurrent-engine.go**，只需要
 ```shell
-go run cmd/start-concurrent-engine.go
+go run cmd/start-concurrent-engine.go -t (aid/bvid/upid) -v (id)
 ```
 首次运行会花时间下一大堆东西，然后按提示操作即可。
 注意，合并视频需要FFmpeg的支持。不然只会下载并不会自动合并。FFmpeg的安装教程请咨询搜索引擎。
 
 3. 编译FastestBibiliDownloader
 ```shell
-go build cmd/start-concurrent-engine.go
+go build cmd/start-concurrent-engine.go -t (aid/bvid/upid) -v (id)
 ```
 之后直接运行./start-concurrent-engine即可。
 
-### 如果你没有Golang编译环境，或者没有FFmeg环境。那么推荐用docker方式运行。已经写好了dockefile和makefile。你只需要：
+#### 如果你没有Golang编译环境，或者没有FFmeg环境。那么推荐用docker方式运行。已经写好了dockefile和makefile。你只需要：
 
    ```shell
    $ cd FastestBilibiliDownloader
@@ -74,9 +94,9 @@ go build cmd/start-concurrent-engine.go
 
 
 
-### 后续有空会打包bin文件到release的。
+#### 后续有空会打包bin文件到release的。
 
-## 感谢
+### 感谢
 
 1. engine部分的框架参考**ccmouse**的思路，后面自己调整了整体架构部分，非常感谢。
 2. [bilibili-downloader](https://github.com/stevenjoezhang/bilibili-downloader)：b站请求视频的API等等都是从这位的代码获得，本身的py代码注释也非常清晰，非常感谢。

cmd/start-concurrent-engine.go

@@ -4,10 +4,12 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"regexp"
 	"simple-golang-crawler/engine"
 	"simple-golang-crawler/parser"
 	"simple-golang-crawler/persist"
 	"simple-golang-crawler/scheduler"
+	"strconv"
 	"sync"
 )
 
@@ -21,27 +23,70 @@ func main() {
 		panic(err)
 	}
 
-	var idType string
-	var id int64
+	var urlInput string
+
+	var idType = "else"
+	var aid int64
+	var upid int64
+	var bvid string
+
+	var params []string
+
 	var req *engine.Request
-	fmt.Println("Please enter your id type(`aid` or `upid`)")
-	fmt.Scan(&idType)
-	fmt.Println("Please enter your id")
-	fmt.Scan(&id)
+
+	fmt.Println("欢迎使用B站视频下载器 v1.0.1")
+	fmt.Println("项目地址：  https://github.com/laorange/FastestBilibiliDownloader")
+	fmt.Println("原项目地址：https://github.com/sodaling/FastestBilibiliDownloader")
+	fmt.Println("\n\n支持以下几种格式的输入：")
+	fmt.Println("·  https://www.bilibili.com/video/旧版的av号/ | av号 是以`av`开头的一串数字")
+	fmt.Println("·  https://www.bilibili.com/video/新版的BV号/ | BV号 是以`BV`开头的一串字符")
+	fmt.Println("·  https://space.bilibili.com/UP主的ID/       | UP主的ID 是一串数字")
+	fmt.Print("\n\n请输入想要下载的视频网址/up主个人主页网址: ")
+	fmt.Scan(&urlInput)
+
+	// bvid
+	bvidRegexp := regexp.MustCompile(`/?(BV\w+)[/?]?`)
+	params = bvidRegexp.FindStringSubmatch(urlInput)
+	if params != nil {
+		idType = "bvid"
+		bvid = params[1]
+	}
+
+	// aid
+	aidRegexp := regexp.MustCompile(`/?(av\d+)/?`)
+	params = aidRegexp.FindStringSubmatch(urlInput)
+	if params != nil {
+		idType = "aid"
+		aid, _ = strconv.ParseInt(params[1], 10, 64)
+	}
+
+	// upid
+	upidRegexp := regexp.MustCompile(`space.bilibili.com/(\d+)/?`)
+	params = upidRegexp.FindStringSubmatch(urlInput)
+	if params != nil {
+		idType = "upid"
+		upid, _ = strconv.ParseInt(params[1], 10, 64)
+	}
 
 	if idType == "aid" {
-		req = parser.GetRequestByAid(id)
+		req = parser.GetRequestByAid(aid)
+	} else if idType == "bvid" {
+		aid = parser.Bv2av(bvid)
+		req = parser.GetRequestByAid(aid)
 	} else if idType == "upid" {
-		req = parser.GetRequestByUpId(id)
+		req = parser.GetRequestByUpId(upid)
 	} else {
-		log.Fatalln("Wrong type you enter")
+		req = nil
+		log.Fatalln("您输入的网址无法解析，请查证后重试")
 		os.Exit(1)
 	}
 
 	queueScheduler := scheduler.NewConcurrentScheduler()
 	conEngine := engine.NewConcurrentEngine(30, queueScheduler, itemChan)
-	log.Println("Start working.")
+	log.Println("开始下载...")
 	conEngine.Run(req)
 	wg.Wait()
-	log.Println("All work has done")
+	log.Print("所有视频均已下载完成。按 Ctrl+C 来退出程序。")
+	var eof string
+	fmt.Scan(&eof)
 }

demo.png

@@ -9,6 +9,7 @@ import (
 	"path/filepath"
 	"simple-golang-crawler/model"
 	"simple-golang-crawler/tool"
+	"time"
 )
 
 var _startUrlTem = "https://api.bilibili.com/x/web-interface/view?aid=%d"
@@ -39,13 +40,13 @@ func GenVideoFetcher(video *model.Video) FetchFun {
 
 		resp, err := client.Do(request)
 		if err != nil {
-			log.Fatalf("Fail to download the video %d,err is %s", video.ParCid.Cid, err)
+			log.Fatalf("下载 %d 时出错, 错误信息：%s", video.ParCid.Cid, err)
 			return nil, err
 		}
 
 		if resp.StatusCode != http.StatusPartialContent {
-			log.Fatalf("Fail to download the video %d,status code is %d", video.ParCid.Cid, resp.StatusCode)
-			return nil, fmt.Errorf("wrong status code: %d", resp.StatusCode)
+			log.Fatalf("下载 %d 时出错, 错误码：%d", video.ParCid.Cid, resp.StatusCode)
+			return nil, fmt.Errorf("错误码： %d", resp.StatusCode)
 		}
 		defer resp.Body.Close()
 
@@ -58,13 +59,18 @@ func GenVideoFetcher(video *model.Video) FetchFun {
 		}
 		defer file.Close()
 
-		log.Println(video.ParCid.ParAid.Title + ":" + filename + " is downloading.")
+		log.Println("正在下载：" + video.ParCid.ParAid.Title + "\\" + filename)
 		_, err = io.Copy(file, resp.Body)
 		if err != nil {
-			log.Printf("Failed to download video %d", video.ParCid.Cid)
+			log.Printf("下载失败 aid: %d, cid: %d, title: %s, part: %s",
+				video.ParCid.ParAid.Aid, video.ParCid.Cid, video.ParCid.ParAid.Title, video.ParCid.Part)
+			log.Println("错误信息：", err)
+
+			// request again
+			go requestLater(file, resp, video)
 			return nil, err
 		}
-		log.Println(video.ParCid.ParAid.Title + ":" + filename + " has finished.")
+		log.Println("下载完成：" + video.ParCid.ParAid.Title + "\\" + filename)
 
 		return nil, nil
 	}
@@ -76,3 +82,16 @@ func genCheckRedirectfun(referer string) func(req *http.Request, via []*http.Req
 		return nil
 	}
 }
+
+func requestLater(file *os.File, resp *http.Response, video *model.Video) error {
+
+	log.Println("连接失败，30秒后重试 (Unable to open the file due to the remote host, request in 30 seconds)")
+	time.Sleep(time.Second * 30)
+
+	_, err := io.Copy(file, resp.Body)
+	if err != nil {
+		log.Printf("下载失败 aid: %d, cid: %d, title: %s, part: %s again",
+			video.ParCid.ParAid.Aid, video.ParCid.Cid, video.ParCid.ParAid.Title, video.ParCid.Part)
+	}
+	return err
+}

fetcher/downloader.go

@@ -43,6 +43,7 @@ type VideoCid struct {
 	Cid      int64
 	ParAid   *VideoAid
 	Page     int64
+	Part     string
 	AllOrder int64
 }
 
@@ -51,6 +52,6 @@ type Video struct {
 	ParCid *VideoCid
 }
 
-func NewVideoCidInfo(cid int64, parAid *VideoAid, page int64) *VideoCid {
-	return &VideoCid{Cid: cid, ParAid: parAid, Page: page}
+func NewVideoCidInfo(cid int64, parAid *VideoAid, page int64, part string) *VideoCid {
+	return &VideoCid{Cid: cid, ParAid: parAid, Page: page, Part: part}
 }

model/bilibili.go

@@ -5,14 +5,24 @@ import (
 	"simple-golang-crawler/engine"
 	"simple-golang-crawler/fetcher"
 	"simple-golang-crawler/model"
-
+	"simple-golang-crawler/tool"
+	"math"
 	"github.com/tidwall/gjson"
 )
 
 var _getAidUrlTemp = "https://api.bilibili.com/x/space/arc/search?mid=%d&ps=30&tid=0&pn=%d&keyword=&order=pubdate&jsonp=jsonp"
-var _getCidUrlTemp = "https://api.bilibili.com/x/player/pagelist?aid=%d"
+var _getCidUrlTemp = "https://api.bilibili.com/x/web-interface/view?aid=%d"
+//var _getCidUrlTemp = "https://api.bilibili.com/x/player/pagelist?aid=%d"
+
+var table string = "fZodR9XQDSUm21yCkr6zBqiveYah8bt4xsWpHnJE7jL5VG3guMTKNPAwcF"
+var s = [6]int{11, 10, 3, 8, 4, 6}
+var xor = 177451812
+var add = 8728348608
+var tr map[string]int
+
 
 func UpSpaceParseFun(contents []byte, url string) engine.ParseResult {
+
 	var retParseResult engine.ParseResult
 	value := gjson.GetManyBytes(contents, "data.list.vlist", "data.page")
 
@@ -25,22 +35,26 @@ func UpSpaceParseFun(contents []byte, url string) engine.ParseResult {
 }
 
 func getAidDetailReqList(pageInfo gjson.Result) ([]*engine.Request, int64) {
+
 	var retRequests []*engine.Request
 	var upid int64
 	for _, i := range pageInfo.Array() {
 		aid := i.Get("aid").Int()
 		upid = i.Get("mid").Int()
 		title := i.Get("title").String()
+		title = tool.TitleEdit(title)  // remove special characters
 		reqUrl := fmt.Sprintf(_getCidUrlTemp, aid)
 		videoAid := model.NewVideoAidInfo(aid, title)
-		reqParseFunction := GenGetAidChildrenParseFun(videoAid)
+		reqParseFunction := GenGetAidChildrenParseFun(videoAid) //子视频
 		req := engine.NewRequest(reqUrl, reqParseFunction, fetcher.DefaultFetcher)
 		retRequests = append(retRequests, req)
 	}
 	return retRequests, upid
 }
 
+// 访问up主的时候 需要翻页
 func getNewBilibiliUpSpaceReqList(pageInfo gjson.Result, upid int64) []*engine.Request {
+
 	var retRequests []*engine.Request
 
 	count := pageInfo.Get("count").Int()
@@ -51,7 +65,7 @@ func getNewBilibiliUpSpaceReqList(pageInfo gjson.Result, upid int64) []*engine.R
 		extraPage = 1
 	}
 	totalPage := count/ps + extraPage
-	for i := int64(1); i < totalPage; i++ {
+	for i := int64(1); i <= totalPage; i++ {
 		if i == pn {
 			continue
 		}
@@ -63,6 +77,20 @@ func getNewBilibiliUpSpaceReqList(pageInfo gjson.Result, upid int64) []*engine.R
 }
 
 func GetRequestByUpId(upid int64) *engine.Request {
+
 	reqUrl := fmt.Sprintf(_getAidUrlTemp, upid, 1)
 	return engine.NewRequest(reqUrl, UpSpaceParseFun, fetcher.DefaultFetcher)
 }
+
+// source code: https://blog.csdn.net/dotastar00/article/details/108805779
+func Bv2av(x string) int64 {
+    tr = make(map[string]int)
+    for i:=0; i<58; i++ {
+        tr[string(table[i])] = i
+    }
+    r := 0
+    for i:=0; i<6; i++ {
+        r += tr[string(x[s[i]])] * int(math.Pow(float64(58), float64(i)))
+    }
+    return int64((r - add) ^ xor)
+}