Qiniu Cloud Storage Upload Tool (Support Breakpoint Resume) | 七牛云文件上传工具（支持断点续传）

qiniu-upload.go

package main

import (
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sync"

	"github.com/pborman/getopt"
	"github.com/qiniu/api.v7/auth/qbox"
	"github.com/qiniu/api.v7/storage"
	"golang.org/x/net/context"
)

var (
	accessKey = "your access key"
	secretKey = "your secret key"
)

func md5Hex(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))
}

type ProgressRecord struct {
	Progresses []storage.BlkputRet `json:"progresses"`
}

func main() {
	localFile := getopt.StringLong("src", 'i', "", "file you want to upload")
	key := getopt.StringLong("dst", 'o', "", "saved path of the file")
	useHTTPS := getopt.BoolLong("https", 't', "use HTTPS")
	useCDN := getopt.BoolLong("cdn", 'c', "use CDN")
	zone := getopt.StringLong("zone", 'z', "z1", "storage zone (default z1)")
	downloadHost := getopt.StringLong("host", 'h', "", "qiniu download host (optional)")
	apiURL := getopt.StringLong("api", 'u', "", "upload token api url (optional)")
	bucket := getopt.StringLong("bucket", 'b', "", "qiniu storage bucket (required if api is empty)")
	accessKey := getopt.StringLong("ak", 'p', "", "qiniu access key (required if api is empty)")
	secretKey := getopt.StringLong("sk", 's', "", "qiniu secret key (required if api is empty)")
	getopt.Parse()

	if *localFile == "" || *key == "" || (*apiURL == "" && (*accessKey == "" || *secretKey == "")) {
		getopt.PrintUsage(os.Stderr)
		return
	}

	upToken := ""
	if *apiURL != "" {
		response, err := http.Get(*apiURL)
		if err != nil {
			fmt.Println("HTTP Request Failed: ", err)
			return
		}

		body, err := ioutil.ReadAll(response.Body)
		result := make(map[string]string)
		err = json.Unmarshal(body, &result)
		if err != nil {
			fmt.Println("Parse Result Failed: ", err, "; ", string(body))
			return
		}

		upToken = result["uptoken"]

		if *downloadHost == "" && result["host"] != "" {
			*downloadHost = result["host"]
		}
	} else {
		putPolicy := storage.PutPolicy{
			Scope: *bucket,
		}
		mac := qbox.NewMac(*accessKey, *secretKey)
		upToken = putPolicy.UploadToken(mac)
	}

	//fmt.Println("upToken: ", upToken)
	cfg := storage.Config{}
	// 空间对应的机房
	region, ok := storage.GetRegionByID(storage.RegionID(*zone))
	if !ok {
		fmt.Println("cannot find zone: ", *zone)
		return
	}
	cfg.Zone = &region
	// 是否使用https域名
	cfg.UseHTTPS = *useHTTPS
	// 上传是否使用CDN上传加速
	cfg.UseCdnDomains = *useCDN
	// 必须仔细选择一个能标志上传唯一性的 recordKey 用来记录上传进度
	// 我们这里采用 md5(bucket+key+local_path+local_file_last_modified)+".progress" 作为记录上传进度的文件名
	fileInfo, statErr := os.Stat(*localFile)
	if statErr != nil {
		fmt.Println(statErr)
		return
	}
	fileSize := fileInfo.Size()
	fileBlockSize := storage.BlockCount(fileSize)
	fileLmd := fileInfo.ModTime().UnixNano()
	recordKey := md5Hex(fmt.Sprintf("%s:%s:%s:%d", *bucket, *key, *localFile, fileLmd)) + ".progress"
	// 指定的进度文件保存目录，实际情况下，请确保该目录存在，而且只用于记录进度文件
	recordDir := "/tmp/qiniu-upload-progress"
	mErr := os.MkdirAll(recordDir, 0755)
	if mErr != nil {
		fmt.Println("mkdir for record dir error,", mErr)
		return
	}
	recordPath := filepath.Join(recordDir, recordKey)
	progressRecord := ProgressRecord{}
	// 尝试从旧的进度文件中读取进度
	var uploadedSize int64
	recordFp, openErr := os.Open(recordPath)
	if openErr == nil {
		progressBytes, readErr := ioutil.ReadAll(recordFp)
		if readErr == nil {
			mErr := json.Unmarshal(progressBytes, &progressRecord)
			if mErr == nil {
				// 检查context 是否过期，避免701错误
				for _, item := range progressRecord.Progresses {
					if storage.IsContextExpired(item) {
						fmt.Println(item.ExpiredAt)
						progressRecord.Progresses = make([]storage.BlkputRet, fileBlockSize)
						break
					}
					uploadedSize += int64(item.Offset)
				}
			}
		}
		recordFp.Close()
	}
	if len(progressRecord.Progresses) == 0 {
		progressRecord.Progresses = make([]storage.BlkputRet, fileBlockSize)
	}
	resumeUploader := storage.NewResumeUploader(&cfg)
	ret := storage.PutRet{}
	progressLock := sync.RWMutex{}
	putExtra := storage.RputExtra{
		Progresses: progressRecord.Progresses,
		Notify: func(blkIdx int, blkSize int, ret *storage.BlkputRet) {
			progressLock.Lock()
			defer progressLock.Unlock()

			uploadedSize += int64(blkSize)
			fmt.Printf("Uploading... %0.3f MB (%0.3f%%)\r", float64(uploadedSize)/float64(1048576), float64(uploadedSize)/float64(fileSize)*float64(100))

			//将进度序列化，然后写入文件
			progressRecord.Progresses[blkIdx] = *ret
			progressBytes, _ := json.Marshal(progressRecord)
			//fmt.Println("write progress file", blkIdx, recordPath)
			wErr := ioutil.WriteFile(recordPath, progressBytes, 0644)
			if wErr != nil {
				fmt.Println("write progress file error,", wErr)
			}
		},
	}
	err := resumeUploader.PutFile(context.Background(), &ret, upToken, *key, *localFile, &putExtra)
	if err != nil {
		fmt.Println(err)
		return
	}
	//上传成功之后，一定记得删除这个进度文件
	os.Remove(recordPath)

	fmt.Println("Uploaded File Hash: ", ret.Hash)

	downloadUrl := ""
	if *downloadHost != "" {
		if cfg.UseHTTPS {
			downloadUrl = "https"
		} else {
			downloadUrl = "http"
		}
		downloadUrl += "://" + *downloadHost + "/"
	}
	downloadUrl += ret.Key
	fmt.Println("Download URL:", downloadUrl)
}

Build Command | 构建命令

go get "github.com/pborman/getopt"
go get "github.com/qiniu/api.v7"
go get "golang.org/x/net/context"
wget https://gist.githubusercontent.com/SwimmingTiger/88078c5a3bfa1668ef0012dc3630e6b6/raw/af1c28f8f52863658479058da3d9ae19b093013d/qiniu-upload.go
go build qiniu-upload.go

For Windows | 在 Windows 上构建时需要注意

Please replace the path in this line to a valid Windows temporary directory (such as C:/tmp/qiniu-upload-progress).
请将这一行代码中的路径替换为合法的Windows临时文件目录（比如C:/tmp/qiniu-upload-progress）。

recordDir := "/tmp/qiniu-upload-progress"

Usage | 使用说明

# upload with remote UpToken API | 通过远程 UpToken API 上传
./qiniu-upload -z z1 -u https://example.com/get-qiniu-uptoken.php -i my.mp4 -o tiger/data/my.mp4

# upload with AK and SK | 通过 AK 和 SK 上传
./qiniu-upload -z z1 -b mybucket --ak="xxx" --sk="xxx" -i my.mp4 -o tiger/data/my.mp4

About the arg -z (--zone) | 关于参数-z (--zone)

Storage area list please refer to: | 存储区域列表请参考：
See here: https://developer.qiniu.com/kodo/manual/1671/region-endpoint