diff --git a/drivers/123_open/driver.go b/drivers/123_open/driver.go new file mode 100644 index 00000000..f51b9090 --- /dev/null +++ b/drivers/123_open/driver.go @@ -0,0 +1,127 @@ +package _123_open + +import ( + "context" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/internal/stream" + "github.com/alist-org/alist/v3/pkg/utils" + "strconv" +) + +type Open123 struct { + model.Storage + Addition +} + +func (d *Open123) Config() driver.Config { + return config +} + +func (d *Open123) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Open123) Init(ctx context.Context) error { + if d.UploadThread < 1 || d.UploadThread > 32 { + d.UploadThread = 3 + } + + return nil +} + +func (d *Open123) Drop(ctx context.Context) error { + return nil +} + +func (d *Open123) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + fileLastId := int64(0) + parentFileId, err := strconv.ParseInt(dir.GetID(), 10, 64) + if err != nil { + return nil, err + } + res := make([]File, 0) + + for fileLastId != -1 { + files, err := d.getFiles(parentFileId, 100, fileLastId) + if err != nil { + return nil, err + } + // 目前123panAPI请求,trashed失效,只能通过遍历过滤 + for i := range files.Data.FileList { + if files.Data.FileList[i].Trashed == 0 { + res = append(res, files.Data.FileList[i]) + } + } + fileLastId = files.Data.LastFileId + + } + return utils.SliceConvert(res, func(src File) (model.Obj, error) { + return src, nil + }) +} + +func (d *Open123) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + fileId, _ := strconv.ParseInt(file.GetID(), 10, 64) + + res, err := d.getDownloadInfo(fileId) + if err != nil { + return nil, err + } + + link := model.Link{URL: res.Data.DownloadUrl} + return &link, nil +} + +func (d *Open123) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + parentFileId, _ := strconv.ParseInt(parentDir.GetID(), 10, 64) + + return d.mkdir(parentFileId, dirName) +} + +func (d *Open123) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + toParentFileID, _ := strconv.ParseInt(dstDir.GetID(), 10, 64) + + return d.move(srcObj.(File).FileId, toParentFileID) +} + +func (d *Open123) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + fileId, _ := strconv.ParseInt(srcObj.GetID(), 10, 64) + + return d.rename(fileId, newName) +} + +func (d *Open123) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + return errs.NotSupport +} + +func (d *Open123) Remove(ctx context.Context, obj model.Obj) error { + fileId, _ := strconv.ParseInt(obj.GetID(), 10, 64) + + return d.trash(fileId) +} + +func (d *Open123) Put(ctx context.Context, dstDir model.Obj, file model.FileStreamer, up driver.UpdateProgress) error { + parentFileId, err := strconv.ParseInt(file.GetID(), 10, 64) + etag := file.GetHash().GetHash(utils.MD5) + + if len(etag) < utils.MD5.Width { + _, etag, err = stream.CacheFullInTempFileAndHash(file, utils.MD5) + if err != nil { + return err + } + } + createResp, err := d.create(parentFileId, file.GetName(), etag, file.GetSize(), 2, false) + if err != nil { + return err + } + if createResp.Data.Reuse { + return nil + } + up(10) + + return d.Upload(ctx, file, createResp, up) +} + +var _ driver.Driver = (*Open123)(nil) diff --git a/drivers/123_open/meta.go b/drivers/123_open/meta.go new file mode 100644 index 00000000..38b1c196 --- /dev/null +++ b/drivers/123_open/meta.go @@ -0,0 +1,37 @@ +package _123_open + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + // refresh_token方式的AccessToken 【对个人开发者暂未开放】 + //RefreshToken string `json:"RefreshToken" required:"false"` + + // 通过 https://www.123pan.com/developer 申请 + ClientID string `json:"ClientID" required:"false"` + ClientSecret string `json:"ClientSecret" required:"false"` + + // 直接写入AccessToken + AccessToken string `json:"AccessToken" required:"false"` + + // 用户名+密码方式登录的AccessToken可以兼容 + //Username string `json:"username" required:"false"` + //Password string `json:"password" required:"false"` + + UploadThread int `json:"UploadThread" type:"number" default:"3" help:"the threads of upload"` + driver.RootID +} + +var config = driver.Config{ + Name: "123 Open", + DefaultRoot: "0", + LocalSort: true, +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Open123{} + }) +} diff --git a/drivers/123_open/types.go b/drivers/123_open/types.go new file mode 100644 index 00000000..f86153ac --- /dev/null +++ b/drivers/123_open/types.go @@ -0,0 +1,225 @@ +package _123_open + +import ( + "fmt" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "net/url" + "path" + "strconv" + "strings" + "time" +) + +type ApiInfo struct { + url string + qps int + token chan struct{} +} + +func (a *ApiInfo) Require() { + if a.qps > 0 { + a.token <- struct{}{} + } +} +func (a *ApiInfo) Release() { + if a.qps > 0 { + time.AfterFunc(time.Duration(1.0/float32(a.qps)*1000)*time.Millisecond, func() { + <-a.token + }) + } +} +func (a *ApiInfo) SetQPS(qps int) { + a.qps = qps + a.token = make(chan struct{}, qps) +} +func (a *ApiInfo) NowLen() int { + return len(a.token) +} +func InitApiInfo(url string, qps int) *ApiInfo { + return &ApiInfo{ + url: url, + qps: qps, + token: make(chan struct{}, qps), + } +} + +type File struct { + FileName string `json:"filename"` + Size int64 `json:"size"` + CreateAt string `json:"createAt"` + UpdateAt string `json:"updateAt"` + FileId int64 `json:"fileId"` + Type int `json:"type"` + Etag string `json:"etag"` + S3KeyFlag string `json:"s3KeyFlag"` + ParentFileId int `json:"parentFileId"` + Category int `json:"category"` + Status int `json:"status"` + Trashed int `json:"trashed"` + DownloadUrl string `json:"DownloadUrl"` +} + +func (f File) GetHash() utils.HashInfo { + return utils.NewHashInfo(utils.MD5, f.Etag) +} + +func (f File) GetPath() string { + return "" +} + +func (f File) GetSize() int64 { + return f.Size +} + +func (f File) GetName() string { + return f.FileName +} + +func (f File) CreateTime() time.Time { + parsedTime, err := time.Parse("2006-01-02 15:04:05", f.CreateAt) + if err != nil { + return time.Now() + } + return parsedTime +} + +func (f File) ModTime() time.Time { + parsedTime, err := time.Parse("2006-01-02 15:04:05", f.UpdateAt) + if err != nil { + return time.Now() + } + return parsedTime +} + +func (f File) IsDir() bool { + return f.Type == 1 +} + +func (f File) GetID() string { + return strconv.FormatInt(f.FileId, 10) +} + +func (f File) Thumb() string { + if f.DownloadUrl == "" { + return "" + } + du, err := url.Parse(f.DownloadUrl) + if err != nil { + return "" + } + du.Path = strings.TrimSuffix(du.Path, "_24_24") + "_70_70" + query := du.Query() + query.Set("w", "70") + query.Set("h", "70") + if !query.Has("type") { + query.Set("type", strings.TrimPrefix(path.Base(f.FileName), ".")) + } + if !query.Has("trade_key") { + query.Set("trade_key", "123pan-thumbnail") + } + du.RawQuery = query.Encode() + fmt.Println(du.String()) + return du.String() +} + +var _ model.Obj = (*File)(nil) +var _ model.Thumb = (*File)(nil) + +type BaseResp struct { + Code int `json:"code"` + Message string `json:"message"` + XTraceID string `json:"x-traceID"` +} + +type AccessTokenResp struct { + BaseResp + Data struct { + AccessToken string `json:"accessToken"` + ExpiredAt string `json:"expiredAt"` + } `json:"data"` +} + +type UserInfoResp struct { + BaseResp + Data struct { + UID int64 `json:"uid"` + Username string `json:"username"` + DisplayName string `json:"displayName"` + HeadImage string `json:"headImage"` + Passport string `json:"passport"` + Mail string `json:"mail"` + SpaceUsed int64 `json:"spaceUsed"` + SpacePermanent int64 `json:"spacePermanent"` + SpaceTemp int64 `json:"spaceTemp"` + SpaceTempExpr string `json:"spaceTempExpr"` + Vip bool `json:"vip"` + DirectTraffic int64 `json:"directTraffic"` + IsHideUID bool `json:"isHideUID"` + } `json:"data"` +} + +type FileListResp struct { + BaseResp + Data struct { + LastFileId int64 `json:"lastFileId"` + FileList []File `json:"fileList"` + } `json:"data"` +} + +type DownloadInfoResp struct { + BaseResp + Data struct { + DownloadUrl string `json:"downloadUrl"` + } `json:"data"` +} + +type UploadCreateResp struct { + BaseResp + Data struct { + FileID int64 `json:"fileID"` + PreuploadID string `json:"preuploadID"` + Reuse bool `json:"reuse"` + SliceSize int64 `json:"sliceSize"` + } `json:"data"` +} + +type UploadUrlResp struct { + BaseResp + Data struct { + PresignedURL string `json:"presignedURL"` + } +} + +type UploadCompleteResp struct { + BaseResp + Data struct { + Async bool `json:"async"` + Completed bool `json:"completed"` + FileID int64 `json:"fileID"` + } `json:"data"` +} + +type UploadAsyncResp struct { + BaseResp + Data struct { + Completed bool `json:"completed"` + FileID int64 `json:"fileID"` + } `json:"data"` +} + +type UploadResp struct { + BaseResp + Data struct { + AccessKeyId string `json:"AccessKeyId"` + Bucket string `json:"Bucket"` + Key string `json:"Key"` + SecretAccessKey string `json:"SecretAccessKey"` + SessionToken string `json:"SessionToken"` + FileId int64 `json:"FileId"` + Reuse bool `json:"Reuse"` + EndPoint string `json:"EndPoint"` + StorageNode string `json:"StorageNode"` + UploadId string `json:"UploadId"` + } `json:"data"` +} diff --git a/drivers/123_open/upload.go b/drivers/123_open/upload.go new file mode 100644 index 00000000..35850577 --- /dev/null +++ b/drivers/123_open/upload.go @@ -0,0 +1,148 @@ +package _123_open + +import ( + "context" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/errgroup" + "github.com/alist-org/alist/v3/pkg/http_range" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/avast/retry-go" + "github.com/go-resty/resty/v2" + "net/http" + "time" +) + +func (d *Open123) create(parentFileID int64, filename string, etag string, size int64, duplicate int, containDir bool) (*UploadCreateResp, error) { + var resp UploadCreateResp + _, err := d.Request(UploadCreate, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "parentFileId": parentFileID, + "filename": filename, + "etag": etag, + "size": size, + "duplicate": duplicate, + "containDir": containDir, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) url(preuploadID string, sliceNo int64) (string, error) { + // get upload url + var resp UploadUrlResp + _, err := d.Request(UploadUrl, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "preuploadId": preuploadID, + "sliceNo": sliceNo, + }) + }, &resp) + if err != nil { + return "", err + } + return resp.Data.PresignedURL, nil +} + +func (d *Open123) complete(preuploadID string) (*UploadCompleteResp, error) { + var resp UploadCompleteResp + _, err := d.Request(UploadComplete, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "preuploadID": preuploadID, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) async(preuploadID string) (*UploadAsyncResp, error) { + var resp UploadAsyncResp + _, err := d.Request(UploadAsync, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "preuploadID": preuploadID, + }) + }, &resp) + if err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Open123) Upload(ctx context.Context, file model.FileStreamer, createResp *UploadCreateResp, up driver.UpdateProgress) error { + size := file.GetSize() + chunkSize := createResp.Data.SliceSize + uploadNums := (size + chunkSize - 1) / chunkSize + threadG, uploadCtx := errgroup.NewGroupWithContext(ctx, d.UploadThread, + retry.Attempts(3), + retry.Delay(time.Second), + retry.DelayType(retry.BackOffDelay)) + for partIndex := int64(0); partIndex < uploadNums; partIndex++ { + if utils.IsCanceled(uploadCtx) { + break + } + + partIndex := partIndex + partNumber := partIndex + 1 // 分片号从1开始 + threadG.Go(func(ctx context.Context) error { + uploadPartUrl, err := d.url(createResp.Data.PreuploadID, partNumber) + if err != nil { + return err + } + + offset := partIndex * chunkSize + size := min(chunkSize, size-offset) + limitedReader, err := file.RangeRead(http_range.Range{ + Start: offset, + Length: size}) + if err != nil { + return err + } + + req, err := http.NewRequestWithContext(ctx, "PUT", uploadPartUrl, limitedReader) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.ContentLength = size + + res, err := base.HttpClient.Do(req) + if err != nil { + return err + } + _ = res.Body.Close() + + progress := 10.0 + 85.0*float64(threadG.Success()+1)/float64(uploadNums) + up(progress) + return nil + }) + } + + if err := threadG.Wait(); err != nil { + return err + } + + uploadCompleteResp, err := d.complete(createResp.Data.PreuploadID) + if err != nil { + return err + } + if uploadCompleteResp.Data.Async == false || uploadCompleteResp.Data.Completed { + return nil + } + + for { + uploadAsyncResp, err := d.async(createResp.Data.PreuploadID) + if err != nil { + return err + } + if uploadAsyncResp.Data.Completed { + break + } + } + up(100) + return nil +} diff --git a/drivers/123_open/util.go b/drivers/123_open/util.go new file mode 100644 index 00000000..64043029 --- /dev/null +++ b/drivers/123_open/util.go @@ -0,0 +1,195 @@ +package _123_open + +import ( + "encoding/json" + "errors" + "fmt" + log "github.com/sirupsen/logrus" + "net/http" + "strconv" + + "github.com/alist-org/alist/v3/drivers/base" + "github.com/go-resty/resty/v2" +) + +var ( //不同情况下获取的AccessTokenQPS限制不同 如下模块化易于拓展 + Api = "https://open-api.123pan.com" + + AccessToken = InitApiInfo(Api+"/api/v1/access_token", 1) + UserInfo = InitApiInfo(Api+"/api/v1/user/info", 1) + FileList = InitApiInfo(Api+"/api/v2/file/list", 2) + DownloadInfo = InitApiInfo(Api+"/api/v1/file/download_info", 0) + Mkdir = InitApiInfo(Api+"/upload/v1/file/mkdir", 2) + Move = InitApiInfo(Api+"/api/v1/file/move", 1) + Rename = InitApiInfo(Api+"/api/v1/file/name", 0) + Trash = InitApiInfo(Api+"/api/v1/file/trash", 0) + UploadCreate = InitApiInfo(Api+"/upload/v1/file/create", 2) + UploadUrl = InitApiInfo(Api+"/upload/v1/file/get_upload_url", 0) + UploadComplete = InitApiInfo(Api+"/upload/v1/file/upload_complete", 0) + UploadAsync = InitApiInfo(Api+"/upload/v1/file/upload_async_result", 1) +) + +func (d *Open123) Request(apiInfo *ApiInfo, method string, callback base.ReqCallback, resp interface{}) ([]byte, error) { + isRetry := false +do: + req := base.RestyClient.R() + req.SetHeaders(map[string]string{ + "authorization": "Bearer " + d.AccessToken, + "platform": "open_platform", + "Content-Type": "application/json", + }) + + if callback != nil { + callback(req) + } + if resp != nil { + req.SetResult(resp) + } + + apiInfo.Require() + defer apiInfo.Release() + log.Debugf("API: %s, QPS: %d, NowLen: %d", apiInfo.url, apiInfo.qps, apiInfo.NowLen()) + + res, err := req.Execute(method, apiInfo.url) + if err != nil { + return nil, err + } + body := res.Body() + + // 解析为通用响应 + var baseResp BaseResp + if err = json.Unmarshal(body, &baseResp); err != nil { + return nil, err + } + + if baseResp.Code != 0 { + if !isRetry && baseResp.Code == 401 { + if d.flushAccessToken() != nil { + return nil, err + } + isRetry = true + goto do + } + return nil, errors.New(baseResp.Message) + } + return body, nil +} + +func (d *Open123) flushAccessToken() error { + var resp AccessTokenResp + _, err := d.Request(AccessToken, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "clientID": d.ClientID, + "clientSecret": d.ClientSecret, + }) + }, &resp) + fmt.Println(resp) + if err != nil { + return err + } + d.AccessToken = resp.Data.AccessToken + return nil +} + +func (d *Open123) getUserInfo() (*UserInfoResp, error) { + var resp UserInfoResp + + if _, err := d.Request(UserInfo, http.MethodGet, nil, &resp); err != nil { + return nil, err + } + + return &resp, nil +} + +func (d *Open123) getFiles(parentFileId int64, limit int, lastFileId int64) (*FileListResp, error) { + var resp FileListResp + + _, err := d.Request(FileList, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams( + map[string]string{ + "parentFileId": strconv.FormatInt(parentFileId, 10), + "limit": strconv.Itoa(limit), + "lastFileId": strconv.FormatInt(lastFileId, 10), + "trashed": "false", + "searchMode": "", + "searchData": "", + }) + }, &resp) + + if err != nil { + return nil, err + } + + return &resp, nil + +} + +func (d *Open123) getDownloadInfo(fileId int64) (*DownloadInfoResp, error) { + var resp DownloadInfoResp + + _, err := d.Request(DownloadInfo, http.MethodGet, func(req *resty.Request) { + req.SetQueryParams(map[string]string{ + "fileId": strconv.FormatInt(fileId, 10), + }) + }, &resp) + if err != nil { + return nil, err + } + + return &resp, nil +} + +func (d *Open123) mkdir(parentID int64, name string) error { + _, err := d.Request(Mkdir, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "parentID": strconv.FormatInt(parentID, 10), + "name": name, + }) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *Open123) move(fileID, toParentFileID int64) error { + _, err := d.Request(Move, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileIDs": []int64{fileID}, + "toParentFileID": toParentFileID, + }) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *Open123) rename(fileId int64, fileName string) error { + _, err := d.Request(Rename, http.MethodPut, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileId": fileId, + "fileName": fileName, + }) + }, nil) + if err != nil { + return err + } + + return nil +} + +func (d *Open123) trash(fileId int64) error { + _, err := d.Request(Trash, http.MethodPost, func(req *resty.Request) { + req.SetBody(base.Json{ + "fileIDs": []int64{fileId}, + }) + }, nil) + if err != nil { + return err + } + + return nil +} diff --git a/drivers/all.go b/drivers/all.go index 224fb8dd..a8c86209 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/115_share" _ "github.com/alist-org/alist/v3/drivers/123" _ "github.com/alist-org/alist/v3/drivers/123_link" + _ "github.com/alist-org/alist/v3/drivers/123_open" _ "github.com/alist-org/alist/v3/drivers/123_share" _ "github.com/alist-org/alist/v3/drivers/139" _ "github.com/alist-org/alist/v3/drivers/189"