From 09084d188561c3aa12884a6b624e20d4a9fefb3c Mon Sep 17 00:00:00 2001 From: Levi-Hope <117010292@link.cuhk.edu.cn> Date: Sun, 25 Jul 2021 16:37:50 +0800 Subject: [PATCH 1/5] baidupan: get the damm OAuth code --- backend/all/all.go | 1 + backend/baidupan/baidupan.go | 154 +++++++++++++++++++++++++++++++++++ lib/oauthutil/oauthutil.go | 2 +- 3 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 backend/baidupan/baidupan.go diff --git a/backend/all/all.go b/backend/all/all.go index fc9dd17912c68..d22211a236d2d 100644 --- a/backend/all/all.go +++ b/backend/all/all.go @@ -6,6 +6,7 @@ import ( _ "github.com/rclone/rclone/backend/amazonclouddrive" _ "github.com/rclone/rclone/backend/azureblob" _ "github.com/rclone/rclone/backend/b2" + _ "github.com/rclone/rclone/backend/baidupan" _ "github.com/rclone/rclone/backend/box" _ "github.com/rclone/rclone/backend/cache" _ "github.com/rclone/rclone/backend/chunker" diff --git a/backend/baidupan/baidupan.go b/backend/baidupan/baidupan.go new file mode 100644 index 0000000000000..dc2916bb1eea7 --- /dev/null +++ b/backend/baidupan/baidupan.go @@ -0,0 +1,154 @@ +package baidupan + +import ( + "context" + "fmt" + "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/config/configmap" + "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/lib/oauthutil" + "golang.org/x/oauth2" + "io" + "time" +) + +const ( + // rcloneAppID = "24599545" + rcloneClientID = "fP5NRUFeA3GfZpc7LuRLRTsWGSm93lmk" + rcloneEncryptedClientSecret = "Q2m4aEy7oRoTyRe0UhWZ5ZSBrqistZAX" +) + +type Fs struct { + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + ci *fs.ConfigInfo // global config + features *fs.Features // optional features + ////srv *rest.Client // the connection to the one drive server + //dirCache *dircache.DirCache // Map of directory path to directory id + //pacer *fs.Pacer // pacer for API calls + //tokenRenewer *oauthutil.Renew // renew the token on expiry +} + +func (f Fs) Name() string { + return f.name +} + +func (f Fs) Root() string { + return f.root +} + +func (f Fs) String() string { + return fmt.Sprintf("Baidu Pan root '%s", f.root) +} + +func (f Fs) Precision() time.Duration { + return time.Second +} + +func (f Fs) Hashes() hash.Set { + return hash.Set(hash.MD5) +} + +func (f Fs) Features() *fs.Features { + return f.features +} + +func (f Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { + panic("implement me") +} + +func (f Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { + panic("implement me") +} + +func (f Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs.Object, error) { + panic("implement me") +} + +func (f Fs) Mkdir(ctx context.Context, dir string) error { + panic("implement me") +} + +func (f Fs) Rmdir(ctx context.Context, dir string) error { + panic("implement me") +} + +type Options struct { + tv oauth2.AuthCodeOption + +} + +type Features struct { + +} + + +var commandHelp = []fs.CommandHelp{{ + +}} + +var Endpoint = oauth2.Endpoint{ + AuthURL: "http://openapi.baidu.com/oauth/2.0/authorize", + TokenURL: "https://openapi.baidu.com/oauth/2.0/token", + //AuthStyle: oauth2.AuthStyleInParams, +} + + +var ( + oauthConfig = &oauth2.Config{ + Scopes: []string{"basic,netdisk"}, + Endpoint: Endpoint, + ClientID: rcloneClientID, + ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), + RedirectURL: oauthutil.RedirectLocalhostURL, + //RedirectURL: "oob", + } +) + +func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { + + + opts := []oauth2.AuthCodeOption{ + oauth2.SetAuthURLParam("display", "tv"), + oauth2.SetAuthURLParam("qrcode", "1"), + oauth2.SetAuthURLParam("force_login", "1"), + } + + return oauthutil.ConfigOut("choose_type", &oauthutil.Options{ + OAuth2Config: oauthConfig, + OAuth2Opts: opts, + }) + return nil, nil +} + +func init(){ + fmt.Println("baidupan init") + fs.Register(&fs.RegInfo{ + Name: "baidupan", + Description: "Baidu Wangpan", + NewFs: NewFs, + CommandHelp: commandHelp, + Config: Config, + //Options: append(oauthutil.SharedOptions), + + }) + +} + +func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, error) { + + opt := new(Options) + ci := fs.GetConfig(ctx) + f := &Fs{ + name: name, + root: root, + opt: *opt, + ci: ci, + } + f.features = (&fs.Features{ + }).Fill(ctx, f) + + return f, nil +} \ No newline at end of file diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index 90cf7b02bbf66..06284a5762d0f 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -647,7 +647,7 @@ func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauth if !auth.OK || auth.Code == "" { return "", auth } - fs.Logf(nil, "Got code\n") + fs.Logf(nil, "Got code %s\n", auth.Code) if opt.CheckAuth != nil { err = opt.CheckAuth(oauthConfig, auth) if err != nil { From 849572a749d0914644977ad5bff60694ddd703c8 Mon Sep 17 00:00:00 2001 From: Levi-Hope <117010292@link.cuhk.edu.cn> Date: Sun, 25 Jul 2021 18:39:57 +0800 Subject: [PATCH 2/5] use http to debug the fscking token exchagne --- backend/baidupan/baidupan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/baidupan/baidupan.go b/backend/baidupan/baidupan.go index dc2916bb1eea7..4dfcd193c7588 100644 --- a/backend/baidupan/baidupan.go +++ b/backend/baidupan/baidupan.go @@ -91,7 +91,7 @@ var commandHelp = []fs.CommandHelp{{ var Endpoint = oauth2.Endpoint{ AuthURL: "http://openapi.baidu.com/oauth/2.0/authorize", - TokenURL: "https://openapi.baidu.com/oauth/2.0/token", + TokenURL: "http://openapi.baidu.com/oauth/2.0/token", //AuthStyle: oauth2.AuthStyleInParams, } From a737383eaa8e18952b7ef9bbf9f7096377dd2195 Mon Sep 17 00:00:00 2001 From: Levi-Hope <117010292@link.cuhk.edu.cn> Date: Sun, 25 Jul 2021 23:22:20 +0800 Subject: [PATCH 3/5] get the freaking token --- backend/baidupan/baidupan.go | 26 ++++++++++++++++---------- lib/oauthutil/oauthutil.go | 22 ++++++++++++++++++++++ 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/backend/baidupan/baidupan.go b/backend/baidupan/baidupan.go index 4dfcd193c7588..51cc626e95346 100644 --- a/backend/baidupan/baidupan.go +++ b/backend/baidupan/baidupan.go @@ -109,18 +109,24 @@ var ( func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { - - opts := []oauth2.AuthCodeOption{ - oauth2.SetAuthURLParam("display", "tv"), - oauth2.SetAuthURLParam("qrcode", "1"), - oauth2.SetAuthURLParam("force_login", "1"), + if config.State == ""{ + opts := []oauth2.AuthCodeOption{ + oauth2.SetAuthURLParam("display", "tv"), + oauth2.SetAuthURLParam("qrcode", "1"), + oauth2.SetAuthURLParam("force_login", "1"), + } + + return oauthutil.ConfigOut("choose_type", &oauthutil.Options{ + OAuth2Config: oauthConfig, + OAuth2Opts: opts, + }) } - return oauthutil.ConfigOut("choose_type", &oauthutil.Options{ - OAuth2Config: oauthConfig, - OAuth2Opts: opts, - }) - return nil, nil + switch config.State { + case "choose_type": + return fs.ConfigGoto(config.Result) + } + return nil, fmt.Errorf("unknown state %q", config.State) } func init(){ diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index 06284a5762d0f..e483ed505735c 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -1,6 +1,7 @@ package oauthutil import ( + "bytes" "context" "encoding/json" "fmt" @@ -660,6 +661,27 @@ func configSetup(ctx context.Context, id, name string, m configmap.Mapper, oauth // Exchange the code for a token func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthConfig *oauth2.Config, code string) error { ctx = Context(ctx, fshttp.NewClient(ctx)) + + // baidupan requires the config be added to the URL directly, which is not supported by golang.org/x/oauth2 + // So hoxfix this thing by adding the config to URL before passing to oauthConfig.Exchange() + // For doc of baidupan, see: https://pan.baidu.com/union/document/entrance#%E6%8E%A5%E5%85%A5%E6%B5%81%E7%A8%8B + if strings.Contains(oauthConfig.Endpoint.TokenURL, "openapi.baidu.com") { + var buf bytes.Buffer + buf.WriteString(oauthConfig.Endpoint.TokenURL) + v := url.Values{ + "grant_type": {"authorization_code"}, + "client_id": {oauthConfig.ClientID}, + // FIXME: Why can't pass oauthConfig.ClientSecret here? + "client_secret": {"Q2m4aEy7oRoTyRe0UhWZ5ZSBrqistZAX"}, + "code": {code}, + "redirect_uri": {oauthConfig.RedirectURL}, + } + buf.WriteByte('?') + buf.WriteString(v.Encode()) + oauthConfig.Endpoint.TokenURL = buf.String() + fs.Debugf(nil, "Retrieving Token from %s", oauthConfig.Endpoint.TokenURL) + } + token, err := oauthConfig.Exchange(ctx, code) if err != nil { return errors.Wrap(err, "failed to get token") From ba55557029b0c441af304f694e766c314079e10d Mon Sep 17 00:00:00 2001 From: Levi-Hope <117010292@link.cuhk.edu.cn> Date: Sun, 25 Jul 2021 23:48:26 +0800 Subject: [PATCH 4/5] slightly refactor codes --- backend/baidupan/baidupan.go | 45 +++++++++++++++--------------------- lib/oauthutil/oauthutil.go | 10 ++++---- 2 files changed, 24 insertions(+), 31 deletions(-) diff --git a/backend/baidupan/baidupan.go b/backend/baidupan/baidupan.go index 51cc626e95346..c78501f1b1ebd 100644 --- a/backend/baidupan/baidupan.go +++ b/backend/baidupan/baidupan.go @@ -14,17 +14,16 @@ import ( ) const ( - // rcloneAppID = "24599545" - rcloneClientID = "fP5NRUFeA3GfZpc7LuRLRTsWGSm93lmk" + rcloneClientID = "fP5NRUFeA3GfZpc7LuRLRTsWGSm93lmk" rcloneEncryptedClientSecret = "Q2m4aEy7oRoTyRe0UhWZ5ZSBrqistZAX" ) type Fs struct { - name string // name of this remote - root string // the path we are working on - opt Options // parsed options - ci *fs.ConfigInfo // global config - features *fs.Features // optional features + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + ci *fs.ConfigInfo // global config + features *fs.Features // optional features ////srv *rest.Client // the connection to the one drive server //dirCache *dircache.DirCache // Map of directory path to directory id //pacer *fs.Pacer // pacer for API calls @@ -76,26 +75,20 @@ func (f Fs) Rmdir(ctx context.Context, dir string) error { } type Options struct { - tv oauth2.AuthCodeOption - + tv oauth2.AuthCodeOption } type Features struct { - } - -var commandHelp = []fs.CommandHelp{{ - -}} +var commandHelp = []fs.CommandHelp{{}} var Endpoint = oauth2.Endpoint{ AuthURL: "http://openapi.baidu.com/oauth/2.0/authorize", TokenURL: "http://openapi.baidu.com/oauth/2.0/token", - //AuthStyle: oauth2.AuthStyleInParams, + AuthStyle: oauth2.AuthStyleInHeader, } - var ( oauthConfig = &oauth2.Config{ Scopes: []string{"basic,netdisk"}, @@ -103,13 +96,14 @@ var ( ClientID: rcloneClientID, ClientSecret: obscure.MustReveal(rcloneEncryptedClientSecret), RedirectURL: oauthutil.RedirectLocalhostURL, - //RedirectURL: "oob", } ) func Config(ctx context.Context, name string, m configmap.Mapper, config fs.ConfigIn) (*fs.ConfigOut, error) { - if config.State == ""{ + if config.State == "" { + + // See: https://pan.baidu.com/union/document/entrance#3%E8%8E%B7%E5%8F%96%E6%8E%88%E6%9D%83 opts := []oauth2.AuthCodeOption{ oauth2.SetAuthURLParam("display", "tv"), oauth2.SetAuthURLParam("qrcode", "1"), @@ -118,7 +112,7 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf return oauthutil.ConfigOut("choose_type", &oauthutil.Options{ OAuth2Config: oauthConfig, - OAuth2Opts: opts, + OAuth2Opts: opts, }) } @@ -129,14 +123,14 @@ func Config(ctx context.Context, name string, m configmap.Mapper, config fs.Conf return nil, fmt.Errorf("unknown state %q", config.State) } -func init(){ +func init() { fmt.Println("baidupan init") fs.Register(&fs.RegInfo{ - Name: "baidupan", + Name: "baidupan", Description: "Baidu Wangpan", - NewFs: NewFs, + NewFs: NewFs, CommandHelp: commandHelp, - Config: Config, + Config: Config, //Options: append(oauthutil.SharedOptions), }) @@ -153,8 +147,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e opt: *opt, ci: ci, } - f.features = (&fs.Features{ - }).Fill(ctx, f) + f.features = (&fs.Features{}).Fill(ctx, f) return f, nil -} \ No newline at end of file +} diff --git a/lib/oauthutil/oauthutil.go b/lib/oauthutil/oauthutil.go index e483ed505735c..422adbfe9017d 100644 --- a/lib/oauthutil/oauthutil.go +++ b/lib/oauthutil/oauthutil.go @@ -664,17 +664,17 @@ func configExchange(ctx context.Context, name string, m configmap.Mapper, oauthC // baidupan requires the config be added to the URL directly, which is not supported by golang.org/x/oauth2 // So hoxfix this thing by adding the config to URL before passing to oauthConfig.Exchange() - // For doc of baidupan, see: https://pan.baidu.com/union/document/entrance#%E6%8E%A5%E5%85%A5%E6%B5%81%E7%A8%8B + // See: https://pan.baidu.com/union/document/entrance#3%E8%8E%B7%E5%8F%96%E6%8E%88%E6%9D%83 if strings.Contains(oauthConfig.Endpoint.TokenURL, "openapi.baidu.com") { var buf bytes.Buffer buf.WriteString(oauthConfig.Endpoint.TokenURL) v := url.Values{ "grant_type": {"authorization_code"}, - "client_id": {oauthConfig.ClientID}, + "client_id": {oauthConfig.ClientID}, // FIXME: Why can't pass oauthConfig.ClientSecret here? - "client_secret": {"Q2m4aEy7oRoTyRe0UhWZ5ZSBrqistZAX"}, - "code": {code}, - "redirect_uri": {oauthConfig.RedirectURL}, + "client_secret": {"Q2m4aEy7oRoTyRe0UhWZ5ZSBrqistZAX"}, + "code": {code}, + "redirect_uri": {oauthConfig.RedirectURL}, } buf.WriteByte('?') buf.WriteString(v.Encode()) From 3d1f1cfa85baf3369529ee6c9e755d3b38ffab7a Mon Sep 17 00:00:00 2001 From: Levi-Hope <117010292@link.cuhk.edu.cn> Date: Tue, 27 Jul 2021 04:02:45 +0800 Subject: [PATCH 5/5] implement a naive version of f.List() --- backend/baidupan/baidupan.go | 108 ++++++++++++++++++++++++++++++++--- fs/walk/walk.go | 3 + 2 files changed, 103 insertions(+), 8 deletions(-) diff --git a/backend/baidupan/baidupan.go b/backend/baidupan/baidupan.go index c78501f1b1ebd..13cbf14dd591d 100644 --- a/backend/baidupan/baidupan.go +++ b/backend/baidupan/baidupan.go @@ -1,15 +1,23 @@ package baidupan +// TODO: Solve the hotfix in lib/oauthutil/oauthutil.go:configExchange() +// TODO: Add support for refreshing Access_token: https://pan.baidu.com/union/document/entrance#3%E8%8E%B7%E5%8F%96%E6%8E%88%E6%9D%83 + import ( "context" + "encoding/json" "fmt" + "github.com/pkg/errors" "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/config/configmap" "github.com/rclone/rclone/fs/config/obscure" + "github.com/rclone/rclone/fs/fshttp" "github.com/rclone/rclone/fs/hash" "github.com/rclone/rclone/lib/oauthutil" "golang.org/x/oauth2" "io" + "io/ioutil" + "net/http" "time" ) @@ -19,11 +27,13 @@ const ( ) type Fs struct { - name string // name of this remote - root string // the path we are working on - opt Options // parsed options - ci *fs.ConfigInfo // global config - features *fs.Features // optional features + name string // name of this remote + root string // the path we are working on + opt Options // parsed options + ci *fs.ConfigInfo // global config + features *fs.Features // optional features + ts *oauthutil.TokenSource // token source for oauth2 + m configmap.Mapper ////srv *rest.Client // the connection to the one drive server //dirCache *dircache.DirCache // Map of directory path to directory id //pacer *fs.Pacer // pacer for API calls @@ -39,7 +49,7 @@ func (f Fs) Root() string { } func (f Fs) String() string { - return fmt.Sprintf("Baidu Pan root '%s", f.root) + return fmt.Sprintf("Baidu Pan root '%s'", f.root) } func (f Fs) Precision() time.Duration { @@ -54,8 +64,80 @@ func (f Fs) Features() *fs.Features { return f.features } +type BaidupanThumbs struct { + Url1 string `json:"url1"` + Url2 string `json:"url2"` + Url3 string `json:"url3"` +} + +type BaidupanList struct { + Category int64 `json:"category"` + Fs_id int64 `json:"fs_id"` + Isdir int64 `json:"isdir"` + Local_ctime int64 `json:"local_ctime"` + Local_mtime int64 `json:"local_mtime"` + Md5 string `json:"md5"` + Path string `json:"path"` + Server_ctime int64 `json:"server_ctime"` + Server_filename string `json:"server_filename"` + Server_mtime int64 `json:"server_mtime"` + Size int64 `json:"size"` + Thumbs []BaidupanThumbs `json:"thumbs"` +} + +type BaidupanAPIResponse struct { + Cursor int64 `json:"cursor"` + Errno int64 `json:"errno"` + Errmsg string `json:"errmsg"` + Has_more int64 `json:"has_more"` + List []BaidupanList `json:"list"` +} + +// List the objects and directories in dir into entries. The +// entries can be returned in any order but should be for a +// complete directory. +// +// dir should be "" to list the root, and should not have +// trailing slashes. +// +// This should return ErrDirNotFound if the directory isn't +// found. func (f Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err error) { - panic("implement me") + + fmt.Println("dir: ", dir) + fmt.Println("f.root: ", f.root) + + token, err := f.ts.Token() + url := fmt.Sprintf("http://pan.baidu.com/rest/2.0/xpan/multimedia?method=listall&path=/%s&access_token=%s&web=1&recursion=1&start=0&limit=50", f.root, token.AccessToken) + fs.Debugf(f, "Getting url: %s", url) + + // TODO: Use rclone API instead of http.Get + res, err := http.Get(url) + if err != nil { + return nil, err + } + + body, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + s := new(BaidupanAPIResponse) + err = json.Unmarshal(body, &s) + // TODO: Why cant unmarshal BaidupanThumbs object + if err.Error() == "json: cannot unmarshal object into Go struct field BaidupanList.list.thumbs of type []baidupan.BaidupanThumbs" { + fs.Debugf(f, "FIXME: %s", err) + } else if err != nil { + return nil, err + } + + fmt.Println(s.Cursor, s.Errno, s.Errmsg, s.Has_more) + for i, file := range s.List { + fmt.Println(i, file) + } + + // TODO: Return entries + return nil, nil } func (f Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { @@ -132,7 +214,6 @@ func init() { CommandHelp: commandHelp, Config: Config, //Options: append(oauthutil.SharedOptions), - }) } @@ -141,11 +222,22 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e opt := new(Options) ci := fs.GetConfig(ctx) + + baseClient := fshttp.NewClient(ctx) + _, ts, err := oauthutil.NewClientWithBaseClient(ctx, name, m, oauthConfig, baseClient) + //fmt.Println("ts: ", ts) + + if err != nil { + return nil, errors.Wrap(err, "failed to configure Box") + } + f := &Fs{ name: name, root: root, opt: *opt, ci: ci, + ts: ts, + m: m, } f.features = (&fs.Features{}).Fill(ctx, f) diff --git a/fs/walk/walk.go b/fs/walk/walk.go index b731fcf5d777c..59443bd0b0bb8 100644 --- a/fs/walk/walk.go +++ b/fs/walk/walk.go @@ -144,6 +144,9 @@ func ListR(ctx context.Context, f fs.Fs, path string, includeAll bool, maxLevel // FIXME disable this with --no-fast-list ??? `--disable ListR` will do it... doListR := f.Features().ListR + // FIXME: Just For Experiment + return listRwalk(ctx, f, path, includeAll, maxLevel, listType, fn) + // Can't use ListR if... if doListR == nil || // ...no ListR fi.HaveFilesFrom() || // ...using --files-from